feat(crawler): 添加抓取地址名配置功能
- 在CrawlerConfigAppraisalDTO中新增urlName字段 - 在DataCrawlView中生成包含租户名和平台的urlName - 将urlName设置到configDTO中传递给爬虫服务 - 修复大众点评平台的urlName拼写错误
This commit is contained in:
@@ -27,4 +27,8 @@ public class CrawlerConfigAppraisalDTO {
|
|||||||
* 平台0美团,1抖音,2饿了么商家端3美团外卖商家版4京东5大众点评
|
* 平台0美团,1抖音,2饿了么商家端3美团外卖商家版4京东5大众点评
|
||||||
*/
|
*/
|
||||||
private Integer platform;
|
private Integer platform;
|
||||||
|
/**
|
||||||
|
* 抓取地址名
|
||||||
|
*/
|
||||||
|
private String urlName;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -158,6 +158,7 @@ public class DataCrawlView {
|
|||||||
private void handleCrawlAction() {
|
private void handleCrawlAction() {
|
||||||
// 获取选择的平台值
|
// 获取选择的平台值
|
||||||
String selectedPlatform = platformComboBox.getValue();
|
String selectedPlatform = platformComboBox.getValue();
|
||||||
|
String urlName = GlobalConfig.tenantName+selectedPlatform+"评价";
|
||||||
int platformValue; // 默认美团
|
int platformValue; // 默认美团
|
||||||
if ("大众点评".equals(selectedPlatform)) {
|
if ("大众点评".equals(selectedPlatform)) {
|
||||||
platformValue = 5;
|
platformValue = 5;
|
||||||
@@ -196,6 +197,7 @@ public class DataCrawlView {
|
|||||||
configDTO.setTenantId(GlobalConfig.tenantId);
|
configDTO.setTenantId(GlobalConfig.tenantId);
|
||||||
configDTO.setTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId);
|
configDTO.setTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId);
|
||||||
configDTO.setPlatform(platformValue);
|
configDTO.setPlatform(platformValue);
|
||||||
|
configDTO.setUrlName(urlName);
|
||||||
System.out.println(JSON.toJSONString(configDTO));
|
System.out.println(JSON.toJSONString(configDTO));
|
||||||
|
|
||||||
// 发送网络请求
|
// 发送网络请求
|
||||||
|
|||||||
@@ -115,7 +115,7 @@ public class FtbCrawlNetFirstMt extends AbstractFtbCrawlNetBase implements FtbCr
|
|||||||
userCookieDTO.setPlatform(5);
|
userCookieDTO.setPlatform(5);
|
||||||
userCookieDTO.setMethod(1);
|
userCookieDTO.setMethod(1);
|
||||||
userCookieDTO.setMessageTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId);
|
userCookieDTO.setMessageTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId);
|
||||||
userCookieDTO.setUrlName(GlobalConfig.tenantName+"美团评价");
|
userCookieDTO.setUrlName(GlobalConfig.tenantName+"大众点评评价");
|
||||||
userCookieDTO.setUrl("https://ecom.meituan.com/emis/gw/TDPFeedbackServiceV2/queryFeedbackPCV2?_tm=1752304359951&yodaReady=h5&csecplatform=4&csecversion=3.2.1");
|
userCookieDTO.setUrl("https://ecom.meituan.com/emis/gw/TDPFeedbackServiceV2/queryFeedbackPCV2?_tm=1752304359951&yodaReady=h5&csecplatform=4&csecversion=3.2.1");
|
||||||
Map<String,String> param = new HashMap<>();
|
Map<String,String> param = new HashMap<>();
|
||||||
param.put("requestBody","{\"platform\":1,\"queryPara\":{\"platform\":1,\"referTag\":0,\"startTime\":1752076800000,\"endTime\":1752163199999,\"starTag\":-1,\"businessTag\":\"-1\",\"poiIdLong\":\"853401195\",\"poiIdStr\":\"853401195\"},\"pageInfo\":{\"offset\":0,\"limit\":10,\"total\":0}}");
|
param.put("requestBody","{\"platform\":1,\"queryPara\":{\"platform\":1,\"referTag\":0,\"startTime\":1752076800000,\"endTime\":1752163199999,\"starTag\":-1,\"businessTag\":\"-1\",\"poiIdLong\":\"853401195\",\"poiIdStr\":\"853401195\"},\"pageInfo\":{\"offset\":0,\"limit\":10,\"total\":0}}");
|
||||||
|
|||||||
Reference in New Issue
Block a user