wangchunxiang/fantaibao-crawler-desktop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(crawler): 新增美团首次接入功能

Browse Source 
- 添加 CrawlerConfigAppriaisalAddDTO 类用于配置爬虫参数
- 实现 FtbCrawlNetFirstMt 类处理美团和大众点评的 Cookie 拦截
- 配置新增的 configInterfaceAddress 地址用于发送配置信息
- 更新 GlobalConfig 类添加租户名称和门店列表接口地址
- 在 PlatformSelectionView 页面增加“美团接入”选项
- 修改模块依赖引入 checkerframework 注解库
- 调整 BASE_NET_URL 默认指向本地开发环境地址
This commit is contained in:
wangchunxiang
2025-12-11 17:05:06 +08:00
parent 31c06dbf78
commit 50c2a742e8
5 changed files with 269 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/com/fantaibao/config/GlobalConfig.java
View File

@@ -7,8 +7,8 @@ public class GlobalConfig {
/**
* 基础网络地址
*/
//public static final String BASE_NET_URL = "http://127.0.0.1:9606";
public static final String BASE_NET_URL = "http://139.155.27.186:9607";
public static final String BASE_NET_URL = "http://127.0.0.1:9606";
//public static final String BASE_NET_URL = "http://139.155.27.186:9607";
/**
* 登录接口地址
@@ -24,6 +24,12 @@ public class GlobalConfig {
* 更新cookie地址
*/
public static final String updateCookieInterfaceAddress = BASE_NET_URL+"/crawler/appraisal/update-user-cookie";
/**
* 配置请求地址
*/
public static final String configInterfaceAddress = BASE_NET_URL+"/crawler/appraisal/config";
/**
* 网络图标接口地址
*/
@@ -34,6 +40,16 @@ public class GlobalConfig {
*/
public static String tenantId;
/**
* 租户Id标识
*/
public static String tenantName;
/**
* 门店列表接口地址
*/
public static String mtShopListInterfaceAddress;
/**
* 美团点评评价接口地址
*/

50
src/main/java/com/fantaibao/model/CrawlerConfigAppriaisalAddDTO.java Normal file
View File

@@ -0,0 +1,50 @@
package com.fantaibao.model;
import com.alibaba.fastjson.JSONObject;
import lombok.Data;
import java.util.Map;
/**
* 参数配置vo
*/
@Data
public class CrawlerConfigAppriaisalAddDTO {
private String url;
/**
* 请求urlName
*/
private String urlName;
/**
* 请求方式 0 get/ 1 post
*/
private Integer method;
/**
* 请求参数
*/
private Map<String,String> param;
/**
* 请求cookie
*/
private Map<String,String> cookie;
/**
* 租户标识
*/
private String tenantId;
/**
* kafka消息topic
*/
private String messageTopic;
/**
* 平台0美团开店宝,1抖音,2饿了么商家端3美团外卖商家版4京东
*/
private Integer platform;
/**
* 美团门店列表参数
*/
private JSONObject storeParam;
}

19
src/main/java/com/fantaibao/page/PlatformSelectionView.java
View File

@@ -48,11 +48,18 @@ public class PlatformSelectionView {
platformOptions.setAlignment(Pos.CENTER);
platformOptions.setSpacing(60);
// 美团开店宝选项
// 美团开店宝cookie选项
VBox meituanOption = createPlatformMtOption(
"美团开店宝",
"美团Cookie更新",
GlobalConfig.mtLogo,
"#FFD700"
"#FFD700","ftbCrawlNetMt"
);
// 美团首次接入
VBox meituanFirstOption = createPlatformMtOption(
"美团接入",
GlobalConfig.mtLogo,
"#FFD700","ftbCrawlNetFirstMt"
);
// 抖音来客选项
@@ -62,7 +69,7 @@ public class PlatformSelectionView {
"#4A90E2"
);
platformOptions.getChildren().addAll(meituanOption, douyinOption);
platformOptions.getChildren().addAll(meituanOption,meituanFirstOption, douyinOption);
// 添加组件到主容器
root.getChildren().addAll(titleLabel, infoLabel, platformOptions);
@@ -73,7 +80,7 @@ public class PlatformSelectionView {
primaryStage.show();
}
private VBox createPlatformMtOption(String title, Image imageUrl, String buttonColor) {
private VBox createPlatformMtOption(String title, Image imageUrl, String buttonColor,String beanName) {
VBox option = new VBox();
option.setAlignment(Pos.CENTER);
option.setSpacing(10);
@@ -93,7 +100,7 @@ public class PlatformSelectionView {
connectButton.setStyle("-fx-background-color: " + buttonColor + "; -fx-text-fill: white; -fx-padding: 10px 20px; -fx-border-radius: 4px;");
connectButton.setOnAction(event -> {
FtbCrawlNetBase ftbCrawlNetMt = SpringContext.getBean("ftbCrawlNetMt");
FtbCrawlNetBase ftbCrawlNetMt = SpringContext.getBean(beanName);
ThreadPoolTaskExecutor poolTaskExecutor = SpringContext.getBean(ThreadPoolTaskExecutor.class);
poolTaskExecutor.execute(() -> {
ftbCrawlNetMt.executeCookieIntercept();

187
src/main/java/com/fantaibao/service/FtbCrawlNetFirstMt.java Normal file
View File

@@ -0,0 +1,187 @@
package com.fantaibao.service;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.fantaibao.base.FtbCrawlNetBase;
import com.fantaibao.config.BrowserStealthConfig;
import com.fantaibao.config.GlobalConfig;
import com.fantaibao.config.PlaywrightManager;
import com.fantaibao.model.CrawlerConfigAppriaisalAddDTO;
import com.microsoft.playwright.Browser;
import com.microsoft.playwright.BrowserContext;
import com.microsoft.playwright.Page;
import com.microsoft.playwright.Request;
import javafx.application.Platform;
import javafx.scene.control.Alert;
import javafx.scene.control.ButtonType;
import javafx.stage.Stage;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@Component(value = "ftbCrawlNetFirstMt")
@Slf4j
public class FtbCrawlNetFirstMt extends AbstractFtbCrawlNetBase implements FtbCrawlNetBase {
@Override
public void executeCookieIntercept() {
// 启动可见浏览器
try (Browser browser = PlaywrightManager.getInstance().getPlaywright()
.chromium().launch(BrowserStealthConfig.getStealthLaunchOptions())) {
// 设置浏览器上下文选项以减少自动化检测
BrowserContext context = browser.newContext(BrowserStealthConfig.getStealthContextOptions());
// 添加反自动化检测脚本
BrowserStealthConfig.configureStealthContext(context);
// 门店抓取
AtomicBoolean isCrawlFinished = new AtomicBoolean(false);
// 美团cookie
AtomicBoolean mtCookie = new AtomicBoolean(false);
// 大众点评cookie
AtomicBoolean dzCookie = new AtomicBoolean(false);
AtomicReference<JSONObject> atomicReference = new AtomicReference<>();
// 监听网络请求
context.onRequest((request -> handleRequest(request,mtCookie,dzCookie,isCrawlFinished,atomicReference)));
Page page = context.newPage();
// 监听浏览器关闭事件释放资源
browser.onDisconnected((brow) -> {
if (page != null && !page.isClosed()) {
log.info("美团浏览器关闭事件");
page.close();
}
});
// 导航到登录页面
page.navigate(GlobalConfig.mtLoginPage,
new Page.NavigateOptions().setTimeout(60000.0));
// 设置30分钟登录超时时间
Page.WaitForConditionOptions waitForConditionOptions = new Page.WaitForConditionOptions();
waitForConditionOptions.setTimeout(30*60*1000);
page.waitForCondition(() -> mtCookie.get() && dzCookie.get() && isCrawlFinished.get(),waitForConditionOptions);
// 关闭页面
showSuccessAlertAndCloset(page,"美团");
} catch (Exception e) {
log.error("执行打开浏览器时发生错误", e);
}
}
/**
* 处理网络请求，提取需要的cookie信息
*
* @param request 网络请求
* @param storeParam
*/
private void handleRequest(Request request, AtomicBoolean mtCookie, AtomicBoolean dzCookie, AtomicBoolean isCrawlFinished, AtomicReference<JSONObject> atomicReference) {
// 美团cookie
if (request.url().contains(GlobalConfig.mtDianPingInterfaceAddress)) {
// 打印请求URL
log.info("请求URL: {}", request.url());
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
log.info("获取到{}的Cookie","美团");
CrawlerConfigAppriaisalAddDTO userCookieDTO = new CrawlerConfigAppriaisalAddDTO();
userCookieDTO.setTenantId(GlobalConfig.tenantId);
userCookieDTO.setPlatform(0);
userCookieDTO.setMethod(1);
userCookieDTO.setMessageTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId);
userCookieDTO.setUrlName(GlobalConfig.tenantName+"美团评价");
userCookieDTO.setUrl("https://ecom.meituan.com/emis/gw/rpc/TFeedbackEcomService/queryFeedback?_tm=1752304094520&yodaReady=h5&csecplatform=4&csecversion=3.2.1");
Map<String,String> param = new HashMap<>();
param.put("requestBody","{\"platform\":0,\"queryPara\":{\"platform\":0,\"referTag\":0,\"startTime\":1751990400000,\"endTime\":1752076799999,\"starTag\":-1,\"businessTag\":\"-1\",\"poiIdLong\":\"853401195\",\"poiIdStr\":\"853401195\"},\"pageInfo\":{\"offset\":0,\"limit\":10,\"total\":0}}");
userCookieDTO.setParam(param);
Map<String,String> cookie = new HashMap<>();
cookie.put("cookie",cookieHeader);
userCookieDTO.setCookie(cookie);
userCookieDTO.setStoreParam(atomicReference.get());
processCookiet(userCookieDTO, "美团",mtCookie);
}
} else if (request.url().contains(GlobalConfig.mtDaZhInterfaceAddress)) {
// 打印请求URL
log.info("请求URL: {}", request.url());
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
log.info("获取到{}的Cookie","大众点评");
CrawlerConfigAppriaisalAddDTO userCookieDTO = new CrawlerConfigAppriaisalAddDTO();
userCookieDTO.setTenantId(GlobalConfig.tenantId);
userCookieDTO.setPlatform(5);
userCookieDTO.setMethod(1);
userCookieDTO.setMessageTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId);
userCookieDTO.setUrlName(GlobalConfig.tenantName+"美团评价");
userCookieDTO.setUrl("https://ecom.meituan.com/emis/gw/TDPFeedbackServiceV2/queryFeedbackPCV2?_tm=1752304359951&yodaReady=h5&csecplatform=4&csecversion=3.2.1");
Map<String,String> param = new HashMap<>();
param.put("requestBody","{\"platform\":1,\"queryPara\":{\"platform\":1,\"referTag\":0,\"startTime\":1752076800000,\"endTime\":1752163199999,\"starTag\":-1,\"businessTag\":\"-1\",\"poiIdLong\":\"853401195\",\"poiIdStr\":\"853401195\"},\"pageInfo\":{\"offset\":0,\"limit\":10,\"total\":0}}");
userCookieDTO.setParam(param);
Map<String,String> cookie = new HashMap<>();
cookie.put("cookie",cookieHeader);
userCookieDTO.setCookie(cookie);
userCookieDTO.setStoreParam(atomicReference.get());
processCookiet(userCookieDTO, "大众点评",dzCookie);
} else if (request.url().contains(GlobalConfig.mtShopListInterfaceAddress)) {
String text = request.response().text();
log.info("请求门店返回内容为: {}", text);
atomicReference.set(JSONObject.parseObject(text));
isCrawlFinished.set(true);
}
}
}
protected void showSuccessAlertAndCloset(Page page, String platformName) {
try {
// 在JavaFX线程中显示弹窗
Platform.runLater(() -> {
Alert alert = new Alert(Alert.AlertType.INFORMATION);
alert.setTitle("美团接入完成");
alert.setHeaderText(null);
alert.setContentText(platformName+"数据已接入完成，请点击\"我已知晓\"关闭");
// 设置弹窗图标
Stage alertStage = (Stage) alert.getDialogPane().getScene().getWindow();
GlobalConfig.addIcon(alertStage);
// 添加自定义按钮
ButtonType acknowledgeButton = new ButtonType("我已知晓");
alert.getButtonTypes().setAll(acknowledgeButton);
// 显示弹窗并等待用户响应
Optional<ButtonType> result = alert.showAndWait();
if (result.isPresent() && result.get() == acknowledgeButton) {
// 用户点击了"我已知晓"按钮，关闭页面
try {
page.close();
} catch (Exception e) {
log.error("关闭页面时出错", e);
}
}
});
} catch (Exception e) {
log.error("显示弹窗或关闭页面时出错", e);
}
}
/**
* 处理cookie并发送到服务器
* @param platformName 平台名称，用于日志记录
*/
protected void processCookiet(CrawlerConfigAppriaisalAddDTO userCookieDTO, String platformName, AtomicBoolean isCrawlFinished) {
try {
String result = HttpUtil.post(GlobalConfig.configInterfaceAddress, JSON.toJSONString(userCookieDTO));
JSONObject parsed = JSON.parseObject(result);
if (parsed.getInteger("code") == 200) {
isCrawlFinished.set(true);
log.info("{} 接入成功", platformName);
} else {
log.warn("{} 接入成功: {}", platformName, parsed.getString("msg"));
}
} catch (Exception e) {
log.error("{} 接入异常", platformName, e);
}
}
}

2
src/main/java/module-info.java
View File

@@ -9,11 +9,11 @@ module fantaibao.crawler.desktop {
requires playwright;
requires org.slf4j;
requires lombok;
requires java.desktop;
requires jakarta.annotation;
requires cn.hutool;
requires fastjson;
requires java.sql;
requires org.checkerframework.checker.qual;
opens com.fantaibao to spring.core, spring.beans, spring.context,
javafx.fxml, javafx.base, javafx.graphics, spring.boot, spring.boot.autoconfigure;