From 50c2a742e889d0aaf0fa79fa6c995f3bad181611 Mon Sep 17 00:00:00 2001 From: wangchunxiang Date: Thu, 11 Dec 2025 17:05:06 +0800 Subject: [PATCH] =?UTF-8?q?feat(crawler):=20=E6=96=B0=E5=A2=9E=E7=BE=8E?= =?UTF-8?q?=E5=9B=A2=E9=A6=96=E6=AC=A1=E6=8E=A5=E5=85=A5=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=20-=20=E6=B7=BB=E5=8A=A0=20CrawlerConfigAppriaisalAddDTO=20?= =?UTF-8?q?=E7=B1=BB=E7=94=A8=E4=BA=8E=E9=85=8D=E7=BD=AE=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E5=8F=82=E6=95=B0=20-=20=E5=AE=9E=E7=8E=B0=20FtbCrawlNetFirstM?= =?UTF-8?q?t=20=E7=B1=BB=E5=A4=84=E7=90=86=E7=BE=8E=E5=9B=A2=E5=92=8C?= =?UTF-8?q?=E5=A4=A7=E4=BC=97=E7=82=B9=E8=AF=84=E7=9A=84=20Cookie=20?= =?UTF-8?q?=E6=8B=A6=E6=88=AA=20-=20=E9=85=8D=E7=BD=AE=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E7=9A=84=20configInterfaceAddress=20=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E5=8F=91=E9=80=81=E9=85=8D=E7=BD=AE=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=20-=20=E6=9B=B4=E6=96=B0=20GlobalConfig=20=E7=B1=BB?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=A7=9F=E6=88=B7=E5=90=8D=E7=A7=B0=E5=92=8C?= =?UTF-8?q?=E9=97=A8=E5=BA=97=E5=88=97=E8=A1=A8=E6=8E=A5=E5=8F=A3=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=20-=20=E5=9C=A8=20PlatformSelectionView=20=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E5=A2=9E=E5=8A=A0=E2=80=9C=E7=BE=8E=E5=9B=A2=E6=8E=A5?= =?UTF-8?q?=E5=85=A5=E2=80=9D=E9=80=89=E9=A1=B9=20-=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E4=BE=9D=E8=B5=96=E5=BC=95=E5=85=A5=20checke?= =?UTF-8?q?rframework=20=E6=B3=A8=E8=A7=A3=E5=BA=93=20-=20=E8=B0=83?= =?UTF-8?q?=E6=95=B4=20BASE=5FNET=5FURL=20=E9=BB=98=E8=AE=A4=E6=8C=87?= =?UTF-8?q?=E5=90=91=E6=9C=AC=E5=9C=B0=E5=BC=80=E5=8F=91=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fantaibao/config/GlobalConfig.java | 20 +- .../model/CrawlerConfigAppriaisalAddDTO.java | 50 +++++ .../fantaibao/page/PlatformSelectionView.java | 19 +- .../fantaibao/service/FtbCrawlNetFirstMt.java | 187 ++++++++++++++++++ src/main/java/module-info.java | 2 +- 5 files changed, 269 insertions(+), 9 deletions(-) create mode 100644 src/main/java/com/fantaibao/model/CrawlerConfigAppriaisalAddDTO.java create mode 100644 src/main/java/com/fantaibao/service/FtbCrawlNetFirstMt.java diff --git a/src/main/java/com/fantaibao/config/GlobalConfig.java b/src/main/java/com/fantaibao/config/GlobalConfig.java index 6a869f5..e7c6aaf 100644 --- a/src/main/java/com/fantaibao/config/GlobalConfig.java +++ b/src/main/java/com/fantaibao/config/GlobalConfig.java @@ -7,8 +7,8 @@ public class GlobalConfig { /** * 基础网络地址 */ - //public static final String BASE_NET_URL = "http://127.0.0.1:9606"; - public static final String BASE_NET_URL = "http://139.155.27.186:9607"; + public static final String BASE_NET_URL = "http://127.0.0.1:9606"; + //public static final String BASE_NET_URL = "http://139.155.27.186:9607"; /** * 登录接口地址 @@ -24,6 +24,12 @@ public class GlobalConfig { * 更新cookie地址 */ public static final String updateCookieInterfaceAddress = BASE_NET_URL+"/crawler/appraisal/update-user-cookie"; + + /** + * 配置请求地址 + */ + public static final String configInterfaceAddress = BASE_NET_URL+"/crawler/appraisal/config"; + /** * 网络图标接口地址 */ @@ -34,6 +40,16 @@ public class GlobalConfig { */ public static String tenantId; + /** + * 租户Id标识 + */ + public static String tenantName; + + /** + * 门店列表接口地址 + */ + public static String mtShopListInterfaceAddress; + /** * 美团点评评价接口地址 */ diff --git a/src/main/java/com/fantaibao/model/CrawlerConfigAppriaisalAddDTO.java b/src/main/java/com/fantaibao/model/CrawlerConfigAppriaisalAddDTO.java new file mode 100644 index 0000000..20b4456 --- /dev/null +++ b/src/main/java/com/fantaibao/model/CrawlerConfigAppriaisalAddDTO.java @@ -0,0 +1,50 @@ +package com.fantaibao.model; + +import com.alibaba.fastjson.JSONObject; +import lombok.Data; + +import java.util.Map; + +/** + * 参数配置vo + */ +@Data +public class CrawlerConfigAppriaisalAddDTO { + + private String url; + /** + * 请求urlName + */ + private String urlName; + /** + * 请求方式 0 get/ 1 post + */ + private Integer method; + /** + * 请求参数 + */ + private Map param; + /** + * 请求cookie + */ + private Map cookie; + /** + * 租户标识 + */ + private String tenantId; + + /** + * kafka消息topic + */ + private String messageTopic; + /** + * 平台0美团开店宝,1抖音,2饿了么商家端3美团外卖商家版4京东 + */ + private Integer platform; + + /** + * 美团门店列表参数 + */ + private JSONObject storeParam; + +} diff --git a/src/main/java/com/fantaibao/page/PlatformSelectionView.java b/src/main/java/com/fantaibao/page/PlatformSelectionView.java index 51a57f0..5750356 100644 --- a/src/main/java/com/fantaibao/page/PlatformSelectionView.java +++ b/src/main/java/com/fantaibao/page/PlatformSelectionView.java @@ -48,11 +48,18 @@ public class PlatformSelectionView { platformOptions.setAlignment(Pos.CENTER); platformOptions.setSpacing(60); - // 美团开店宝选项 + // 美团开店宝cookie选项 VBox meituanOption = createPlatformMtOption( - "美团开店宝", + "美团Cookie更新", GlobalConfig.mtLogo, - "#FFD700" + "#FFD700","ftbCrawlNetMt" + ); + + // 美团首次接入 + VBox meituanFirstOption = createPlatformMtOption( + "美团接入", + GlobalConfig.mtLogo, + "#FFD700","ftbCrawlNetFirstMt" ); // 抖音来客选项 @@ -62,7 +69,7 @@ public class PlatformSelectionView { "#4A90E2" ); - platformOptions.getChildren().addAll(meituanOption, douyinOption); + platformOptions.getChildren().addAll(meituanOption,meituanFirstOption, douyinOption); // 添加组件到主容器 root.getChildren().addAll(titleLabel, infoLabel, platformOptions); @@ -73,7 +80,7 @@ public class PlatformSelectionView { primaryStage.show(); } - private VBox createPlatformMtOption(String title, Image imageUrl, String buttonColor) { + private VBox createPlatformMtOption(String title, Image imageUrl, String buttonColor,String beanName) { VBox option = new VBox(); option.setAlignment(Pos.CENTER); option.setSpacing(10); @@ -93,7 +100,7 @@ public class PlatformSelectionView { connectButton.setStyle("-fx-background-color: " + buttonColor + "; -fx-text-fill: white; -fx-padding: 10px 20px; -fx-border-radius: 4px;"); connectButton.setOnAction(event -> { - FtbCrawlNetBase ftbCrawlNetMt = SpringContext.getBean("ftbCrawlNetMt"); + FtbCrawlNetBase ftbCrawlNetMt = SpringContext.getBean(beanName); ThreadPoolTaskExecutor poolTaskExecutor = SpringContext.getBean(ThreadPoolTaskExecutor.class); poolTaskExecutor.execute(() -> { ftbCrawlNetMt.executeCookieIntercept(); diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetFirstMt.java b/src/main/java/com/fantaibao/service/FtbCrawlNetFirstMt.java new file mode 100644 index 0000000..2484b77 --- /dev/null +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetFirstMt.java @@ -0,0 +1,187 @@ +package com.fantaibao.service; + +import cn.hutool.http.HttpUtil; +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONObject; +import com.fantaibao.base.FtbCrawlNetBase; +import com.fantaibao.config.BrowserStealthConfig; +import com.fantaibao.config.GlobalConfig; +import com.fantaibao.config.PlaywrightManager; +import com.fantaibao.model.CrawlerConfigAppriaisalAddDTO; +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Request; +import javafx.application.Platform; +import javafx.scene.control.Alert; +import javafx.scene.control.ButtonType; +import javafx.stage.Stage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +@Component(value = "ftbCrawlNetFirstMt") +@Slf4j +public class FtbCrawlNetFirstMt extends AbstractFtbCrawlNetBase implements FtbCrawlNetBase { + + @Override + public void executeCookieIntercept() { + // 启动可见浏览器 + try (Browser browser = PlaywrightManager.getInstance().getPlaywright() + .chromium().launch(BrowserStealthConfig.getStealthLaunchOptions())) { + // 设置浏览器上下文选项以减少自动化检测 + BrowserContext context = browser.newContext(BrowserStealthConfig.getStealthContextOptions()); + + // 添加反自动化检测脚本 + BrowserStealthConfig.configureStealthContext(context); + + // 门店抓取 + AtomicBoolean isCrawlFinished = new AtomicBoolean(false); + // 美团cookie + AtomicBoolean mtCookie = new AtomicBoolean(false); + // 大众点评cookie + AtomicBoolean dzCookie = new AtomicBoolean(false); + AtomicReference atomicReference = new AtomicReference<>(); + // 监听网络请求 + context.onRequest((request -> handleRequest(request,mtCookie,dzCookie,isCrawlFinished,atomicReference))); + + Page page = context.newPage(); + // 监听浏览器关闭事件释放资源 + browser.onDisconnected((brow) -> { + if (page != null && !page.isClosed()) { + log.info("美团浏览器关闭事件"); + page.close(); + } + }); + + // 导航到登录页面 + page.navigate(GlobalConfig.mtLoginPage, + new Page.NavigateOptions().setTimeout(60000.0)); + // 设置30分钟登录超时时间 + Page.WaitForConditionOptions waitForConditionOptions = new Page.WaitForConditionOptions(); + waitForConditionOptions.setTimeout(30*60*1000); + page.waitForCondition(() -> mtCookie.get() && dzCookie.get() && isCrawlFinished.get(),waitForConditionOptions); + // 关闭页面 + showSuccessAlertAndCloset(page,"美团"); + } catch (Exception e) { + log.error("执行打开浏览器时发生错误", e); + } + } + + /** + * 处理网络请求，提取需要的cookie信息 + * + * @param request 网络请求 + * @param storeParam + */ + private void handleRequest(Request request, AtomicBoolean mtCookie, AtomicBoolean dzCookie, AtomicBoolean isCrawlFinished, AtomicReference atomicReference) { + // 美团cookie + if (request.url().contains(GlobalConfig.mtDianPingInterfaceAddress)) { + // 打印请求URL + log.info("请求URL: {}", request.url()); + // 获取请求头中的Cookie + String cookieHeader = request.headerValue("cookie"); + if (cookieHeader != null && !cookieHeader.isEmpty()) { + log.info("获取到{}的Cookie","美团"); + CrawlerConfigAppriaisalAddDTO userCookieDTO = new CrawlerConfigAppriaisalAddDTO(); + userCookieDTO.setTenantId(GlobalConfig.tenantId); + userCookieDTO.setPlatform(0); + userCookieDTO.setMethod(1); + userCookieDTO.setMessageTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId); + userCookieDTO.setUrlName(GlobalConfig.tenantName+"美团评价"); + userCookieDTO.setUrl("https://ecom.meituan.com/emis/gw/rpc/TFeedbackEcomService/queryFeedback?_tm=1752304094520&yodaReady=h5&csecplatform=4&csecversion=3.2.1"); + Map param = new HashMap<>(); + param.put("requestBody","{\"platform\":0,\"queryPara\":{\"platform\":0,\"referTag\":0,\"startTime\":1751990400000,\"endTime\":1752076799999,\"starTag\":-1,\"businessTag\":\"-1\",\"poiIdLong\":\"853401195\",\"poiIdStr\":\"853401195\"},\"pageInfo\":{\"offset\":0,\"limit\":10,\"total\":0}}"); + userCookieDTO.setParam(param); + Map cookie = new HashMap<>(); + cookie.put("cookie",cookieHeader); + userCookieDTO.setCookie(cookie); + userCookieDTO.setStoreParam(atomicReference.get()); + processCookiet(userCookieDTO, "美团",mtCookie); + } + } else if (request.url().contains(GlobalConfig.mtDaZhInterfaceAddress)) { + // 打印请求URL + log.info("请求URL: {}", request.url()); + // 获取请求头中的Cookie + String cookieHeader = request.headerValue("cookie"); + if (cookieHeader != null && !cookieHeader.isEmpty()) { + log.info("获取到{}的Cookie","大众点评"); + CrawlerConfigAppriaisalAddDTO userCookieDTO = new CrawlerConfigAppriaisalAddDTO(); + userCookieDTO.setTenantId(GlobalConfig.tenantId); + userCookieDTO.setPlatform(5); + userCookieDTO.setMethod(1); + userCookieDTO.setMessageTopic("ftb-crawler-mt-evaluate-"+GlobalConfig.tenantId); + userCookieDTO.setUrlName(GlobalConfig.tenantName+"美团评价"); + userCookieDTO.setUrl("https://ecom.meituan.com/emis/gw/TDPFeedbackServiceV2/queryFeedbackPCV2?_tm=1752304359951&yodaReady=h5&csecplatform=4&csecversion=3.2.1"); + Map param = new HashMap<>(); + param.put("requestBody","{\"platform\":1,\"queryPara\":{\"platform\":1,\"referTag\":0,\"startTime\":1752076800000,\"endTime\":1752163199999,\"starTag\":-1,\"businessTag\":\"-1\",\"poiIdLong\":\"853401195\",\"poiIdStr\":\"853401195\"},\"pageInfo\":{\"offset\":0,\"limit\":10,\"total\":0}}"); + userCookieDTO.setParam(param); + Map cookie = new HashMap<>(); + cookie.put("cookie",cookieHeader); + userCookieDTO.setCookie(cookie); + userCookieDTO.setStoreParam(atomicReference.get()); + processCookiet(userCookieDTO, "大众点评",dzCookie); + } else if (request.url().contains(GlobalConfig.mtShopListInterfaceAddress)) { + String text = request.response().text(); + log.info("请求门店返回内容为: {}", text); + atomicReference.set(JSONObject.parseObject(text)); + isCrawlFinished.set(true); + } + } + } + + protected void showSuccessAlertAndCloset(Page page, String platformName) { + try { + // 在JavaFX线程中显示弹窗 + Platform.runLater(() -> { + Alert alert = new Alert(Alert.AlertType.INFORMATION); + alert.setTitle("美团接入完成"); + alert.setHeaderText(null); + alert.setContentText(platformName+"数据已接入完成，请点击\"我已知晓\"关闭"); + // 设置弹窗图标 + Stage alertStage = (Stage) alert.getDialogPane().getScene().getWindow(); + GlobalConfig.addIcon(alertStage); + // 添加自定义按钮 + ButtonType acknowledgeButton = new ButtonType("我已知晓"); + alert.getButtonTypes().setAll(acknowledgeButton); + + // 显示弹窗并等待用户响应 + Optional result = alert.showAndWait(); + if (result.isPresent() && result.get() == acknowledgeButton) { + // 用户点击了"我已知晓"按钮，关闭页面 + try { + page.close(); + } catch (Exception e) { + log.error("关闭页面时出错", e); + } + } + }); + } catch (Exception e) { + log.error("显示弹窗或关闭页面时出错", e); + } + } + + /** + * 处理cookie并发送到服务器 + * @param platformName 平台名称，用于日志记录 + */ + protected void processCookiet(CrawlerConfigAppriaisalAddDTO userCookieDTO, String platformName, AtomicBoolean isCrawlFinished) { + try { + String result = HttpUtil.post(GlobalConfig.configInterfaceAddress, JSON.toJSONString(userCookieDTO)); + JSONObject parsed = JSON.parseObject(result); + if (parsed.getInteger("code") == 200) { + isCrawlFinished.set(true); + log.info("{} 接入成功", platformName); + } else { + log.warn("{} 接入成功: {}", platformName, parsed.getString("msg")); + } + } catch (Exception e) { + log.error("{} 接入异常", platformName, e); + } + } +} \ No newline at end of file diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index b3a8802..7c0c6f3 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -9,11 +9,11 @@ module fantaibao.crawler.desktop { requires playwright; requires org.slf4j; requires lombok; - requires java.desktop; requires jakarta.annotation; requires cn.hutool; requires fastjson; requires java.sql; + requires org.checkerframework.checker.qual; opens com.fantaibao to spring.core, spring.beans, spring.context, javafx.fxml, javafx.base, javafx.graphics, spring.boot, spring.boot.autoconfigure;