From 9fa2e10049c3449a445ce554d9f1fc5443397392 Mon Sep 17 00:00:00 2001 From: wangchunxiang <526328077@qq.com> Date: Mon, 13 Oct 2025 16:57:52 +0800 Subject: [PATCH] =?UTF-8?q?```=20feat(playwright):=20=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=B5=8F=E8=A7=88=E5=99=A8=E9=85=8D=E7=BD=AE=E7=B1=BB=E5=B9=B6?= =?UTF-8?q?=E9=9B=86=E6=88=90Playwright?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 新增BrowserConfig配置类，用于创建和管理Playwright实例。 移除DesktopApplication中冗余的UI示例代码和旧版PlaywrightService。 添加FtbCrawlNetBase接口及其实现类FtbCrawlNetMt，用于执行Cookie拦截逻辑。更新PlatformSelectionView以支持美团平台的连接功能，并通过SpringContext获取爬虫服务。 调整module-info.java以开放和导出新增模块包路径，确保Spring能够正确注入依赖。 ``` --- .../com/fantaibao/DesktopApplication.java | 57 ------- .../com/fantaibao/base/FtbCrawlNetBase.java | 16 ++ .../com/fantaibao/config/BrowserConfig.java | 15 ++ .../com/fantaibao/config/SpringContext.java | 44 ++++++ .../fantaibao/page/PlatformSelectionView.java | 45 ++++-- .../com/fantaibao/service/FtbCrawlNetMt.java | 52 +++++++ .../fantaibao/service/PlaywrightService.java | 143 ------------------ src/main/java/module-info.java | 7 + 8 files changed, 170 insertions(+), 209 deletions(-) create mode 100644 src/main/java/com/fantaibao/base/FtbCrawlNetBase.java create mode 100644 src/main/java/com/fantaibao/config/BrowserConfig.java create mode 100644 src/main/java/com/fantaibao/config/SpringContext.java create mode 100644 src/main/java/com/fantaibao/service/FtbCrawlNetMt.java delete mode 100644 src/main/java/com/fantaibao/service/PlaywrightService.java diff --git a/src/main/java/com/fantaibao/DesktopApplication.java b/src/main/java/com/fantaibao/DesktopApplication.java index aef7e90..33d53f2 100644 --- a/src/main/java/com/fantaibao/DesktopApplication.java +++ b/src/main/java/com/fantaibao/DesktopApplication.java @@ -1,14 +1,7 @@ package com.fantaibao; import com.fantaibao.page.LoginView; -import com.fantaibao.service.PlaywrightService; import javafx.application.Application; -import javafx.application.Platform; -import javafx.geometry.Insets; -import javafx.scene.Scene; -import javafx.scene.control.Button; -import javafx.scene.control.TextArea; -import javafx.scene.layout.VBox; import javafx.stage.Stage; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @@ -36,57 +29,7 @@ public class DesktopApplication extends Application { public void start(Stage primaryStage) { // 直接启动登录界面 new LoginView(primaryStage); - - /* - primaryStage.setTitle("饭太煲爬虫桌面程序"); - - textArea = new TextArea(); - textArea.setPrefRowCount(20); - textArea.setPrefColumnCount(50); - textArea.setEditable(false); - - Button playwrightButton = new Button("运行Cookie拦截示例"); - playwrightButton.setOnAction(e -> runPlaywrightCookieExample()); - - Button loginButton = new Button("运行登录示例"); - loginButton.setOnAction(e -> runLoginExample()); - - VBox vBox = new VBox(10, playwrightButton, loginButton, textArea); - vBox.setPadding(new Insets(10)); - - Scene scene = new Scene(vBox, 800, 600); - primaryStage.setScene(scene); - primaryStage.setOnCloseRequest(e -> { - if (applicationContext != null) { - applicationContext.close(); - } - Platform.exit(); - }); - primaryStage.show(); - */ } -/* - private void runPlaywrightCookieExample() { - appendText("开始运行Playwright Cookie拦截示例...\n"); - Thread thread = new Thread(() -> - playwrightService.interceptCookieExample(this::appendText) - ); - thread.setDaemon(true); - thread.start(); - } - - private void runLoginExample() { - appendText("开始运行登录示例...\n"); - Thread thread = new Thread(() -> - playwrightService.loginExample(this::appendText) - ); - thread.setDaemon(true); - thread.start(); - } - - private void appendText(String text) { - Platform.runLater(() -> textArea.appendText(text + "\n")); - }*/ @Override public void stop() { diff --git a/src/main/java/com/fantaibao/base/FtbCrawlNetBase.java b/src/main/java/com/fantaibao/base/FtbCrawlNetBase.java new file mode 100644 index 0000000..903076c --- /dev/null +++ b/src/main/java/com/fantaibao/base/FtbCrawlNetBase.java @@ -0,0 +1,16 @@ +package com.fantaibao.base; + +/** + * 爬虫基类 + * + * @author wangchunxiang + * @date 2025/10/13 + */ +public interface FtbCrawlNetBase { + + /** + * 执行爬虫cookie拦截 + */ + void executeCookieIntercept(); + +} diff --git a/src/main/java/com/fantaibao/config/BrowserConfig.java b/src/main/java/com/fantaibao/config/BrowserConfig.java new file mode 100644 index 0000000..1bc5ad7 --- /dev/null +++ b/src/main/java/com/fantaibao/config/BrowserConfig.java @@ -0,0 +1,15 @@ +package com.fantaibao.config; + +import com.microsoft.playwright.Playwright; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class BrowserConfig { + + @Bean + public Playwright playwright() { + return Playwright.create(); + } + +} diff --git a/src/main/java/com/fantaibao/config/SpringContext.java b/src/main/java/com/fantaibao/config/SpringContext.java new file mode 100644 index 0000000..3e13877 --- /dev/null +++ b/src/main/java/com/fantaibao/config/SpringContext.java @@ -0,0 +1,44 @@ +package com.fantaibao.config; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Component; + +@Component +public class SpringContext implements ApplicationContextAware { + + + private static ApplicationContext applicationContext; + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + SpringContext.applicationContext = applicationContext; + } + + public static ApplicationContext getApplicationContext() { + assertApplicationContext(); + return applicationContext; + } + + public static T getBean(String beanName) { + assertApplicationContext(); + try { + return (T) applicationContext.getBean(beanName); + } catch (Exception e) { + return null; + } + } + + public static T getBean(Class beanName) { + assertApplicationContext(); + return (T) applicationContext.getBean(beanName); + } + + private static void assertApplicationContext() { + if (SpringContext.applicationContext == null) { + throw new RuntimeException("applicaitonContext属性为null,请检查是否注入了SpringContextHolder!"); + } + } + +} diff --git a/src/main/java/com/fantaibao/page/PlatformSelectionView.java b/src/main/java/com/fantaibao/page/PlatformSelectionView.java index 3a04e8b..fa8dae1 100644 --- a/src/main/java/com/fantaibao/page/PlatformSelectionView.java +++ b/src/main/java/com/fantaibao/page/PlatformSelectionView.java @@ -1,5 +1,7 @@ package com.fantaibao.page; +import com.fantaibao.base.FtbCrawlNetBase; +import com.fantaibao.config.SpringContext; import javafx.geometry.Insets; import javafx.geometry.Pos; import javafx.scene.Scene; @@ -35,7 +37,7 @@ public class PlatformSelectionView { titleLabel.setStyle("-fx-font-size: 18px; -fx-font-weight: bold;"); // 描述信息 - Label infoLabel = new Label("连接您的电商平台，智能化采集商品数据"); + Label infoLabel = new Label("连接您的电商平台，智能化采集中差评数据"); infoLabel.setStyle("-fx-font-size: 14px;"); // 平台选项容器 @@ -44,18 +46,16 @@ public class PlatformSelectionView { platformOptions.setSpacing(20); // 美团开店宝选项 - VBox meituanOption = createPlatformOption( + VBox meituanOption = createPlatformMtOption( "美团开店宝", "https://placehold.co/64x64?text=MT&bg=FFD700&fg=333", - "采集店铺数据、获取商品信息", "#FFD700" ); // 抖音来客选项 - VBox douyinOption = createPlatformOption( + VBox douyinOption = createPlatformDyOption( "抖音来客", "https://placehold.co/64x64?text=DY&bg=4A90E2&fg=FFF", - "获取直播数据、用户评论、商品指标", "#4A90E2" ); @@ -70,7 +70,7 @@ public class PlatformSelectionView { primaryStage.show(); } - private VBox createPlatformOption(String title, String imageUrl, String description, String buttonColor) { + private VBox createPlatformMtOption(String title, String imageUrl, String buttonColor) { VBox option = new VBox(); option.setAlignment(Pos.CENTER); option.setSpacing(10); @@ -84,14 +84,41 @@ public class PlatformSelectionView { Label titleLabel = new Label(title); titleLabel.setStyle("-fx-font-size: 16px; -fx-font-weight: bold;"); - Label descLabel = new Label(description); - descLabel.setStyle("-fx-font-size: 12px;"); Button connectButton = new Button("立即连接"); connectButton.setStyle("-fx-background-color: " + buttonColor + "; -fx-text-fill: white; -fx-padding: 10px 20px; -fx-border-radius: 4px;"); - option.getChildren().addAll(imageView, titleLabel, descLabel, connectButton); + connectButton.setOnAction(event -> { + FtbCrawlNetBase ftbCrawlNetMt = SpringContext.getBean("ftbCrawlNetMt"); + ftbCrawlNetMt.executeCookieIntercept(); + + }); + + option.getChildren().addAll(imageView, titleLabel, connectButton); return option; } + + private VBox createPlatformDyOption(String title, String imageUrl, String buttonColor) { + VBox option = new VBox(); + option.setAlignment(Pos.CENTER); + option.setSpacing(10); + option.setPadding(new Insets(20)); + option.setStyle("-fx-background-color: white; -fx-border-radius: 8px; -fx-padding: 20px;"); + + ImageView imageView = new ImageView(new Image(imageUrl)); + imageView.setFitWidth(64); + imageView.setFitHeight(64); + + Label titleLabel = new Label(title); + titleLabel.setStyle("-fx-font-size: 16px; -fx-font-weight: bold;"); + + Button connectButton = new Button("立即连接"); + connectButton.setStyle("-fx-background-color: " + buttonColor + "; -fx-text-fill: white; -fx-padding: 10px 20px; -fx-border-radius: 4px;"); + + option.getChildren().addAll(imageView, titleLabel, connectButton); + + return option; + } + } \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java new file mode 100644 index 0000000..f94f291 --- /dev/null +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java @@ -0,0 +1,52 @@ +package com.fantaibao.service; + +import com.fantaibao.base.FtbCrawlNetBase; +import com.microsoft.playwright.*; +import com.microsoft.playwright.options.Cookie; +import com.microsoft.playwright.options.WaitForSelectorState; +import jakarta.annotation.Resource; +import org.springframework.stereotype.Component; +import java.util.List; + + +@Component(value = "ftbCrawlNetMt") +public class FtbCrawlNetMt implements FtbCrawlNetBase { + + @Resource + private Playwright playwright; + + @Override + public void executeCookieIntercept() { + // 启动可见浏览器 + try (Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) { + + BrowserContext context = browser.newContext(); + // 监听网络请求 + context.onRequest(request -> { + // 打印请求URL + System.out.println("请求URL: " + request.url()); + // 获取请求头中的Cookie + String cookieHeader = request.headerValue("cookie"); + if (cookieHeader != null && !cookieHeader.isEmpty()) { + System.out.println("请求中的Cookie: " + cookieHeader); + } + }); + Page page = context.newPage(); + + // 导航到登录页面 + page.navigate("https://ecom.meituan.com/bizaccount/login.html?loginByPhoneNumber=true&isProduction=true&epassportParams=%3Fbg_source%3D1%26service%3Dcom.sankuai.meishi.fe.ecom%26part_type%3D0%26feconfig%3Dbssoify%26biz_line%3D1%26continue%3Dhttps%253A%252F%252Fecom.meituan.com%252Fbizaccount%252Fbiz-choice.html%253Fredirect_uri%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%2526_t%253D1759399140148%2526target%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%26leftBottomLink%3D%26signUpTarget%3Dself", + new Page.NavigateOptions().setTimeout(6000000)); + + // 获取登录后的Cookie + List cookies = context.cookies(); + for (Cookie cookie : cookies) { + System.out.println(cookie.name + "=" + cookie.value); + } + // 保持浏览器打开一段时间以便查看 + System.out.println("浏览器将在10秒后关闭..."); + page.waitForTimeout(10000); + } + + } + +} diff --git a/src/main/java/com/fantaibao/service/PlaywrightService.java b/src/main/java/com/fantaibao/service/PlaywrightService.java deleted file mode 100644 index 2fc9e13..0000000 --- a/src/main/java/com/fantaibao/service/PlaywrightService.java +++ /dev/null @@ -1,143 +0,0 @@ -package com.fantaibao.service; - -import com.microsoft.playwright.*; -import com.microsoft.playwright.options.Cookie; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Service; - -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.Consumer; - -@Service -@Slf4j -public class PlaywrightService { - - public void interceptCookieExample(Consumer logger) { - try (Playwright playwright = Playwright.create()) { - // 启动可见浏览器 - Browser browser = playwright.chromium().launch( - new BrowserType.LaunchOptions().setHeadless(false) - ); - - // 创建浏览器上下文 - BrowserContext context = browser.newContext(); - - // 用于存储拦截到的Cookie - ConcurrentMap interceptedCookies = new ConcurrentHashMap<>(); - - // 监听网络请求 - context.onRequest(request -> { - // 打印请求URL - String logMessage = "请求URL: " + request.url(); - logger.accept(logMessage); - log.info(logMessage); - - // 获取请求头中的Cookie - String cookieHeader = request.headerValue("cookie"); - if (cookieHeader != null && !cookieHeader.isEmpty()) { - String cookieMessage = "请求中的Cookie: " + cookieHeader; - logger.accept(cookieMessage); - log.info(cookieMessage); - interceptedCookies.put(request.url(), cookieHeader); - } - }); - - Page page = context.newPage(); - - // 导航到示例页面（这里使用百度作为示例） - page.navigate("https://www.baidu.com"); - - logger.accept("页面加载完成，请执行点击操作..."); - logger.accept("点击按钮后按回车键继续获取Cookie..."); - - // 等待用户执行点击操作 (在实际桌面应用中需要替换为UI交互) - try { - Thread.sleep(10000); // 等待10秒模拟用户操作 - } catch (InterruptedException e) { - log.warn("等待过程中被中断"); - } - - // 获取当前页面的所有Cookie - List cookies = context.cookies(); - logger.accept("\n当前页面的Cookie:"); - for (Cookie cookie : cookies) { - String cookieInfo = cookie.name + "=" + cookie.value + - " (域名: " + cookie.domain + ", 路径: " + cookie.path + ")"; - logger.accept(cookieInfo); - log.info("Cookie: {}", cookieInfo); - } - - // 显示拦截到的请求Cookie - logger.accept("\n拦截到的请求Cookie:"); - interceptedCookies.forEach((url, cookie) -> { - String info = "URL: " + url + "\nCookie: " + cookie + "\n"; - logger.accept(info); - log.info("拦截Cookie - {}: {}", url, cookie); - }); - - // 保持浏览器打开一段时间以便查看 - logger.accept("浏览器将在5秒后关闭..."); - page.waitForTimeout(5000); - } catch (Exception e) { - String errorMsg = "执行过程中出现错误: " + e.getMessage(); - logger.accept(errorMsg); - log.error("Playwright执行出错", e); - } - } - - public void loginExample(Consumer logger) { - try (Playwright playwright = Playwright.create(); - Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) { - - BrowserContext context = browser.newContext(); - // 监听网络请求 - context.onRequest(request -> { - // 打印请求URL - String logMessage = "请求URL: " + request.url(); - logger.accept(logMessage); - log.info(logMessage); - - // 获取请求头中的Cookie - String cookieHeader = request.headerValue("cookie"); - if (cookieHeader != null && !cookieHeader.isEmpty()) { - String cookieMessage = "请求中的Cookie: " + cookieHeader; - logger.accept(cookieMessage); - log.info(cookieMessage); - } - }); - - Page page = context.newPage(); - - // 导航到登录页面 - page.navigate("https://ecom.meituan.com/bizaccount/login.html?loginByPhoneNumber=true&isProduction=true&epassportParams=%3Fbg_source%3D1%26service%3Dcom.sankuai.meishi.fe.ecom%26part_type%3D0%26feconfig%3Dbssoify%26biz_line%3D1%26continue%3Dhttps%253A%252F%252Fecom.meituan.com%252Fbizaccount%252Fbiz-choice.html%253Fredirect_uri%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%2526_t%253D1759399140148%2526target%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%26leftBottomLink%3D%26signUpTarget%3Dself"); - - logger.accept("请在浏览器中手动登录..."); - logger.accept("登录完成后程序将继续执行..."); - - // 在真实应用中，这里应该通过UI事件触发而不是固定等待 - try { - Thread.sleep(15000); // 等待15秒让用户登录 - } catch (InterruptedException e) { - log.warn("等待登录过程中被中断"); - } - - // 获取登录后的Cookie - List cookies = context.cookies(); - logger.accept("登录后的Cookie信息:"); - for (Cookie cookie : cookies) { - String cookieInfo = cookie.name + "=" + cookie.value; - logger.accept(cookieInfo); - log.info("登录Cookie: {}", cookieInfo); - } - - logger.accept("浏览器将在5秒后关闭..."); - page.waitForTimeout(5000); - } catch (Exception e) { - String errorMsg = "执行过程中出现错误: " + e.getMessage(); - logger.accept(errorMsg); - log.error("Playwright登录示例执行出错", e); - } - } -} \ No newline at end of file diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index 39ccc70..d7ecb31 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -10,11 +10,18 @@ module fantaibao.crawler.desktop { requires org.slf4j; requires lombok; requires java.desktop; + requires jakarta.annotation; opens com.fantaibao to spring.core, spring.beans, spring.context, javafx.fxml, javafx.base, javafx.graphics, spring.boot, spring.boot.autoconfigure; + opens com.fantaibao.config to spring.core,spring.beans, spring.context; + opens com.fantaibao.service to spring.core,spring.beans, spring.context; + + exports com.fantaibao; exports com.fantaibao.service; exports com.fantaibao.page; + exports com.fantaibao.config; + exports com.fantaibao.base; } \ No newline at end of file