wangchunxiang/fantaibao-crawler-desktop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

```

Browse Source 
feat(playwright): 添加浏览器配置类并集成Playwright

新增BrowserConfig配置类，用于创建和管理Playwright实例。
移除DesktopApplication中冗余的UI示例代码和旧版PlaywrightService。
添加FtbCrawlNetBase接口及其实现类FtbCrawlNetMt，用于执行Cookie拦截逻辑。更新PlatformSelectionView以支持美团平台的连接功能，并通过SpringContext获取爬虫服务。
调整module-info.java以开放和导出新增模块包路径，确保Spring能够正确注入依赖。
```
This commit is contained in:
wangchunxiang
2025-10-13 16:57:52 +08:00
parent a5fcb54815
commit 9fa2e10049
8 changed files with 170 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/main/java/com/fantaibao/DesktopApplication.java
View File

@@ -1,14 +1,7 @@
package com.fantaibao;
import com.fantaibao.page.LoginView;
import com.fantaibao.service.PlaywrightService;
import javafx.application.Application;
import javafx.application.Platform;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.Button;
import javafx.scene.control.TextArea;
import javafx.scene.layout.VBox;
import javafx.stage.Stage;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -36,57 +29,7 @@ public class DesktopApplication extends Application {
public void start(Stage primaryStage) {
// 直接启动登录界面
new LoginView(primaryStage);
/*
primaryStage.setTitle("饭太煲爬虫桌面程序");
textArea = new TextArea();
textArea.setPrefRowCount(20);
textArea.setPrefColumnCount(50);
textArea.setEditable(false);
Button playwrightButton = new Button("运行Cookie拦截示例");
playwrightButton.setOnAction(e -> runPlaywrightCookieExample());
Button loginButton = new Button("运行登录示例");
loginButton.setOnAction(e -> runLoginExample());
VBox vBox = new VBox(10, playwrightButton, loginButton, textArea);
vBox.setPadding(new Insets(10));
Scene scene = new Scene(vBox, 800, 600);
primaryStage.setScene(scene);
primaryStage.setOnCloseRequest(e -> {
if (applicationContext != null) {
applicationContext.close();
}
Platform.exit();
});
primaryStage.show();
*/
}
/*
private void runPlaywrightCookieExample() {
appendText("开始运行Playwright Cookie拦截示例...\n");
Thread thread = new Thread(() ->
playwrightService.interceptCookieExample(this::appendText)
);
thread.setDaemon(true);
thread.start();
}
private void runLoginExample() {
appendText("开始运行登录示例...\n");
Thread thread = new Thread(() ->
playwrightService.loginExample(this::appendText)
);
thread.setDaemon(true);
thread.start();
}
private void appendText(String text) {
Platform.runLater(() -> textArea.appendText(text + "\n"));
}*/
@Override
public void stop() {

16
src/main/java/com/fantaibao/base/FtbCrawlNetBase.java Normal file
View File

@@ -0,0 +1,16 @@
package com.fantaibao.base;
/**
* 爬虫基类
*
* @author wangchunxiang
* @date 2025/10/13
*/
public interface FtbCrawlNetBase {
/**
* 执行爬虫cookie拦截
*/
void executeCookieIntercept();
}

15
src/main/java/com/fantaibao/config/BrowserConfig.java Normal file
View File

@@ -0,0 +1,15 @@
package com.fantaibao.config;
import com.microsoft.playwright.Playwright;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class BrowserConfig {
@Bean
public Playwright playwright() {
return Playwright.create();
}
}

44
src/main/java/com/fantaibao/config/SpringContext.java Normal file
View File

@@ -0,0 +1,44 @@
package com.fantaibao.config;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.stereotype.Component;
@Component
public class SpringContext implements ApplicationContextAware {
private static ApplicationContext applicationContext;
@Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
SpringContext.applicationContext = applicationContext;
}
public static ApplicationContext getApplicationContext() {
assertApplicationContext();
return applicationContext;
}
public static <T> T getBean(String beanName) {
assertApplicationContext();
try {
return (T) applicationContext.getBean(beanName);
} catch (Exception e) {
return null;
}
}
public static <T> T getBean(Class<T> beanName) {
assertApplicationContext();
return (T) applicationContext.getBean(beanName);
}
private static void assertApplicationContext() {
if (SpringContext.applicationContext == null) {
throw new RuntimeException("applicaitonContext属性为null,请检查是否注入了SpringContextHolder!");
}
}
}

45
src/main/java/com/fantaibao/page/PlatformSelectionView.java
View File

@@ -1,5 +1,7 @@
package com.fantaibao.page;
import com.fantaibao.base.FtbCrawlNetBase;
import com.fantaibao.config.SpringContext;
import javafx.geometry.Insets;
import javafx.geometry.Pos;
import javafx.scene.Scene;
@@ -35,7 +37,7 @@ public class PlatformSelectionView {
titleLabel.setStyle("-fx-font-size: 18px; -fx-font-weight: bold;");
// 描述信息
Label infoLabel = new Label("连接您的电商平台，智能化采集商品数据");
Label infoLabel = new Label("连接您的电商平台，智能化采集中差评数据");
infoLabel.setStyle("-fx-font-size: 14px;");
// 平台选项容器
@@ -44,18 +46,16 @@ public class PlatformSelectionView {
platformOptions.setSpacing(20);
// 美团开店宝选项
VBox meituanOption = createPlatformOption(
VBox meituanOption = createPlatformMtOption(
"美团开店宝",
"https://placehold.co/64x64?text=MT&bg=FFD700&fg=333",
"采集店铺数据、获取商品信息",
"#FFD700"
);
// 抖音来客选项
VBox douyinOption = createPlatformOption(
VBox douyinOption = createPlatformDyOption(
"抖音来客",
"https://placehold.co/64x64?text=DY&bg=4A90E2&fg=FFF",
"获取直播数据、用户评论、商品指标",
"#4A90E2"
);
@@ -70,7 +70,7 @@ public class PlatformSelectionView {
primaryStage.show();
}
private VBox createPlatformOption(String title, String imageUrl, String description, String buttonColor) {
private VBox createPlatformMtOption(String title, String imageUrl, String buttonColor) {
VBox option = new VBox();
option.setAlignment(Pos.CENTER);
option.setSpacing(10);
@@ -84,14 +84,41 @@ public class PlatformSelectionView {
Label titleLabel = new Label(title);
titleLabel.setStyle("-fx-font-size: 16px; -fx-font-weight: bold;");
Label descLabel = new Label(description);
descLabel.setStyle("-fx-font-size: 12px;");
Button connectButton = new Button("立即连接");
connectButton.setStyle("-fx-background-color: " + buttonColor + "; -fx-text-fill: white; -fx-padding: 10px 20px; -fx-border-radius: 4px;");
option.getChildren().addAll(imageView, titleLabel, descLabel, connectButton);
connectButton.setOnAction(event -> {
FtbCrawlNetBase ftbCrawlNetMt = SpringContext.getBean("ftbCrawlNetMt");
ftbCrawlNetMt.executeCookieIntercept();
});
option.getChildren().addAll(imageView, titleLabel, connectButton);
return option;
}
private VBox createPlatformDyOption(String title, String imageUrl, String buttonColor) {
VBox option = new VBox();
option.setAlignment(Pos.CENTER);
option.setSpacing(10);
option.setPadding(new Insets(20));
option.setStyle("-fx-background-color: white; -fx-border-radius: 8px; -fx-padding: 20px;");
ImageView imageView = new ImageView(new Image(imageUrl));
imageView.setFitWidth(64);
imageView.setFitHeight(64);
Label titleLabel = new Label(title);
titleLabel.setStyle("-fx-font-size: 16px; -fx-font-weight: bold;");
Button connectButton = new Button("立即连接");
connectButton.setStyle("-fx-background-color: " + buttonColor + "; -fx-text-fill: white; -fx-padding: 10px 20px; -fx-border-radius: 4px;");
option.getChildren().addAll(imageView, titleLabel, connectButton);
return option;
}
}

52
src/main/java/com/fantaibao/service/FtbCrawlNetMt.java Normal file
View File

@@ -0,0 +1,52 @@
package com.fantaibao.service;
import com.fantaibao.base.FtbCrawlNetBase;
import com.microsoft.playwright.*;
import com.microsoft.playwright.options.Cookie;
import com.microsoft.playwright.options.WaitForSelectorState;
import jakarta.annotation.Resource;
import org.springframework.stereotype.Component;
import java.util.List;
@Component(value = "ftbCrawlNetMt")
public class FtbCrawlNetMt implements FtbCrawlNetBase {
@Resource
private Playwright playwright;
@Override
public void executeCookieIntercept() {
// 启动可见浏览器
try (Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) {
BrowserContext context = browser.newContext();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
System.out.println("请求URL: " + request.url());
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
System.out.println("请求中的Cookie: " + cookieHeader);
}
});
Page page = context.newPage();
// 导航到登录页面
page.navigate("https://ecom.meituan.com/bizaccount/login.html?loginByPhoneNumber=true&isProduction=true&epassportParams=%3Fbg_source%3D1%26service%3Dcom.sankuai.meishi.fe.ecom%26part_type%3D0%26feconfig%3Dbssoify%26biz_line%3D1%26continue%3Dhttps%253A%252F%252Fecom.meituan.com%252Fbizaccount%252Fbiz-choice.html%253Fredirect_uri%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%2526_t%253D1759399140148%2526target%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%26leftBottomLink%3D%26signUpTarget%3Dself",
new Page.NavigateOptions().setTimeout(6000000));
// 获取登录后的Cookie
List<Cookie> cookies = context.cookies();
for (Cookie cookie : cookies) {
System.out.println(cookie.name + "=" + cookie.value);
}
// 保持浏览器打开一段时间以便查看
System.out.println("浏览器将在10秒后关闭...");
page.waitForTimeout(10000);
}
}
}

143
src/main/java/com/fantaibao/service/PlaywrightService.java
View File

@@ -1,143 +0,0 @@
package com.fantaibao.service;
import com.microsoft.playwright.*;
import com.microsoft.playwright.options.Cookie;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Consumer;
@Service
@Slf4j
public class PlaywrightService {
public void interceptCookieExample(Consumer<String> logger) {
try (Playwright playwright = Playwright.create()) {
// 启动可见浏览器
Browser browser = playwright.chromium().launch(
new BrowserType.LaunchOptions().setHeadless(false)
);
// 创建浏览器上下文
BrowserContext context = browser.newContext();
// 用于存储拦截到的Cookie
ConcurrentMap<String, String> interceptedCookies = new ConcurrentHashMap<>();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
String logMessage = "请求URL: " + request.url();
logger.accept(logMessage);
log.info(logMessage);
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
String cookieMessage = "请求中的Cookie: " + cookieHeader;
logger.accept(cookieMessage);
log.info(cookieMessage);
interceptedCookies.put(request.url(), cookieHeader);
}
});
Page page = context.newPage();
// 导航到示例页面（这里使用百度作为示例）
page.navigate("https://www.baidu.com");
logger.accept("页面加载完成，请执行点击操作...");
logger.accept("点击按钮后按回车键继续获取Cookie...");
// 等待用户执行点击操作 (在实际桌面应用中需要替换为UI交互)
try {
Thread.sleep(10000); // 等待10秒模拟用户操作
} catch (InterruptedException e) {
log.warn("等待过程中被中断");
}
// 获取当前页面的所有Cookie
List<Cookie> cookies = context.cookies();
logger.accept("\n当前页面的Cookie:");
for (Cookie cookie : cookies) {
String cookieInfo = cookie.name + "=" + cookie.value +
" (域名: " + cookie.domain + ", 路径: " + cookie.path + ")";
logger.accept(cookieInfo);
log.info("Cookie: {}", cookieInfo);
}
// 显示拦截到的请求Cookie
logger.accept("\n拦截到的请求Cookie:");
interceptedCookies.forEach((url, cookie) -> {
String info = "URL: " + url + "\nCookie: " + cookie + "\n";
logger.accept(info);
log.info("拦截Cookie - {}: {}", url, cookie);
});
// 保持浏览器打开一段时间以便查看
logger.accept("浏览器将在5秒后关闭...");
page.waitForTimeout(5000);
} catch (Exception e) {
String errorMsg = "执行过程中出现错误: " + e.getMessage();
logger.accept(errorMsg);
log.error("Playwright执行出错", e);
}
}
public void loginExample(Consumer<String> logger) {
try (Playwright playwright = Playwright.create();
Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) {
BrowserContext context = browser.newContext();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
String logMessage = "请求URL: " + request.url();
logger.accept(logMessage);
log.info(logMessage);
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
String cookieMessage = "请求中的Cookie: " + cookieHeader;
logger.accept(cookieMessage);
log.info(cookieMessage);
}
});
Page page = context.newPage();
// 导航到登录页面
page.navigate("https://ecom.meituan.com/bizaccount/login.html?loginByPhoneNumber=true&isProduction=true&epassportParams=%3Fbg_source%3D1%26service%3Dcom.sankuai.meishi.fe.ecom%26part_type%3D0%26feconfig%3Dbssoify%26biz_line%3D1%26continue%3Dhttps%253A%252F%252Fecom.meituan.com%252Fbizaccount%252Fbiz-choice.html%253Fredirect_uri%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%2526_t%253D1759399140148%2526target%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%26leftBottomLink%3D%26signUpTarget%3Dself");
logger.accept("请在浏览器中手动登录...");
logger.accept("登录完成后程序将继续执行...");
// 在真实应用中，这里应该通过UI事件触发而不是固定等待
try {
Thread.sleep(15000); // 等待15秒让用户登录
} catch (InterruptedException e) {
log.warn("等待登录过程中被中断");
}
// 获取登录后的Cookie
List<Cookie> cookies = context.cookies();
logger.accept("登录后的Cookie信息:");
for (Cookie cookie : cookies) {
String cookieInfo = cookie.name + "=" + cookie.value;
logger.accept(cookieInfo);
log.info("登录Cookie: {}", cookieInfo);
}
logger.accept("浏览器将在5秒后关闭...");
page.waitForTimeout(5000);
} catch (Exception e) {
String errorMsg = "执行过程中出现错误: " + e.getMessage();
logger.accept(errorMsg);
log.error("Playwright登录示例执行出错", e);
}
}
}

7
src/main/java/module-info.java
View File

@@ -10,11 +10,18 @@ module fantaibao.crawler.desktop {
requires org.slf4j;
requires lombok;
requires java.desktop;
requires jakarta.annotation;
opens com.fantaibao to spring.core, spring.beans, spring.context,
javafx.fxml, javafx.base, javafx.graphics, spring.boot, spring.boot.autoconfigure;
opens com.fantaibao.config to spring.core,spring.beans, spring.context;
opens com.fantaibao.service to spring.core,spring.beans, spring.context;
exports com.fantaibao;
exports com.fantaibao.service;
exports com.fantaibao.page;
exports com.fantaibao.config;
exports com.fantaibao.base;
}