wangchunxiang/fantaibao-crawler-desktop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(playwright): 添加Cookie拦截示例和登录示例功能新增PlaywrightService服务，提供Cookie拦截与手动登录示例。新增DesktopApplication桌面应用入口，集成JavaFX界面。添加相关依赖：Playwright、JavaFX、Spring Boot等。

Browse Source 
配置Maven插件支持JavaFX应用打包。
完善README文档，说明项目结构与运行方式。
This commit is contained in:
wangchunxiang
2025-10-11 15:31:24 +08:00
parent 7e11a6fcf8
commit c1e6dc67b1
16 changed files with 662 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,8 @@
# Default ignored files
/shelf/
/workspace.xml
# Editor-based HTTP Client requests
/httpRequests/
# Datasource local storage ignored files
/dataSources/
/dataSources.local.xml

27
.idea/compiler.xml generated Normal file
View File

@@ -0,0 +1,27 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CompilerConfiguration">
<annotationProcessing>
<profile default="true" name="Default" enabled="true" />
<profile name="Maven default annotation processors profile" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
</profile>
<profile name="Annotation profile for fantaibao-crawler-desktop" enabled="true">
<sourceOutputDir name="target/generated-sources/annotations" />
<sourceTestOutputDir name="target/generated-test-sources/test-annotations" />
<outputRelativeToContentRoot value="true" />
<processorPath useClasspath="false">
<entry name="$PROJECT_DIR$/../../maven_repository/org/projectlombok/lombok/1.18.30/lombok-1.18.30.jar" />
</processorPath>
<module name="fantaibao-crawler-desktop" />
</profile>
</annotationProcessing>
</component>
<component name="JavacSettings">
<option name="ADDITIONAL_OPTIONS_OVERRIDE">
<module name="fantaibao-crawler-desktop" options="-parameters" />
</option>
</component>
</project>

7
.idea/encodings.xml generated Normal file
View File

@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="Encoding">
<file url="file://$PROJECT_DIR$/src/main/java" charset="UTF-8" />
<file url="file://$PROJECT_DIR$/src/main/resources" charset="UTF-8" />
</component>
</project>

6
.idea/git_toolbox_blame.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="GitToolBoxBlameSettings">
<option name="version" value="2" />
</component>
</project>

5
.idea/inspectionProfiles/Project_Default.xml generated Normal file
View File

@@ -0,0 +1,5 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
</profile>
</component>

20
.idea/jarRepositories.xml generated Normal file
View File

@@ -0,0 +1,20 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="RemoteRepositoriesConfiguration">
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Maven Central repository" />
<option name="url" value="https://repo1.maven.org/maven2" />
</remote-repository>
<remote-repository>
<option name="id" value="central" />
<option name="name" value="Central Repository" />
<option name="url" value="http://maven.aliyun.com/nexus/content/groups/public" />
</remote-repository>
<remote-repository>
<option name="id" value="jboss.community" />
<option name="name" value="JBoss Community repository" />
<option name="url" value="https://repository.jboss.org/nexus/content/repositories/public/" />
</remote-repository>
</component>
</project>

15
.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ExternalStorageConfigurationManager" enabled="true" />
<component name="MavenProjectsManager">
<option name="originalFiles">
<list>
<option value="$PROJECT_DIR$/pom.xml" />
</list>
</option>
<option name="workspaceImportForciblyTurnedOn" value="true" />
</component>
<component name="ProjectRootManager" version="2" languageLevel="JDK_17" default="true" project-jdk-name="17" project-jdk-type="JavaSDK">
<output url="file://$PROJECT_DIR$/out" />
</component>
</project>

6
.idea/vcs.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="" vcs="Git" />
</component>
</project>

34
README.md
View File

@@ -1,3 +1,35 @@
# fantaibao-crawler-desktop
爬虫桌面
饭太煲爬虫桌面程序
## 项目介绍
这是一个基于Spring Boot和JavaFX的Windows桌面应用程序，集成了Playwright爬虫功能。
## 功能特性
1. 桌面应用程序界面
2. Playwright网页爬虫功能
3. Cookie拦截和管理
4. 网站登录模拟
## 技术栈
- Spring Boot 3.4
- JavaFX 17
- Playwright 1.40
- Maven 3.x
## 运行方式
```bash
mvn clean spring-boot:run
```
## 打包
```bash
mvn clean package
```
运行打包后的程序需要Java 17+环境。

142
pom.xml Normal file
View File

@@ -0,0 +1,142 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.ftb</groupId>
<artifactId>fantaibao-crawler-desktop</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>fantaibao-crawler-desktop</name>
<description>fantaibao-crawler-desktop</description>
<properties>
<java.version>17</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>3.4.0</spring-boot.version>
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
<playwright.version>1.40.0</playwright.version>
<javafx.version>17.0.2</javafx.version>
<javafx.maven.plugin.version>0.0.8</javafx.maven.plugin.version>
</properties>
<dependencies>
<!-- Spring Boot Starters -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-logging</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<!-- SLF4J API -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
<exclusions>
<exclusion>
<artifactId>error_prone_annotations</artifactId>
<groupId>com.google.errorprone</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.83</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.8.22</version>
</dependency>
<dependency>
<groupId>com.microsoft.playwright</groupId>
<artifactId>playwright</artifactId>
<version>${playwright.version}</version>
</dependency>
<!-- JavaFX Dependencies -->
<dependency>
<groupId>org.openjfx</groupId>
<artifactId>javafx-controls</artifactId>
<version>${javafx.version}</version>
</dependency>
<dependency>
<groupId>org.openjfx</groupId>
<artifactId>javafx-fxml</artifactId>
<version>${javafx.version}</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>${maven-compiler-plugin.version}</version>
<configuration>
<release>${java.version}</release>
<compilerArgs>
<compilerArg>-parameters</compilerArg>
</compilerArgs>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.30</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
<!-- JavaFX Maven Plugin -->
<plugin>
<groupId>org.openjfx</groupId>
<artifactId>javafx-maven-plugin</artifactId>
<version>${javafx.maven.plugin.version}</version>
<configuration>
<mainClass>com.fantaibao.JavaFxApplication</mainClass>
<launcher>app</launcher>
<jlinkImageName>fxapp</jlinkImageName>
<jlinkZipName>fxapp-zip</jlinkZipName>
</configuration>
</plugin>
</plugins>
</build>
</project>

95
src/main/java/com/fantaibao/DesktopApplication.java Normal file
View File

@@ -0,0 +1,95 @@
package com.fantaibao;
import com.fantaibao.service.PlaywrightService;
import javafx.application.Application;
import javafx.application.Platform;
import javafx.geometry.Insets;
import javafx.scene.Scene;
import javafx.scene.control.Button;
import javafx.scene.control.TextArea;
import javafx.scene.layout.VBox;
import javafx.stage.Stage;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.ConfigurableApplicationContext;
//@SpringBootApplication
public class DesktopApplication extends Application {
private TextArea textArea;
private ConfigurableApplicationContext applicationContext;
private PlaywrightService playwrightService;
public static void main(String[] args) {
// 设置JavaFX系统属性
System.setProperty("javafx.application.platform", "egl");
SpringApplication.run(DesktopApplication.class, args);
launch(args);
}
@Override
public void init() throws Exception {
super.init();
applicationContext = (ConfigurableApplicationContext) SpringApplication.run(DesktopApplication.class);
//playwrightService = applicationContext.getBean(PlaywrightService.class);
playwrightService = new PlaywrightService();
}
@Override
public void start(Stage primaryStage) {
primaryStage.setTitle("饭太煲爬虫桌面程序");
textArea = new TextArea();
textArea.setPrefRowCount(20);
textArea.setPrefColumnCount(50);
textArea.setEditable(false);
Button playwrightButton = new Button("运行Cookie拦截示例");
playwrightButton.setOnAction(e -> runPlaywrightCookieExample());
Button loginButton = new Button("运行登录示例");
loginButton.setOnAction(e -> runLoginExample());
VBox vBox = new VBox(10, playwrightButton, loginButton, textArea);
vBox.setPadding(new Insets(10));
Scene scene = new Scene(vBox, 800, 600);
primaryStage.setScene(scene);
primaryStage.setOnCloseRequest(e -> {
if (applicationContext != null) {
applicationContext.close();
}
Platform.exit();
});
primaryStage.show();
}
private void runPlaywrightCookieExample() {
appendText("开始运行Playwright Cookie拦截示例...\n");
Thread thread = new Thread(() ->
playwrightService.interceptCookieExample(this::appendText)
);
thread.setDaemon(true);
thread.start();
}
private void runLoginExample() {
appendText("开始运行登录示例...\n");
Thread thread = new Thread(() ->
playwrightService.loginExample(this::appendText)
);
thread.setDaemon(true);
thread.start();
}
private void appendText(String text) {
Platform.runLater(() -> textArea.appendText(text + "\n"));
}
@Override
public void stop() {
if (applicationContext != null) {
applicationContext.close();
}
}
}

68
src/main/java/com/fantaibao/PlaywrightInterceptCookieExample.java Normal file
View File

@@ -0,0 +1,68 @@
package com.fantaibao;
import com.microsoft.playwright.*;
import com.microsoft.playwright.options.Cookie;
import java.util.List;
import java.util.Scanner;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
public class PlaywrightInterceptCookieExample {
public static void main(String[] args) {
try (Playwright playwright = Playwright.create()) {
// 启动可见浏览器
Browser browser = playwright.chromium().launch(
new BrowserType.LaunchOptions().setHeadless(false)
);
// 创建浏览器上下文
BrowserContext context = browser.newContext();
// 用于存储拦截到的Cookie
ConcurrentMap<String, String> interceptedCookies = new ConcurrentHashMap<>();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
System.out.println("请求URL: " + request.url());
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
System.out.println("请求中的Cookie: " + cookieHeader);
interceptedCookies.put(request.url(), cookieHeader);
}
});
Page page = context.newPage();
// 导航到示例页面（这里使用百度作为示例）
page.navigate("https://www.baidu.com");
System.out.println("页面加载完成，请执行点击操作...");
System.out.println("点击按钮后按回车键继续获取Cookie...");
// 等待用户执行点击操作
Scanner scanner = new Scanner(System.in);
scanner.nextLine();
// 获取当前页面的所有Cookie
List<Cookie> cookies = context.cookies();
System.out.println("\n当前页面的Cookie:");
for (Cookie cookie : cookies) {
System.out.println(cookie.name + "=" + cookie.value +
" (域名: " + cookie.domain + ", 路径: " + cookie.path + ")");
}
// 显示拦截到的请求Cookie
System.out.println("\n拦截到的请求Cookie:");
interceptedCookies.forEach((url, cookie) ->
System.out.println("URL: " + url + "\nCookie: " + cookie + "\n"));
// 保持浏览器打开一段时间以便查看
System.out.println("浏览器将在10秒后关闭...");
page.waitForTimeout(10000);
}
}
}

55
src/main/java/com/fantaibao/Test.java Normal file
View File

@@ -0,0 +1,55 @@
package com.fantaibao;
import com.microsoft.playwright.*;
import com.microsoft.playwright.options.Cookie;
import com.microsoft.playwright.options.WaitForSelectorState;
import java.util.List;
import java.util.Scanner;
public class Test {
public static void main(String[] args) {
// 启动可见浏览器
try (Playwright playwright = Playwright.create();
Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) {
BrowserContext context = browser.newContext();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
System.out.println("请求URL: " + request.url());
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
System.out.println("请求中的Cookie: " + cookieHeader);
}
});
Page page = context.newPage();
// 导航到登录页面
page.navigate("https://ecom.meituan.com/bizaccount/login.html?loginByPhoneNumber=true&isProduction=true&epassportParams=%3Fbg_source%3D1%26service%3Dcom.sankuai.meishi.fe.ecom%26part_type%3D0%26feconfig%3Dbssoify%26biz_line%3D1%26continue%3Dhttps%253A%252F%252Fecom.meituan.com%252Fbizaccount%252Fbiz-choice.html%253Fredirect_uri%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%2526_t%253D1759399140148%2526target%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%26leftBottomLink%3D%26signUpTarget%3Dself");
// page.navigate("https://playwright.dev", new Page.NavigateOptions().setTimeout(6000000));
// 等待用户手动登录
System.out.println("请在浏览器中手动登录...");
System.out.println("登录完成后按回车键继续...");
Scanner scanner = new Scanner(System.in);
scanner.nextLine();
// 等待元素可见并启用后再点击
page.locator(".getStarted_Sjon").waitFor(new Locator.WaitForOptions().setState(WaitForSelectorState.VISIBLE));
page.locator(".getStarted_Sjon").click();
// 获取登录后的Cookie
List<Cookie> cookies = context.cookies();
for (Cookie cookie : cookies) {
System.out.println(cookie.name + "=" + cookie.value);
}
// 保持浏览器打开一段时间以便查看
System.out.println("浏览器将在10秒后关闭...");
page.waitForTimeout(10000);
}
}
}

143
src/main/java/com/fantaibao/service/PlaywrightService.java Normal file
View File

@@ -0,0 +1,143 @@
package com.fantaibao.service;
import com.microsoft.playwright.*;
import com.microsoft.playwright.options.Cookie;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Consumer;
@Service
@Slf4j
public class PlaywrightService {
public void interceptCookieExample(Consumer<String> logger) {
try (Playwright playwright = Playwright.create()) {
// 启动可见浏览器
Browser browser = playwright.chromium().launch(
new BrowserType.LaunchOptions().setHeadless(false)
);
// 创建浏览器上下文
BrowserContext context = browser.newContext();
// 用于存储拦截到的Cookie
ConcurrentMap<String, String> interceptedCookies = new ConcurrentHashMap<>();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
String logMessage = "请求URL: " + request.url();
logger.accept(logMessage);
log.info(logMessage);
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
String cookieMessage = "请求中的Cookie: " + cookieHeader;
logger.accept(cookieMessage);
log.info(cookieMessage);
interceptedCookies.put(request.url(), cookieHeader);
}
});
Page page = context.newPage();
// 导航到示例页面（这里使用百度作为示例）
page.navigate("https://www.baidu.com");
logger.accept("页面加载完成，请执行点击操作...");
logger.accept("点击按钮后按回车键继续获取Cookie...");
// 等待用户执行点击操作 (在实际桌面应用中需要替换为UI交互)
try {
Thread.sleep(10000); // 等待10秒模拟用户操作
} catch (InterruptedException e) {
log.warn("等待过程中被中断");
}
// 获取当前页面的所有Cookie
List<Cookie> cookies = context.cookies();
logger.accept("\n当前页面的Cookie:");
for (Cookie cookie : cookies) {
String cookieInfo = cookie.name + "=" + cookie.value +
" (域名: " + cookie.domain + ", 路径: " + cookie.path + ")";
logger.accept(cookieInfo);
log.info("Cookie: {}", cookieInfo);
}
// 显示拦截到的请求Cookie
logger.accept("\n拦截到的请求Cookie:");
interceptedCookies.forEach((url, cookie) -> {
String info = "URL: " + url + "\nCookie: " + cookie + "\n";
logger.accept(info);
log.info("拦截Cookie - {}: {}", url, cookie);
});
// 保持浏览器打开一段时间以便查看
logger.accept("浏览器将在5秒后关闭...");
page.waitForTimeout(5000);
} catch (Exception e) {
String errorMsg = "执行过程中出现错误: " + e.getMessage();
logger.accept(errorMsg);
log.error("Playwright执行出错", e);
}
}
public void loginExample(Consumer<String> logger) {
try (Playwright playwright = Playwright.create();
Browser browser = playwright.chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) {
BrowserContext context = browser.newContext();
// 监听网络请求
context.onRequest(request -> {
// 打印请求URL
String logMessage = "请求URL: " + request.url();
logger.accept(logMessage);
log.info(logMessage);
// 获取请求头中的Cookie
String cookieHeader = request.headerValue("cookie");
if (cookieHeader != null && !cookieHeader.isEmpty()) {
String cookieMessage = "请求中的Cookie: " + cookieHeader;
logger.accept(cookieMessage);
log.info(cookieMessage);
}
});
Page page = context.newPage();
// 导航到登录页面
page.navigate("https://ecom.meituan.com/bizaccount/login.html?loginByPhoneNumber=true&isProduction=true&epassportParams=%3Fbg_source%3D1%26service%3Dcom.sankuai.meishi.fe.ecom%26part_type%3D0%26feconfig%3Dbssoify%26biz_line%3D1%26continue%3Dhttps%253A%252F%252Fecom.meituan.com%252Fbizaccount%252Fbiz-choice.html%253Fredirect_uri%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%2526_t%253D1759399140148%2526target%253Dhttps%25253A%25252F%25252Fecom.meituan.com%25252Fmeishi%25252F%26leftBottomLink%3D%26signUpTarget%3Dself");
logger.accept("请在浏览器中手动登录...");
logger.accept("登录完成后程序将继续执行...");
// 在真实应用中，这里应该通过UI事件触发而不是固定等待
try {
Thread.sleep(15000); // 等待15秒让用户登录
} catch (InterruptedException e) {
log.warn("等待登录过程中被中断");
}
// 获取登录后的Cookie
List<Cookie> cookies = context.cookies();
logger.accept("登录后的Cookie信息:");
for (Cookie cookie : cookies) {
String cookieInfo = cookie.name + "=" + cookie.value;
logger.accept(cookieInfo);
log.info("登录Cookie: {}", cookieInfo);
}
logger.accept("浏览器将在5秒后关闭...");
page.waitForTimeout(5000);
} catch (Exception e) {
String errorMsg = "执行过程中出现错误: " + e.getMessage();
logger.accept(errorMsg);
log.error("Playwright登录示例执行出错", e);
}
}
}

16
src/main/java/module-info.java Normal file
View File

@@ -0,0 +1,16 @@
module fantaibao.crawler.desktop {
requires javafx.controls;
requires javafx.fxml;
requires spring.boot;
requires spring.boot.autoconfigure;
requires spring.context;
requires spring.beans;
requires playwright;
requires org.slf4j;
requires lombok;
opens com.fantaibao to spring.core, spring.beans, spring.context,
javafx.fxml, javafx.base, javafx.graphics, spring.boot, spring.boot.autoconfigure;
exports com.fantaibao;
}

16
src/main/resources/application.yml Normal file
View File

@@ -0,0 +1,16 @@
logging:
level:
com.fantaibao: INFO
org.springframework: INFO
pattern:
console: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n"
file: "%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n"
file:
name: logs/application.log
spring:
main:
banner-mode: console
server:
port: 8080