From 481e110566403494f39185ce485ee083c3eed533 Mon Sep 17 00:00:00 2001 From: wangchunxiang <526328077@qq.com> Date: Thu, 16 Oct 2025 20:42:04 +0800 Subject: [PATCH] =?UTF-8?q?feat(playwright):=20=E5=A2=9E=E5=BC=BA=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E5=99=A8=E5=8F=8D=E8=87=AA=E5=8A=A8=E5=8C=96=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E8=83=BD=E5=8A=9B=20=E4=B8=BA=20FtbCrawlNetDy=20?= =?UTF-8?q?=E5=92=8C=20FtbCrawlNetMt=20=E6=9C=8D=E5=8A=A1=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=BA=86=E5=8F=8D=E8=87=AA=E5=8A=A8=E5=8C=96=E6=A3=80?= =?UTF-8?q?=E6=B5=8B=E9=85=8D=E7=BD=AE=EF=BC=8C=E5=8C=85=E6=8B=AC=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E5=8F=82=E6=95=B0=E3=80=81HTTP=E5=A4=B4=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E4=BB=A5=E5=8F=8A=E5=88=9D=E5=A7=8B=E5=8C=96=E8=84=9A?= =?UTF-8?q?=E6=9C=AC=E3=80=82=E5=90=8C=E6=97=B6=E6=89=A9=E5=B1=95=20Playwr?= =?UTF-8?q?ightManager=20=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E9=9A=90=E8=BA=AB=E6=A8=A1=E5=BC=8F=E4=B8=8A?= =?UTF-8?q?=E4=B8=8B=E6=96=87=E9=80=89=E9=A1=B9=E5=92=8C=E9=87=8D=E6=96=B0?= =?UTF-8?q?=E5=88=9B=E5=BB=BA=E5=AE=9E=E4=BE=8B=EF=BC=8C=E6=8F=90=E9=AB=98?= =?UTF-8?q?=E7=88=AC=E8=99=AB=E7=A8=B3=E5=AE=9A=E6=80=A7=E4=B8=8E=E9=9A=90?= =?UTF-8?q?=E8=94=BD=E6=80=A7=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fantaibao/config/PlaywrightManager.java | 32 ++++++++- .../com/fantaibao/service/FtbCrawlNetDy.java | 66 +++++++++++++++++-- .../com/fantaibao/service/FtbCrawlNetMt.java | 62 ++++++++++++++++- 3 files changed, 150 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/fantaibao/config/PlaywrightManager.java b/src/main/java/com/fantaibao/config/PlaywrightManager.java index 63830ff..bd62c69 100644 --- a/src/main/java/com/fantaibao/config/PlaywrightManager.java +++ b/src/main/java/com/fantaibao/config/PlaywrightManager.java @@ -57,7 +57,11 @@ public class PlaywrightManager { public void close() { if (playwright != null) { try { + log.info("开始关闭Playwright实例"); + long startTime = System.currentTimeMillis(); playwright.close(); + long endTime = System.currentTimeMillis(); + log.info("Playwright实例关闭完成，耗时: {}ms", (endTime - startTime)); } catch (Exception e) { log.error("关闭Playwright实例时出错", e); } finally { @@ -66,5 +70,31 @@ public class PlaywrightManager { } } } - + + /** + * 重新创建Playwright实例 + */ + public void recreate() { + close(); + try { + this.playwright = Playwright.create(); + } catch (Exception e) { + log.error("重新创建Playwright实例失败", e); + throw new RuntimeException("重新创建Playwright实例失败", e); + } + } + + /** + * 创建带反检测配置的浏览器上下文选项 + * @return 浏览器上下文选项 + */ + public com.microsoft.playwright.Browser.NewContextOptions createStealthContextOptions() { + com.microsoft.playwright.Browser.NewContextOptions options = new com.microsoft.playwright.Browser.NewContextOptions() + .setViewportSize(1680, 900); + + // 设置用户代理（可以设置为常用浏览器的用户代理） + // options.setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"); + + return options; + } } \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java b/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java index 6ecca94..8d12cf3 100644 --- a/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java @@ -8,6 +8,9 @@ import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -19,11 +22,34 @@ public class FtbCrawlNetDy extends AbstractFtbCrawlNetBase implements FtbCrawlNe public void executeCookieIntercept() { // 启动可见浏览器 try (Browser browser = PlaywrightManager.getInstance().getPlaywright() - .chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) { - // 设置浏览器窗口大小为1920x1080 - Browser.NewContextOptions contextOptions = new Browser.NewContextOptions() - .setViewportSize(1680, 900); + .chromium().launch(getLaunchOptions())) { + // 设置浏览器上下文选项以减少自动化检测 + Browser.NewContextOptions contextOptions = PlaywrightManager.getInstance().createStealthContextOptions(); + // 添加额外的反检测配置 + Map extraHTTPHeaders = new HashMap<>(); + extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + contextOptions.setExtraHTTPHeaders(extraHTTPHeaders); + BrowserContext context = browser.newContext(contextOptions); + + // 添加反自动化检测脚本 + context.addInitScript(""" + () => { + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + + // 添加一些常见的浏览器属性 + Object.defineProperty(navigator, 'languages', { + get: () => ['zh-CN', 'zh', 'en'], + }); + + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + } + """); + Page page = context.newPage(); AtomicBoolean dyCookie = new AtomicBoolean(false); // 监听网络请求 @@ -53,7 +79,35 @@ public class FtbCrawlNetDy extends AbstractFtbCrawlNetBase implements FtbCrawlNe } } - - + + /** + * 获取浏览器启动选项，减少自动化检测 + * @return 浏览器启动选项 + */ + private BrowserType.LaunchOptions getLaunchOptions() { + return new BrowserType.LaunchOptions() + .setHeadless(false) + // 添加这些参数减少自动化检测 + .setArgs(Arrays.asList( + "--no-sandbox", + "--disable-blink-features=AutomationControlled", + "--disable-extensions", + "--disable-infobars", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--disable-background-networking", + "--disable-default-apps", + "--disable-features=TranslateUI", + "--disable-hang-monitor", + "--disable-prompt-on-repost", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--password-store=basic", + "--use-mock-keychain" + )); + } } \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java index 536a866..3b44895 100644 --- a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java @@ -8,6 +8,9 @@ import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @Component(value = "ftbCrawlNetMt") @@ -18,11 +21,34 @@ public class FtbCrawlNetMt extends AbstractFtbCrawlNetBase implements FtbCrawlNe public void executeCookieIntercept() { // 启动可见浏览器 try (Browser browser = PlaywrightManager.getInstance().getPlaywright() - .chromium().launch(new BrowserType.LaunchOptions().setHeadless(false))) { + .chromium().launch(getLaunchOptions())) { // 设置浏览器窗口大小为1920x1080 - Browser.NewContextOptions contextOptions = new Browser.NewContextOptions() - .setViewportSize(1680, 900); + Browser.NewContextOptions contextOptions = PlaywrightManager.getInstance().createStealthContextOptions(); + // 添加额外的反检测配置 + Map extraHTTPHeaders = new HashMap<>(); + extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + contextOptions.setExtraHTTPHeaders(extraHTTPHeaders); + BrowserContext context = browser.newContext(contextOptions); + + // 添加反自动化检测脚本 + context.addInitScript(""" + () => { + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + + // 添加一些常见的浏览器属性 + Object.defineProperty(navigator, 'languages', { + get: () => ['zh-CN', 'zh', 'en'], + }); + + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + } + """); + // 美团cookie AtomicBoolean mtCookie = new AtomicBoolean(false); // 大众点评cookie @@ -54,6 +80,36 @@ public class FtbCrawlNetMt extends AbstractFtbCrawlNetBase implements FtbCrawlNe } } + /** + * 获取浏览器启动选项，减少自动化检测 + * @return 浏览器启动选项 + */ + private BrowserType.LaunchOptions getLaunchOptions() { + return new BrowserType.LaunchOptions() + .setHeadless(false) + // 添加这些参数减少自动化检测 + .setArgs(Arrays.asList( + "--no-sandbox", + "--disable-blink-features=AutomationControlled", + "--disable-extensions", + "--disable-infobars", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--disable-background-networking", + "--disable-default-apps", + "--disable-features=TranslateUI", + "--disable-hang-monitor", + "--disable-prompt-on-repost", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--password-store=basic", + "--use-mock-keychain" + )); + } + /** * 处理网络请求，提取需要的cookie信息 * @param request 网络请求