From 763444b9e8a59f1220e5b3cd6fdbaab77148802a Mon Sep 17 00:00:00 2001 From: wangchunxiang <526328077@qq.com> Date: Thu, 16 Oct 2025 20:47:06 +0800 Subject: [PATCH] =?UTF-8?q?feat(playwright):=20=E5=BC=95=E5=85=A5=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E5=99=A8=E5=8F=8D=E6=A3=80=E6=B5=8B=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E7=B1=BB=20=E6=96=B0=E5=A2=9E=20`BrowserStea?= =?UTF-8?q?lthConfig`=20=E5=B7=A5=E5=85=B7=E7=B1=BB=EF=BC=8C=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E7=AE=A1=E7=90=86=20Playwright=20=E7=9A=84=E5=8F=8D?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=8C=96=E6=A3=80=E6=B5=8B=E9=85=8D=E7=BD=AE?= =?UTF-8?q?=E3=80=82=20=E8=AF=A5=E5=8F=98=E6=9B=B4=E5=B0=86=E5=8E=9F?= =?UTF-8?q?=E6=9C=AC=E5=88=86=E6=95=A3=E5=9C=A8=20Dy=20=E5=92=8C=20Mt=20?= =?UTF-8?q?=E7=88=AC=E8=99=AB=E6=9C=8D=E5=8A=A1=E4=B8=AD=E7=9A=84=E6=B5=8F?= =?UTF-8?q?=E8=A7=88=E5=99=A8=E5=90=AF=E5=8A=A8=E9=80=89=E9=A1=B9=E3=80=81?= =?UTF-8?q?=E4=B8=8A=E4=B8=8B=E6=96=87=E9=85=8D=E7=BD=AE=E5=8F=8A=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96=E8=84=9A=E6=9C=AC=E8=BF=9B=E8=A1=8C=E6=8A=BD?= =?UTF-8?q?=E5=8F=96=EF=BC=8C=20=E9=9B=86=E4=B8=AD=E5=88=B0=E6=96=B0?= =?UTF-8?q?=E7=9A=84=E9=85=8D=E7=BD=AE=E7=B1=BB=E4=B8=AD=EF=BC=8C=E6=8F=90?= =?UTF-8?q?=E5=8D=87=E4=BB=A3=E7=A0=81=E5=A4=8D=E7=94=A8=E6=80=A7=E5=92=8C?= =?UTF-8?q?=E5=8F=AF=E7=BB=B4=E6=8A=A4=E6=80=A7=E3=80=82-=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=20`getStealthLaunchOptions()`=20=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E7=94=A8=E4=BA=8E=E8=8E=B7=E5=8F=96=E6=A0=87=E5=87=86=E7=9A=84?= =?UTF-8?q?=E5=8F=8D=E6=A3=80=E6=B5=8B=E6=B5=8F=E8=A7=88=E5=99=A8=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E5=8F=82=E6=95=B0-=20=E6=B7=BB=E5=8A=A0=20`configureS?= =?UTF-8?q?tealthContext()`=20=E6=96=B9=E6=B3=95=E5=B0=81=E8=A3=85=20navig?= =?UTF-8?q?ator=20=E5=B1=9E=E6=80=A7=E4=BC=AA=E8=A3=85=E9=80=BB=E8=BE=91?= =?UTF-8?q?=20-=20=E6=B7=BB=E5=8A=A0=20`getStealthContextOptions()`=20?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E6=8F=90=E4=BE=9B=E5=B8=A6=E8=AF=AD=E8=A8=80?= =?UTF-8?q?=E5=A4=B4=E7=9A=84=E4=B8=8A=E4=B8=8B=E6=96=87=E9=85=8D=E7=BD=AE?= =?UTF-8?q?-=20=E5=9C=A8=20Dy=20=E5=92=8C=20Mt=20=E7=88=AC=E8=99=AB?= =?UTF-8?q?=E6=9C=8D=E5=8A=A1=E4=B8=AD=E7=A7=BB=E9=99=A4=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E4=BB=A3=E7=A0=81=EF=BC=8C=E6=94=B9=E4=B8=BA?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=96=B0=E5=B7=A5=E5=85=B7=E7=B1=BB=E6=96=B9?= =?UTF-8?q?=E6=B3=95-=20=E7=A7=BB=E9=99=A4=E4=BA=86=E4=B8=8D=E5=86=8D?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=9A=84=20import=E8=AF=AD=E5=8F=A5=E5=8F=8A?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E5=86=97=E4=BD=99=E5=8F=98=E9=87=8F=E5=A3=B0?= =?UTF-8?q?=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/BrowserStealthConfig.java | 86 +++++++++++++++++++ .../com/fantaibao/service/FtbCrawlNetDy.java | 62 +------------ .../com/fantaibao/service/FtbCrawlNetMt.java | 64 ++------------ 3 files changed, 95 insertions(+), 117 deletions(-) create mode 100644 src/main/java/com/fantaibao/config/BrowserStealthConfig.java diff --git a/src/main/java/com/fantaibao/config/BrowserStealthConfig.java b/src/main/java/com/fantaibao/config/BrowserStealthConfig.java new file mode 100644 index 0000000..830528e --- /dev/null +++ b/src/main/java/com/fantaibao/config/BrowserStealthConfig.java @@ -0,0 +1,86 @@ +package com.fantaibao.config; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.BrowserType; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * 浏览器反检测配置工具类 + * 用于提供统一的浏览器反自动化检测配置 + */ +public class BrowserStealthConfig { + + /** + * 获取浏览器启动选项，减少自动化检测 + * + * @return 浏览器启动选项 + */ + public static BrowserType.LaunchOptions getStealthLaunchOptions() { + return new BrowserType.LaunchOptions() + .setHeadless(false) + // 添加这些参数减少自动化检测 + .setArgs(Arrays.asList( + "--no-sandbox", + "--disable-blink-features=AutomationControlled", + "--disable-extensions", + "--disable-infobars", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--disable-background-networking", + "--disable-default-apps", + "--disable-features=TranslateUI", + "--disable-hang-monitor", + "--disable-prompt-on-repost", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--password-store=basic", + "--use-mock-keychain" + )); + } + + /** + * 配置浏览器上下文，添加反检测设置 + * + * @param context 浏览器上下文 + */ + public static void configureStealthContext(BrowserContext context) { + // 添加反自动化检测脚本 + context.addInitScript(""" + () => { + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + + // 添加一些常见的浏览器属性 + Object.defineProperty(navigator, 'languages', { + get: () => ['zh-CN', 'zh', 'en'], + }); + + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + } + """); + } + + /** + * 获取带反检测配置的浏览器上下文选项 + * + * @return 浏览器上下文选项 + */ + public static Browser.NewContextOptions getStealthContextOptions() { + Browser.NewContextOptions options = PlaywrightManager.getInstance().createStealthContextOptions(); + // 添加额外的反检测配置 + Map extraHTTPHeaders = new HashMap<>(); + extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + options.setExtraHTTPHeaders(extraHTTPHeaders); + return options; + } +} \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java b/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java index 8d12cf3..2ff21ae 100644 --- a/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java @@ -1,16 +1,13 @@ package com.fantaibao.service; import com.fantaibao.base.FtbCrawlNetBase; +import com.fantaibao.config.BrowserStealthConfig; import com.fantaibao.config.GlobalConfig; import com.fantaibao.config.PlaywrightManager; import com.microsoft.playwright.*; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -22,33 +19,12 @@ public class FtbCrawlNetDy extends AbstractFtbCrawlNetBase implements FtbCrawlNe public void executeCookieIntercept() { // 启动可见浏览器 try (Browser browser = PlaywrightManager.getInstance().getPlaywright() - .chromium().launch(getLaunchOptions())) { + .chromium().launch(BrowserStealthConfig.getStealthLaunchOptions())) { // 设置浏览器上下文选项以减少自动化检测 - Browser.NewContextOptions contextOptions = PlaywrightManager.getInstance().createStealthContextOptions(); - // 添加额外的反检测配置 - Map extraHTTPHeaders = new HashMap<>(); - extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - contextOptions.setExtraHTTPHeaders(extraHTTPHeaders); - - BrowserContext context = browser.newContext(contextOptions); + BrowserContext context = browser.newContext(BrowserStealthConfig.getStealthContextOptions()); // 添加反自动化检测脚本 - context.addInitScript(""" - () => { - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined, - }); - - // 添加一些常见的浏览器属性 - Object.defineProperty(navigator, 'languages', { - get: () => ['zh-CN', 'zh', 'en'], - }); - - Object.defineProperty(navigator, 'plugins', { - get: () => [1, 2, 3, 4, 5], - }); - } - """); + BrowserStealthConfig.configureStealthContext(context); Page page = context.newPage(); AtomicBoolean dyCookie = new AtomicBoolean(false); @@ -80,34 +56,4 @@ public class FtbCrawlNetDy extends AbstractFtbCrawlNetBase implements FtbCrawlNe } - /** - * 获取浏览器启动选项，减少自动化检测 - * @return 浏览器启动选项 - */ - private BrowserType.LaunchOptions getLaunchOptions() { - return new BrowserType.LaunchOptions() - .setHeadless(false) - // 添加这些参数减少自动化检测 - .setArgs(Arrays.asList( - "--no-sandbox", - "--disable-blink-features=AutomationControlled", - "--disable-extensions", - "--disable-infobars", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--disable-background-networking", - "--disable-default-apps", - "--disable-features=TranslateUI", - "--disable-hang-monitor", - "--disable-prompt-on-repost", - "--force-color-profile=srgb", - "--metrics-recording-only", - "--no-first-run", - "--password-store=basic", - "--use-mock-keychain" - )); - } - } \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java index 3b44895..74b1b50 100644 --- a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java @@ -1,16 +1,13 @@ package com.fantaibao.service; import com.fantaibao.base.FtbCrawlNetBase; +import com.fantaibao.config.BrowserStealthConfig; import com.fantaibao.config.GlobalConfig; import com.fantaibao.config.PlaywrightManager; import com.microsoft.playwright.*; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @Component(value = "ftbCrawlNetMt") @@ -21,33 +18,12 @@ public class FtbCrawlNetMt extends AbstractFtbCrawlNetBase implements FtbCrawlNe public void executeCookieIntercept() { // 启动可见浏览器 try (Browser browser = PlaywrightManager.getInstance().getPlaywright() - .chromium().launch(getLaunchOptions())) { - // 设置浏览器窗口大小为1920x1080 - Browser.NewContextOptions contextOptions = PlaywrightManager.getInstance().createStealthContextOptions(); - // 添加额外的反检测配置 - Map extraHTTPHeaders = new HashMap<>(); - extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - contextOptions.setExtraHTTPHeaders(extraHTTPHeaders); - - BrowserContext context = browser.newContext(contextOptions); + .chromium().launch(BrowserStealthConfig.getStealthLaunchOptions())) { + // 设置浏览器上下文选项以减少自动化检测 + BrowserContext context = browser.newContext(BrowserStealthConfig.getStealthContextOptions()); // 添加反自动化检测脚本 - context.addInitScript(""" - () => { - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined, - }); - - // 添加一些常见的浏览器属性 - Object.defineProperty(navigator, 'languages', { - get: () => ['zh-CN', 'zh', 'en'], - }); - - Object.defineProperty(navigator, 'plugins', { - get: () => [1, 2, 3, 4, 5], - }); - } - """); + BrowserStealthConfig.configureStealthContext(context); // 美团cookie AtomicBoolean mtCookie = new AtomicBoolean(false); @@ -80,36 +56,6 @@ public class FtbCrawlNetMt extends AbstractFtbCrawlNetBase implements FtbCrawlNe } } - /** - * 获取浏览器启动选项，减少自动化检测 - * @return 浏览器启动选项 - */ - private BrowserType.LaunchOptions getLaunchOptions() { - return new BrowserType.LaunchOptions() - .setHeadless(false) - // 添加这些参数减少自动化检测 - .setArgs(Arrays.asList( - "--no-sandbox", - "--disable-blink-features=AutomationControlled", - "--disable-extensions", - "--disable-infobars", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--disable-background-networking", - "--disable-default-apps", - "--disable-features=TranslateUI", - "--disable-hang-monitor", - "--disable-prompt-on-repost", - "--force-color-profile=srgb", - "--metrics-recording-only", - "--no-first-run", - "--password-store=basic", - "--use-mock-keychain" - )); - } - /** * 处理网络请求，提取需要的cookie信息 * @param request 网络请求