diff --git a/src/main/java/com/fantaibao/config/BrowserStealthConfig.java b/src/main/java/com/fantaibao/config/BrowserStealthConfig.java new file mode 100644 index 0000000..830528e --- /dev/null +++ b/src/main/java/com/fantaibao/config/BrowserStealthConfig.java @@ -0,0 +1,86 @@ +package com.fantaibao.config; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.BrowserType; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * 浏览器反检测配置工具类 + * 用于提供统一的浏览器反自动化检测配置 + */ +public class BrowserStealthConfig { + + /** + * 获取浏览器启动选项，减少自动化检测 + * + * @return 浏览器启动选项 + */ + public static BrowserType.LaunchOptions getStealthLaunchOptions() { + return new BrowserType.LaunchOptions() + .setHeadless(false) + // 添加这些参数减少自动化检测 + .setArgs(Arrays.asList( + "--no-sandbox", + "--disable-blink-features=AutomationControlled", + "--disable-extensions", + "--disable-infobars", + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + "--disable-ipc-flooding-protection", + "--disable-background-networking", + "--disable-default-apps", + "--disable-features=TranslateUI", + "--disable-hang-monitor", + "--disable-prompt-on-repost", + "--force-color-profile=srgb", + "--metrics-recording-only", + "--no-first-run", + "--password-store=basic", + "--use-mock-keychain" + )); + } + + /** + * 配置浏览器上下文，添加反检测设置 + * + * @param context 浏览器上下文 + */ + public static void configureStealthContext(BrowserContext context) { + // 添加反自动化检测脚本 + context.addInitScript(""" + () => { + Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + }); + + // 添加一些常见的浏览器属性 + Object.defineProperty(navigator, 'languages', { + get: () => ['zh-CN', 'zh', 'en'], + }); + + Object.defineProperty(navigator, 'plugins', { + get: () => [1, 2, 3, 4, 5], + }); + } + """); + } + + /** + * 获取带反检测配置的浏览器上下文选项 + * + * @return 浏览器上下文选项 + */ + public static Browser.NewContextOptions getStealthContextOptions() { + Browser.NewContextOptions options = PlaywrightManager.getInstance().createStealthContextOptions(); + // 添加额外的反检测配置 + Map extraHTTPHeaders = new HashMap<>(); + extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); + options.setExtraHTTPHeaders(extraHTTPHeaders); + return options; + } +} \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java b/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java index 8d12cf3..2ff21ae 100644 --- a/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetDy.java @@ -1,16 +1,13 @@ package com.fantaibao.service; import com.fantaibao.base.FtbCrawlNetBase; +import com.fantaibao.config.BrowserStealthConfig; import com.fantaibao.config.GlobalConfig; import com.fantaibao.config.PlaywrightManager; import com.microsoft.playwright.*; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -22,33 +19,12 @@ public class FtbCrawlNetDy extends AbstractFtbCrawlNetBase implements FtbCrawlNe public void executeCookieIntercept() { // 启动可见浏览器 try (Browser browser = PlaywrightManager.getInstance().getPlaywright() - .chromium().launch(getLaunchOptions())) { + .chromium().launch(BrowserStealthConfig.getStealthLaunchOptions())) { // 设置浏览器上下文选项以减少自动化检测 - Browser.NewContextOptions contextOptions = PlaywrightManager.getInstance().createStealthContextOptions(); - // 添加额外的反检测配置 - Map extraHTTPHeaders = new HashMap<>(); - extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - contextOptions.setExtraHTTPHeaders(extraHTTPHeaders); - - BrowserContext context = browser.newContext(contextOptions); + BrowserContext context = browser.newContext(BrowserStealthConfig.getStealthContextOptions()); // 添加反自动化检测脚本 - context.addInitScript(""" - () => { - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined, - }); - - // 添加一些常见的浏览器属性 - Object.defineProperty(navigator, 'languages', { - get: () => ['zh-CN', 'zh', 'en'], - }); - - Object.defineProperty(navigator, 'plugins', { - get: () => [1, 2, 3, 4, 5], - }); - } - """); + BrowserStealthConfig.configureStealthContext(context); Page page = context.newPage(); AtomicBoolean dyCookie = new AtomicBoolean(false); @@ -80,34 +56,4 @@ public class FtbCrawlNetDy extends AbstractFtbCrawlNetBase implements FtbCrawlNe } - /** - * 获取浏览器启动选项，减少自动化检测 - * @return 浏览器启动选项 - */ - private BrowserType.LaunchOptions getLaunchOptions() { - return new BrowserType.LaunchOptions() - .setHeadless(false) - // 添加这些参数减少自动化检测 - .setArgs(Arrays.asList( - "--no-sandbox", - "--disable-blink-features=AutomationControlled", - "--disable-extensions", - "--disable-infobars", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--disable-background-networking", - "--disable-default-apps", - "--disable-features=TranslateUI", - "--disable-hang-monitor", - "--disable-prompt-on-repost", - "--force-color-profile=srgb", - "--metrics-recording-only", - "--no-first-run", - "--password-store=basic", - "--use-mock-keychain" - )); - } - } \ No newline at end of file diff --git a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java index 3b44895..74b1b50 100644 --- a/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java +++ b/src/main/java/com/fantaibao/service/FtbCrawlNetMt.java @@ -1,16 +1,13 @@ package com.fantaibao.service; import com.fantaibao.base.FtbCrawlNetBase; +import com.fantaibao.config.BrowserStealthConfig; import com.fantaibao.config.GlobalConfig; import com.fantaibao.config.PlaywrightManager; import com.microsoft.playwright.*; -import jakarta.annotation.Resource; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @Component(value = "ftbCrawlNetMt") @@ -21,33 +18,12 @@ public class FtbCrawlNetMt extends AbstractFtbCrawlNetBase implements FtbCrawlNe public void executeCookieIntercept() { // 启动可见浏览器 try (Browser browser = PlaywrightManager.getInstance().getPlaywright() - .chromium().launch(getLaunchOptions())) { - // 设置浏览器窗口大小为1920x1080 - Browser.NewContextOptions contextOptions = PlaywrightManager.getInstance().createStealthContextOptions(); - // 添加额外的反检测配置 - Map extraHTTPHeaders = new HashMap<>(); - extraHTTPHeaders.put("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8"); - contextOptions.setExtraHTTPHeaders(extraHTTPHeaders); - - BrowserContext context = browser.newContext(contextOptions); + .chromium().launch(BrowserStealthConfig.getStealthLaunchOptions())) { + // 设置浏览器上下文选项以减少自动化检测 + BrowserContext context = browser.newContext(BrowserStealthConfig.getStealthContextOptions()); // 添加反自动化检测脚本 - context.addInitScript(""" - () => { - Object.defineProperty(navigator, 'webdriver', { - get: () => undefined, - }); - - // 添加一些常见的浏览器属性 - Object.defineProperty(navigator, 'languages', { - get: () => ['zh-CN', 'zh', 'en'], - }); - - Object.defineProperty(navigator, 'plugins', { - get: () => [1, 2, 3, 4, 5], - }); - } - """); + BrowserStealthConfig.configureStealthContext(context); // 美团cookie AtomicBoolean mtCookie = new AtomicBoolean(false); @@ -80,36 +56,6 @@ public class FtbCrawlNetMt extends AbstractFtbCrawlNetBase implements FtbCrawlNe } } - /** - * 获取浏览器启动选项，减少自动化检测 - * @return 浏览器启动选项 - */ - private BrowserType.LaunchOptions getLaunchOptions() { - return new BrowserType.LaunchOptions() - .setHeadless(false) - // 添加这些参数减少自动化检测 - .setArgs(Arrays.asList( - "--no-sandbox", - "--disable-blink-features=AutomationControlled", - "--disable-extensions", - "--disable-infobars", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-ipc-flooding-protection", - "--disable-background-networking", - "--disable-default-apps", - "--disable-features=TranslateUI", - "--disable-hang-monitor", - "--disable-prompt-on-repost", - "--force-color-profile=srgb", - "--metrics-recording-only", - "--no-first-run", - "--password-store=basic", - "--use-mock-keychain" - )); - } - /** * 处理网络请求，提取需要的cookie信息 * @param request 网络请求