From ee2f70d838f8dd5f26b6bf579370bbf76e304472 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 15 Sep 2026 10:06:56 +0200 Subject: [PATCH 1/2] #2094 Decouple ignoring HTTPS errors from the proxy settings in the Playwright protocol and make it configurable via playwright.ignore.https.errors (default false). --- docs/src/main/asciidoc/configuration.adoc | 1 + external/playwright/README.md | 3 + external/playwright/playwright-conf.yaml | 9 ++ .../protocol/playwright/HttpProtocol.java | 77 +++++++++++------ .../playwright/ContextOptionsTest.java | 86 +++++++++++++++++++ 5 files changed, 152 insertions(+), 24 deletions(-) create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 093f60e5a..dea2d4a53 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -609,6 +609,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/playwrigh | playwright.remote.ws | - | Remote WebSocket URL for Playwright (alternative to CDP, e.g. `ws://localhost:3000/`). | playwright.skip.download | false | Skip automatic browser download. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set. | playwright.load.event | load | Page load event to wait for. One of `load`, `domcontentloaded`, `networkidle`. +| playwright.ignore.https.errors | false | If `true`, the browser context accepts any TLS certificate, including self-signed, expired or otherwise invalid ones, for every page, navigation and subresource. The servers are then not authenticated: anyone able to answer for the host name can serve content that the browser renders and whose JavaScript it executes. Independent of `http.proxy`: deployments behind a TLS-intercepting proxy, which previously relied on certificate validation being disabled whenever a proxy was set, now need this key or the proxy CA installed in the browser image. | playwright.skip.resource.types | - | List of resource types aborted during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`). | playwright.evaluations | - | List of JavaScript expressions evaluated after load; each JSON-serialised result is stored in response metadata under the expression itself. | playwright.capture.content.on.error | false | If `true`, also capture `page.content()` for non-2xx responses — useful for SPAs that return a stub then hydrate via JS. diff --git a/external/playwright/README.md b/external/playwright/README.md index 8083d8fd0..91c943c67 100644 --- a/external/playwright/README.md +++ b/external/playwright/README.md @@ -36,12 +36,15 @@ The setting `playwright.skip.download` to `true` in the configuration will assum | `playwright.remote.ws` | _unset_ | If set, connect to a remote Playwright server over WebSocket (e.g. `ws://localhost:3000/`). Mutually exclusive with `playwright.cdp.url`. | | `playwright.skip.download` | `false` | If `true`, sets `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true` so Playwright will not install browsers. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set. | | `playwright.load.event` | `load` | The Playwright `WaitUntilState` to wait for before considering the page ready. Accepts `load`, `domcontentloaded`, or `networkidle`. | +| `playwright.ignore.https.errors` | `false` | If `true`, the browser context accepts any TLS certificate, including self-signed, expired or otherwise invalid ones, for every page, navigation and subresource. The servers are then not authenticated: anyone able to answer for the host name can serve content that the browser renders and whose JavaScript it executes. A warning is logged when enabled. Independent of `http.proxy`, see the compatibility note below. | | `playwright.skip.resource.types` | _empty_ | List of resource types to abort during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`). | | `playwright.evaluations` | _empty_ | List of JavaScript expressions evaluated on the page after load. Each result is JSON-serialized and stored in the response metadata under the expression itself as the key. | | `playwright.capture.content.on.error` | `false` | By default the rendered DOM is only captured when the origin returns a 2xx status. Set to `true` to also capture `page.content()` for non-2xx responses — useful for Single-Page Applications that return a non-2xx stub document and then hydrate the real content via JavaScript. | | `playwright.override.status.on.content` | `false` | When the rendered DOM was captured for a non-2xx response, override the reported HTTP status with `200` so downstream components treat the URL as `FETCHED`. The original origin status is preserved in the response metadata under the key `playwright.origin.status`. No-op unless `playwright.capture.content.on.error` is also `true`. | | `playwright.page.actions.config.file` | _unset_ | Path to a JSON file declaring an ordered chain of `PageAction` implementations applied after `page.navigate()` succeeds and before `page.content()` is captured. Use this to plug site-specific post-navigate behaviour (tab/accordion expansion, cookie-banner dismissal, scroll-to-bottom, custom `evaluate()` calls, ...) into the protocol without subclassing it. The chain runs only when content would otherwise be captured (i.e. on 2xx, or on non-2xx if `playwright.capture.content.on.error` is `true`). | +**Compatibility note:** earlier versions implicitly disabled certificate validation whenever `http.proxy` was set. This is no longer the case. Deployments running behind a TLS-intercepting proxy which relied on that behaviour now need either `playwright.ignore.https.errors: true` or the proxy CA installed in the browser image. + Per-URL metadata triggers: | Metadata key | Effect | diff --git a/external/playwright/playwright-conf.yaml b/external/playwright/playwright-conf.yaml index 79f232af6..355fc63cb 100644 --- a/external/playwright/playwright-conf.yaml +++ b/external/playwright/playwright-conf.yaml @@ -24,6 +24,15 @@ config: # com.microsoft.playwright.options.WaitUntilState # playwright.load.event: "domcontentloaded" + # If true, the browser context accepts any TLS certificate, including + # self-signed, expired or otherwise invalid ones, for every page, navigation + # and subresource. The servers are then not authenticated. This is + # independent of http.proxy: configuring a proxy no longer disables + # certificate validation. Deployments behind a TLS-intercepting proxy need + # either this key or the proxy CA installed in the browser image. + # A warning is logged when enabled. + # playwright.ignore.https.errors: false + # By default the rendered DOM is only captured when the origin returns a 2xx # status. Enable this to also capture page.content() for non-2xx responses # (useful for SPAs that return e.g. a 404 stub and then hydrate via JS). diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index 3e947e917..762b034cb 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -63,6 +63,13 @@ public class HttpProtocol extends AbstractHttpProtocol { public static final String MD_EVALUATIONS = "playwright.evaluations"; public static final String MD_SKIPS = "playwright.skip.resource.types"; + /** + * If true, the browser context accepts any TLS certificate, including self-signed, expired or + * otherwise invalid ones. Applies to every page, navigation and subresource of the context, + * independently of whether a proxy is configured. + */ + public static final String IGNORE_HTTPS_ERRORS_KEY = "playwright.ignore.https.errors"; + private int timeout = 10000; private boolean captureContentOnError = false; @@ -145,29 +152,7 @@ public void configure(final Config conf) { overrideStatusOnContent = ConfUtils.getBoolean(conf, "playwright.override.status.on.content", false); - final String ua = getAgentString(conf); - - NewContextOptions b_c_options = - new Browser.NewContextOptions().setIsMobile(false).setUserAgent(ua); - - // set Accept-Language if configured, as done by the other protocol implementations; - // an explicitly empty value overrides the browser's default with an empty header, - // only an absent key leaves the browser's default untouched - final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language"); - if (acceptLanguage != null) { - b_c_options.setExtraHTTPHeaders(Map.of("Accept-Language", acceptLanguage)); - } - - // global proxy - String proxyServer = ConfUtils.getString(conf, "http.proxy"); - String proxyUser = ConfUtils.getString(conf, "http.proxy.username"); - String proxyPwd = ConfUtils.getString(conf, "http.proxy.password"); - - final Proxy globalProxy = getProxy(proxyServer, proxyUser, proxyPwd); - if (globalProxy != null) { - b_c_options.setProxy(globalProxy); - b_c_options.setIgnoreHTTPSErrors(true); - } + final NewContextOptions b_c_options = buildContextOptions(conf, getAgentString(conf)); context = browser.newContext(b_c_options); @@ -190,6 +175,50 @@ public void configure(final Config conf) { pageActions = PageActions.fromConf(conf); } + /** + * Builds the options of the browser context shared by all fetches, whether the browser is + * launched locally or reached via CDP or a remote Playwright server. + * + * @param conf the configuration + * @param userAgent the user agent string sent by the browser + * @return the context options + */ + static NewContextOptions buildContextOptions(final Config conf, final String userAgent) { + final NewContextOptions options = + new Browser.NewContextOptions().setIsMobile(false).setUserAgent(userAgent); + + // set Accept-Language if configured, as done by the other protocol implementations; + // an explicitly empty value overrides the browser's default with an empty header, + // only an absent key leaves the browser's default untouched + final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language"); + if (acceptLanguage != null) { + options.setExtraHTTPHeaders(Map.of("Accept-Language", acceptLanguage)); + } + + // global proxy + final String proxyServer = ConfUtils.getString(conf, "http.proxy"); + final String proxyUser = ConfUtils.getString(conf, "http.proxy.username"); + final String proxyPwd = ConfUtils.getString(conf, "http.proxy.password"); + + final Proxy globalProxy = getProxy(proxyServer, proxyUser, proxyPwd); + if (globalProxy != null) { + options.setProxy(globalProxy); + } + + // certificate validation is independent of the proxy settings + final boolean ignoreHTTPSErrors = + ConfUtils.getBoolean(conf, IGNORE_HTTPS_ERRORS_KEY, false); + if (ignoreHTTPSErrors) { + LOG.warn( + "{} is true: TLS certificates are not validated by the browser, any server" + + " able to answer for a host name is accepted", + IGNORE_HTTPS_ERRORS_KEY); + } + options.setIgnoreHTTPSErrors(ignoreHTTPSErrors); + + return options; + } + @Override public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception { @@ -392,7 +421,7 @@ private void storeVerbatimHeaders( } /** Returns a proxy object if required * */ - private Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { + private static Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { if (proxyserver == null) { return null; } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java new file mode 100644 index 000000000..ee2294f37 --- /dev/null +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.playwright; + +import com.microsoft.playwright.Browser.NewContextOptions; +import org.apache.storm.Config; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the browser context options built from the configuration. They do not need a + * browser so they always run. + */ +class ContextOptionsTest { + + private static final String USER_AGENT = "StormCrawlerTest"; + + private static final String PROXY = "http://proxy.example.com:3128"; + + @Test + void certificatesValidatedByDefault() { + final NewContextOptions options = + HttpProtocol.buildContextOptions(new Config(), USER_AGENT); + Assertions.assertNull(options.proxy); + Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); + Assertions.assertEquals(USER_AGENT, options.userAgent); + } + + @Test + void proxyDoesNotDisableCertificateValidation() { + final Config conf = new Config(); + conf.put("http.proxy", PROXY); + conf.put("http.proxy.username", "user"); + conf.put("http.proxy.password", "secret"); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNotNull(options.proxy); + Assertions.assertEquals(PROXY, options.proxy.server); + Assertions.assertEquals("user", options.proxy.username); + Assertions.assertEquals("secret", options.proxy.password); + Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); + } + + @Test + void ignoreHttpsErrorsWithoutProxy() { + final Config conf = new Config(); + conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNull(options.proxy); + Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); + } + + @Test + void ignoreHttpsErrorsWithProxy() { + final Config conf = new Config(); + conf.put("http.proxy", PROXY); + conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNotNull(options.proxy); + Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); + } + + @Test + void explicitFalseKeepsCertificateValidationWithProxy() { + final Config conf = new Config(); + conf.put("http.proxy", PROXY); + conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, false); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNotNull(options.proxy); + Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); + } +} From 69d035fb79fed22cb7402ea56daff244beb5fe98 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 15 Sep 2026 10:16:30 +0200 Subject: [PATCH 2/2] Use instance methods for building the Playwright context options. --- .../protocol/playwright/HttpProtocol.java | 18 +++++------------- .../playwright/ContextOptionsTest.java | 16 ++++++++-------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index 762b034cb..c24bab169 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -63,13 +63,6 @@ public class HttpProtocol extends AbstractHttpProtocol { public static final String MD_EVALUATIONS = "playwright.evaluations"; public static final String MD_SKIPS = "playwright.skip.resource.types"; - /** - * If true, the browser context accepts any TLS certificate, including self-signed, expired or - * otherwise invalid ones. Applies to every page, navigation and subresource of the context, - * independently of whether a proxy is configured. - */ - public static final String IGNORE_HTTPS_ERRORS_KEY = "playwright.ignore.https.errors"; - private int timeout = 10000; private boolean captureContentOnError = false; @@ -183,7 +176,7 @@ public void configure(final Config conf) { * @param userAgent the user agent string sent by the browser * @return the context options */ - static NewContextOptions buildContextOptions(final Config conf, final String userAgent) { + NewContextOptions buildContextOptions(final Config conf, final String userAgent) { final NewContextOptions options = new Browser.NewContextOptions().setIsMobile(false).setUserAgent(userAgent); @@ -207,12 +200,11 @@ static NewContextOptions buildContextOptions(final Config conf, final String use // certificate validation is independent of the proxy settings final boolean ignoreHTTPSErrors = - ConfUtils.getBoolean(conf, IGNORE_HTTPS_ERRORS_KEY, false); + ConfUtils.getBoolean(conf, "playwright.ignore.https.errors", false); if (ignoreHTTPSErrors) { LOG.warn( - "{} is true: TLS certificates are not validated by the browser, any server" - + " able to answer for a host name is accepted", - IGNORE_HTTPS_ERRORS_KEY); + "playwright.ignore.https.errors is true: TLS certificates are not validated by" + + " the browser, any server able to answer for a host name is accepted"); } options.setIgnoreHTTPSErrors(ignoreHTTPSErrors); @@ -421,7 +413,7 @@ private void storeVerbatimHeaders( } /** Returns a proxy object if required * */ - private static Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { + private Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { if (proxyserver == null) { return null; } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java index ee2294f37..e81a69465 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java @@ -35,7 +35,7 @@ class ContextOptionsTest { @Test void certificatesValidatedByDefault() { final NewContextOptions options = - HttpProtocol.buildContextOptions(new Config(), USER_AGENT); + new HttpProtocol().buildContextOptions(new Config(), USER_AGENT); Assertions.assertNull(options.proxy); Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); Assertions.assertEquals(USER_AGENT, options.userAgent); @@ -47,7 +47,7 @@ void proxyDoesNotDisableCertificateValidation() { conf.put("http.proxy", PROXY); conf.put("http.proxy.username", "user"); conf.put("http.proxy.password", "secret"); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNotNull(options.proxy); Assertions.assertEquals(PROXY, options.proxy.server); Assertions.assertEquals("user", options.proxy.username); @@ -58,8 +58,8 @@ void proxyDoesNotDisableCertificateValidation() { @Test void ignoreHttpsErrorsWithoutProxy() { final Config conf = new Config(); - conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + conf.put("playwright.ignore.https.errors", true); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNull(options.proxy); Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); } @@ -68,8 +68,8 @@ void ignoreHttpsErrorsWithoutProxy() { void ignoreHttpsErrorsWithProxy() { final Config conf = new Config(); conf.put("http.proxy", PROXY); - conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + conf.put("playwright.ignore.https.errors", true); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNotNull(options.proxy); Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); } @@ -78,8 +78,8 @@ void ignoreHttpsErrorsWithProxy() { void explicitFalseKeepsCertificateValidationWithProxy() { final Config conf = new Config(); conf.put("http.proxy", PROXY); - conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, false); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + conf.put("playwright.ignore.https.errors", false); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNotNull(options.proxy); Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); }