From ee2f70d838f8dd5f26b6bf579370bbf76e304472 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 15 Sep 2026 10:06:56 +0200 Subject: [PATCH 1/4] #2094 Decouple ignoring HTTPS errors from the proxy settings in the Playwright protocol and make it configurable via playwright.ignore.https.errors (default false). --- docs/src/main/asciidoc/configuration.adoc | 1 + external/playwright/README.md | 3 + external/playwright/playwright-conf.yaml | 9 ++ .../protocol/playwright/HttpProtocol.java | 77 +++++++++++------ .../playwright/ContextOptionsTest.java | 86 +++++++++++++++++++ 5 files changed, 152 insertions(+), 24 deletions(-) create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 093f60e5a..dea2d4a53 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -609,6 +609,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/playwrigh | playwright.remote.ws | - | Remote WebSocket URL for Playwright (alternative to CDP, e.g. `ws://localhost:3000/`). | playwright.skip.download | false | Skip automatic browser download. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set. | playwright.load.event | load | Page load event to wait for. One of `load`, `domcontentloaded`, `networkidle`. +| playwright.ignore.https.errors | false | If `true`, the browser context accepts any TLS certificate, including self-signed, expired or otherwise invalid ones, for every page, navigation and subresource. The servers are then not authenticated: anyone able to answer for the host name can serve content that the browser renders and whose JavaScript it executes. Independent of `http.proxy`: deployments behind a TLS-intercepting proxy, which previously relied on certificate validation being disabled whenever a proxy was set, now need this key or the proxy CA installed in the browser image. | playwright.skip.resource.types | - | List of resource types aborted during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`). | playwright.evaluations | - | List of JavaScript expressions evaluated after load; each JSON-serialised result is stored in response metadata under the expression itself. | playwright.capture.content.on.error | false | If `true`, also capture `page.content()` for non-2xx responses — useful for SPAs that return a stub then hydrate via JS. diff --git a/external/playwright/README.md b/external/playwright/README.md index 8083d8fd0..91c943c67 100644 --- a/external/playwright/README.md +++ b/external/playwright/README.md @@ -36,12 +36,15 @@ The setting `playwright.skip.download` to `true` in the configuration will assum | `playwright.remote.ws` | _unset_ | If set, connect to a remote Playwright server over WebSocket (e.g. `ws://localhost:3000/`). Mutually exclusive with `playwright.cdp.url`. | | `playwright.skip.download` | `false` | If `true`, sets `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true` so Playwright will not install browsers. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set. | | `playwright.load.event` | `load` | The Playwright `WaitUntilState` to wait for before considering the page ready. Accepts `load`, `domcontentloaded`, or `networkidle`. | +| `playwright.ignore.https.errors` | `false` | If `true`, the browser context accepts any TLS certificate, including self-signed, expired or otherwise invalid ones, for every page, navigation and subresource. The servers are then not authenticated: anyone able to answer for the host name can serve content that the browser renders and whose JavaScript it executes. A warning is logged when enabled. Independent of `http.proxy`, see the compatibility note below. | | `playwright.skip.resource.types` | _empty_ | List of resource types to abort during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`). | | `playwright.evaluations` | _empty_ | List of JavaScript expressions evaluated on the page after load. Each result is JSON-serialized and stored in the response metadata under the expression itself as the key. | | `playwright.capture.content.on.error` | `false` | By default the rendered DOM is only captured when the origin returns a 2xx status. Set to `true` to also capture `page.content()` for non-2xx responses — useful for Single-Page Applications that return a non-2xx stub document and then hydrate the real content via JavaScript. | | `playwright.override.status.on.content` | `false` | When the rendered DOM was captured for a non-2xx response, override the reported HTTP status with `200` so downstream components treat the URL as `FETCHED`. The original origin status is preserved in the response metadata under the key `playwright.origin.status`. No-op unless `playwright.capture.content.on.error` is also `true`. | | `playwright.page.actions.config.file` | _unset_ | Path to a JSON file declaring an ordered chain of `PageAction` implementations applied after `page.navigate()` succeeds and before `page.content()` is captured. Use this to plug site-specific post-navigate behaviour (tab/accordion expansion, cookie-banner dismissal, scroll-to-bottom, custom `evaluate()` calls, ...) into the protocol without subclassing it. The chain runs only when content would otherwise be captured (i.e. on 2xx, or on non-2xx if `playwright.capture.content.on.error` is `true`). | +**Compatibility note:** earlier versions implicitly disabled certificate validation whenever `http.proxy` was set. This is no longer the case. Deployments running behind a TLS-intercepting proxy which relied on that behaviour now need either `playwright.ignore.https.errors: true` or the proxy CA installed in the browser image. + Per-URL metadata triggers: | Metadata key | Effect | diff --git a/external/playwright/playwright-conf.yaml b/external/playwright/playwright-conf.yaml index 79f232af6..355fc63cb 100644 --- a/external/playwright/playwright-conf.yaml +++ b/external/playwright/playwright-conf.yaml @@ -24,6 +24,15 @@ config: # com.microsoft.playwright.options.WaitUntilState # playwright.load.event: "domcontentloaded" + # If true, the browser context accepts any TLS certificate, including + # self-signed, expired or otherwise invalid ones, for every page, navigation + # and subresource. The servers are then not authenticated. This is + # independent of http.proxy: configuring a proxy no longer disables + # certificate validation. Deployments behind a TLS-intercepting proxy need + # either this key or the proxy CA installed in the browser image. + # A warning is logged when enabled. + # playwright.ignore.https.errors: false + # By default the rendered DOM is only captured when the origin returns a 2xx # status. Enable this to also capture page.content() for non-2xx responses # (useful for SPAs that return e.g. a 404 stub and then hydrate via JS). diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index 3e947e917..762b034cb 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -63,6 +63,13 @@ public class HttpProtocol extends AbstractHttpProtocol { public static final String MD_EVALUATIONS = "playwright.evaluations"; public static final String MD_SKIPS = "playwright.skip.resource.types"; + /** + * If true, the browser context accepts any TLS certificate, including self-signed, expired or + * otherwise invalid ones. Applies to every page, navigation and subresource of the context, + * independently of whether a proxy is configured. + */ + public static final String IGNORE_HTTPS_ERRORS_KEY = "playwright.ignore.https.errors"; + private int timeout = 10000; private boolean captureContentOnError = false; @@ -145,29 +152,7 @@ public void configure(final Config conf) { overrideStatusOnContent = ConfUtils.getBoolean(conf, "playwright.override.status.on.content", false); - final String ua = getAgentString(conf); - - NewContextOptions b_c_options = - new Browser.NewContextOptions().setIsMobile(false).setUserAgent(ua); - - // set Accept-Language if configured, as done by the other protocol implementations; - // an explicitly empty value overrides the browser's default with an empty header, - // only an absent key leaves the browser's default untouched - final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language"); - if (acceptLanguage != null) { - b_c_options.setExtraHTTPHeaders(Map.of("Accept-Language", acceptLanguage)); - } - - // global proxy - String proxyServer = ConfUtils.getString(conf, "http.proxy"); - String proxyUser = ConfUtils.getString(conf, "http.proxy.username"); - String proxyPwd = ConfUtils.getString(conf, "http.proxy.password"); - - final Proxy globalProxy = getProxy(proxyServer, proxyUser, proxyPwd); - if (globalProxy != null) { - b_c_options.setProxy(globalProxy); - b_c_options.setIgnoreHTTPSErrors(true); - } + final NewContextOptions b_c_options = buildContextOptions(conf, getAgentString(conf)); context = browser.newContext(b_c_options); @@ -190,6 +175,50 @@ public void configure(final Config conf) { pageActions = PageActions.fromConf(conf); } + /** + * Builds the options of the browser context shared by all fetches, whether the browser is + * launched locally or reached via CDP or a remote Playwright server. + * + * @param conf the configuration + * @param userAgent the user agent string sent by the browser + * @return the context options + */ + static NewContextOptions buildContextOptions(final Config conf, final String userAgent) { + final NewContextOptions options = + new Browser.NewContextOptions().setIsMobile(false).setUserAgent(userAgent); + + // set Accept-Language if configured, as done by the other protocol implementations; + // an explicitly empty value overrides the browser's default with an empty header, + // only an absent key leaves the browser's default untouched + final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language"); + if (acceptLanguage != null) { + options.setExtraHTTPHeaders(Map.of("Accept-Language", acceptLanguage)); + } + + // global proxy + final String proxyServer = ConfUtils.getString(conf, "http.proxy"); + final String proxyUser = ConfUtils.getString(conf, "http.proxy.username"); + final String proxyPwd = ConfUtils.getString(conf, "http.proxy.password"); + + final Proxy globalProxy = getProxy(proxyServer, proxyUser, proxyPwd); + if (globalProxy != null) { + options.setProxy(globalProxy); + } + + // certificate validation is independent of the proxy settings + final boolean ignoreHTTPSErrors = + ConfUtils.getBoolean(conf, IGNORE_HTTPS_ERRORS_KEY, false); + if (ignoreHTTPSErrors) { + LOG.warn( + "{} is true: TLS certificates are not validated by the browser, any server" + + " able to answer for a host name is accepted", + IGNORE_HTTPS_ERRORS_KEY); + } + options.setIgnoreHTTPSErrors(ignoreHTTPSErrors); + + return options; + } + @Override public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception { @@ -392,7 +421,7 @@ private void storeVerbatimHeaders( } /** Returns a proxy object if required * */ - private Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { + private static Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { if (proxyserver == null) { return null; } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java new file mode 100644 index 000000000..ee2294f37 --- /dev/null +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.playwright; + +import com.microsoft.playwright.Browser.NewContextOptions; +import org.apache.storm.Config; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Unit tests for the browser context options built from the configuration. They do not need a + * browser so they always run. + */ +class ContextOptionsTest { + + private static final String USER_AGENT = "StormCrawlerTest"; + + private static final String PROXY = "http://proxy.example.com:3128"; + + @Test + void certificatesValidatedByDefault() { + final NewContextOptions options = + HttpProtocol.buildContextOptions(new Config(), USER_AGENT); + Assertions.assertNull(options.proxy); + Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); + Assertions.assertEquals(USER_AGENT, options.userAgent); + } + + @Test + void proxyDoesNotDisableCertificateValidation() { + final Config conf = new Config(); + conf.put("http.proxy", PROXY); + conf.put("http.proxy.username", "user"); + conf.put("http.proxy.password", "secret"); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNotNull(options.proxy); + Assertions.assertEquals(PROXY, options.proxy.server); + Assertions.assertEquals("user", options.proxy.username); + Assertions.assertEquals("secret", options.proxy.password); + Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); + } + + @Test + void ignoreHttpsErrorsWithoutProxy() { + final Config conf = new Config(); + conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNull(options.proxy); + Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); + } + + @Test + void ignoreHttpsErrorsWithProxy() { + final Config conf = new Config(); + conf.put("http.proxy", PROXY); + conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNotNull(options.proxy); + Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); + } + + @Test + void explicitFalseKeepsCertificateValidationWithProxy() { + final Config conf = new Config(); + conf.put("http.proxy", PROXY); + conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, false); + final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + Assertions.assertNotNull(options.proxy); + Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); + } +} From 69d035fb79fed22cb7402ea56daff244beb5fe98 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 15 Sep 2026 10:16:30 +0200 Subject: [PATCH 2/4] Use instance methods for building the Playwright context options. --- .../protocol/playwright/HttpProtocol.java | 18 +++++------------- .../playwright/ContextOptionsTest.java | 16 ++++++++-------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index 762b034cb..c24bab169 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -63,13 +63,6 @@ public class HttpProtocol extends AbstractHttpProtocol { public static final String MD_EVALUATIONS = "playwright.evaluations"; public static final String MD_SKIPS = "playwright.skip.resource.types"; - /** - * If true, the browser context accepts any TLS certificate, including self-signed, expired or - * otherwise invalid ones. Applies to every page, navigation and subresource of the context, - * independently of whether a proxy is configured. - */ - public static final String IGNORE_HTTPS_ERRORS_KEY = "playwright.ignore.https.errors"; - private int timeout = 10000; private boolean captureContentOnError = false; @@ -183,7 +176,7 @@ public void configure(final Config conf) { * @param userAgent the user agent string sent by the browser * @return the context options */ - static NewContextOptions buildContextOptions(final Config conf, final String userAgent) { + NewContextOptions buildContextOptions(final Config conf, final String userAgent) { final NewContextOptions options = new Browser.NewContextOptions().setIsMobile(false).setUserAgent(userAgent); @@ -207,12 +200,11 @@ static NewContextOptions buildContextOptions(final Config conf, final String use // certificate validation is independent of the proxy settings final boolean ignoreHTTPSErrors = - ConfUtils.getBoolean(conf, IGNORE_HTTPS_ERRORS_KEY, false); + ConfUtils.getBoolean(conf, "playwright.ignore.https.errors", false); if (ignoreHTTPSErrors) { LOG.warn( - "{} is true: TLS certificates are not validated by the browser, any server" - + " able to answer for a host name is accepted", - IGNORE_HTTPS_ERRORS_KEY); + "playwright.ignore.https.errors is true: TLS certificates are not validated by" + + " the browser, any server able to answer for a host name is accepted"); } options.setIgnoreHTTPSErrors(ignoreHTTPSErrors); @@ -421,7 +413,7 @@ private void storeVerbatimHeaders( } /** Returns a proxy object if required * */ - private static Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { + private Proxy getProxy(String proxyserver, String proxyuser, String proxypwd) { if (proxyserver == null) { return null; } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java index ee2294f37..e81a69465 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/ContextOptionsTest.java @@ -35,7 +35,7 @@ class ContextOptionsTest { @Test void certificatesValidatedByDefault() { final NewContextOptions options = - HttpProtocol.buildContextOptions(new Config(), USER_AGENT); + new HttpProtocol().buildContextOptions(new Config(), USER_AGENT); Assertions.assertNull(options.proxy); Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); Assertions.assertEquals(USER_AGENT, options.userAgent); @@ -47,7 +47,7 @@ void proxyDoesNotDisableCertificateValidation() { conf.put("http.proxy", PROXY); conf.put("http.proxy.username", "user"); conf.put("http.proxy.password", "secret"); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNotNull(options.proxy); Assertions.assertEquals(PROXY, options.proxy.server); Assertions.assertEquals("user", options.proxy.username); @@ -58,8 +58,8 @@ void proxyDoesNotDisableCertificateValidation() { @Test void ignoreHttpsErrorsWithoutProxy() { final Config conf = new Config(); - conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + conf.put("playwright.ignore.https.errors", true); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNull(options.proxy); Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); } @@ -68,8 +68,8 @@ void ignoreHttpsErrorsWithoutProxy() { void ignoreHttpsErrorsWithProxy() { final Config conf = new Config(); conf.put("http.proxy", PROXY); - conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, true); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + conf.put("playwright.ignore.https.errors", true); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNotNull(options.proxy); Assertions.assertEquals(Boolean.TRUE, options.ignoreHTTPSErrors); } @@ -78,8 +78,8 @@ void ignoreHttpsErrorsWithProxy() { void explicitFalseKeepsCertificateValidationWithProxy() { final Config conf = new Config(); conf.put("http.proxy", PROXY); - conf.put(HttpProtocol.IGNORE_HTTPS_ERRORS_KEY, false); - final NewContextOptions options = HttpProtocol.buildContextOptions(conf, USER_AGENT); + conf.put("playwright.ignore.https.errors", false); + final NewContextOptions options = new HttpProtocol().buildContextOptions(conf, USER_AGENT); Assertions.assertNotNull(options.proxy); Assertions.assertEquals(Boolean.FALSE, options.ignoreHTTPSErrors); } From ff9425926461e809b90fc4e42c318882ca308fd3 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Tue, 15 Sep 2026 12:56:28 +0200 Subject: [PATCH 3/4] #2095 Apply the IP address filter to the requests made by the Playwright browser. --- core/src/main/resources/crawler-default.yaml | 3 + docs/src/main/asciidoc/configuration.adoc | 10 ++- .../protocol/playwright/HttpProtocol.java | 73 ++++++++++++++++++ .../protocol/playwright/IPFilterTest.java | 74 +++++++++++++++++++ 4 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 1f3a0be2a..8230af38d 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -220,6 +220,9 @@ config: # Fetches through a proxy are not filtered: the proxy resolves and connects # to the target, so only its address is known here and a proxy on a private # address is fine. The proxy's own egress rules decide what it may reach. + # Applies to the okhttp and playwright protocols. Playwright checks each + # request of the browser (page, redirects, subresources) against an address + # resolved separately from the browser; WebSockets are not checked. http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal,anylocal,multicast,100.64.0.0/10,0.0.0.0/8,fc00::/7,::/128" # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden): diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index dea2d4a53..05afe1f71 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -260,8 +260,8 @@ header. | http.accept.language | en-us,en-gb,en;q=0.7,*;q=0.3 | HTTP Accept-Language header. | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP. -| http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. -| http.filter.ipaddress.exclude | localhost, sitelocal, linklocal, anylocal, multicast, 100.64.0.0/10, 0.0.0.0/8, fc00::/7, ::/128 | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. Set to `""` to crawl private address space. See <>. +| http.filter.ipaddress.include | - | (OkHttp and Playwright) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <>. +| http.filter.ipaddress.exclude | localhost, sitelocal, linklocal, anylocal, multicast, 100.64.0.0/10, 0.0.0.0/8, fc00::/7, ::/128 | (OkHttp and Playwright) Comma-separated list (or YAML list) of blocked IP ranges. Set to `""` to crawl private address space. See <>. | http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true. | http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. Credentials are then withheld unless `http.credentials.allow.insecure` is set to true. | http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true or `http.verify.hostnames` is false, https:// urls as well. Redirect hops to another origin (scheme, host or port) never carry them. @@ -286,6 +286,12 @@ Fetches through a proxy are not filtered. The proxy resolves the target host and the protocol only sees the address of the proxy, which may well be private. What the fetcher can reach through a proxy is then decided by the egress rules of the proxy. +The Playwright protocol (`org.apache.stormcrawler.protocol.playwright.HttpProtocol`) applies the same +rules to every request the browser makes, including redirects and subresources, and aborts the +rejected ones. It cannot see the address the browser connects to: the host is resolved separately, +so short-lived DNS records or a remote browser using another resolver can lead to a different +address. WebSocket connections are not checked. + Two properties control the behaviour: * `http.filter.ipaddress.include` — defines the allowed IP ranges. If empty, all addresses are diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index c24bab169..d93366a97 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -30,7 +30,10 @@ import com.microsoft.playwright.options.HttpHeader; import com.microsoft.playwright.options.Proxy; import com.microsoft.playwright.options.WaitUntilState; +import java.net.InetAddress; import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -46,6 +49,7 @@ import org.apache.stormcrawler.Metadata; import org.apache.stormcrawler.persistence.Status; import org.apache.stormcrawler.protocol.AbstractHttpProtocol; +import org.apache.stormcrawler.protocol.IPFilterRules; import org.apache.stormcrawler.protocol.Protocol; import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.ConfUtils; @@ -84,6 +88,8 @@ public class HttpProtocol extends AbstractHttpProtocol { private PageActions pageActions = PageActions.emptyPageActions; + private IPFilterRules ipFilterRules; + @Override public void configure(final Config conf) { super.configure(conf); @@ -166,6 +172,68 @@ public void configure(final Config conf) { // optional chain of page actions applied after navigate, before content capture pageActions = PageActions.fromConf(conf); + + configureIPFilter(conf); + } + + /** + * Sets up the filtering of the requests made by the browser against http.filter.ipaddress.*. As + * with the OkHttp protocol, fetches through a proxy are not filtered. + */ + void configureIPFilter(final Config conf) { + final IPFilterRules rules = new IPFilterRules(conf); + if (rules.isEmpty()) { + ipFilterRules = null; + } else if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.proxy"))) { + ipFilterRules = null; + LOG.info( + "http.filter.ipaddress.* do not apply to fetches through a proxy, the proxy" + + " resolves the target host and its own egress rules decide which" + + " addresses are reached"); + } else { + ipFilterRules = rules; + } + } + + /** + * Checks the host of a request made by the browser against the IP filter rules. The host is + * resolved here, separately from the browser, so the address the browser connects to can + * differ, e.g. with short-lived DNS records or a remote browser using another resolver. + * + * @return false if any address of the host is rejected or the host cannot be resolved + */ + boolean isAllowedAddress(final String url) { + if (ipFilterRules == null) { + return true; + } + final URI uri; + try { + uri = new URI(url); + } catch (URISyntaxException e) { + return false; + } + final String scheme = uri.getScheme(); + if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) { + // data:, blob: and the like do not open a connection + return true; + } + String host = uri.getHost(); + if (host == null) { + return false; + } + if (host.startsWith("[") && host.endsWith("]")) { + host = host.substring(1, host.length() - 1); + } + try { + for (InetAddress address : InetAddress.getAllByName(host)) { + if (!ipFilterRules.accept(address)) { + return false; + } + } + return true; + } catch (UnknownHostException e) { + return false; + } } /** @@ -272,6 +340,11 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except } else if (resourceTypesToSkip.contains( route.request().resourceType())) { route.abort(); + } else if (!isAllowedAddress(route.request().url())) { + LOG.warn( + "Blocked request to forbidden IP address: {}", + route.request().url()); + route.abort("addressunreachable"); } else { route.resume(); } diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java new file mode 100644 index 000000000..cdb16cdb6 --- /dev/null +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol.playwright; + +import org.apache.storm.Config; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** Checks the IP filter applied to the requests of the browser. Does not need a browser. */ +class IPFilterTest { + + private HttpProtocol protocol(final Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configureIPFilter(conf); + return protocol; + } + + private Config excludeLocal() { + final Config conf = new Config(); + conf.put("http.filter.ipaddress.exclude", "localhost,sitelocal"); + return conf; + } + + @Test + void everythingAllowedWithoutRules() { + final HttpProtocol protocol = protocol(new Config()); + Assertions.assertTrue(protocol.isAllowedAddress("http://127.0.0.1/")); + } + + @Test + void excludedAddressesAreRejected() { + final HttpProtocol protocol = protocol(excludeLocal()); + Assertions.assertFalse(protocol.isAllowedAddress("http://127.0.0.1:8080/page.html")); + Assertions.assertFalse(protocol.isAllowedAddress("https://localhost/")); + Assertions.assertFalse(protocol.isAllowedAddress("http://[::1]/")); + Assertions.assertFalse(protocol.isAllowedAddress("http://192.168.1.1/")); + Assertions.assertTrue(protocol.isAllowedAddress("http://8.8.8.8/")); + } + + @Test + void urlsWithoutConnectionAreAllowed() { + final HttpProtocol protocol = protocol(excludeLocal()); + Assertions.assertTrue(protocol.isAllowedAddress("data:text/plain,hello")); + } + + @Test + void unresolvableHostsAreRejected() { + final HttpProtocol protocol = protocol(excludeLocal()); + Assertions.assertFalse(protocol.isAllowedAddress("http://does-not-exist.invalid/")); + } + + @Test + void notAppliedThroughProxy() { + final Config conf = excludeLocal(); + conf.put("http.proxy", "http://proxy.example.com:3128"); + final HttpProtocol protocol = protocol(conf); + Assertions.assertTrue(protocol.isAllowedAddress("http://127.0.0.1/")); + } +} From e1c88f6cfd7f708a0f970390e880e6e228a74208 Mon Sep 17 00:00:00 2001 From: Richard Zowalla Date: Wed, 16 Sep 2026 20:08:34 +0200 Subject: [PATCH 4/4] Parse browser URLs leniently, cover popups and block service workers in the Playwright IP filter. --- core/src/main/resources/crawler-default.yaml | 8 +-- docs/src/main/asciidoc/configuration.adoc | 14 +++-- .../protocol/playwright/HttpProtocol.java | 59 +++++++++++-------- .../protocol/playwright/IPFilterTest.java | 19 ++++++ 4 files changed, 64 insertions(+), 36 deletions(-) diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 8230af38d..bc6692e9d 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -194,7 +194,7 @@ config: # like it does when redirects are not followed at all. http.allow.redirects.max: 5 - # IP address filtering (okhttp protocol only). Optionally limit or block + # IP address filtering (okhttp and playwright protocols). Optionally limit or block # connections to IP address ranges once the host name has been resolved. This # prevents information leakage to a public index when a DNS entry points to a # private or loopback address. Rules can be given as a comma-separated string @@ -220,9 +220,9 @@ config: # Fetches through a proxy are not filtered: the proxy resolves and connects # to the target, so only its address is known here and a proxy on a private # address is fine. The proxy's own egress rules decide what it may reach. - # Applies to the okhttp and playwright protocols. Playwright checks each - # request of the browser (page, redirects, subresources) against an address - # resolved separately from the browser; WebSockets are not checked. + # Playwright checks the page, its subresources and popups against an address + # resolved separately from the browser and blocks service workers; redirect + # hops and WebSockets are not checked. http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal,anylocal,multicast,100.64.0.0/10,0.0.0.0/8,fc00::/7,::/128" # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden): diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 05afe1f71..614f306cb 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -287,10 +287,11 @@ the protocol only sees the address of the proxy, which may well be private. What reach through a proxy is then decided by the egress rules of the proxy. The Playwright protocol (`org.apache.stormcrawler.protocol.playwright.HttpProtocol`) applies the same -rules to every request the browser makes, including redirects and subresources, and aborts the -rejected ones. It cannot see the address the browser connects to: the host is resolved separately, -so short-lived DNS records or a remote browser using another resolver can lead to a different -address. WebSocket connections are not checked. +rules to the requests of the page, its subresources and popups, and aborts the rejected ones. +Service workers are blocked while the filter is active, as their requests cannot be intercepted. +Redirect hops and WebSocket connections are not checked. The host is resolved separately from the +browser, so short-lived DNS records or a remote browser using another resolver can lead to a +different address. Two properties control the behaviour: @@ -320,8 +321,9 @@ http.filter.ipaddress.include: "10.0.0.0/8" grade NAT (`100.64.0.0/10`) and IPv6 unique local addresses (`fd00::/8`) have no keyword and are added as CIDR blocks when they are also to be blocked. -When a connection to a blocked address is attempted, the fetch fails with an `IOException` and a -warning is logged. If neither property is set, no IP filtering is performed. +When a connection to a blocked address is attempted, a warning is logged. With OkHttp the fetch +fails with an `IOException`. With Playwright the request is aborted: a blocked page fails the fetch, +a blocked subresource is missing from the rendered page. If neither property is set, no IP filtering is performed. NOTE: When a proxy is configured, the connection is established to the proxy and the filter sees the proxy's IP address rather than the target host's resolved address. IP filtering is therefore diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index d93366a97..929d5c89c 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -29,10 +29,10 @@ import com.microsoft.playwright.Tracing; import com.microsoft.playwright.options.HttpHeader; import com.microsoft.playwright.options.Proxy; +import com.microsoft.playwright.options.ServiceWorkerPolicy; import com.microsoft.playwright.options.WaitUntilState; import java.net.InetAddress; import java.net.URI; -import java.net.URISyntaxException; import java.net.UnknownHostException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import okhttp3.HttpUrl; import org.apache.commons.lang3.StringUtils; import org.apache.storm.Config; import org.apache.storm.utils.MutableInt; @@ -151,12 +152,30 @@ public void configure(final Config conf) { overrideStatusOnContent = ConfUtils.getBoolean(conf, "playwright.override.status.on.content", false); + configureIPFilter(conf); + final NewContextOptions b_c_options = buildContextOptions(conf, getAgentString(conf)); context = browser.newContext(b_c_options); context.setDefaultTimeout(timeout); + // on the context so that popups are covered too, the page route falls back to it + if (ipFilterRules != null) { + context.route( + lambdaUrl -> true, + route -> { + if (isAllowedAddress(route.request().url())) { + route.resume(); + } else { + LOG.warn( + "Blocked request to forbidden IP address: {}", + route.request().url()); + route.abort("addressunreachable"); + } + }); + } + // list of resource types to skip // document, stylesheet, image, media, font, script, texttrack, xhr, fetch, // eventsource, websocket, manifest, other @@ -172,8 +191,6 @@ public void configure(final Config conf) { // optional chain of page actions applied after navigate, before content capture pageActions = PageActions.fromConf(conf); - - configureIPFilter(conf); } /** @@ -206,26 +223,15 @@ boolean isAllowedAddress(final String url) { if (ipFilterRules == null) { return true; } - final URI uri; - try { - uri = new URI(url); - } catch (URISyntaxException e) { - return false; - } - final String scheme = uri.getScheme(); - if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) { + // lenient like the browser, e.g. with '|' in the path or '_' in the host name + final HttpUrl parsed = HttpUrl.parse(url); + if (parsed == null) { // data:, blob: and the like do not open a connection - return true; - } - String host = uri.getHost(); - if (host == null) { - return false; - } - if (host.startsWith("[") && host.endsWith("]")) { - host = host.substring(1, host.length() - 1); + return !StringUtils.startsWithIgnoreCase(url, "http:") + && !StringUtils.startsWithIgnoreCase(url, "https:"); } try { - for (InetAddress address : InetAddress.getAllByName(host)) { + for (InetAddress address : InetAddress.getAllByName(parsed.host())) { if (!ipFilterRules.accept(address)) { return false; } @@ -276,6 +282,11 @@ NewContextOptions buildContextOptions(final Config conf, final String userAgent) } options.setIgnoreHTTPSErrors(ignoreHTTPSErrors); + // requests of a service worker are not routed, so they would bypass the IP filter + if (ipFilterRules != null) { + options.setServiceWorkers(ServiceWorkerPolicy.BLOCK); + } + return options; } @@ -340,13 +351,9 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except } else if (resourceTypesToSkip.contains( route.request().resourceType())) { route.abort(); - } else if (!isAllowedAddress(route.request().url())) { - LOG.warn( - "Blocked request to forbidden IP address: {}", - route.request().url()); - route.abort("addressunreachable"); } else { - route.resume(); + // lets the IP filter on the context check the request + route.fallback(); } }); diff --git a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java index cdb16cdb6..e6b6a6338 100644 --- a/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java +++ b/external/playwright/src/test/java/org/apache/stormcrawler/protocol/playwright/IPFilterTest.java @@ -17,6 +17,7 @@ package org.apache.stormcrawler.protocol.playwright; +import com.microsoft.playwright.options.ServiceWorkerPolicy; import org.apache.storm.Config; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -58,6 +59,24 @@ void urlsWithoutConnectionAreAllowed() { Assertions.assertTrue(protocol.isAllowedAddress("data:text/plain,hello")); } + @Test + void urlsChromiumSendsAsIsAreParsed() { + final HttpProtocol protocol = protocol(excludeLocal()); + Assertions.assertFalse(protocol.isAllowedAddress("http://127.0.0.1/a|b[c]?q={x}^")); + Assertions.assertTrue(protocol.isAllowedAddress("http://8.8.8.8/a|b[c]?q={x}^")); + } + + @Test + void serviceWorkersBlockedOnlyWithRules() { + Assertions.assertEquals( + ServiceWorkerPolicy.BLOCK, + protocol(excludeLocal()) + .buildContextOptions(excludeLocal(), "test") + .serviceWorkers); + Assertions.assertNull( + protocol(new Config()).buildContextOptions(new Config(), "test").serviceWorkers); + } + @Test void unresolvableHostsAreRejected() { final HttpProtocol protocol = protocol(excludeLocal());