#2095 Apply the IP address filter to the requests made by the Playwright browser

core/src/main/resources/crawler-default.yaml

@@ -194,7 +194,7 @@ config:
   # like it does when redirects are not followed at all.
   http.allow.redirects.max: 5
 
-  # IP address filtering (okhttp protocol only). Optionally limit or block
+  # IP address filtering (okhttp and playwright protocols). Optionally limit or block
   # connections to IP address ranges once the host name has been resolved. This
   # prevents information leakage to a public index when a DNS entry points to a
   # private or loopback address. Rules can be given as a comma-separated string
@@ -220,6 +220,9 @@ config:
   # Fetches through a proxy are not filtered: the proxy resolves and connects
   # to the target, so only its address is known here and a proxy on a private
   # address is fine. The proxy's own egress rules decide what it may reach.
+  # Playwright checks the page, its subresources and popups against an address
+  # resolved separately from the browser and blocks service workers; redirect
+  # hops and WebSockets are not checked.
   http.filter.ipaddress.exclude: "localhost,sitelocal,linklocal,anylocal,multicast,100.64.0.0/10,0.0.0.0/8,fc00::/7,::/128"
 
   # Allow all if robots.txt cannot be parsed due to code 403 (Forbidden):

docs/src/main/asciidoc/configuration.adoc

@@ -260,8 +260,8 @@ header.
 | http.accept.language | en-us,en-gb,en;q=0.7,*;q=0.3 | HTTP Accept-Language
 header.
 | http.content.partial.as.trimmed | false | Accepts partially fetched content in OKHTTP.
-| http.filter.ipaddress.include | - | (OkHttp only) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <<IP Address Filtering>>.
-| http.filter.ipaddress.exclude | localhost, sitelocal, linklocal, anylocal, multicast, 100.64.0.0/10, 0.0.0.0/8, fc00::/7, ::/128 | (OkHttp only) Comma-separated list (or YAML list) of blocked IP ranges. Set to `""` to crawl private address space. See <<IP Address Filtering>>.
+| http.filter.ipaddress.include | - | (OkHttp and Playwright) Comma-separated list (or YAML list) of allowed IP ranges. If empty, all addresses are allowed unless excluded. See <<IP Address Filtering>>.
+| http.filter.ipaddress.exclude | localhost, sitelocal, linklocal, anylocal, multicast, 100.64.0.0/10, 0.0.0.0/8, fc00::/7, ::/128 | (OkHttp and Playwright) Comma-separated list (or YAML list) of blocked IP ranges. Set to `""` to crawl private address space. See <<IP Address Filtering>>.
 | http.trust.everything | false | (OkHttp only) If true, accept any TLS certificate, including self-signed, expired or otherwise invalid ones. The servers are then not authenticated: anyone able to answer for the host name receives everything sent to them. Credentials (basic auth, credential headers, cookies) are withheld from cleartext http:// requests and from https:// requests on such connections unless `http.credentials.allow.insecure` is set to true.
 | http.verify.hostnames | true | (OkHttp only) If true, check that the certificate presented by the server matches the host name contacted. Independent of `http.trust.everything`: a valid certificate for a different name is accepted when this is false. Credentials are then withheld unless `http.credentials.allow.insecure` is set to true.
 | http.credentials.allow.insecure | false | (OkHttp only) If true, send credentials (basic auth, credential headers, cookies) even on connections which do not authenticate the server: cleartext http:// urls and, when `http.trust.everything` is true or `http.verify.hostnames` is false, https:// urls as well. Redirect hops to another origin (scheme, host or port) never carry them.
@@ -286,6 +286,13 @@ Fetches through a proxy are not filtered. The proxy resolves the target host and
 the protocol only sees the address of the proxy, which may well be private. What the fetcher can
 reach through a proxy is then decided by the egress rules of the proxy.
 
+The Playwright protocol (`org.apache.stormcrawler.protocol.playwright.HttpProtocol`) applies the same
+rules to the requests of the page, its subresources and popups, and aborts the rejected ones.
+Service workers are blocked while the filter is active, as their requests cannot be intercepted.
+Redirect hops and WebSocket connections are not checked. The host is resolved separately from the
+browser, so short-lived DNS records or a remote browser using another resolver can lead to a
+different address.
+
 Two properties control the behaviour:
 
 * `http.filter.ipaddress.include` — defines the allowed IP ranges. If empty, all addresses are
@@ -314,8 +321,9 @@ http.filter.ipaddress.include: "10.0.0.0/8"
 grade NAT (`100.64.0.0/10`) and IPv6 unique local addresses (`fd00::/8`) have no keyword and are added
 as CIDR blocks when they are also to be blocked.
 
-When a connection to a blocked address is attempted, the fetch fails with an `IOException` and a
-warning is logged. If neither property is set, no IP filtering is performed.
+When a connection to a blocked address is attempted, a warning is logged. With OkHttp the fetch
+fails with an `IOException`. With Playwright the request is aborted: a blocked page fails the fetch,
+a blocked subresource is missing from the rendered page. If neither property is set, no IP filtering is performed.
 
 NOTE: When a proxy is configured, the connection is established to the proxy and the filter sees the
 proxy's IP address rather than the target host's resolved address. IP filtering is therefore
@@ -609,6 +617,7 @@ See the link:https://github.com/apache/stormcrawler/tree/main/external/playwrigh
 | playwright.remote.ws | - | Remote WebSocket URL for Playwright (alternative to CDP, e.g. `ws://localhost:3000/`).
 | playwright.skip.download | false | Skip automatic browser download. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set.
 | playwright.load.event | load | Page load event to wait for. One of `load`, `domcontentloaded`, `networkidle`.
+| playwright.ignore.https.errors | false | If `true`, the browser context accepts any TLS certificate, including self-signed, expired or otherwise invalid ones, for every page, navigation and subresource. The servers are then not authenticated: anyone able to answer for the host name can serve content that the browser renders and whose JavaScript it executes. Independent of `http.proxy`: deployments behind a TLS-intercepting proxy, which previously relied on certificate validation being disabled whenever a proxy was set, now need this key or the proxy CA installed in the browser image.
 | playwright.skip.resource.types | - | List of resource types aborted during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`).
 | playwright.evaluations | - | List of JavaScript expressions evaluated after load; each JSON-serialised result is stored in response metadata under the expression itself.
 | playwright.capture.content.on.error | false | If `true`, also capture `page.content()` for non-2xx responses — useful for SPAs that return a stub then hydrate via JS.

external/playwright/README.md

@@ -36,12 +36,15 @@ The setting `playwright.skip.download` to `true` in the configuration will assum
 | `playwright.remote.ws` | _unset_ | If set, connect to a remote Playwright server over WebSocket (e.g. `ws://localhost:3000/`). Mutually exclusive with `playwright.cdp.url`. |
 | `playwright.skip.download` | `false` | If `true`, sets `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=true` so Playwright will not install browsers. Implicitly forced to `true` when `playwright.cdp.url` or `playwright.remote.ws` is set. |
 | `playwright.load.event` | `load` | The Playwright `WaitUntilState` to wait for before considering the page ready. Accepts `load`, `domcontentloaded`, or `networkidle`. |
+| `playwright.ignore.https.errors` | `false` | If `true`, the browser context accepts any TLS certificate, including self-signed, expired or otherwise invalid ones, for every page, navigation and subresource. The servers are then not authenticated: anyone able to answer for the host name can serve content that the browser renders and whose JavaScript it executes. A warning is logged when enabled. Independent of `http.proxy`, see the compatibility note below. |
 | `playwright.skip.resource.types` | _empty_ | List of resource types to abort during navigation (`document`, `stylesheet`, `image`, `media`, `font`, `script`, `texttrack`, `xhr`, `fetch`, `eventsource`, `websocket`, `manifest`, `other`). |
 | `playwright.evaluations` | _empty_ | List of JavaScript expressions evaluated on the page after load. Each result is JSON-serialized and stored in the response metadata under the expression itself as the key. |
 | `playwright.capture.content.on.error` | `false` | By default the rendered DOM is only captured when the origin returns a 2xx status. Set to `true` to also capture `page.content()` for non-2xx responses — useful for Single-Page Applications that return a non-2xx stub document and then hydrate the real content via JavaScript. |
 | `playwright.override.status.on.content` | `false` | When the rendered DOM was captured for a non-2xx response, override the reported HTTP status with `200` so downstream components treat the URL as `FETCHED`. The original origin status is preserved in the response metadata under the key `playwright.origin.status`. No-op unless `playwright.capture.content.on.error` is also `true`. |
 | `playwright.page.actions.config.file` | _unset_ | Path to a JSON file declaring an ordered chain of `PageAction` implementations applied after `page.navigate()` succeeds and before `page.content()` is captured. Use this to plug site-specific post-navigate behaviour (tab/accordion expansion, cookie-banner dismissal, scroll-to-bottom, custom `evaluate()` calls, ...) into the protocol without subclassing it. The chain runs only when content would otherwise be captured (i.e. on 2xx, or on non-2xx if `playwright.capture.content.on.error` is `true`). |
 
+**Compatibility note:** earlier versions implicitly disabled certificate validation whenever `http.proxy` was set. This is no longer the case. Deployments running behind a TLS-intercepting proxy which relied on that behaviour now need either `playwright.ignore.https.errors: true` or the proxy CA installed in the browser image.
+
 Per-URL metadata triggers:
 
 | Metadata key | Effect |

external/playwright/playwright-conf.yaml

@@ -24,6 +24,15 @@ config:
   # com.microsoft.playwright.options.WaitUntilState
   # playwright.load.event: "domcontentloaded"
 
+  # If true, the browser context accepts any TLS certificate, including
+  # self-signed, expired or otherwise invalid ones, for every page, navigation
+  # and subresource. The servers are then not authenticated. This is
+  # independent of http.proxy: configuring a proxy no longer disables
+  # certificate validation. Deployments behind a TLS-intercepting proxy need
+  # either this key or the proxy CA installed in the browser image.
+  # A warning is logged when enabled.
+  # playwright.ignore.https.errors: false
+
   # By default the rendered DOM is only captured when the origin returns a 2xx
   # status. Enable this to also capture page.content() for non-2xx responses
   # (useful for SPAs that return e.g. a 404 stub and then hydrate via JS).

external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java

@@ -29,8 +29,11 @@
 import com.microsoft.playwright.Tracing;
 import com.microsoft.playwright.options.HttpHeader;
 import com.microsoft.playwright.options.Proxy;
+import com.microsoft.playwright.options.ServiceWorkerPolicy;
 import com.microsoft.playwright.options.WaitUntilState;
+import java.net.InetAddress;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -40,12 +43,14 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import okhttp3.HttpUrl;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.storm.Config;
 import org.apache.storm.utils.MutableInt;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.persistence.Status;
 import org.apache.stormcrawler.protocol.AbstractHttpProtocol;
+import org.apache.stormcrawler.protocol.IPFilterRules;
 import org.apache.stormcrawler.protocol.Protocol;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.ConfUtils;
@@ -84,6 +89,8 @@ public class HttpProtocol extends AbstractHttpProtocol {
 
     private PageActions pageActions = PageActions.emptyPageActions;
 
+    private IPFilterRules ipFilterRules;
+
     @Override
     public void configure(final Config conf) {
         super.configure(conf);
@@ -145,34 +152,30 @@ public void configure(final Config conf) {
         overrideStatusOnContent =
                 ConfUtils.getBoolean(conf, "playwright.override.status.on.content", false);
 
-        final String ua = getAgentString(conf);
-
-        NewContextOptions b_c_options =
-                new Browser.NewContextOptions().setIsMobile(false).setUserAgent(ua);
+        configureIPFilter(conf);
 
-        // set Accept-Language if configured, as done by the other protocol implementations;
-        // an explicitly empty value overrides the browser's default with an empty header,
-        // only an absent key leaves the browser's default untouched
-        final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language");
-        if (acceptLanguage != null) {
-            b_c_options.setExtraHTTPHeaders(Map.of("Accept-Language", acceptLanguage));
-        }
-
-        // global proxy
-        String proxyServer = ConfUtils.getString(conf, "http.proxy");
-        String proxyUser = ConfUtils.getString(conf, "http.proxy.username");
-        String proxyPwd = ConfUtils.getString(conf, "http.proxy.password");
-
-        final Proxy globalProxy = getProxy(proxyServer, proxyUser, proxyPwd);
-        if (globalProxy != null) {
-            b_c_options.setProxy(globalProxy);
-            b_c_options.setIgnoreHTTPSErrors(true);
-        }
+        final NewContextOptions b_c_options = buildContextOptions(conf, getAgentString(conf));
 
         context = browser.newContext(b_c_options);
 
         context.setDefaultTimeout(timeout);
 
+        // on the context so that popups are covered too, the page route falls back to it
+        if (ipFilterRules != null) {
+            context.route(
+                    lambdaUrl -> true,
+                    route -> {
+                        if (isAllowedAddress(route.request().url())) {
+                            route.resume();
+                        } else {
+                            LOG.warn(
+                                    "Blocked request to forbidden IP address: {}",
+                                    route.request().url());
+                            route.abort("addressunreachable");
+                        }
+                    });
+        }
+
         // list of resource types to skip
         // document, stylesheet, image, media, font, script, texttrack, xhr, fetch,
         // eventsource, websocket, manifest, other
@@ -190,6 +193,103 @@ public void configure(final Config conf) {
         pageActions = PageActions.fromConf(conf);
     }
 
+    /**
+     * Sets up the filtering of the requests made by the browser against http.filter.ipaddress.*. As
+     * with the OkHttp protocol, fetches through a proxy are not filtered.
+     */
+    void configureIPFilter(final Config conf) {
+        final IPFilterRules rules = new IPFilterRules(conf);
+        if (rules.isEmpty()) {
+            ipFilterRules = null;
+        } else if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.proxy"))) {
+            ipFilterRules = null;
+            LOG.info(
+                    "http.filter.ipaddress.* do not apply to fetches through a proxy, the proxy"
+                            + " resolves the target host and its own egress rules decide which"
+                            + " addresses are reached");
+        } else {
+            ipFilterRules = rules;
+        }
+    }
+
+    /**
+     * Checks the host of a request made by the browser against the IP filter rules. The host is
+     * resolved here, separately from the browser, so the address the browser connects to can
+     * differ, e.g. with short-lived DNS records or a remote browser using another resolver.
+     *
+     * @return false if any address of the host is rejected or the host cannot be resolved
+     */
+    boolean isAllowedAddress(final String url) {
+        if (ipFilterRules == null) {
+            return true;
+        }
+        // lenient like the browser, e.g. with '|' in the path or '_' in the host name
+        final HttpUrl parsed = HttpUrl.parse(url);
+        if (parsed == null) {
+            // data:, blob: and the like do not open a connection
+            return !StringUtils.startsWithIgnoreCase(url, "http:")
+                    && !StringUtils.startsWithIgnoreCase(url, "https:");
+        }
+        try {
+            for (InetAddress address : InetAddress.getAllByName(parsed.host())) {
+                if (!ipFilterRules.accept(address)) {
+                    return false;
+                }
+            }
+            return true;
+        } catch (UnknownHostException e) {
+            return false;
+        }
+    }
+
+    /**
+     * Builds the options of the browser context shared by all fetches, whether the browser is
+     * launched locally or reached via CDP or a remote Playwright server.
+     *
+     * @param conf the configuration
+     * @param userAgent the user agent string sent by the browser
+     * @return the context options
+     */
+    NewContextOptions buildContextOptions(final Config conf, final String userAgent) {
+        final NewContextOptions options =
+                new Browser.NewContextOptions().setIsMobile(false).setUserAgent(userAgent);
+
+        // set Accept-Language if configured, as done by the other protocol implementations;
+        // an explicitly empty value overrides the browser's default with an empty header,
+        // only an absent key leaves the browser's default untouched
+        final String acceptLanguage = ConfUtils.getString(conf, "http.accept.language");
+        if (acceptLanguage != null) {
+            options.setExtraHTTPHeaders(Map.of("Accept-Language", acceptLanguage));
+        }
+
+        // global proxy
+        final String proxyServer = ConfUtils.getString(conf, "http.proxy");
+        final String proxyUser = ConfUtils.getString(conf, "http.proxy.username");
+        final String proxyPwd = ConfUtils.getString(conf, "http.proxy.password");
+
+        final Proxy globalProxy = getProxy(proxyServer, proxyUser, proxyPwd);
+        if (globalProxy != null) {
+            options.setProxy(globalProxy);
+        }
+
+        // certificate validation is independent of the proxy settings
+        final boolean ignoreHTTPSErrors =
+                ConfUtils.getBoolean(conf, "playwright.ignore.https.errors", false);
+        if (ignoreHTTPSErrors) {
+            LOG.warn(
+                    "playwright.ignore.https.errors is true: TLS certificates are not validated by"
+                            + " the browser, any server able to answer for a host name is accepted");
+        }
+        options.setIgnoreHTTPSErrors(ignoreHTTPSErrors);
+
+        // requests of a service worker are not routed, so they would bypass the IP filter
+        if (ipFilterRules != null) {
+            options.setServiceWorkers(ServiceWorkerPolicy.BLOCK);
+        }
+
+        return options;
+    }
+
     @Override
     public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Exception {
 
@@ -252,7 +352,8 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except
                                         route.request().resourceType())) {
                                     route.abort();
                                 } else {
-                                    route.resume();
+                                    // lets the IP filter on the context check the request
+                                    route.fallback();
                                 }
                             });