#2162 Send credential headers from http.custom.headers only to the hosts in http.custom.headers.hosts

core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java

@@ -87,11 +87,16 @@ public void setConf(Config conf) {
      */
     private static void logForwardedRequestHeaders(Config conf) {
         List<String> sources = new ArrayList<>();
-        if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))) {
-            sources.add("http.basicauth.user");
+        if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))
+                && !ConfUtils.loadListFromConf("http.basicauth.hosts", conf).isEmpty()) {
+            // only reaches a target whose host is listed in http.basicauth.hosts
+            sources.add("http.basicauth.* (if the target is listed in http.basicauth.hosts)");
         }
         if (!ConfUtils.loadListFromConf("http.custom.headers", conf).isEmpty()) {
-            sources.add("http.custom.headers");
+            // credential headers only reach a target listed in http.custom.headers.hosts
+            sources.add(
+                    "http.custom.headers (credential headers if the target is listed in "
+                            + "http.custom.headers.hosts)");
         }
         if (!sources.isEmpty()) {
             LOG.warn(

core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java

@@ -145,9 +145,20 @@ public class HttpProtocol extends AbstractHttpProtocol {
     // lower case header names considered to carry credentials
     private Set<String> credentialHeaders = DEFAULT_CREDENTIAL_HEADERS;
 
-    // request headers withheld from servers which were not authenticated
+    // credential headers of http.custom.headers, only sent to customCredentialHosts and withheld
+    // from servers which were not authenticated
     private final List<KeyValue> credentialRequestHeaders = new LinkedList<>();
 
+    // http.custom.headers.hosts: hosts (canonical form, see HttpUrl#host) the
+    // credentialRequestHeaders are sent to
+    private final Set<String> customCredentialHosts = new HashSet<>();
+
+    // value of the Authorization header built from http.basicauth.*, null when not sent at all
+    private String basicAuthorization;
+
+    // http.basicauth.hosts: hosts (canonical form, see HttpUrl#host) basicAuthorization is sent to
+    private final Set<String> basicAuthHosts = new HashSet<>();
+
     // http.trust.everything: accept any certificate chain
     private boolean trustEverything = false;
 
@@ -321,27 +332,56 @@ public void configure(Config conf) {
 
         final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null);
 
-        // use a basic auth? the header is withheld unless the server was authenticated
+        // use a basic auth? the header is only sent to the hosts listed in http.basicauth.hosts
+        // and withheld unless the server was authenticated
         if (StringUtils.isNotBlank(basicAuthUser)) {
-            final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", "");
-            final String encoding =
-                    Base64.getEncoder()
-                            .encodeToString(
-                                    (basicAuthUser + ":" + basicAuthPass)
-                                            .getBytes(StandardCharsets.UTF_8));
-            credentialRequestHeaders.add(
-                    new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding));
+            addHosts(conf, "http.basicauth.hosts", basicAuthHosts);
+            if (basicAuthHosts.isEmpty()) {
+                // fail closed: before 4.0.0 the credentials went to every host crawled
+                LOG.warn(
+                        "http.basicauth.user is set but http.basicauth.hosts lists no host, the "
+                                + "credentials are not sent to any server. Since 4.0.0 they are "
+                                + "only sent to the hosts listed in http.basicauth.hosts; add the "
+                                + "host names of the sites which require them there.");
+            } else {
+                final String basicAuthPass =
+                        ConfUtils.getString(conf, "http.basicauth.password", "");
+                final String encoding =
+                        Base64.getEncoder()
+                                .encodeToString(
+                                        (basicAuthUser + ":" + basicAuthPass)
+                                                .getBytes(StandardCharsets.UTF_8));
+                basicAuthorization = "Basic " + encoding;
+            }
         }
 
+        // credential headers are only sent to the hosts listed in http.custom.headers.hosts, the
+        // other custom headers to every host
+        final List<KeyValue> credentialCustomHeaders = new ArrayList<>();
         for (KeyValue customHeader : customHeaders) {
             if (isCredentialHeader(customHeader.getKey())) {
-                credentialRequestHeaders.add(customHeader);
+                credentialCustomHeaders.add(customHeader);
             } else {
                 customRequestHeaders.add(customHeader);
             }
         }
+        if (!credentialCustomHeaders.isEmpty()) {
+            addHosts(conf, "http.custom.headers.hosts", customCredentialHosts);
+            if (customCredentialHosts.isEmpty()) {
+                // fail closed: before 4.0.0 the credential headers went to every host crawled
+                LOG.warn(
+                        "http.custom.headers sets the credential headers {} but "
+                                + "http.custom.headers.hosts lists no host, they are not sent to "
+                                + "any server. Since 4.0.0 they are only sent to the hosts listed "
+                                + "in http.custom.headers.hosts; add the host names of the sites "
+                                + "which require them there.",
+                        credentialCustomHeaders.stream().map(KeyValue::getKey).toList());
+            } else {
+                credentialRequestHeaders.addAll(credentialCustomHeaders);
+            }
+        }
 
-        if (!credentialRequestHeaders.isEmpty() || useCookies) {
+        if (basicAuthorization != null || !credentialRequestHeaders.isEmpty() || useCookies) {
             if (trustEverything && !insecureCredentialsAllowed) {
                 LOG.warn(
                         "Credentials configured with http.basicauth.*, credential headers in "
@@ -499,6 +539,63 @@ private URL getCookieOrigin(Metadata md, String url) {
         }
     }
 
+    /**
+     * Adds the hosts configured under the key, a list or a comma-separated string, to the set of
+     * hosts a credential is sent to.
+     */
+    private void addHosts(Config conf, String key, Set<String> hosts) {
+        for (String entry : ConfUtils.loadListFromConf(key, conf)) {
+            for (String host : entry.split(",")) {
+                addHost(key, host, hosts);
+            }
+        }
+    }
+
+    /**
+     * Adds a host configured under the key in the canonical form OkHttp reports for the host of a
+     * request, i.e. lower case and IDNs in punycode. An IPv6 address may be given with or without
+     * brackets. An entry with a scheme, port, user info, path, query or wildcard is not a plain
+     * host and is ignored.
+     */
+    private void addHost(String key, String entry, Set<String> hosts) {
+        if (StringUtils.isBlank(entry)) {
+            return;
+        }
+        String host = entry.trim();
+        if (!host.startsWith("[") && StringUtils.countMatches(host, ':') > 1) {
+            // an IPv6 address without brackets
+            host = "[" + host + "]";
+        }
+        // checked on the entry itself: HttpUrl keeps '*' as part of the host name and drops the
+        // default port 80, so neither would be noticed on the parsed url
+        final String afterAddress =
+                host.startsWith("[") ? StringUtils.substringAfter(host, "]") : host;
+        final HttpUrl parsed =
+                host.contains("*") || afterAddress.contains(":")
+                        ? null
+                        : HttpUrl.parse("http://" + host + "/");
+        if (parsed == null
+                || !parsed.equals(
+                        new HttpUrl.Builder().scheme("http").host(parsed.host()).build())) {
+            LOG.warn(
+                    "Ignoring '{}' in {}, expected a host name or IP address "
+                            + "without scheme, port, path or wildcard",
+                    entry,
+                    key);
+            return;
+        }
+        hosts.add(parsed.host());
+    }
+
+    /**
+     * Whether the host of the url is one of the hosts. Hosts match exactly, ignoring case:
+     * subdomains of a listed host are not included.
+     */
+    private boolean isListedHost(Set<String> hosts, String url) {
+        final HttpUrl parsed = HttpUrl.parse(url);
+        return parsed != null && hosts.contains(parsed.host());
+    }
+
     /** Whether the header carries credentials, see http.credentials.headers. */
     private boolean isCredentialHeader(String name) {
         if (name == null) {
@@ -624,18 +721,29 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata)
         }
 
         final boolean sendCredentials = credentialsAllowed(url);
+        final boolean basicAuthForHost =
+                basicAuthorization != null && isListedHost(basicAuthHosts, url);
+        final boolean customCredentialsForHost =
+                !credentialRequestHeaders.isEmpty() && isListedHost(customCredentialHosts, url);
 
         final Builder rb = new Request.Builder().url(url);
         customRequestHeaders.forEach(
                 (k) -> {
                     rb.header(k.getKey(), k.getValue());
                 });
         if (sendCredentials) {
-            credentialRequestHeaders.forEach(
-                    (k) -> {
-                        rb.header(k.getKey(), k.getValue());
-                    });
-        } else if (!credentialRequestHeaders.isEmpty()
+            if (basicAuthForHost) {
+                // set first so that an Authorization in http.custom.headers still replaces it on
+                // the hosts listed in http.custom.headers.hosts
+                rb.header(HttpHeaders.AUTHORIZATION, basicAuthorization);
+            }
+            if (customCredentialsForHost) {
+                credentialRequestHeaders.forEach(
+                        (k) -> {
+                            rb.header(k.getKey(), k.getValue());
+                        });
+            }
+        } else if ((basicAuthForHost || customCredentialsForHost)
                 && withheldRequestHeadersLogged.compareAndSet(false, true)) {
             LOG.warn(
                     "Configured credential headers (http.basicauth.*, http.custom.headers) are "

core/src/main/resources/crawler-default.yaml

@@ -188,6 +188,40 @@ config:
   # http.credentials.headers:
   # - "x-auth-token"
 
+  # Basic authentication for the sites which require it. The Authorization
+  # header built from http.basicauth.user and http.basicauth.password is only
+  # sent to the hosts listed in http.basicauth.hosts (a list or a
+  # comma-separated string): host names or IP addresses without scheme, port,
+  # path or wildcard, matched exactly and case-insensitively; other entries are
+  # ignored with a warning. Subdomains are not included, list each host
+  # separately. The host of every request is checked, robots.txt fetches and
+  # redirect targets included; a redirect followed with http.allow.redirects
+  # never carries the header to another scheme, host or port. The header is
+  # also withheld from servers which were not authenticated, see
+  # http.credentials.allow.insecure.
+  # Breaking change in 4.0.0: the credentials used to be sent to every host
+  # crawled. If http.basicauth.hosts is empty they are not sent at all and a
+  # warning is logged at startup.
+  # http.basicauth.user:
+  # http.basicauth.password:
+  # http.basicauth.hosts:
+  # - "intranet.example.com"
+
+  # The headers of http.custom.headers which carry credentials (see
+  # http.credentials.headers) are only sent to the hosts listed in
+  # http.custom.headers.hosts, which takes the same entries and is matched in
+  # the same way as http.basicauth.hosts; the other custom headers are sent to
+  # every host. The two host lists are independent. On a host listed in both,
+  # an Authorization header of http.custom.headers replaces the one built from
+  # http.basicauth.*.
+  # Breaking change in 4.0.0: the credential headers used to be sent to every
+  # host crawled. If http.custom.headers.hosts is empty they are not sent at
+  # all and a warning is logged at startup.
+  # http.custom.headers:
+  # - "X-Api-Key=secret"
+  # http.custom.headers.hosts:
+  # - "api.example.com"
+
   # Maximum number of redirect hops followed when http.allow.redirects is
   # enabled (okhttp protocol only). A chain which does not end within this
   # many hops returns its last redirect response, which the caller handles
@@ -234,8 +268,10 @@ config:
   # while staying on the same host and port is always followed, whatever this is
   # set to. When this is enabled, the robots.txt is re-fetched from the host
   # named in the Location header, for up to 5 hops, and each of these requests
-  # carries the headers configured through http.basicauth.* and
-  # http.custom.headers as any other request does; http.filter.ipaddress.exclude
+  # carries the headers configured through http.custom.headers, the credential
+  # ones only if the host is listed in http.custom.headers.hosts, and those of
+  # http.basicauth.* if the host is listed in http.basicauth.hosts, as any other
+  # request does; http.filter.ipaddress.exclude
   # can be used to restrict the addresses those fetches may reach.
   http.robots.redirect.crossorigin.allow: false
 

core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java

@@ -288,6 +288,7 @@ void testHeadersNotSentToAnUnfollowedRedirectTarget() {
         authConf.putAll(conf);
         authConf.put("http.basicauth.user", "this_is_only_a_test");
         authConf.put("http.basicauth.password", "this_is_only_a_test");
+        authConf.put("http.basicauth.hosts", "localhost");
         // the test verifies where the Authorization header is sent, not the
         // withholding on unauthenticated connections: opt in
         authConf.put("http.credentials.allow.insecure", true);