diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index c1419718e..45cec3c2b 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -87,8 +87,10 @@ public void setConf(Config conf) { */ private static void logForwardedRequestHeaders(Config conf) { List sources = new ArrayList<>(); - if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))) { - sources.add("http.basicauth.user"); + if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null)) + && !ConfUtils.loadListFromConf("http.basicauth.hosts", conf).isEmpty()) { + // only reaches a target whose host is listed in http.basicauth.hosts + sources.add("http.basicauth.* (if the target is listed in http.basicauth.hosts)"); } if (!ConfUtils.loadListFromConf("http.custom.headers", conf).isEmpty()) { sources.add("http.custom.headers"); diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 42fdf9158..eda210324 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -148,6 +148,12 @@ public class HttpProtocol extends AbstractHttpProtocol { // request headers withheld from servers which were not authenticated private final List credentialRequestHeaders = new LinkedList<>(); + // value of the Authorization header built from http.basicauth.*, null when not sent at all + private String basicAuthorization; + + // http.basicauth.hosts: hosts (canonical form, see HttpUrl#host) basicAuthorization is sent to + private final Set basicAuthHosts = new HashSet<>(); + // http.trust.everything: accept any certificate chain private boolean trustEverything = false; @@ -321,27 +327,54 @@ public void configure(Config conf) { final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null); - // use a basic auth? the header is withheld unless the server was authenticated + // use a basic auth? the header is only sent to the hosts listed in http.basicauth.hosts + // and withheld unless the server was authenticated if (StringUtils.isNotBlank(basicAuthUser)) { - final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); - final String encoding = - Base64.getEncoder() - .encodeToString( - (basicAuthUser + ":" + basicAuthPass) - .getBytes(StandardCharsets.UTF_8)); - credentialRequestHeaders.add( - new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + for (String entry : ConfUtils.loadListFromConf("http.basicauth.hosts", conf)) { + for (String host : entry.split(",")) { + addBasicAuthHost(host); + } + } + if (basicAuthHosts.isEmpty()) { + // fail closed: before 4.0.0 the credentials went to every host crawled + LOG.warn( + "http.basicauth.user is set but http.basicauth.hosts lists no host, the " + + "credentials are not sent to any server. Since 4.0.0 they are " + + "only sent to the hosts listed in http.basicauth.hosts; add the " + + "host names of the sites which require them there."); + } else { + final String basicAuthPass = + ConfUtils.getString(conf, "http.basicauth.password", ""); + final String encoding = + Base64.getEncoder() + .encodeToString( + (basicAuthUser + ":" + basicAuthPass) + .getBytes(StandardCharsets.UTF_8)); + basicAuthorization = "Basic " + encoding; + } } + final List credentialCustomHeaders = new ArrayList<>(); for (KeyValue customHeader : customHeaders) { if (isCredentialHeader(customHeader.getKey())) { credentialRequestHeaders.add(customHeader); + credentialCustomHeaders.add(customHeader.getKey()); } else { customRequestHeaders.add(customHeader); } } + if (!credentialCustomHeaders.isEmpty()) { + LOG.warn( + "http.custom.headers sets the credential headers {}, which are sent to every " + + "host the crawl reaches, including the targets of outlinks. Use " + + "http.basicauth.* with http.basicauth.hosts for Basic authentication, " + + "or set the headers per site with {}{} in the metadata of its urls.", + credentialCustomHeaders, + protocolMetadataPrefix, + SET_HEADER_BY_REQUEST); + } - if (!credentialRequestHeaders.isEmpty() || useCookies) { + if (basicAuthorization != null || !credentialRequestHeaders.isEmpty() || useCookies) { if (trustEverything && !insecureCredentialsAllowed) { LOG.warn( "Credentials configured with http.basicauth.*, credential headers in " @@ -499,6 +532,50 @@ private URL getCookieOrigin(Metadata md, String url) { } } + /** + * Adds a host of http.basicauth.hosts in the canonical form OkHttp reports for the host of a + * request, i.e. lower case and IDNs in punycode. An IPv6 address may be given with or without + * brackets. An entry with a scheme, port, user info, path, query or wildcard is not a plain + * host and is ignored. + */ + private void addBasicAuthHost(String entry) { + if (StringUtils.isBlank(entry)) { + return; + } + String host = entry.trim(); + if (!host.startsWith("[") && StringUtils.countMatches(host, ':') > 1) { + // an IPv6 address without brackets + host = "[" + host + "]"; + } + // checked on the entry itself: HttpUrl keeps '*' as part of the host name and drops the + // default port 80, so neither would be noticed on the parsed url + final String afterAddress = + host.startsWith("[") ? StringUtils.substringAfter(host, "]") : host; + final HttpUrl parsed = + host.contains("*") || afterAddress.contains(":") + ? null + : HttpUrl.parse("http://" + host + "/"); + if (parsed == null + || !parsed.equals( + new HttpUrl.Builder().scheme("http").host(parsed.host()).build())) { + LOG.warn( + "Ignoring '{}' in http.basicauth.hosts, expected a host name or IP address " + + "without scheme, port, path or wildcard", + entry); + return; + } + basicAuthHosts.add(parsed.host()); + } + + /** + * Whether the Authorization header built from http.basicauth.* is meant for the host of the + * url. Hosts match exactly, ignoring case: subdomains of a listed host are not included. + */ + private boolean isBasicAuthHost(String url) { + final HttpUrl parsed = HttpUrl.parse(url); + return parsed != null && basicAuthHosts.contains(parsed.host()); + } + /** Whether the header carries credentials, see http.credentials.headers. */ private boolean isCredentialHeader(String name) { if (name == null) { @@ -624,6 +701,7 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } final boolean sendCredentials = credentialsAllowed(url); + final boolean basicAuthForHost = basicAuthorization != null && isBasicAuthHost(url); final Builder rb = new Request.Builder().url(url); customRequestHeaders.forEach( @@ -631,11 +709,15 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) rb.header(k.getKey(), k.getValue()); }); if (sendCredentials) { + if (basicAuthForHost) { + // set first so that an Authorization in http.custom.headers still replaces it + rb.header(HttpHeaders.AUTHORIZATION, basicAuthorization); + } credentialRequestHeaders.forEach( (k) -> { rb.header(k.getKey(), k.getValue()); }); - } else if (!credentialRequestHeaders.isEmpty() + } else if ((basicAuthForHost || !credentialRequestHeaders.isEmpty()) && withheldRequestHeadersLogged.compareAndSet(false, true)) { LOG.warn( "Configured credential headers (http.basicauth.*, http.custom.headers) are " diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 1f3a0be2a..da5e32265 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -188,6 +188,28 @@ config: # http.credentials.headers: # - "x-auth-token" + # Basic authentication for the sites which require it. The Authorization + # header built from http.basicauth.user and http.basicauth.password is only + # sent to the hosts listed in http.basicauth.hosts (a list or a + # comma-separated string): host names or IP addresses without scheme, port, + # path or wildcard, matched exactly and case-insensitively; other entries are + # ignored with a warning. Subdomains are not included, list each host + # separately. The host of every request is checked, robots.txt fetches and + # redirect targets included; a redirect followed with http.allow.redirects + # never carries the header to another scheme, host or port. The header is + # also withheld from servers which were not authenticated, see + # http.credentials.allow.insecure. + # Breaking change in 4.0.0: the credentials used to be sent to every host + # crawled. If http.basicauth.hosts is empty they are not sent at all and a + # warning is logged at startup. + # Credential headers configured with http.custom.headers are still sent to + # every host; set them per site with protocol.set-header in the metadata of + # the URLs instead. + # http.basicauth.user: + # http.basicauth.password: + # http.basicauth.hosts: + # - "intranet.example.com" + # Maximum number of redirect hops followed when http.allow.redirects is # enabled (okhttp protocol only). A chain which does not end within this # many hops returns its last redirect response, which the caller handles @@ -234,8 +256,9 @@ config: # while staying on the same host and port is always followed, whatever this is # set to. When this is enabled, the robots.txt is re-fetched from the host # named in the Location header, for up to 5 hops, and each of these requests - # carries the headers configured through http.basicauth.* and - # http.custom.headers as any other request does; http.filter.ipaddress.exclude + # carries the headers configured through http.custom.headers, and those of + # http.basicauth.* if the host is listed in http.basicauth.hosts, as any other + # request does; http.filter.ipaddress.exclude # can be used to restrict the addresses those fetches may reach. http.robots.redirect.crossorigin.allow: false diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java index 19d7527ce..4ed68e407 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java @@ -288,6 +288,7 @@ void testHeadersNotSentToAnUnfollowedRedirectTarget() { authConf.putAll(conf); authConf.put("http.basicauth.user", "this_is_only_a_test"); authConf.put("http.basicauth.password", "this_is_only_a_test"); + authConf.put("http.basicauth.hosts", "localhost"); // the test verifies where the Authorization header is sent, not the // withholding on unauthenticated connections: opt in authConf.put("http.credentials.allow.insecure", true); diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpBasicAuthScopeTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpBasicAuthScopeTest.java new file mode 100644 index 000000000..b4775fa59 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpBasicAuthScopeTest.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.okhttp.HttpProtocol; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * The Authorization header built from http.basicauth.* is only sent to the hosts listed in + * http.basicauth.hosts. The local server is reached as localhost and as 127.0.0.1, which the + * protocol treats as two different hosts. + */ +class OkHttpBasicAuthScopeTest extends AbstractProtocolTest { + + private static final String EXPECTED = + "Basic " + + Base64.getEncoder() + .encodeToString("wikiuser:wikipass".getBytes(StandardCharsets.UTF_8)); + + /** Authorization header seen per requested "host/path", "null" when absent. */ + static final Map authorizationSeen = new ConcurrentHashMap<>(); + + /** Location the robots.txt of localhost redirects to, none when null. */ + static volatile String robotsRedirect; + + @Override + protected Handler[] getHandlers() { + return new Handler[] { + new AbstractHandler() { + @Override + public void handle( + String target, + Request baseRequest, + jakarta.servlet.http.HttpServletRequest request, + HttpServletResponse response) + throws IOException { + baseRequest.setHandled(true); + authorizationSeen.put( + request.getServerName() + target, + String.valueOf(request.getHeader("Authorization"))); + final String location; + if (target.equals("/redirect")) { + location = "http://127.0.0.1:" + HTTP_PORT + "/target"; + } else if (target.equals("/redirect-same-origin")) { + location = "/target"; + } else if (target.equals("/robots.txt") + && "localhost".equals(request.getServerName())) { + location = robotsRedirect; + } else { + location = null; + } + if (location != null) { + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", location); + response.setContentLength(0); + response.getOutputStream().close(); + return; + } + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType("text/plain"); + final byte[] content = "hello".getBytes(StandardCharsets.UTF_8); + response.setContentLength(content.length); + try (OutputStream out = response.getOutputStream()) { + out.write(content); + } + } + } + }; + } + + @BeforeEach + void reset() { + authorizationSeen.clear(); + robotsRedirect = null; + } + + private Config config(Object hosts) { + final Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + // the local server is cleartext HTTP: opt in so that only the host list decides + conf.put("http.credentials.allow.insecure", true); + conf.put("http.basicauth.user", "wikiuser"); + conf.put("http.basicauth.password", "wikipass"); + if (hosts != null) { + conf.put("http.basicauth.hosts", hosts); + } + return conf; + } + + private HttpProtocol protocol(Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private void fetch(HttpProtocol protocol, String host, String path) throws Exception { + protocol.getProtocolOutput("http://" + host + ":" + HTTP_PORT + path, new Metadata()); + } + + @Test + void credentialsOnlyGoToTheListedHost() throws Exception { + // the listed host is matched regardless of case + final HttpProtocol protocol = protocol(config("LocalHost")); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/page.html")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/page.html"), + "a host which is not listed must not receive the credentials"); + } + + @Test + void robotsTxtIsScopedLikeAnyOtherFetch() throws Exception { + final Config conf = config(List.of("localhost")); + final HttpProtocol protocol = protocol(conf); + try { + final HttpRobotRulesParser parser = new HttpRobotRulesParser(conf); + parser.getRobotRulesSet(protocol, "http://localhost:" + HTTP_PORT + "/page.html"); + parser.getRobotRulesSet(protocol, "http://127.0.0.1:" + HTTP_PORT + "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/robots.txt")); + Assertions.assertEquals("null", authorizationSeen.get("127.0.0.1/robots.txt")); + } + + @Test + void crossHostRobotsTxtRedirectDoesNotCarryTheCredentials() throws Exception { + robotsRedirect = "http://127.0.0.1:" + HTTP_PORT + "/robots.txt"; + final Config conf = config("localhost"); + conf.put("http.robots.redirect.crossorigin.allow", true); + final HttpProtocol protocol = protocol(conf); + try { + new HttpRobotRulesParser(conf) + .getRobotRulesSet(protocol, "http://localhost:" + HTTP_PORT + "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/robots.txt")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/robots.txt"), + "the redirect is followed but its target is not a listed host"); + } + + @Test + void crossHostRedirectDoesNotCarryTheCredentials() throws Exception { + // regression guard: a change of origin already stripped the credentials before the host + // list was introduced + final Config conf = config("localhost"); + conf.put("http.allow.redirects", true); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "localhost", "/redirect"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/redirect")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/target"), + "the redirect is followed but its target is not a listed host"); + } + + @Test + void sameOriginRedirectOnHostWhichIsNotListedDoesNotCarryTheCredentials() throws Exception { + final Config conf = config("localhost"); + conf.put("http.allow.redirects", true); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "127.0.0.1", "/redirect-same-origin"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", authorizationSeen.get("127.0.0.1/redirect-same-origin")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/target"), + "a redirect within the origin keeps the headers, which must not include them"); + } + + @Test + void credentialsAreNotSentWithoutHosts() throws Exception { + final HttpProtocol protocol = protocol(config(null)); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", authorizationSeen.get("localhost/page.html")); + Assertions.assertEquals("null", authorizationSeen.get("127.0.0.1/page.html")); + } + + @Test + void entriesWhichAreNotPlainHostsAreIgnored() throws Exception { + final HttpProtocol protocol = + protocol( + config( + List.of( + "http://localhost", + "localhost:" + HTTP_PORT, + // the default port, which HttpUrl drops when parsing + "localhost:80", + "localhost/page.html", + "*.localhost"))); + try { + fetch(protocol, "localhost", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", authorizationSeen.get("localhost/page.html")); + } +} diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java index 942be408b..e88084ed5 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java @@ -206,6 +206,7 @@ void credentialsSurviveASameOriginHop() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "127.0.0.1"); conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret")); HttpProtocol protocol = protocol(conf); protocol.getProtocolOutput("http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata()); @@ -229,6 +230,8 @@ void credentialsAreStrippedOnACrossOriginHop() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + // both hosts are listed: only the origin change strips the header + conf.put("http.basicauth.hosts", java.util.List.of("127.0.0.1", "localhost")); conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret")); HttpProtocol protocol = protocol(conf); ProtocolResponse response = @@ -259,6 +262,7 @@ void credentialsAreStrippedOnACrossPortHop() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "127.0.0.1"); conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret")); HttpProtocol protocol = protocol(conf); ProtocolResponse response = diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index 9dd9c586b..e8da8b855 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -152,6 +152,7 @@ void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { conf.put("http.trust.everything", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/basicauth"); server.verify( @@ -165,6 +166,7 @@ void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/basicauth"); final String expected = @@ -235,6 +237,7 @@ void basicAuthIsWithheldOverCleartextHttp() throws Exception { final Config conf = config(); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); final ProtocolResponse response = fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); @@ -249,6 +252,7 @@ void basicAuthIsSentOverCleartextHttpWhenExplicitlyAllowed() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); final String expected = "Basic " + base64("user:secret"); @@ -307,6 +311,7 @@ void credentialsAreWithheldWhenHostnameVerificationIsDisabled() throws Exception conf.put("http.verify.hostnames", false); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(OTHERHOST_KEYSTORE); final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); assertEquals(200, response.getStatusCode(), "the connection succeeds as configured"); @@ -324,6 +329,7 @@ void credentialsAreSentWhenHostnameVerificationIsDisabledAndInsecureAllowed() th conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(OTHERHOST_KEYSTORE); fetch(protocol(conf), "/nohostnamecheckinsecure"); final String expected = "Basic " + base64("user:secret"); @@ -403,6 +409,7 @@ void credentialsAreStrippedOnHttpsToHttpRedirect() throws Exception { conf.put("http.allow.redirects", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(conf); @@ -441,6 +448,7 @@ void credentialsAreStrippedOnHttpsToHttpRedirectEvenWhenInsecureAllowed() throws conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(conf); diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 093f60e5a..a8d57ea78 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -198,8 +198,9 @@ is defined. | http.agent.name | - | Name in User-Agent header. | http.agent.url | - | URL in User-Agent header. | http.agent.version | - | Version in User-Agent header. +| http.basicauth.hosts | - | Hosts the Basic Authentication credentials of `http.basicauth.user` are sent to, as a list or a comma-separated string. Host names or IP addresses (IPv6 with or without brackets) without scheme, port, path or wildcard, matched exactly and case-insensitively against the host of each request, robots.txt fetches and redirect targets included; subdomains are not included and have to be listed separately. *Breaking change in 4.0.0:* the credentials used to be sent to every host; if this is empty they are not sent at all and a warning is logged at startup. | http.basicauth.password | - | Password for http.basicauth.user. -| http.basicauth.user | - | Username for Basic Authentication. +| http.basicauth.user | - | Username for Basic Authentication. Only sent to the hosts listed in `http.basicauth.hosts`. | http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit. | http.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol implementation. @@ -220,7 +221,7 @@ implementation. | http.robots.file.skip | false | Ignore robots.txt rules entirely. | http.robots.headers.skip | false | Ignore robots directives from HTTP headers. | http.robots.meta.skip | false | Ignore robots directives from HTML meta tags. -| http.robots.redirect.crossorigin.allow | false | Follow a robots.txt redirect pointing at a different scheme, host or port than the URL it was reached from. Redirects to schemes other than http and https are never followed. A redirect which only replaces http with https while staying on the same host and port is always followed, whatever this is set to. When enabled, the robots.txt is re-fetched from the host named in the `Location` header, for up to 5 hops, and each of these requests carries the headers configured through `http.basicauth.*` and `http.custom.headers` as any other request does; see `http.filter.ipaddress.exclude` to restrict the addresses those fetches may reach. +| http.robots.redirect.crossorigin.allow | false | Follow a robots.txt redirect pointing at a different scheme, host or port than the URL it was reached from. Redirects to schemes other than http and https are never followed. A redirect which only replaces http with https while staying on the same host and port is always followed, whatever this is set to. When enabled, the robots.txt is re-fetched from the host named in the `Location` header, for up to 5 hops, and each of these requests carries the headers configured through `http.custom.headers`, and those of `http.basicauth.*` if the host is listed in `http.basicauth.hosts`, as any other request does; see `http.filter.ipaddress.exclude` to restrict the addresses those fetches may reach. | http.robots.redirect.refused.allow | false | Allow crawling when a robots.txt redirect was not followed because of `http.robots.redirect.crossorigin.allow`. If false, nothing is crawled on that host until the redirect is followed or this is set to true. | http.skip.robots | false | Deprecated (replaced by http.robots.file.skip). | robots.noFollow.strict | true | If true, remove all outlinks from pages marked as noFollow. @@ -254,7 +255,7 @@ implementation. | robots.error.cache.spec | maximumSize=10000,expireAfterWrite=1h | CacheBuilder configuration for error cache. | okhttp.protocol.connection.pool.max.idle.connections | 5 | OkHttp maximum number of idle connections. | okhttp.protocol.connection.pool.connection.keep.alive | 300 | OkHttp connection keep-alive time (seconds). -| http.custom.headers | - | Custom HTTP headers. +| http.custom.headers | - | Custom HTTP headers, sent to every host. A warning is logged for credential headers (see `http.credentials.headers`); use `http.basicauth.hosts` or the `protocol.set-header` metadata to send credentials to specific sites only. | http.accept | text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 | HTTP Accept header. | http.accept.language | en-us,en-gb,en;q=0.7,*;q=0.3 | HTTP Accept-Language diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 25f65ab7e..64a787dd7 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -402,9 +402,15 @@ By default, StormCrawler's OkHttp protocol validates the certificate chains of H ---- http.basicauth.user: "username" http.basicauth.password: "password" +http.basicauth.hosts: + - "intranet.example.com" ---- -These credentials are sent as an `Authorization` header with requests to servers authenticated by their TLS certificate; they are withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled or `http.verify.hostnames` is disabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header only to the hosts listed in `http.basicauth.hosts`. Entries are host names or IP addresses without scheme, port, path or wildcard; an entry which is not is ignored with a warning. Hosts are matched exactly and case-insensitively against the host of each request, including robots.txt fetches and redirect targets; subdomains are not included and have to be listed separately. Redirects followed with `http.allow.redirects` never carry the header to another scheme, host or port. Within these hosts, the header is sent with requests to servers authenticated by their TLS certificate; it is withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled or `http.verify.hostnames` is disabled, unless `http.credentials.allow.insecure` is set to `true`. + +WARNING: Breaking change in 4.0.0: before, the credentials were sent to every host crawled, including the targets of outlinks. If `http.basicauth.hosts` is empty they are now not sent at all, and a warning is logged at startup. + +Headers configured with `http.custom.headers` are sent to every host; a warning is logged when one of them carries credentials (see `http.credentials.headers`). For other kinds of per-site authentication, use metadata-driven headers instead (see <> in the Internals section). ==== Proxy Authentication