diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java index c1419718e..808982140 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java @@ -87,11 +87,16 @@ public void setConf(Config conf) { */ private static void logForwardedRequestHeaders(Config conf) { List sources = new ArrayList<>(); - if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null))) { - sources.add("http.basicauth.user"); + if (StringUtils.isNotBlank(ConfUtils.getString(conf, "http.basicauth.user", null)) + && !ConfUtils.loadListFromConf("http.basicauth.hosts", conf).isEmpty()) { + // only reaches a target whose host is listed in http.basicauth.hosts + sources.add("http.basicauth.* (if the target is listed in http.basicauth.hosts)"); } if (!ConfUtils.loadListFromConf("http.custom.headers", conf).isEmpty()) { - sources.add("http.custom.headers"); + // credential headers only reach a target listed in http.custom.headers.hosts + sources.add( + "http.custom.headers (credential headers if the target is listed in " + + "http.custom.headers.hosts)"); } if (!sources.isEmpty()) { LOG.warn( diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 42fdf9158..a452c390a 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -145,9 +145,20 @@ public class HttpProtocol extends AbstractHttpProtocol { // lower case header names considered to carry credentials private Set credentialHeaders = DEFAULT_CREDENTIAL_HEADERS; - // request headers withheld from servers which were not authenticated + // credential headers of http.custom.headers, only sent to customCredentialHosts and withheld + // from servers which were not authenticated private final List credentialRequestHeaders = new LinkedList<>(); + // http.custom.headers.hosts: hosts (canonical form, see HttpUrl#host) the + // credentialRequestHeaders are sent to + private final Set customCredentialHosts = new HashSet<>(); + + // value of the Authorization header built from http.basicauth.*, null when not sent at all + private String basicAuthorization; + + // http.basicauth.hosts: hosts (canonical form, see HttpUrl#host) basicAuthorization is sent to + private final Set basicAuthHosts = new HashSet<>(); + // http.trust.everything: accept any certificate chain private boolean trustEverything = false; @@ -321,27 +332,56 @@ public void configure(Config conf) { final String basicAuthUser = ConfUtils.getString(conf, "http.basicauth.user", null); - // use a basic auth? the header is withheld unless the server was authenticated + // use a basic auth? the header is only sent to the hosts listed in http.basicauth.hosts + // and withheld unless the server was authenticated if (StringUtils.isNotBlank(basicAuthUser)) { - final String basicAuthPass = ConfUtils.getString(conf, "http.basicauth.password", ""); - final String encoding = - Base64.getEncoder() - .encodeToString( - (basicAuthUser + ":" + basicAuthPass) - .getBytes(StandardCharsets.UTF_8)); - credentialRequestHeaders.add( - new KeyValue(HttpHeaders.AUTHORIZATION, "Basic " + encoding)); + addHosts(conf, "http.basicauth.hosts", basicAuthHosts); + if (basicAuthHosts.isEmpty()) { + // fail closed: before 4.0.0 the credentials went to every host crawled + LOG.warn( + "http.basicauth.user is set but http.basicauth.hosts lists no host, the " + + "credentials are not sent to any server. Since 4.0.0 they are " + + "only sent to the hosts listed in http.basicauth.hosts; add the " + + "host names of the sites which require them there."); + } else { + final String basicAuthPass = + ConfUtils.getString(conf, "http.basicauth.password", ""); + final String encoding = + Base64.getEncoder() + .encodeToString( + (basicAuthUser + ":" + basicAuthPass) + .getBytes(StandardCharsets.UTF_8)); + basicAuthorization = "Basic " + encoding; + } } + // credential headers are only sent to the hosts listed in http.custom.headers.hosts, the + // other custom headers to every host + final List credentialCustomHeaders = new ArrayList<>(); for (KeyValue customHeader : customHeaders) { if (isCredentialHeader(customHeader.getKey())) { - credentialRequestHeaders.add(customHeader); + credentialCustomHeaders.add(customHeader); } else { customRequestHeaders.add(customHeader); } } + if (!credentialCustomHeaders.isEmpty()) { + addHosts(conf, "http.custom.headers.hosts", customCredentialHosts); + if (customCredentialHosts.isEmpty()) { + // fail closed: before 4.0.0 the credential headers went to every host crawled + LOG.warn( + "http.custom.headers sets the credential headers {} but " + + "http.custom.headers.hosts lists no host, they are not sent to " + + "any server. Since 4.0.0 they are only sent to the hosts listed " + + "in http.custom.headers.hosts; add the host names of the sites " + + "which require them there.", + credentialCustomHeaders.stream().map(KeyValue::getKey).toList()); + } else { + credentialRequestHeaders.addAll(credentialCustomHeaders); + } + } - if (!credentialRequestHeaders.isEmpty() || useCookies) { + if (basicAuthorization != null || !credentialRequestHeaders.isEmpty() || useCookies) { if (trustEverything && !insecureCredentialsAllowed) { LOG.warn( "Credentials configured with http.basicauth.*, credential headers in " @@ -499,6 +539,63 @@ private URL getCookieOrigin(Metadata md, String url) { } } + /** + * Adds the hosts configured under the key, a list or a comma-separated string, to the set of + * hosts a credential is sent to. + */ + private void addHosts(Config conf, String key, Set hosts) { + for (String entry : ConfUtils.loadListFromConf(key, conf)) { + for (String host : entry.split(",")) { + addHost(key, host, hosts); + } + } + } + + /** + * Adds a host configured under the key in the canonical form OkHttp reports for the host of a + * request, i.e. lower case and IDNs in punycode. An IPv6 address may be given with or without + * brackets. An entry with a scheme, port, user info, path, query or wildcard is not a plain + * host and is ignored. + */ + private void addHost(String key, String entry, Set hosts) { + if (StringUtils.isBlank(entry)) { + return; + } + String host = entry.trim(); + if (!host.startsWith("[") && StringUtils.countMatches(host, ':') > 1) { + // an IPv6 address without brackets + host = "[" + host + "]"; + } + // checked on the entry itself: HttpUrl keeps '*' as part of the host name and drops the + // default port 80, so neither would be noticed on the parsed url + final String afterAddress = + host.startsWith("[") ? StringUtils.substringAfter(host, "]") : host; + final HttpUrl parsed = + host.contains("*") || afterAddress.contains(":") + ? null + : HttpUrl.parse("http://" + host + "/"); + if (parsed == null + || !parsed.equals( + new HttpUrl.Builder().scheme("http").host(parsed.host()).build())) { + LOG.warn( + "Ignoring '{}' in {}, expected a host name or IP address " + + "without scheme, port, path or wildcard", + entry, + key); + return; + } + hosts.add(parsed.host()); + } + + /** + * Whether the host of the url is one of the hosts. Hosts match exactly, ignoring case: + * subdomains of a listed host are not included. + */ + private boolean isListedHost(Set hosts, String url) { + final HttpUrl parsed = HttpUrl.parse(url); + return parsed != null && hosts.contains(parsed.host()); + } + /** Whether the header carries credentials, see http.credentials.headers. */ private boolean isCredentialHeader(String name) { if (name == null) { @@ -624,6 +721,10 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) } final boolean sendCredentials = credentialsAllowed(url); + final boolean basicAuthForHost = + basicAuthorization != null && isListedHost(basicAuthHosts, url); + final boolean customCredentialsForHost = + !credentialRequestHeaders.isEmpty() && isListedHost(customCredentialHosts, url); final Builder rb = new Request.Builder().url(url); customRequestHeaders.forEach( @@ -631,11 +732,18 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) rb.header(k.getKey(), k.getValue()); }); if (sendCredentials) { - credentialRequestHeaders.forEach( - (k) -> { - rb.header(k.getKey(), k.getValue()); - }); - } else if (!credentialRequestHeaders.isEmpty() + if (basicAuthForHost) { + // set first so that an Authorization in http.custom.headers still replaces it on + // the hosts listed in http.custom.headers.hosts + rb.header(HttpHeaders.AUTHORIZATION, basicAuthorization); + } + if (customCredentialsForHost) { + credentialRequestHeaders.forEach( + (k) -> { + rb.header(k.getKey(), k.getValue()); + }); + } + } else if ((basicAuthForHost || customCredentialsForHost) && withheldRequestHeadersLogged.compareAndSet(false, true)) { LOG.warn( "Configured credential headers (http.basicauth.*, http.custom.headers) are " diff --git a/core/src/main/resources/crawler-default.yaml b/core/src/main/resources/crawler-default.yaml index 1f3a0be2a..581669fed 100644 --- a/core/src/main/resources/crawler-default.yaml +++ b/core/src/main/resources/crawler-default.yaml @@ -188,6 +188,40 @@ config: # http.credentials.headers: # - "x-auth-token" + # Basic authentication for the sites which require it. The Authorization + # header built from http.basicauth.user and http.basicauth.password is only + # sent to the hosts listed in http.basicauth.hosts (a list or a + # comma-separated string): host names or IP addresses without scheme, port, + # path or wildcard, matched exactly and case-insensitively; other entries are + # ignored with a warning. Subdomains are not included, list each host + # separately. The host of every request is checked, robots.txt fetches and + # redirect targets included; a redirect followed with http.allow.redirects + # never carries the header to another scheme, host or port. The header is + # also withheld from servers which were not authenticated, see + # http.credentials.allow.insecure. + # Breaking change in 4.0.0: the credentials used to be sent to every host + # crawled. If http.basicauth.hosts is empty they are not sent at all and a + # warning is logged at startup. + # http.basicauth.user: + # http.basicauth.password: + # http.basicauth.hosts: + # - "intranet.example.com" + + # The headers of http.custom.headers which carry credentials (see + # http.credentials.headers) are only sent to the hosts listed in + # http.custom.headers.hosts, which takes the same entries and is matched in + # the same way as http.basicauth.hosts; the other custom headers are sent to + # every host. The two host lists are independent. On a host listed in both, + # an Authorization header of http.custom.headers replaces the one built from + # http.basicauth.*. + # Breaking change in 4.0.0: the credential headers used to be sent to every + # host crawled. If http.custom.headers.hosts is empty they are not sent at + # all and a warning is logged at startup. + # http.custom.headers: + # - "X-Api-Key=secret" + # http.custom.headers.hosts: + # - "api.example.com" + # Maximum number of redirect hops followed when http.allow.redirects is # enabled (okhttp protocol only). A chain which does not end within this # many hops returns its last redirect response, which the caller handles @@ -234,8 +268,10 @@ config: # while staying on the same host and port is always followed, whatever this is # set to. When this is enabled, the robots.txt is re-fetched from the host # named in the Location header, for up to 5 hops, and each of these requests - # carries the headers configured through http.basicauth.* and - # http.custom.headers as any other request does; http.filter.ipaddress.exclude + # carries the headers configured through http.custom.headers, the credential + # ones only if the host is listed in http.custom.headers.hosts, and those of + # http.basicauth.* if the host is listed in http.basicauth.hosts, as any other + # request does; http.filter.ipaddress.exclude # can be used to restrict the addresses those fetches may reach. http.robots.redirect.crossorigin.allow: false diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java index 19d7527ce..4ed68e407 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/HttpRobotRulesParserRedirectTest.java @@ -288,6 +288,7 @@ void testHeadersNotSentToAnUnfollowedRedirectTarget() { authConf.putAll(conf); authConf.put("http.basicauth.user", "this_is_only_a_test"); authConf.put("http.basicauth.password", "this_is_only_a_test"); + authConf.put("http.basicauth.hosts", "localhost"); // the test verifies where the Authorization header is sent, not the // withholding on unauthenticated connections: opt in authConf.put("http.credentials.allow.insecure", true); diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpBasicAuthScopeTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpBasicAuthScopeTest.java new file mode 100644 index 000000000..0c9fcac04 --- /dev/null +++ b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpBasicAuthScopeTest.java @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.stormcrawler.protocol; + +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.storm.Config; +import org.apache.stormcrawler.Metadata; +import org.apache.stormcrawler.protocol.okhttp.HttpProtocol; +import org.eclipse.jetty.server.Handler; +import org.eclipse.jetty.server.Request; +import org.eclipse.jetty.server.handler.AbstractHandler; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * The Authorization header built from http.basicauth.* is only sent to the hosts listed in + * http.basicauth.hosts, the credential headers of http.custom.headers only to the hosts listed in + * http.custom.headers.hosts. The local server is reached as localhost and as 127.0.0.1, which the + * protocol treats as two different hosts. + */ +class OkHttpBasicAuthScopeTest extends AbstractProtocolTest { + + private static final String EXPECTED = + "Basic " + + Base64.getEncoder() + .encodeToString("wikiuser:wikipass".getBytes(StandardCharsets.UTF_8)); + + /** Authorization header seen per requested "host/path", "null" when absent. */ + static final Map authorizationSeen = new ConcurrentHashMap<>(); + + /** X-Api-Key header, a credential, seen per requested "host/path", "null" when absent. */ + static final Map apiKeySeen = new ConcurrentHashMap<>(); + + /** X-Trace header, not a credential, seen per requested "host/path", "null" when absent. */ + static final Map traceSeen = new ConcurrentHashMap<>(); + + /** Location the robots.txt of localhost redirects to, none when null. */ + static volatile String robotsRedirect; + + @Override + protected Handler[] getHandlers() { + return new Handler[] { + new AbstractHandler() { + @Override + public void handle( + String target, + Request baseRequest, + jakarta.servlet.http.HttpServletRequest request, + HttpServletResponse response) + throws IOException { + baseRequest.setHandled(true); + final String seen = request.getServerName() + target; + authorizationSeen.put(seen, String.valueOf(request.getHeader("Authorization"))); + apiKeySeen.put(seen, String.valueOf(request.getHeader("X-Api-Key"))); + traceSeen.put(seen, String.valueOf(request.getHeader("X-Trace"))); + final String location; + if (target.equals("/redirect")) { + location = "http://127.0.0.1:" + HTTP_PORT + "/target"; + } else if (target.equals("/redirect-same-origin")) { + location = "/target"; + } else if (target.equals("/robots.txt") + && "localhost".equals(request.getServerName())) { + location = robotsRedirect; + } else { + location = null; + } + if (location != null) { + response.setStatus(HttpServletResponse.SC_MOVED_PERMANENTLY); + response.setHeader("Location", location); + response.setContentLength(0); + response.getOutputStream().close(); + return; + } + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType("text/plain"); + final byte[] content = "hello".getBytes(StandardCharsets.UTF_8); + response.setContentLength(content.length); + try (OutputStream out = response.getOutputStream()) { + out.write(content); + } + } + } + }; + } + + @BeforeEach + void reset() { + authorizationSeen.clear(); + apiKeySeen.clear(); + traceSeen.clear(); + robotsRedirect = null; + } + + private Config baseConfig() { + final Config conf = new Config(); + conf.put("http.agent.name", "this_is_only_a_test"); + // the local server is cleartext HTTP: opt in so that only the host list decides + conf.put("http.credentials.allow.insecure", true); + return conf; + } + + private Config config(Object hosts) { + final Config conf = baseConfig(); + conf.put("http.basicauth.user", "wikiuser"); + conf.put("http.basicauth.password", "wikipass"); + if (hosts != null) { + conf.put("http.basicauth.hosts", hosts); + } + return conf; + } + + private Config customHeadersConfig(Object hosts) { + final Config conf = baseConfig(); + conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + if (hosts != null) { + conf.put("http.custom.headers.hosts", hosts); + } + return conf; + } + + private HttpProtocol protocol(Config conf) { + final HttpProtocol protocol = new HttpProtocol(); + protocol.configure(conf); + return protocol; + } + + private void fetch(HttpProtocol protocol, String host, String path) throws Exception { + protocol.getProtocolOutput("http://" + host + ":" + HTTP_PORT + path, new Metadata()); + } + + @Test + void credentialsOnlyGoToTheListedHost() throws Exception { + // the listed host is matched regardless of case + final HttpProtocol protocol = protocol(config("LocalHost")); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/page.html")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/page.html"), + "a host which is not listed must not receive the credentials"); + } + + @Test + void robotsTxtIsScopedLikeAnyOtherFetch() throws Exception { + final Config conf = config(List.of("localhost")); + final HttpProtocol protocol = protocol(conf); + try { + final HttpRobotRulesParser parser = new HttpRobotRulesParser(conf); + parser.getRobotRulesSet(protocol, "http://localhost:" + HTTP_PORT + "/page.html"); + parser.getRobotRulesSet(protocol, "http://127.0.0.1:" + HTTP_PORT + "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/robots.txt")); + Assertions.assertEquals("null", authorizationSeen.get("127.0.0.1/robots.txt")); + } + + @Test + void crossHostRobotsTxtRedirectDoesNotCarryTheCredentials() throws Exception { + robotsRedirect = "http://127.0.0.1:" + HTTP_PORT + "/robots.txt"; + final Config conf = config("localhost"); + conf.put("http.robots.redirect.crossorigin.allow", true); + final HttpProtocol protocol = protocol(conf); + try { + new HttpRobotRulesParser(conf) + .getRobotRulesSet(protocol, "http://localhost:" + HTTP_PORT + "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/robots.txt")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/robots.txt"), + "the redirect is followed but its target is not a listed host"); + } + + @Test + void crossHostRedirectDoesNotCarryTheCredentials() throws Exception { + // regression guard: a change of origin already stripped the credentials before the host + // list was introduced + final Config conf = config("localhost"); + conf.put("http.allow.redirects", true); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "localhost", "/redirect"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/redirect")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/target"), + "the redirect is followed but its target is not a listed host"); + } + + @Test + void sameOriginRedirectOnHostWhichIsNotListedDoesNotCarryTheCredentials() throws Exception { + final Config conf = config("localhost"); + conf.put("http.allow.redirects", true); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "127.0.0.1", "/redirect-same-origin"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", authorizationSeen.get("127.0.0.1/redirect-same-origin")); + Assertions.assertEquals( + "null", + authorizationSeen.get("127.0.0.1/target"), + "a redirect within the origin keeps the headers, which must not include them"); + } + + @Test + void credentialsAreNotSentWithoutHosts() throws Exception { + final HttpProtocol protocol = protocol(config(null)); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", authorizationSeen.get("localhost/page.html")); + Assertions.assertEquals("null", authorizationSeen.get("127.0.0.1/page.html")); + } + + @Test + void entriesWhichAreNotPlainHostsAreIgnored() throws Exception { + final HttpProtocol protocol = + protocol( + config( + List.of( + "http://localhost", + "localhost:" + HTTP_PORT, + // the default port, which HttpUrl drops when parsing + "localhost:80", + "localhost/page.html", + "*.localhost"))); + try { + fetch(protocol, "localhost", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", authorizationSeen.get("localhost/page.html")); + } + + @Test + void customCredentialHeadersOnlyGoToTheListedHost() throws Exception { + // the listed host is matched regardless of case + final HttpProtocol protocol = protocol(customHeadersConfig("LocalHost")); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("s3cret", apiKeySeen.get("localhost/page.html")); + Assertions.assertEquals( + "null", + apiKeySeen.get("127.0.0.1/page.html"), + "a host which is not listed must not receive the credential header"); + Assertions.assertEquals("public", traceSeen.get("localhost/page.html")); + Assertions.assertEquals( + "public", + traceSeen.get("127.0.0.1/page.html"), + "a header which is not a credential is sent to every host"); + } + + @Test + void robotsTxtIsScopedLikeAnyOtherFetchForCustomCredentialHeaders() throws Exception { + final Config conf = customHeadersConfig(List.of("localhost")); + final HttpProtocol protocol = protocol(conf); + try { + final HttpRobotRulesParser parser = new HttpRobotRulesParser(conf); + parser.getRobotRulesSet(protocol, "http://localhost:" + HTTP_PORT + "/page.html"); + parser.getRobotRulesSet(protocol, "http://127.0.0.1:" + HTTP_PORT + "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("s3cret", apiKeySeen.get("localhost/robots.txt")); + Assertions.assertEquals("null", apiKeySeen.get("127.0.0.1/robots.txt")); + Assertions.assertEquals("public", traceSeen.get("127.0.0.1/robots.txt")); + } + + @Test + void crossHostRedirectDoesNotCarryCustomCredentialHeaders() throws Exception { + // both hosts are listed: only the change of host strips the header + final Config conf = customHeadersConfig("localhost,127.0.0.1"); + conf.put("http.allow.redirects", true); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "localhost", "/redirect"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("s3cret", apiKeySeen.get("localhost/redirect")); + Assertions.assertEquals("null", apiKeySeen.get("127.0.0.1/target")); + Assertions.assertEquals("public", traceSeen.get("127.0.0.1/target")); + } + + @Test + void customCredentialHeadersAreNotSentWithoutHosts() throws Exception { + // http.basicauth.hosts does not apply to http.custom.headers + final Config conf = customHeadersConfig(null); + conf.put("http.basicauth.user", "wikiuser"); + conf.put("http.basicauth.password", "wikipass"); + conf.put("http.basicauth.hosts", "localhost"); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("null", apiKeySeen.get("localhost/page.html")); + Assertions.assertEquals("null", apiKeySeen.get("127.0.0.1/page.html")); + Assertions.assertEquals("public", traceSeen.get("localhost/page.html")); + Assertions.assertEquals("public", traceSeen.get("127.0.0.1/page.html")); + Assertions.assertEquals(EXPECTED, authorizationSeen.get("localhost/page.html")); + } + + @Test + void customAuthorizationOnlyReplacesBasicAuthOnItsHosts() throws Exception { + final Config conf = config("localhost,127.0.0.1"); + conf.put("http.custom.headers", List.of("Authorization=Bearer token")); + conf.put("http.custom.headers.hosts", "localhost"); + final HttpProtocol protocol = protocol(conf); + try { + fetch(protocol, "localhost", "/page.html"); + fetch(protocol, "127.0.0.1", "/page.html"); + } finally { + protocol.cleanup(); + } + Assertions.assertEquals("Bearer token", authorizationSeen.get("localhost/page.html")); + Assertions.assertEquals( + EXPECTED, + authorizationSeen.get("127.0.0.1/page.html"), + "the basic auth credentials are kept where the custom header is not sent"); + } +} diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java index 942be408b..40965516a 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/OkHttpFollowRedirectsTest.java @@ -206,7 +206,9 @@ void credentialsSurviveASameOriginHop() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "127.0.0.1"); conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret")); + conf.put("http.custom.headers.hosts", "127.0.0.1"); HttpProtocol protocol = protocol(conf); protocol.getProtocolOutput("http://127.0.0.1:" + HTTP_PORT + "/start", new Metadata()); protocol.cleanup(); @@ -229,7 +231,10 @@ void credentialsAreStrippedOnACrossOriginHop() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + // both hosts are listed: only the origin change strips the header + conf.put("http.basicauth.hosts", java.util.List.of("127.0.0.1", "localhost")); conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret")); + conf.put("http.custom.headers.hosts", java.util.List.of("127.0.0.1", "localhost")); HttpProtocol protocol = protocol(conf); ProtocolResponse response = protocol.getProtocolOutput( @@ -259,7 +264,9 @@ void credentialsAreStrippedOnACrossPortHop() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "127.0.0.1"); conf.put("http.custom.headers", java.util.List.of("X-Api-Key=s3cret")); + conf.put("http.custom.headers.hosts", "127.0.0.1"); HttpProtocol protocol = protocol(conf); ProtocolResponse response = protocol.getProtocolOutput( diff --git a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java index 9dd9c586b..0685542a3 100644 --- a/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java +++ b/core/src/test/java/org/apache/stormcrawler/protocol/okhttp/OkHttpTrustEverythingTest.java @@ -152,6 +152,7 @@ void basicAuthIsWithheldFromUnauthenticatedServers() throws Exception { conf.put("http.trust.everything", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/basicauth"); server.verify( @@ -165,6 +166,7 @@ void basicAuthIsSentWhenExplicitlyAllowed() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/basicauth"); final String expected = @@ -182,6 +184,7 @@ void credentialCustomHeadersAreWithheldFromUnauthenticatedServers() throws Excep final Config conf = config(); conf.put("http.trust.everything", true); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/customheaders"); server.verify( @@ -198,6 +201,7 @@ void credentialCustomHeadersAreSentWhenExplicitlyAllowed() throws Exception { conf.put("http.trust.everything", true); conf.put("http.credentials.allow.insecure", true); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret", "X-Trace=public")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/customheaders"); server.verify( @@ -235,6 +239,7 @@ void basicAuthIsWithheldOverCleartextHttp() throws Exception { final Config conf = config(); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); final ProtocolResponse response = fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); @@ -249,6 +254,7 @@ void basicAuthIsSentOverCleartextHttpWhenExplicitlyAllowed() throws Exception { conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetchUrl(protocol(conf), httpUrl("/cleartext"), new Metadata()); final String expected = "Basic " + base64("user:secret"); @@ -264,6 +270,7 @@ void credentialHeaderNamesCanBeConfigured() throws Exception { conf.put("http.trust.everything", true); conf.put("http.credentials.headers", List.of("X-Auth-Token")); conf.put("http.custom.headers", List.of("X-Auth-Token=token1", "X-Other=plain")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/configuredheaders"); server.verify( @@ -285,6 +292,7 @@ void configuredHeaderNamesDoNotDropTheBuiltInOnes() throws Exception { conf.put( "http.custom.headers", List.of("Authorization=Basic c2VjcmV0", "X-Api-Key=key1", "Cookie=sid=x")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); fetch(protocol(conf), "/builtinheaders"); server.verify( @@ -307,6 +315,7 @@ void credentialsAreWithheldWhenHostnameVerificationIsDisabled() throws Exception conf.put("http.verify.hostnames", false); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(OTHERHOST_KEYSTORE); final ProtocolResponse response = fetch(protocol(conf), "/nohostnamecheck"); assertEquals(200, response.getStatusCode(), "the connection succeeds as configured"); @@ -324,6 +333,7 @@ void credentialsAreSentWhenHostnameVerificationIsDisabledAndInsecureAllowed() th conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); startServer(OTHERHOST_KEYSTORE); fetch(protocol(conf), "/nohostnamecheckinsecure"); final String expected = "Basic " + base64("user:secret"); @@ -403,7 +413,9 @@ void credentialsAreStrippedOnHttpsToHttpRedirect() throws Exception { conf.put("http.allow.redirects", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(conf); trustTestKeystore(protocol, LOCALHOST_KEYSTORE); @@ -441,7 +453,9 @@ void credentialsAreStrippedOnHttpsToHttpRedirectEvenWhenInsecureAllowed() throws conf.put("http.credentials.allow.insecure", true); conf.put("http.basicauth.user", "user"); conf.put("http.basicauth.password", "secret"); + conf.put("http.basicauth.hosts", "localhost"); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(conf); trustTestKeystore(protocol, LOCALHOST_KEYSTORE); @@ -473,6 +487,7 @@ void storedRequestHeadersMatchTheStrippedHop() throws Exception { conf.put("http.allow.redirects", true); conf.put("http.store.headers", true); conf.put("http.custom.headers", List.of("X-Api-Key=s3cret")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(conf); trustTestKeystore(protocol, LOCALHOST_KEYSTORE); @@ -506,6 +521,7 @@ void staticProxyAuthorizationIsStrippedOnHttpsToHttpRedirect() throws Exception final Config conf = config(); conf.put("http.allow.redirects", true); conf.put("http.custom.headers", List.of("Proxy-Authorization=Basic cHJveHk6c2VjcmV0")); + conf.put("http.custom.headers.hosts", "localhost"); startServer(LOCALHOST_KEYSTORE); final HttpProtocol protocol = protocol(conf); trustTestKeystore(protocol, LOCALHOST_KEYSTORE); diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc index 093f60e5a..0763869aa 100644 --- a/docs/src/main/asciidoc/configuration.adoc +++ b/docs/src/main/asciidoc/configuration.adoc @@ -198,8 +198,9 @@ is defined. | http.agent.name | - | Name in User-Agent header. | http.agent.url | - | URL in User-Agent header. | http.agent.version | - | Version in User-Agent header. +| http.basicauth.hosts | - | Hosts the Basic Authentication credentials of `http.basicauth.user` are sent to, as a list or a comma-separated string. Host names or IP addresses (IPv6 with or without brackets) without scheme, port, path or wildcard, matched exactly and case-insensitively against the host of each request, robots.txt fetches and redirect targets included; subdomains are not included and have to be listed separately. *Breaking change in 4.0.0:* the credentials used to be sent to every host; if this is empty they are not sent at all and a warning is logged at startup. | http.basicauth.password | - | Password for http.basicauth.user. -| http.basicauth.user | - | Username for Basic Authentication. +| http.basicauth.user | - | Username for Basic Authentication. Only sent to the hosts listed in `http.basicauth.hosts`. | http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit. | http.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol implementation. @@ -220,7 +221,7 @@ implementation. | http.robots.file.skip | false | Ignore robots.txt rules entirely. | http.robots.headers.skip | false | Ignore robots directives from HTTP headers. | http.robots.meta.skip | false | Ignore robots directives from HTML meta tags. -| http.robots.redirect.crossorigin.allow | false | Follow a robots.txt redirect pointing at a different scheme, host or port than the URL it was reached from. Redirects to schemes other than http and https are never followed. A redirect which only replaces http with https while staying on the same host and port is always followed, whatever this is set to. When enabled, the robots.txt is re-fetched from the host named in the `Location` header, for up to 5 hops, and each of these requests carries the headers configured through `http.basicauth.*` and `http.custom.headers` as any other request does; see `http.filter.ipaddress.exclude` to restrict the addresses those fetches may reach. +| http.robots.redirect.crossorigin.allow | false | Follow a robots.txt redirect pointing at a different scheme, host or port than the URL it was reached from. Redirects to schemes other than http and https are never followed. A redirect which only replaces http with https while staying on the same host and port is always followed, whatever this is set to. When enabled, the robots.txt is re-fetched from the host named in the `Location` header, for up to 5 hops, and each of these requests carries the headers configured through `http.custom.headers`, except for credential headers if the host is not listed in `http.custom.headers.hosts`, and those of `http.basicauth.*` if the host is listed in `http.basicauth.hosts`, as any other request does; see `http.filter.ipaddress.exclude` to restrict the addresses those fetches may reach. | http.robots.redirect.refused.allow | false | Allow crawling when a robots.txt redirect was not followed because of `http.robots.redirect.crossorigin.allow`. If false, nothing is crawled on that host until the redirect is followed or this is set to true. | http.skip.robots | false | Deprecated (replaced by http.robots.file.skip). | robots.noFollow.strict | true | If true, remove all outlinks from pages marked as noFollow. @@ -254,7 +255,8 @@ implementation. | robots.error.cache.spec | maximumSize=10000,expireAfterWrite=1h | CacheBuilder configuration for error cache. | okhttp.protocol.connection.pool.max.idle.connections | 5 | OkHttp maximum number of idle connections. | okhttp.protocol.connection.pool.connection.keep.alive | 300 | OkHttp connection keep-alive time (seconds). -| http.custom.headers | - | Custom HTTP headers. +| http.custom.headers | - | Custom HTTP headers. Credential headers (see `http.credentials.headers`) are only sent to the hosts listed in `http.custom.headers.hosts`, the other headers to every host. +| http.custom.headers.hosts | - | Hosts the credential headers of `http.custom.headers` are sent to, with the same syntax and matching as `http.basicauth.hosts` and independent of it. On a host listed in both, an `Authorization` header of `http.custom.headers` replaces the one of `http.basicauth.*`. *Breaking change in 4.0.0:* the credential headers used to be sent to every host; if this is empty they are not sent at all and a warning is logged at startup. | http.accept | text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 | HTTP Accept header. | http.accept.language | en-us,en-gb,en;q=0.7,*;q=0.3 | HTTP Accept-Language diff --git a/docs/src/main/asciidoc/extending.adoc b/docs/src/main/asciidoc/extending.adoc index 25f65ab7e..dce33e001 100644 --- a/docs/src/main/asciidoc/extending.adoc +++ b/docs/src/main/asciidoc/extending.adoc @@ -402,9 +402,26 @@ By default, StormCrawler's OkHttp protocol validates the certificate chains of H ---- http.basicauth.user: "username" http.basicauth.password: "password" +http.basicauth.hosts: + - "intranet.example.com" ---- -These credentials are sent as an `Authorization` header with requests to servers authenticated by their TLS certificate; they are withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled or `http.verify.hostnames` is disabled, unless `http.credentials.allow.insecure` is set to `true`. For per-site authentication, use metadata-driven headers instead (see <> in the Internals section). +These credentials are sent as an `Authorization` header only to the hosts listed in `http.basicauth.hosts`. Entries are host names or IP addresses without scheme, port, path or wildcard; an entry which is not is ignored with a warning. Hosts are matched exactly and case-insensitively against the host of each request, including robots.txt fetches and redirect targets; subdomains are not included and have to be listed separately. Redirects followed with `http.allow.redirects` never carry the header to another scheme, host or port. Within these hosts, the header is sent with requests to servers authenticated by their TLS certificate; it is withheld from cleartext http:// requests and from https:// requests when `http.trust.everything` is enabled or `http.verify.hostnames` is disabled, unless `http.credentials.allow.insecure` is set to `true`. + +**Other credential headers**, e.g. a bearer token or an API key, can be configured with `http.custom.headers`: + +[source,yaml] +---- +http.custom.headers: + - "Authorization=Bearer token" + - "X-Trace=crawler" +http.custom.headers.hosts: + - "api.example.com" +---- + +The headers of `http.custom.headers` which carry credentials (see `http.credentials.headers`) are only sent to the hosts listed in `http.custom.headers.hosts`, which takes entries of the same form and is matched in the same way as `http.basicauth.hosts`; the other headers, like `X-Trace` above, are sent to every host. The two host lists are independent of each other. On a host listed in both, an `Authorization` header of `http.custom.headers` replaces the one built from `http.basicauth.*`; on a host listed only in `http.basicauth.hosts`, the Basic Authentication header is sent. Credential headers are withheld from servers which were not authenticated in the same way as the Basic Authentication credentials. For per-URL authentication, use metadata-driven headers instead (see <> in the Internals section). + +WARNING: Breaking change in 4.0.0: before, the credentials of `http.basicauth.*` and the credential headers of `http.custom.headers` were sent to every host crawled, including the targets of outlinks. If `http.basicauth.hosts` or `http.custom.headers.hosts` is empty, the corresponding credentials are now not sent at all, and a warning is logged at startup. ==== Proxy Authentication