From 5c0dc82a38abbc5c9a8b645a75b712b1d9aeb3ff Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Wed, 17 Jun 2026 21:26:27 +0100 Subject: [PATCH 1/2] feat: make request URL available to the caller. This could be useful when the URL is sanitized by the underlying library. --- .../stormcrawler/protocol/ProtocolResponse.java | 12 ++++++++++++ .../stormcrawler/protocol/file/FileResponse.java | 5 +++-- .../stormcrawler/protocol/okhttp/HttpProtocol.java | 2 +- .../protocol/playwright/HttpProtocol.java | 3 ++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index c465ff000..68d405b29 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -19,6 +19,8 @@ import org.apache.stormcrawler.Metadata; +import java.net.URL; + public class ProtocolResponse { /** @@ -81,11 +83,17 @@ public enum TrimmedContentReason { private final byte[] content; private final int statusCode; private final Metadata metadata; + private final URL url; public ProtocolResponse(byte[] c, int s, Metadata md) { + this(c, s, md, null); + } + + public ProtocolResponse(byte[] c, int s, Metadata md, URL url) { content = c; statusCode = s; metadata = md == null ? new Metadata() : md; + this.url = url; } public byte[] getContent() { @@ -99,4 +107,8 @@ public int getStatusCode() { public Metadata getMetadata() { return metadata; } + + public URL getUrl() { + return url; + } } diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java index 028984deb..6a53a6c70 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java @@ -44,6 +44,7 @@ public class FileResponse { private byte[] content; private int statusCode; private final Metadata metadata; + private final URL url; public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOException { @@ -51,7 +52,7 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE content = new byte[0]; statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR; - URL url = URLUtil.toURL(u); + this.url = URLUtil.toURL(u); if (!url.getPath().equals(url.getFile())) { LOG.warn("url.getPath() != url.getFile(): {}.", url); @@ -125,7 +126,7 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE } public ProtocolResponse toProtocolResponse() { - return new ProtocolResponse(content, statusCode, metadata); + return new ProtocolResponse(content, statusCode, metadata, this.url); } private void getFileAsHttpResponse(File file) { diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 927508b43..1563acc5d 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -520,7 +520,7 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) responsemetadata.setValue("metrics.dns.resolution.msec", dnsResolution.toString()); } - return new ProtocolResponse(bytes, response.code(), responsemetadata); + return new ProtocolResponse(bytes, response.code(), responsemetadata, request.url().url()); } } diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index 3e947e917..e7df3d2e7 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -49,6 +49,7 @@ import org.apache.stormcrawler.protocol.Protocol; import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.URLUtil; import org.slf4j.LoggerFactory; public class HttpProtocol extends AbstractHttpProtocol { @@ -318,7 +319,7 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except responseMetaData.addValue(MD_KEY_END, Instant.now().toString()); - return new ProtocolResponse(content, status.get(), responseMetaData); + return new ProtocolResponse(content, status.get(), responseMetaData, URLUtil.toURL(url)); } finally { if (isTracing && responseMetaData.getFirstValue(MD_TRACE) == null) { From 918e95125766c2a4f2c697cfdeea297334002e26 Mon Sep 17 00:00:00 2001 From: Luca Foppiano Date: Mon, 22 Jun 2026 08:05:03 +0100 Subject: [PATCH 2/2] chore: reformat --- .../org/apache/stormcrawler/protocol/ProtocolResponse.java | 3 +-- .../org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java | 3 ++- .../apache/stormcrawler/protocol/playwright/HttpProtocol.java | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index 68d405b29..97e102ecd 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -17,9 +17,8 @@ package org.apache.stormcrawler.protocol; -import org.apache.stormcrawler.Metadata; - import java.net.URL; +import org.apache.stormcrawler.Metadata; public class ProtocolResponse { diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 1563acc5d..cfd57c15d 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -520,7 +520,8 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) responsemetadata.setValue("metrics.dns.resolution.msec", dnsResolution.toString()); } - return new ProtocolResponse(bytes, response.code(), responsemetadata, request.url().url()); + return new ProtocolResponse( + bytes, response.code(), responsemetadata, request.url().url()); } } diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index e7df3d2e7..462c63ba6 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -319,7 +319,8 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except responseMetaData.addValue(MD_KEY_END, Instant.now().toString()); - return new ProtocolResponse(content, status.get(), responseMetaData, URLUtil.toURL(url)); + return new ProtocolResponse( + content, status.get(), responseMetaData, URLUtil.toURL(url)); } finally { if (isTracing && responseMetaData.getFirstValue(MD_TRACE) == null) {