diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java index c465ff000..97e102ecd 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/ProtocolResponse.java @@ -17,6 +17,7 @@ package org.apache.stormcrawler.protocol; +import java.net.URL; import org.apache.stormcrawler.Metadata; public class ProtocolResponse { @@ -81,11 +82,17 @@ public enum TrimmedContentReason { private final byte[] content; private final int statusCode; private final Metadata metadata; + private final URL url; public ProtocolResponse(byte[] c, int s, Metadata md) { + this(c, s, md, null); + } + + public ProtocolResponse(byte[] c, int s, Metadata md, URL url) { content = c; statusCode = s; metadata = md == null ? new Metadata() : md; + this.url = url; } public byte[] getContent() { @@ -99,4 +106,8 @@ public int getStatusCode() { public Metadata getMetadata() { return metadata; } + + public URL getUrl() { + return url; + } } diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java index 028984deb..6a53a6c70 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/file/FileResponse.java @@ -44,6 +44,7 @@ public class FileResponse { private byte[] content; private int statusCode; private final Metadata metadata; + private final URL url; public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOException { @@ -51,7 +52,7 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE content = new byte[0]; statusCode = HttpStatus.SC_INTERNAL_SERVER_ERROR; - URL url = URLUtil.toURL(u); + this.url = URLUtil.toURL(u); if (!url.getPath().equals(url.getFile())) { LOG.warn("url.getPath() != url.getFile(): {}.", url); @@ -125,7 +126,7 @@ public FileResponse(String u, Metadata md, FileProtocol fileProtocol) throws IOE } public ProtocolResponse toProtocolResponse() { - return new ProtocolResponse(content, statusCode, metadata); + return new ProtocolResponse(content, statusCode, metadata, this.url); } private void getFileAsHttpResponse(File file) { diff --git a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java index 927508b43..cfd57c15d 100644 --- a/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java +++ b/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java @@ -520,7 +520,8 @@ public ProtocolResponse getProtocolOutput(String url, final Metadata metadata) responsemetadata.setValue("metrics.dns.resolution.msec", dnsResolution.toString()); } - return new ProtocolResponse(bytes, response.code(), responsemetadata); + return new ProtocolResponse( + bytes, response.code(), responsemetadata, request.url().url()); } } diff --git a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java index 3e947e917..462c63ba6 100644 --- a/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java +++ b/external/playwright/src/main/java/org/apache/stormcrawler/protocol/playwright/HttpProtocol.java @@ -49,6 +49,7 @@ import org.apache.stormcrawler.protocol.Protocol; import org.apache.stormcrawler.protocol.ProtocolResponse; import org.apache.stormcrawler.util.ConfUtils; +import org.apache.stormcrawler.util.URLUtil; import org.slf4j.LoggerFactory; public class HttpProtocol extends AbstractHttpProtocol { @@ -318,7 +319,8 @@ public ProtocolResponse getProtocolOutput(String url, Metadata md) throws Except responseMetaData.addValue(MD_KEY_END, Instant.now().toString()); - return new ProtocolResponse(content, status.get(), responseMetaData); + return new ProtocolResponse( + content, status.get(), responseMetaData, URLUtil.toURL(url)); } finally { if (isTracing && responseMetaData.getFirstValue(MD_TRACE) == null) {