Fix deletion of redirected URLs

core/src/main/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBolt.java

@@ -71,6 +71,9 @@ public abstract class AbstractStatusUpdaterBolt extends BaseRichBolt {
      */
     public static String roundDateParamName = "status.updater.unit.round.date";
 
+    /** Parameter name to enable deletion of URLs with permanent redirects. */
+    public static String deleteRedirectionsParamName = "status.updater.delete.redirections";
+
     /**
      * Key used to pass a preset Date to use as nextFetchDate. The value must represent a valid
      * instant in UTC and be parsable using {@link DateTimeFormatter#ISO_INSTANT}. This also
@@ -94,6 +97,8 @@ public abstract class AbstractStatusUpdaterBolt extends BaseRichBolt {
 
     private int roundDateUnit = Calendar.SECOND;
 
+    private boolean deleteRedirections = false;
+
     @Override
     public void prepare(
             Map<String, Object> stormConf, TopologyContext context, OutputCollector collector) {
@@ -104,6 +109,7 @@ public void prepare(
         mdTransfer = MetadataTransfer.getInstance(stormConf);
 
         useCache = ConfUtils.getBoolean(stormConf, useCacheParamName, true);
+        deleteRedirections = ConfUtils.getBoolean(stormConf, deleteRedirectionsParamName, false);
 
         if (useCache) {
             String spec = ConfUtils.getString(stormConf, cacheConfigParamName);
@@ -119,6 +125,7 @@ public void prepare(
                         return v;
                     },
                     30);
+
             CrawlerMetrics.registerGauge(
                     context,
                     stormConf,
@@ -129,6 +136,7 @@ public void prepare(
                         return v;
                     },
                     30);
+
             CrawlerMetrics.registerGauge(
                     context, stormConf, "cache.size", cache::estimatedSize, 30);
         }
@@ -157,7 +165,7 @@ public void execute(Tuple tuple) {
         // store it again
         if (potentiallyNew && useCache) {
             if (cache.getIfPresent(url) != null) {
-                // no need to add it to the queue
+                // no need to add the URL to the queue
                 LOG.debug("URL {} already in cache", url);
                 cacheHits++;
                 collector.ack(tuple);
@@ -227,22 +235,38 @@ public void execute(Tuple tuple) {
         if (!status.equals(Status.FETCH_ERROR)) {
             metadata.remove(Constants.fetchErrorCountParamName);
         }
+
         // https://github.com/apache/stormcrawler/issues/415
         // remove error related key values in case of success
         if (status.equals(Status.FETCHED) || status.equals(Status.REDIRECTION)) {
             metadata.remove(Constants.STATUS_ERROR_CAUSE);
             metadata.remove(Constants.STATUS_ERROR_MESSAGE);
             metadata.remove(Constants.STATUS_ERROR_SOURCE);
-        } else if (status == Status.ERROR) {
+        }
+
+        if (status == Status.ERROR) {
             // gone? notify any deleters. Doesn't need to be anchored
             collector.emit(Constants.DELETION_STREAM_NAME, new Values(url, metadata));
+        } else if (status == Status.REDIRECTION && deleteRedirections) {
+            String statusCode = metadata.getFirstValue("fetch.statusCode");
+
+            if (statusCode != null) {
+                try {
+                    // Delete URLs that have been permanently redirected.
+                    if (Status.isPermanentRedirect(Integer.parseInt(statusCode))) {
+                        collector.emit(Constants.DELETION_STREAM_NAME, new Values(url, metadata));
+                    }
+                } catch (NumberFormatException e) {
+                    LOG.debug("Invalid HTTP status code: {}", statusCode);
+                }
+            }
         }
 
         // determine the value of the next fetch based on the status
         Optional<Date> nextFetch = scheduler.schedule(status, metadata);
 
-        // filter metadata just before storing it, so that non-persisted
-        // metadata is available to fetch schedulers
+        // filter metadata just before storing it, so that non-persisted metadata is available
+        // to fetch schedulers
         metadata = mdTransfer.filter(metadata);
 
         // round next fetch date - unless it is never

core/src/main/java/org/apache/stormcrawler/persistence/Status.java

@@ -38,4 +38,9 @@ public static Status fromHTTPCode(int code) {
         // error otherwise
         return Status.FETCH_ERROR;
     }
+
+    /** Returns true if the HTTP code indicates a permanent redirect. */
+    public static boolean isPermanentRedirect(int code) {
+        return code == 301 || code == 308;
+    }
 }

core/src/main/resources/crawler-default.yaml

@@ -336,6 +336,10 @@ config:
   # Can also take "MINUTE" or "HOUR"
   status.updater.unit.round.date: "SECOND"
 
+  # Emit permanently redirected URLs (HTTP 301/308) on the deletion stream
+  # so that they can be removed from the index. Requires redirections.allowed.
+  status.updater.delete.redirections: false
+
   # configuration for the classes extending AbstractIndexerBolt
   # indexer.md.filter: "someKey=aValue"
   indexer.md.docid: ""

core/src/test/java/org/apache/stormcrawler/persistence/AbstractStatusUpdaterBoltTest.java

@@ -25,22 +25,23 @@
 
 import java.util.Date;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.tuple.Tuple;
+import org.apache.stormcrawler.Constants;
 import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.TestOutputCollector;
 import org.apache.stormcrawler.TestUtil;
 import org.apache.stormcrawler.util.MetadataTransfer;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
-/**
- * The date passed in {@link AbstractStatusUpdaterBolt#AS_IS_NEXTFETCHDATE_METADATA} comes from the
- * metadata and can be anything, so it must be parsed defensively.
- */
 class AbstractStatusUpdaterBoltTest {
 
+    // The date passed in AS_IS_NEXTFETCHDATE_METADATA comes from the metadata and can be anything,
+    // so it must be parsed defensively.
     private static final String URL = "http://example.com/";
 
     /** Records what the bolt asked to store. */
@@ -119,4 +120,219 @@ void outOfRangeNextFetchDateIsIgnoredAndTheUrlIsScheduled() {
         assertEquals(1, bolt.stored, "the URL must still be stored");
         assertNotNull(bolt.nextFetch);
     }
+
+    @Test
+    void testPermanentRedirect301IsEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        Map<String, Object> config = createConfig();
+        config.put(AbstractStatusUpdaterBolt.deleteRedirectionsParamName, true);
+
+        bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/old-page";
+        Metadata metadata = new Metadata();
+        metadata.setValue("fetch.statusCode", "301");
+
+        Tuple tuple = createTuple(url, Status.REDIRECTION, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(1, deletions.size());
+        assertEquals(url, deletions.get(0).get(0));
+
+        Metadata emittedMetadata = (Metadata) deletions.get(0).get(1);
+        assertEquals("301", emittedMetadata.getFirstValue("fetch.statusCode"));
+    }
+
+    @Test
+    void testPermanentRedirect308IsEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        Map<String, Object> config = createConfig();
+        config.put(AbstractStatusUpdaterBolt.deleteRedirectionsParamName, true);
+
+        bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/old-page";
+        Metadata metadata = new Metadata();
+        metadata.setValue("fetch.statusCode", "308");
+
+        Tuple tuple = createTuple(url, Status.REDIRECTION, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(1, deletions.size());
+        assertEquals(url, deletions.get(0).get(0));
+
+        Metadata emittedMetadata = (Metadata) deletions.get(0).get(1);
+        assertEquals("308", emittedMetadata.getFirstValue("fetch.statusCode"));
+    }
+
+    @Test
+    void testTemporaryRedirect302IsNotEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        Map<String, Object> config = createConfig();
+        config.put(AbstractStatusUpdaterBolt.deleteRedirectionsParamName, true);
+
+        bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/old-page";
+        Metadata metadata = new Metadata();
+        metadata.setValue("fetch.statusCode", "302");
+
+        Tuple tuple = createTuple(url, Status.REDIRECTION, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(0, deletions.size());
+    }
+
+    @Test
+    void testMetaRefreshRedirectIsNotEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        Map<String, Object> config = createConfig();
+        config.put(AbstractStatusUpdaterBolt.deleteRedirectionsParamName, true);
+
+        bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/old-page";
+        Metadata metadata = new Metadata();
+        metadata.setValue("fetch.statusCode", "200");
+        metadata.setValue("_redirTo", "http://example.com/new-page");
+
+        Tuple tuple = createTuple(url, Status.REDIRECTION, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(0, deletions.size());
+    }
+
+    @Test
+    void testRedirectionWithoutStatusCodeIsNotEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        Map<String, Object> config = createConfig();
+        config.put(AbstractStatusUpdaterBolt.deleteRedirectionsParamName, true);
+
+        bolt.prepare(config, TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/old-page";
+        Metadata metadata = new Metadata();
+
+        Tuple tuple = createTuple(url, Status.REDIRECTION, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(0, deletions.size());
+    }
+
+    @Test
+    void testPermanentRedirectIsNotDeletedByDefault() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        bolt.prepare(
+                createConfig(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/old-page";
+        Metadata metadata = new Metadata();
+        metadata.setValue("fetch.statusCode", "301");
+
+        Tuple tuple = createTuple(url, Status.REDIRECTION, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(0, deletions.size());
+    }
+
+    @Test
+    void testFetchedUrlIsNotEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        bolt.prepare(
+                createConfig(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/page";
+        Metadata metadata = new Metadata();
+        metadata.setValue("fetch.statusCode", "200");
+
+        Tuple tuple = createTuple(url, Status.FETCHED, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(0, deletions.size());
+    }
+
+    @Test
+    void testErrorIsEmittedToDeletionStream() {
+        TestOutputCollector output = new TestOutputCollector();
+        TestStatusUpdaterBolt bolt = new TestStatusUpdaterBolt();
+
+        bolt.prepare(
+                createConfig(), TestUtil.getMockedTopologyContext(), new OutputCollector(output));
+
+        String url = "http://example.com/error";
+        Metadata metadata = new Metadata();
+
+        Tuple tuple = createTuple(url, Status.ERROR, metadata);
+
+        bolt.execute(tuple);
+
+        List<List<Object>> deletions = output.getEmitted(Constants.DELETION_STREAM_NAME);
+
+        assertEquals(1, deletions.size());
+        assertEquals(url, deletions.get(0).get(0));
+    }
+
+    private static Map<String, Object> createConfig() {
+        Map<String, Object> config = new HashMap<>();
+        config.put(AbstractStatusUpdaterBolt.useCacheParamName, false);
+        config.put("scheduler.class", "org.apache.stormcrawler.persistence.DefaultScheduler");
+        return config;
+    }
+
+    private static Tuple createTuple(String url, Status status, Metadata metadata) {
+        Map<String, Object> tupleValues = new HashMap<>();
+        tupleValues.put("url", url);
+        tupleValues.put("status", status);
+        tupleValues.put("metadata", metadata);
+
+        return TestUtil.getMockedTestTuple(tupleValues);
+    }
+
+    private static class TestStatusUpdaterBolt extends AbstractStatusUpdaterBolt {
+
+        @Override
+        protected void store(
+                String url,
+                Status status,
+                Metadata metadata,
+                java.util.Optional<java.util.Date> nextFetch,
+                Tuple tuple) {
+            collector.ack(tuple);
+        }
+    }
 }