From 2aec52eb3db7e16ba4ce282f1bf7e867b2d2b1bb Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Tue, 15 Sep 2026 09:24:57 +0200 Subject: [PATCH 1/2] fix: bound the look-ahead for a meta charset cut by the detection window (#2092) getCharsetFromMeta retried itself with a window ten bytes larger, decoding the whole window again each time, until the closing quote of a thrown = new AtomicReference<>(); + Thread thread = + new Thread( + null, + () -> { + try { + CharsetIdentification.getCharset(new Metadata(), content, maxlength); + } catch (Throwable t) { + thrown.set(t); + } + }, + "charset-detection", + 1024 * 1024); + thread.start(); + thread.join(); + return thrown.get(); + } + + @Test + void largeDocumentWithUnterminatedMetaCharsetDoesNotOverflowTheStack() throws Exception { + // 400 KB is an ordinary page size, fetched whole under the default http.content.limit + Throwable thrown = detectOnSmallStack(unterminatedMetaCharset(400_000), MAXLENGTH); + Assertions.assertNull(thrown, "charset detection threw " + thrown); + } + + @Test + void smallDocumentWithUnterminatedMetaCharsetIsHandled() throws Exception { + Throwable thrown = detectOnSmallStack(unterminatedMetaCharset(20_000), MAXLENGTH); + Assertions.assertNull(thrown, "charset detection threw " + thrown); + } + + @Test + void unterminatedMetaCharsetIsHandledWithFullContentDetection() throws Exception { + Throwable thrown = detectOnSmallStack(unterminatedMetaCharset(400_000), -1); + Assertions.assertNull(thrown, "charset detection threw " + thrown); + } + + /** A declaration cut by the detection window is still read, see #870. */ + @Test + void metaCharsetCutByTheDetectionWindowIsStillRead() { + // ASCII content, no BOM and no HTTP header: only the meta tag can yield this charset + String declaration = ""; + StringBuilder page = new StringBuilder(""); + while (page.length() < MAXLENGTH) { + page.append(""); + } + // the window ends inside the charset name, the buffer ends right after the tag + int cut = page.length() + ""); + byte[] content = page.toString().getBytes(StandardCharsets.US_ASCII); + + String charset = CharsetIdentification.getCharsetFast(new Metadata(), content, cut); + + Assertions.assertEquals("windows-1251", charset); + } +} From 701d687d69f19d14fa22d47f7e1fd370fff40e52 Mon Sep 17 00:00:00 2001 From: Davide Polato Date: Tue, 15 Sep 2026 09:24:58 +0200 Subject: [PATCH 2/2] fix: report a charset detection failure as a parse error in JSoupParserBolt (#2092) Charset detection ran before the try block whose catch (Throwable) routes parse failures through handleException, so an error thrown there escaped execute() and took the worker down; the URL stayed scheduled and the replay took the restarted worker down again. The detection now runs inside that try, so a failure ends as a Status.ERROR for that URL like any other parse failure. A test injects an Error into the detection and checks the status tuple, the ack and that no document is emitted. Signed-off-by: Davide Polato --- .../stormcrawler/bolt/JSoupParserBolt.java | 33 ++++++++++--------- .../bolt/JSoupParserBoltTest.java | 28 ++++++++++++++++ .../util/CharsetIdentificationTest.java | 3 +- 3 files changed, 47 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java index 43a591c74..dc8c32c92 100644 --- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java +++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java @@ -269,22 +269,6 @@ public void execute(Tuple tuple) { long start = System.currentTimeMillis(); - String charset; - - if (fastCharsetDetection) { - charset = - CharsetIdentification.getCharsetFast( - metadata, content, maxLengthCharsetDetection); - } else { - charset = - CharsetIdentification.getCharset(metadata, content, maxLengthCharsetDetection); - } - - LOG.debug( - "Charset identified as {} in {} msec", - charset, - (System.currentTimeMillis() - start)); - RobotsTags robotsTags = new RobotsTags(); // get the robots tags from the fetch metadata @@ -295,8 +279,25 @@ public void execute(Tuple tuple) { Map> slinks; String text; final org.jsoup.nodes.Document jsoupDoc; + String charset; try { + // inside the try: a failure here is a parse error of this URL, not a dead worker + if (fastCharsetDetection) { + charset = + CharsetIdentification.getCharsetFast( + metadata, content, maxLengthCharsetDetection); + } else { + charset = + CharsetIdentification.getCharset( + metadata, content, maxLengthCharsetDetection); + } + + LOG.debug( + "Charset identified as {} in {} msec", + charset, + (System.currentTimeMillis() - start)); + String html = Charset.forName(charset).decode(ByteBuffer.wrap(content)).toString(); if (isPlainText) { diff --git a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java index 05ae595cc..ba28bc98c 100644 --- a/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java +++ b/core/src/test/java/org/apache/stormcrawler/bolt/JSoupParserBoltTest.java @@ -28,10 +28,13 @@ import org.apache.stormcrawler.TestUtil; import org.apache.stormcrawler.parse.ParsingTester; import org.apache.stormcrawler.persistence.Status; +import org.apache.stormcrawler.util.CharsetIdentification; import org.apache.stormcrawler.util.RobotsTags; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; class JSoupParserBoltTest extends ParsingTester { @@ -115,6 +118,31 @@ void setupParserBolt() { setupParserBolt(bolt); } + /** A failure inside charset detection is a parse error of that URL, not a dead worker. */ + @Test + void charsetDetectionFailureIsReportedAsParseError() throws IOException { + bolt.prepare( + new HashMap<>(), TestUtil.getMockedTopologyContext(), new OutputCollector(output)); + try (MockedStatic detection = + Mockito.mockStatic(CharsetIdentification.class)) { + detection + .when( + () -> + CharsetIdentification.getCharset( + Mockito.any(), Mockito.any(), Mockito.anyInt())) + .thenThrow(new StackOverflowError()); + parse("https://stormcrawler.apache.org", "stormcrawler.apache.org.html"); + } + List> statusTuples = output.getEmitted(Constants.StatusStreamName); + Assertions.assertEquals(1, statusTuples.size()); + Assertions.assertEquals(Status.ERROR, statusTuples.get(0).get(2)); + Metadata metadata = (Metadata) statusTuples.get(0).get(1); + Assertions.assertEquals( + "content parsing", metadata.getFirstValue(Constants.STATUS_ERROR_SOURCE)); + Assertions.assertEquals(1, output.getAckedTuples().size()); + Assertions.assertTrue(output.getEmitted().isEmpty(), "no document must be emitted"); + } + /** Checks that content in script is not included in the text representation. */ @Test void testNoScriptInText() throws IOException { diff --git a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java index 730068f91..3b9615508 100644 --- a/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java +++ b/core/src/test/java/org/apache/stormcrawler/util/CharsetIdentificationTest.java @@ -50,7 +50,8 @@ private static Throwable detectOnSmallStack(byte[] content, int maxlength) null, () -> { try { - CharsetIdentification.getCharset(new Metadata(), content, maxlength); + CharsetIdentification.getCharset( + new Metadata(), content, maxlength); } catch (Throwable t) { thrown.set(t); }