From 573632ff8f957363e9da171ef1fa7ad947c3d6cc Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 16:30:56 +0800 Subject: [PATCH 1/3] fix: address CodeQL error findings and size overflow fallouts - add the missing Tablet.hashCode and make the extract-filter equals() type check explicit - guard the copyDirectory containment check with a path separator so a sibling directory is not mistaken for a child - widen size arithmetic to long in the encoders and TimeDuration so byte size estimates and month durations cannot overflow int - remove unused containers/variables flagged in tests and use pytest.importorskip for pyarrow --- .../tsfile/utils/RamUsageEstimator.java | 18 ++++++++++--- .../encoding/encoder/BitmapEncoder.java | 2 +- .../encoding/encoder/DeltaBinaryEncoder.java | 4 +-- .../encoding/encoder/IntRleEncoder.java | 2 +- .../encoding/encoder/IntZigzagEncoder.java | 2 +- .../encoding/encoder/LongRleEncoder.java | 2 +- .../encoding/encoder/LongZigzagEncoder.java | 2 +- .../encoding/encoder/RegularDataEncoder.java | 4 +-- .../tsfile/external/commons/io/FileUtils.java | 10 ++++++-- .../operator/ExtractValueFilterOperators.java | 10 +++++--- .../org/apache/tsfile/utils/TimeDuration.java | 6 ++--- .../apache/tsfile/write/record/Tablet.java | 8 ++++++ .../encoding/decoder/BitmapDecoderTest.java | 9 ------- .../tsfile/read/ReadInPartitionTest.java | 7 ------ .../read/TsFileV4ReadWriteInterfacesTest.java | 5 ---- .../tsfile/tableview/TableViewTest.java | 4 +-- python/tests/test_batch_arrow.py | 25 ++++--------------- 17 files changed, 54 insertions(+), 66 deletions(-) diff --git a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java index e70a00f404..c57c309984 100644 --- a/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java +++ b/java/common/src/main/java/org/apache/tsfile/utils/RamUsageEstimator.java @@ -201,13 +201,11 @@ private RamUsageEstimator() {} // get min/max value of cached Long class instances: long longCacheMinValue = 0; - while (longCacheMinValue > Long.MIN_VALUE - && Long.valueOf(longCacheMinValue - 1) == Long.valueOf(longCacheMinValue - 1)) { + while (longCacheMinValue > Long.MIN_VALUE && isCachedLong(longCacheMinValue - 1)) { longCacheMinValue -= 1; } long longCacheMaxValue = -1; - while (longCacheMaxValue < Long.MAX_VALUE - && Long.valueOf(longCacheMaxValue + 1) == Long.valueOf(longCacheMaxValue + 1)) { + while (longCacheMaxValue < Long.MAX_VALUE && isCachedLong(longCacheMaxValue + 1)) { longCacheMaxValue += 1; } LONG_CACHE_MIN_VALUE = longCacheMinValue; @@ -231,6 +229,18 @@ public static long alignObjectSize(long size) { return (size + ALIGN_MASK) & ~ALIGN_MASK; } + /** + * Returns true when {@link Long#valueOf(long)} hands out the same cached instance for {@code + * value}. The two references are compared on purpose — identity is exactly what identifies a + * value inside the JVM's {@code Long} cache — and they are typed as {@link Object} so this is not + * a boxed-value comparison. + */ + private static boolean isCachedLong(long value) { + final Object first = Long.valueOf(value); + final Object second = Long.valueOf(value); + return first == second; + } + /** * Return the size of the provided {@link Long} object, returning 0 if it is cached by the JVM and * its shallow size otherwise. diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java index bb728e67ff..5116e770fe 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java @@ -127,6 +127,6 @@ public int getOneItemMaxSize() { @Override public long getMaxByteSize() { // byteCacheSize + byteDictSize + (byte array + array length) * byteDictSize - return (long) 4 + 4 + ((values.size() + 7) / 8 + 4) * values.size(); + return (long) 4 + 4 + ((long) (values.size() + 7) / 8 + 4) * values.size(); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DeltaBinaryEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DeltaBinaryEncoder.java index 17ac444525..8172d8e99e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DeltaBinaryEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/DeltaBinaryEncoder.java @@ -219,7 +219,7 @@ public int getOneItemMaxSize() { @Override public long getMaxByteSize() { // The meaning of 24 is: index(4)+width(4)+minDeltaBase(4)+firstValue(4) - return (long) 24 + writeIndex * 4; + return (long) 24 + (long) writeIndex * 4; } } @@ -294,7 +294,7 @@ public int getOneItemMaxSize() { @Override public long getMaxByteSize() { // The meaning of 24 is: index(4)+width(4)+minDeltaBase(8)+firstValue(8) - return (long) 24 + writeIndex * 8; + return (long) 24 + (long) writeIndex * 8; } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntRleEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntRleEncoder.java index a9fd2e8fc8..a744af6623 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntRleEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntRleEncoder.java @@ -124,6 +124,6 @@ public long getMaxByteSize() { } // try to calculate max value int groupNum = (values.size() / 8 + 1) / 63 + 1; - return (long) 8 + groupNum * 5 + values.size() * 4; + return (long) 8 + groupNum * 5L + (long) values.size() * 4; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntZigzagEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntZigzagEncoder.java index b056167d0b..4628834a83 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntZigzagEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/IntZigzagEncoder.java @@ -97,6 +97,6 @@ public long getMaxByteSize() { return 0; } // try to calculate max value - return (long) 8 + values.size() * 4; + return (long) 8 + (long) values.size() * 4; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongRleEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongRleEncoder.java index f9e9c55702..d7473f4714 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongRleEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongRleEncoder.java @@ -117,6 +117,6 @@ public long getMaxByteSize() { } // try to calculate max value int groupNum = (values.size() / 8 + 1) / 63 + 1; - return (long) 8 + groupNum * 5 + values.size() * 8; + return (long) 8 + groupNum * 5L + (long) values.size() * 8; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongZigzagEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongZigzagEncoder.java index 130cf9bae3..1d1e702722 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongZigzagEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/LongZigzagEncoder.java @@ -108,6 +108,6 @@ public long getMaxByteSize() { return 0; } // try to calculate max value - return (long) 8 + values.size() * 4; + return (long) 8 + (long) values.size() * 4; } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/RegularDataEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/RegularDataEncoder.java index ad74699678..1f40784bab 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/RegularDataEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/RegularDataEncoder.java @@ -164,7 +164,7 @@ public int getOneItemMaxSize() { public long getMaxByteSize() { // The meaning of 20 is: // identifier(4)+bitmapLength(4)+index(4)+minDeltaBase(4)+firstValue(4) - return (long) 20 + (writeIndex * 2 / 8) + (writeIndex * 4); + return (long) 20 + ((long) writeIndex * 2 / 8) + ((long) writeIndex * 4); } @Override @@ -291,7 +291,7 @@ public int getOneItemMaxSize() { public long getMaxByteSize() { // The meaning of 20 is: // identifier(4)+bitmapLength(4)+index(4)+minDeltaBase(8)+firstValue(8) - return (long) 28 + (writeIndex * 2 / 8) + (writeIndex * 8); + return (long) 28 + ((long) writeIndex * 2 / 8) + ((long) writeIndex * 8); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/external/commons/io/FileUtils.java b/java/tsfile/src/main/java/org/apache/tsfile/external/commons/io/FileUtils.java index 35e7f7b9e3..9bc5f3a56e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/external/commons/io/FileUtils.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/external/commons/io/FileUtils.java @@ -784,11 +784,17 @@ public static void copyDirectory( requireDirectory(srcDir, "srcDir"); requireCanonicalPathsNotEquals(srcDir, destDir); - // Cater for destination being directory within the source directory (see IO-141) + // Cater for destination being directory within the source directory (see IO-141). + // Compare against a path prefix that ends with the separator, so a sibling + // such as "/a/bc" is not mistaken for a child of "/a/b". List exclusionList = null; final String srcDirCanonicalPath = srcDir.getCanonicalPath(); final String destDirCanonicalPath = destDir.getCanonicalPath(); - if (destDirCanonicalPath.startsWith(srcDirCanonicalPath)) { + final String srcDirPrefix = + srcDirCanonicalPath.endsWith(File.separator) + ? srcDirCanonicalPath + : srcDirCanonicalPath + File.separator; + if (destDirCanonicalPath.startsWith(srcDirPrefix)) { final File[] srcFiles = listFiles(srcDir, fileFilter); if (srcFiles.length > 0) { exclusionList = new ArrayList<>(srcFiles.length); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ExtractValueFilterOperators.java b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ExtractValueFilterOperators.java index 05a39f0a73..ce2d6961cf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ExtractValueFilterOperators.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/filter/operator/ExtractValueFilterOperators.java @@ -96,15 +96,17 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || getClass() != o.getClass()) { + if (!(o instanceof ExtractValueCompareFilter)) { + return false; + } + if (getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } - ExtractTimeFilterOperators.ExtractTimeCompareFilter thatDelegate = - ((ExtractValueCompareFilter) o).delegate; - return delegate.equals(thatDelegate); + ExtractValueCompareFilter that = (ExtractValueCompareFilter) o; + return delegate.equals(that.delegate); } @Override diff --git a/java/tsfile/src/main/java/org/apache/tsfile/utils/TimeDuration.java b/java/tsfile/src/main/java/org/apache/tsfile/utils/TimeDuration.java index 9e835423d3..4c2490ebce 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/utils/TimeDuration.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/utils/TimeDuration.java @@ -54,13 +54,13 @@ public boolean containsMonth() { * @return the total duration of this timeDuration in current precision */ public long getTotalDuration(TimeUnit currPrecision) { - return currPrecision.convert(monthDuration * 30 * 86400_000L, TimeUnit.MILLISECONDS) + return currPrecision.convert(monthDuration * 30L * 86400_000L, TimeUnit.MILLISECONDS) + nonMonthDuration; } /** Think month as 31 days. */ public long getMaxTotalDuration(TimeUnit currPrecision) { - return currPrecision.convert(monthDuration * 31 * 86400_000L, TimeUnit.MILLISECONDS) + return currPrecision.convert(monthDuration * 31L * 86400_000L, TimeUnit.MILLISECONDS) + nonMonthDuration; } @@ -84,7 +84,7 @@ public TimeDuration multiple(long times) { /** Think month as 28 days. */ public long getMinTotalDuration(TimeUnit currPrecision) { - return currPrecision.convert(monthDuration * 28 * 86400_000L, TimeUnit.MILLISECONDS) + return currPrecision.convert(monthDuration * 28L * 86400_000L, TimeUnit.MILLISECONDS) + nonMonthDuration; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index 953dbf6d6e..ad7f6222e5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -986,6 +986,14 @@ public boolean equals(Object o) { return true; } + @Override + public int hashCode() { + // Must stay consistent with equals(): only the fields compared before the + // timestamp/bitmap/value checks are hashed. Equal tablets share them, and + // unequal tablets may still collide, which is allowed. + return Objects.hash(rowSize, insertTargetName, schemas, columnCategories, measurementIndex); + } + private boolean isTimestampsEqual(long[] thisTimestamps, long[] thatTimestamps, int rowSize) { if (thisTimestamps == thatTimestamps) { return true; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/BitmapDecoderTest.java b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/BitmapDecoderTest.java index 52adc33e3f..ac922ee3dd 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/BitmapDecoderTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/encoding/decoder/BitmapDecoderTest.java @@ -41,7 +41,6 @@ public class BitmapDecoderTest { private static final Logger logger = LoggerFactory.getLogger(BitmapDecoderTest.class); private List intList; - private List booleanList; @Before public void setUp() { @@ -52,14 +51,6 @@ public void setUp() { for (int i = 0; i < int_num; i++) { intList.add(int_array[i % int_len]); } - - booleanList = new ArrayList(); - boolean[] boolean_array = {true, false, true, true, false, true, false, false}; - int boolean_len = boolean_array.length; - int boolean_num = 100000; - for (int i = 0; i < boolean_num; i++) { - booleanList.add(boolean_array[i % boolean_len]); - } } @After diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/ReadInPartitionTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/ReadInPartitionTest.java index f0a5fd1bf8..b493e929b2 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/ReadInPartitionTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/ReadInPartitionTest.java @@ -62,7 +62,6 @@ public class ReadInPartitionTest { private static final String FILE_PATH = TsFileGeneratorForTest.outputDataFile; private static TsFileReader roTsFile = null; private ArrayList d1s6timeRangeList = new ArrayList<>(); - private ArrayList d2s1timeRangeList = new ArrayList<>(); private ArrayList d1chunkGroupMetaDataOffsetList = new ArrayList<>(); @Before @@ -91,12 +90,6 @@ public void before() throws IOException { + chunkMetaData.getStatistics().getSerializedSize(); d1chunkGroupMetaDataOffsetList.add(startEndOffsets); } - - List d2s1List = reader.getChunkMetadataList(new Path("d2", "s1", true)); - for (ChunkMetadata chunkMetaData : d2s1List) { - d2s1timeRangeList.add( - new TimeRange(chunkMetaData.getStartTime(), chunkMetaData.getEndTime())); - } } @After diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java index bbcc3278b8..9a1c9dd164 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TsFileV4ReadWriteInterfacesTest.java @@ -24,7 +24,6 @@ import org.apache.tsfile.exception.write.WriteProcessException; import org.apache.tsfile.file.metadata.AbstractAlignedChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; -import org.apache.tsfile.file.metadata.StringArrayDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.read.v4.DeviceTableModelReader; import org.apache.tsfile.read.v4.ITsFileReader; @@ -45,7 +44,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -207,7 +205,6 @@ public void testGetTableDeviceMethods() throws Exception { startValue, chunkGroupSize, pageSize); - List deviceIDList = new ArrayList<>(); TableSchema tableSchema = new TableSchema( "t1", @@ -245,8 +242,6 @@ public void testGetTableDeviceMethods() throws Exception { tablet.addValue("id1", i, ids[i][0]); tablet.addValue("id2", i, ids[i][1]); tablet.addValue("id3", i, ids[i][2]); - deviceIDList.add( - new StringArrayDeviceID(tableSchema.getTableName(), ids[i][0], ids[i][1], ids[i][2])); tablet.addValue("s1", i, i); } tablet.setRowSize(ids.length); diff --git a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java index 434fd50cf6..8c0364f9d0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/tableview/TableViewTest.java @@ -30,7 +30,6 @@ import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.read.common.Path; -import org.apache.tsfile.read.common.RowRecord; import org.apache.tsfile.read.common.block.TsBlock; import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.read.common.type.service.TypeService; @@ -497,9 +496,8 @@ public void testHybridWrite() throws Exception { final QueryExpression queryExpression = QueryExpression.create(selectedSeries, null); final QueryDataSet queryDataSet = queryExecutor.execute(queryExpression); cnt = 0; - List rowRecords = new ArrayList<>(); while (queryDataSet.hasNext()) { - rowRecords.add(queryDataSet.next()); + queryDataSet.next(); cnt++; } assertEquals(10, cnt); diff --git a/python/tests/test_batch_arrow.py b/python/tests/test_batch_arrow.py index 70911efe0f..6103ce71c2 100644 --- a/python/tests/test_batch_arrow.py +++ b/python/tests/test_batch_arrow.py @@ -56,10 +56,7 @@ def test_batch_read_arrow_basic(): tablet.add_value_by_name("value1", i, i * 10) tablet.add_value_by_name("value2", i, i * 1.5) writer.write_table(tablet) - try: - import pyarrow as pa - except ImportError: - pytest.skip("pyarrow is not installed") + pa = pytest.importorskip("pyarrow") reader = TsFileReader(file_path) result_set = reader.query_table( @@ -134,10 +131,7 @@ def test_batch_read_arrow_compare_with_dataframe(): tablet.add_value_by_name("value3", i, i % 2 == 0) writer.write_table(tablet) - try: - import pyarrow as pa - except ImportError: - pytest.skip("pyarrow is not installed") + pa = pytest.importorskip("pyarrow") reader1 = TsFileReader(file_path) result_set1 = reader1.query_table( @@ -242,10 +236,7 @@ def test_batch_read_arrow_empty_result(): tablet.add_value_by_name("value", i, i) writer.write_table(tablet) - try: - import pyarrow as pa - except ImportError: - pytest.skip("pyarrow is not installed") + pa = pytest.importorskip("pyarrow") reader = TsFileReader(file_path) result_set = reader.query_table( @@ -294,10 +285,7 @@ def test_batch_read_arrow_time_range(): tablet.add_value_by_name("value", i, i) writer.write_table(tablet) - try: - import pyarrow as pa - except ImportError: - pytest.skip("pyarrow is not installed") + pa = pytest.importorskip("pyarrow") reader = TsFileReader(file_path) result_set = reader.query_table( @@ -384,10 +372,7 @@ def test_batch_read_arrow_all_datatypes(): tablet.add_value_by_name("date_val", i, date(2025, 1, (i % 28) + 1)) writer.write_table(tablet) - try: - import pyarrow as pa - except ImportError: - pytest.skip("pyarrow is not installed") + pa = pytest.importorskip("pyarrow") reader = TsFileReader(file_path) result_set = reader.query_table( From 19c44da7d641c9831284f5d3eb62976288671203 Mon Sep 17 00:00:00 2001 From: Colin Lee <105656576+ColinLeeo@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:21:06 +0800 Subject: [PATCH 2/3] Update hash calculation in Tablet class Removed 'schemas' from hash calculation to maintain consistency with equals() method. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../src/main/java/org/apache/tsfile/write/record/Tablet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index ad7f6222e5..fe4c4b8905 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -991,7 +991,7 @@ public int hashCode() { // Must stay consistent with equals(): only the fields compared before the // timestamp/bitmap/value checks are hashed. Equal tablets share them, and // unequal tablets may still collide, which is allowed. - return Objects.hash(rowSize, insertTargetName, schemas, columnCategories, measurementIndex); + return Objects.hash(rowSize, insertTargetName, columnCategories, measurementIndex); } private boolean isTimestampsEqual(long[] thisTimestamps, long[] thatTimestamps, int rowSize) { From a44493bbb73071b70d9d9040f2eb8bc4f72689c1 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 17:41:45 +0800 Subject: [PATCH 3/3] fix: complete the overflow and hash fixes from review - BitmapEncoder now evaluates `values.size() + 7` in long, so the addition itself cannot wrap before the byte-size estimate is widened. - VectorMeasurementSchema hashed its type/encoding/compressor arrays by identity while equals() compares them by content; hash the contents so equal schemas share a hash. - Tablet.hashCode keeps its partial-field contract and no longer hashes `schemas`, whose pre-fix hash was the only inconsistent input. --- .../apache/tsfile/encoding/encoder/BitmapEncoder.java | 2 +- .../java/org/apache/tsfile/write/record/Tablet.java | 5 +++-- .../tsfile/write/schema/VectorMeasurementSchema.java | 10 +++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java index 5116e770fe..e3dd15d0b5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/encoding/encoder/BitmapEncoder.java @@ -127,6 +127,6 @@ public int getOneItemMaxSize() { @Override public long getMaxByteSize() { // byteCacheSize + byteDictSize + (byte array + array length) * byteDictSize - return (long) 4 + 4 + ((long) (values.size() + 7) / 8 + 4) * values.size(); + return (long) 4 + 4 + ((values.size() + 7L) / 8 + 4) * values.size(); } } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java index fe4c4b8905..1b1010c019 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/record/Tablet.java @@ -988,8 +988,9 @@ public boolean equals(Object o) { @Override public int hashCode() { - // Must stay consistent with equals(): only the fields compared before the - // timestamp/bitmap/value checks are hashed. Equal tablets share them, and + // Must stay consistent with equals(): `schemas` is left out so the hash + // does not depend on the List's element hash implementations; the + // remaining partial fields keep equal tablets on the same hash, and // unequal tablets may still collide, which is allowed. return Objects.hash(rowSize, insertTargetName, columnCategories, measurementIndex); } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/VectorMeasurementSchema.java b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/VectorMeasurementSchema.java index 6cee451e98..44ad4ef61b 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/write/schema/VectorMeasurementSchema.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/write/schema/VectorMeasurementSchema.java @@ -483,7 +483,15 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(deviceId, types, encodings, unifiedCompressor, compressors); + // The arrays are compared by content in equals(), so hash their contents + // as well: array identity hashing would give equal schemas different + // hashes. + return Objects.hash( + deviceId, + Arrays.hashCode(types), + Arrays.hashCode(encodings), + unifiedCompressor, + Arrays.hashCode(compressors)); } /** compare by vector name */