From e2691fcec7f6c44a158d670dacb742f5dc41d707 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Tue, 17 Nov 2026 16:45:34 +0800 Subject: [PATCH 1/8] Add a selective metadata reader for a set of measurements MetadataIndexNode.getChildIndexEntries() searches the floor child entry of a sorted key set in one pass, choosing between a monotonic binary search and a merge scan and reusing one pair instance per child. The new TsFileSequenceReader.readTimeseriesMetadata(device, offset, measurements, returnedMeasurements, ...) uses it to visit only the leaf groups that hold a searched measurement, reads each visited group once, and returns the metadata of the returned measurements. A caller can resolve a whole projection without one index walk and one leaf deserialization per column, and a single-measurement call can warm its group siblings through returnedMeasurements. Follows the batch index-search approach of #550. --- .../file/metadata/MetadataIndexNode.java | 95 ++++++++- .../tsfile/read/TsFileSequenceReader.java | 198 ++++++++++++++++++ .../file/metadata/MetadataIndexNodeTest.java | 50 +++++ .../read/TimeSeriesMetadataReadTest.java | 83 ++++++++ 4 files changed, 425 insertions(+), 1 deletion(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 693b4ffb18..26aed827c0 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -33,10 +33,12 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; public class MetadataIndexNode { + private static final double LOG2 = Math.log(2); protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); protected final List children; protected long endOffset; @@ -126,7 +128,10 @@ public static MetadataIndexNode deserializeFrom( } public Pair getChildIndexEntry(Comparable key, boolean exactSearch) { - int index = binarySearchInChildren(key, exactSearch); + return getChildIndexEntry(binarySearchInChildren(key, exactSearch)); + } + + private Pair getChildIndexEntry(int index) { if (index == -1) { return null; } @@ -139,6 +144,33 @@ public Pair getChildIndexEntry(Comparable key, boolea return new Pair<>(children.get(index), childEndOffset); } + /** + * Searches the floor child entry of every key in one pass. The same pair instance is returned for + * consecutive keys that belong to the same child, so callers can group keys by identity. + * + * @param keys keys sorted in ascending order + * @param exactSearch whether to return null instead of the floor child when the key is missing + */ + public List> getChildIndexEntries( + List keys, boolean exactSearch) { + int[] indexes = + keys.size() >= children.size() + || keys.size() * Math.log(children.size()) / LOG2 > keys.size() + children.size() + ? mergeSearchInChildren(keys, exactSearch) + : binarySearchInChildren(keys, exactSearch); + List> pairs = new ArrayList<>(indexes.length); + Pair previousPair = null; + int previousIndex = -1; + for (int index : indexes) { + if (index != previousIndex) { + previousPair = getChildIndexEntry(index); + previousIndex = index; + } + pairs.add(previousPair); + } + return pairs; + } + int binarySearchInChildren(Comparable key, boolean exactSearch) { int low = 0; int high = children.size() - 1; @@ -165,6 +197,67 @@ int binarySearchInChildren(Comparable key, boolean exactSearch) { } } + int[] binarySearchInChildren(List keys, boolean exactSearch) { + int[] results = new int[keys.size()]; + Arrays.fill(results, -1); + int currentLow = 0; + int high = children.size() - 1; + + for (int i = 0; i < keys.size(); i++) { + Comparable key = keys.get(i); + if (currentLow > high) { + Arrays.fill(results, i, keys.size(), exactSearch ? -1 : currentLow - 1); + return results; + } + + int foundIndex = -1; + int start = currentLow; + int end = high; + while (start <= end) { + int mid = (start + end) >>> 1; + int cmp = children.get(mid).getCompareKey().compareTo(key); + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; + } else { + foundIndex = mid; + break; + } + } + + if (foundIndex >= 0) { + results[i] = foundIndex; + currentLow = foundIndex + 1; + } else { + results[i] = exactSearch ? -1 : start - 1; + currentLow = start; + } + } + return results; + } + + int[] mergeSearchInChildren(List keys, boolean exactSearch) { + int[] results = new int[keys.size()]; + int i = 0; + int j = 0; + while (i < keys.size() && j < children.size()) { + int cmp = keys.get(i).compareTo(children.get(j).getCompareKey()); + if (cmp == 0) { + results[i] = j; + i++; + j++; + } else if (cmp > 0) { + j++; + } else { + results[i] = exactSearch ? -1 : j - 1; + i++; + } + } + Arrays.fill(results, i, keys.size(), exactSearch ? -1 : children.size() - 1); + return results; + } + public boolean isDeviceLevel() { return this.nodeType == MetadataIndexNodeType.INTERNAL_DEVICE || this.nodeType == MetadataIndexNodeType.LEAF_DEVICE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index a3137149c7..8cddf4c46e 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -95,10 +95,12 @@ import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -110,6 +112,7 @@ import java.util.Queue; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -1092,6 +1095,201 @@ public List readTimeseriesMetadata( return timeseriesMetadataList; } + /** + * Reads the {@link TimeseriesMetadata} of the given measurements of a device, visiting each + * measurement leaf group that holds a requested measurement at most once. + * + * @param deviceMetadataIndexNodeOffset measurement metadata index node offset of an external + * TsFile; null to locate the device through the table metadata index + * @param measurements measurements to read, in any order + * @param ignoreNotExistDevice whether to return an empty map instead of throwing when the device + * is not recorded in the file metadata + * @return the metadata of the measurements found in this file; absent measurements are missing + * from the map + */ + public Map readTimeseriesMetadata( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Collection measurements, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { + return readTimeseriesMetadata( + device, + deviceMetadataIndexNodeOffset, + measurements, + null, + ignoreNotExistDevice, + ioSizeRecorder); + } + + /** + * Reads the {@link TimeseriesMetadata} of the measurements that share a leaf group with one of + * {@code measurements}. Leaf groups that no searched measurement maps into are not read, and each + * visited leaf group is read once. + * + * @param measurements measurements that select the leaf groups to visit + * @param returnedMeasurements measurements to deserialize from the visited groups; null means + * {@code measurements}. It may contain measurements outside {@code measurements}, which is + * how a caller warms the metadata cache with the siblings of a single requested measurement + * @return the metadata of the returned measurements found in this file; absent measurements are + * missing from the map + */ + public Map readTimeseriesMetadata( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Collection measurements, + Set returnedMeasurements, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { + if (measurements.isEmpty()) { + return Collections.emptyMap(); + } + MetadataIndexNode measurementIndexNode = + getMeasurementMetadataIndexNode(device, deviceMetadataIndexNodeOffset, ioSizeRecorder); + if (measurementIndexNode == null) { + if (ignoreNotExistDevice) { + return Collections.emptyMap(); + } + throw new IOException( + Messages.format("error.read.device_not_in_metadata_file", device, file)); + } + List sortedMeasurements = new ArrayList<>(new TreeSet<>(measurements)); + Set returned = + returnedMeasurements == null ? new HashSet<>(sortedMeasurements) : returnedMeasurements; + Map result = new HashMap<>(); + readMeasurementsInIndexNode( + measurementIndexNode, sortedMeasurements, returned, result, ioSizeRecorder); + return result; + } + + public Map readTimeseriesMetadata( + IDeviceID device, Collection measurements, boolean ignoreNotExistDevice) + throws IOException { + return readTimeseriesMetadata(device, null, measurements, null, ignoreNotExistDevice, null); + } + + private MetadataIndexNode getMeasurementMetadataIndexNode( + IDeviceID device, long[] deviceMetadataIndexNodeOffset, LongConsumer ioSizeRecorder) + throws IOException { + if (deviceMetadataIndexNodeOffset != null) { + ByteBuffer buffer = + readData( + deviceMetadataIndexNodeOffset[0], deviceMetadataIndexNodeOffset[1], ioSizeRecorder); + return deserializeMeasurementMetadataIndexNode(buffer); + } + readFileMetadata(ioSizeRecorder); + MetadataIndexNode deviceIndexNode = + tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); + Pair deviceEntry = + getMetadataAndEndOffsetOfDeviceNode(deviceIndexNode, device, true, ioSizeRecorder); + if (deviceEntry == null) { + return null; + } + ByteBuffer buffer = readData(deviceEntry.left.getOffset(), deviceEntry.right, ioSizeRecorder); + if (deviceIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + return deviceIndexNode; + } + return deserializeMeasurementMetadataIndexNode(buffer); + } + + private MetadataIndexNode deserializeMeasurementMetadataIndexNode(ByteBuffer buffer) + throws IOException { + try { + return deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( + buffer, deserializeConfig); + } catch (Exception e) { + logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file); + throw e; + } + } + + private void readMeasurementsInIndexNode( + MetadataIndexNode node, + List sortedMeasurements, + Set requested, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + List> entries = + node.getChildIndexEntries(sortedMeasurements, false); + int groupStart = 0; + for (int i = 1; i <= sortedMeasurements.size(); i++) { + if (i < sortedMeasurements.size() && entries.get(i) == entries.get(groupStart)) { + continue; + } + Pair entry = entries.get(groupStart); + if (entry != null) { + if (node.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { + readLeafMeasurementGroup(entry, requested, result, ioSizeRecorder); + } else { + ByteBuffer buffer = readData(entry.left.getOffset(), entry.right, ioSizeRecorder); + readMeasurementsInIndexNode( + deserializeMeasurementMetadataIndexNode(buffer), + sortedMeasurements.subList(groupStart, i), + requested, + result, + ioSizeRecorder); + } + } + groupStart = i; + } + } + + private void readLeafMeasurementGroup( + Pair entry, + Set requested, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + long startOffset = entry.left.getOffset(); + long endOffset = entry.right; + if (endOffset - startOffset < Integer.MAX_VALUE) { + ByteBuffer buffer = readData(startOffset, endOffset, ioSizeRecorder); + while (buffer.hasRemaining()) { + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(buffer); + if (requested.contains(metadata.getMeasurementId())) { + result.put(metadata.getMeasurementId(), metadata); + } + } + return; + } + // a group larger than Integer.MAX_VALUE cannot be held in one buffer + if (ioSizeRecorder != null) { + ioSizeRecorder.accept(endOffset - startOffset); + } + synchronized (this) { + tsFileInput.position(startOffset); + while (tsFileInput.position() < endOffset) { + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput); + if (requested.contains(metadata.getMeasurementId())) { + result.put(metadata.getMeasurementId(), metadata); + } + } + } + } + + private TimeseriesMetadata deserializeTimeseriesMetadata(ByteBuffer buffer) throws IOException { + try { + return TimeseriesMetadata.deserializeFrom(buffer, true); + } catch (Exception e) { + logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); + throw e; + } + } + + private TimeseriesMetadata deserializeTimeseriesMetadata(TsFileInput input) throws IOException { + try { + return TimeseriesMetadata.deserializeFrom(input, true); + } catch (StopReadTsFileByInterruptException e) { + throw e; + } catch (Exception e) { + logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); + throw e; + } + } + /* Get leaf MetadataIndexPair which contains path */ private Pair getLeafMetadataIndexPair( IDeviceID device, diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java index df18f6f39d..736eed3746 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java @@ -21,11 +21,14 @@ import org.apache.tsfile.file.IMetadataIndexEntry; import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType; +import org.apache.tsfile.utils.Pair; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; public class MetadataIndexNodeTest { @@ -48,4 +51,51 @@ public void testBinarySearchInChildren() { Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s13", true)); Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s23", true)); } + + @Test + public void testGetChildIndexEntries() { + List list = new ArrayList<>(); + list.add(new MeasurementMetadataIndexEntry("s0", 0L)); + list.add(new MeasurementMetadataIndexEntry("s5", 10L)); + list.add(new MeasurementMetadataIndexEntry("s10", 20L)); + list.add(new MeasurementMetadataIndexEntry("s15", 30L)); + list.add(new MeasurementMetadataIndexEntry("s20", 40L)); + MetadataIndexNode node = + new MetadataIndexNode(list, 50L, MetadataIndexNodeType.LEAF_MEASUREMENT); + + // few keys: monotonic binary search, each key gets the floor child and its end offset + List> pairs = + node.getChildIndexEntries(Arrays.asList("s1", "s12"), false); + Assert.assertEquals(2, pairs.size()); + Assert.assertEquals(0L, pairs.get(0).getLeft().getOffset()); + Assert.assertEquals(10L, pairs.get(0).getRight().longValue()); + Assert.assertEquals(20L, pairs.get(1).getLeft().getOffset()); + Assert.assertEquals(30L, pairs.get(1).getRight().longValue()); + + // many keys: merge search, consecutive keys of one child share the same pair instance + List> merged = + node.getChildIndexEntries(Arrays.asList("s0", "s1", "s2", "s3", "s4", "s5", "s6"), false); + Assert.assertEquals(7, merged.size()); + Assert.assertSame(merged.get(0), merged.get(4)); + Assert.assertNotSame(merged.get(0), merged.get(5)); + Assert.assertEquals(10L, merged.get(5).getLeft().getOffset()); + Assert.assertEquals(20L, merged.get(5).getRight().longValue()); + + // a key before the first child has no floor, and the last child ends at the node end offset + Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("a"), false).get(0)); + Assert.assertEquals( + 50L, + node.getChildIndexEntries(Collections.singletonList("zz"), false) + .get(0) + .getRight() + .longValue()); + + // exact search returns null for a missing key + Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("s13"), true).get(0)); + + // an empty node has no floor entry + MetadataIndexNode empty = + new MetadataIndexNode(new ArrayList<>(), 0L, MetadataIndexNodeType.LEAF_MEASUREMENT); + Assert.assertNull(empty.getChildIndexEntries(Collections.singletonList("s1"), false).get(0)); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index e4bfbb44de..f76962bc2e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -21,6 +21,7 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.utils.FileGenerator; @@ -32,8 +33,10 @@ import java.io.File; import java.io.IOException; +import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class TimeSeriesMetadataReadTest { @@ -93,4 +96,84 @@ public void testReadTimeseriesMetadata() throws IOException { Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 5).getMeasurementId()); } } + + @Test + public void testReadTimeseriesMetadataInBatch() throws IOException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + IDeviceID device = new Path("d1", "s1", true).getIDeviceID(); + Set requested = + new HashSet<>(Arrays.asList("s1", "s2", "s3", "s4", "s5", "s6", "s8")); + + Map batch = + reader.readTimeseriesMetadata(device, requested, false); + // s8 does not exist in the file + Assert.assertEquals(6, batch.size()); + Assert.assertNull(batch.get("s8")); + for (String measurement : Arrays.asList("s1", "s2", "s3", "s4", "s5", "s6")) { + TimeseriesMetadata single = reader.readTimeseriesMetadata(device, measurement, false, null); + TimeseriesMetadata batched = batch.get(measurement); + Assert.assertNotNull(measurement, single); + Assert.assertNotNull(measurement, batched); + Assert.assertEquals(measurement, single.getMeasurementId(), batched.getMeasurementId()); + Assert.assertEquals( + measurement, single.getStatistics().getCount(), batched.getStatistics().getCount()); + Assert.assertEquals( + measurement, + single.getStatistics().getStartTime(), + batched.getStatistics().getStartTime()); + Assert.assertEquals( + measurement, single.getStatistics().getEndTime(), batched.getStatistics().getEndTime()); + Assert.assertEquals( + measurement, + single.getChunkMetadataList().size(), + batched.getChunkMetadataList().size()); + } + + // a single searched measurement can warm the cache with its group siblings + Map withSiblings = + reader.readTimeseriesMetadata( + device, + null, + java.util.Collections.singletonList("s1"), + new HashSet<>(Arrays.asList("s1", "s2", "s3")), + false, + null); + Assert.assertEquals(new HashSet<>(Arrays.asList("s1", "s2", "s3")), withSiblings.keySet()); + for (Map.Entry entry : withSiblings.entrySet()) { + Assert.assertEquals( + entry.getKey(), + batch.get(entry.getKey()).getStatistics().getCount(), + entry.getValue().getStatistics().getCount()); + } + + // an absent device is tolerated only when ignoreNotExistDevice is set + IDeviceID absentDevice = new Path("d_not_exist", "s1", true).getIDeviceID(); + Assert.assertTrue(reader.readTimeseriesMetadata(absentDevice, requested, true).isEmpty()); + Assert.assertThrows( + IOException.class, () -> reader.readTimeseriesMetadata(absentDevice, requested, false)); + + // the offset-addressed variant used by external tsfiles reads the same metadata + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + IDeviceID current = null; + long[] offset = null; + while (iterator.hasNext()) { + IDeviceID next = iterator.next(); + if (next.equals(device)) { + current = next; + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + break; + } + } + Assert.assertNotNull(offset); + Map viaOffset = + reader.readTimeseriesMetadata(current, offset, requested, false, null); + Assert.assertEquals(batch.keySet(), viaOffset.keySet()); + for (String measurement : batch.keySet()) { + Assert.assertEquals( + measurement, + batch.get(measurement).getStatistics().getCount(), + viaOffset.get(measurement).getStatistics().getCount()); + } + } + } } From 1967d857d089f1497c3d31d7dd56151ef3309ef9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 18 Nov 2026 09:35:09 +0800 Subject: [PATCH 2/8] Fix batch index search and filter before decoding chunk metadata Compare the index child to the key in the merge search, like the binary searches do, so a plain and a string-array device id can be compared without throwing. Keep the matched child eligible for duplicate keys in both batch strategies. Decode the chunk metadata of a leaf group only for the measurements the call returns, using the existing filter-aware TimeseriesMetadata deserializer (and a stream counterpart for groups larger than one buffer), so a sparse request does not build chunk metadata for every series it walks past. Cover the internal measurement node with a 20-measurement fixture, assert that absent names add no reads, and make the index-node tests use lexicographically ordered children plus duplicate and mixed-device-id cases. --- .../file/metadata/MetadataIndexNode.java | 11 ++- .../file/metadata/TimeseriesMetadata.java | 59 +++++++++++- .../tsfile/read/TsFileSequenceReader.java | 15 ++-- .../file/metadata/MetadataIndexNodeTest.java | 89 +++++++++++++------ .../read/TimeSeriesMetadataReadTest.java | 88 ++++++++++++++++++ 5 files changed, 223 insertions(+), 39 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 26aed827c0..54797ad799 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -228,7 +228,8 @@ int[] binarySearchInChildren(List keys, boolean exactSearc if (foundIndex >= 0) { results[i] = foundIndex; - currentLow = foundIndex + 1; + // keep the matched child eligible: the key list may contain duplicates + currentLow = foundIndex; } else { results[i] = exactSearch ? -1 : start - 1; currentLow = start; @@ -242,12 +243,14 @@ int[] mergeSearchInChildren(List keys, boolean exactSearch int i = 0; int j = 0; while (i < keys.size() && j < children.size()) { - int cmp = keys.get(i).compareTo(children.get(j).getCompareKey()); + // compare in the same direction as binarySearchInChildren: the key and the child compare + // key may be different implementations that only support child.compareTo(key) + int cmp = children.get(j).getCompareKey().compareTo(keys.get(i)); if (cmp == 0) { results[i] = j; + // keep the child eligible for the next key, which may duplicate this one i++; - j++; - } else if (cmp > 0) { + } else if (cmp < 0) { j++; } else { results[i] = exactSearch ? -1 : j - 1; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java index 790f457d11..5d4a64fcdb 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java @@ -44,6 +44,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Predicate; import static org.apache.tsfile.utils.Preconditions.checkArgument; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfCharArray; @@ -194,6 +195,44 @@ public static TimeseriesMetadata deserializeFrom( return timeseriesMetaData; } + /** + * Deserializes one TimeseriesMetadata from a file input and decodes its chunk metadata only when + * {@code decodeChunkMetadata} accepts the measurement. + */ + public static TimeseriesMetadata deserializeFrom( + TsFileInput tsFileInput, + Predicate decodeChunkMetadata, + boolean needChunkMetadataForDataTypeWithValuesInStatistics, + boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) + throws IOException { + InputStream inputStream = tsFileInput.wrapAsInputStream(); + TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata(); + timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(inputStream)); + timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(inputStream)); + timeseriesMetaData.setTsDataType(ReadWriteIOUtils.readDataType(inputStream)); + int chunkMetaDataListDataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream); + timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize); + timeseriesMetaData.setStatistics( + Statistics.deserialize(inputStream, timeseriesMetaData.dataType)); + long startOffset = tsFileInput.position(); + if (decodeChunkMetadata.test(timeseriesMetaData.getMeasurementId()) + && ((!timeseriesMetaData.getTsDataType().hasNoValueInStatistics() + && needChunkMetadataForDataTypeWithValuesInStatistics) + || (timeseriesMetaData.getTsDataType().hasNoValueInStatistics() + && needChunkMetadataForDataTypeWithoutValuesInStatistics))) { + timeseriesMetaData.chunkMetadataList = new ArrayList<>(); + while (tsFileInput.position() < startOffset + chunkMetaDataListDataSize) { + timeseriesMetaData.chunkMetadataList.add( + ChunkMetadata.deserializeFrom(inputStream, timeseriesMetaData)); + } + // minimize the storage of an ArrayList instance. + timeseriesMetaData.chunkMetadataList.trimToSize(); + } else { + tsFileInput.position(startOffset + chunkMetaDataListDataSize); + } + return timeseriesMetaData; + } + /** * Return timeseries metadata without deserializing chunk metadatas if excludedMeasurements * contains the measurementId of this timeseries metadata or needChunkMetadata is false. @@ -208,6 +247,23 @@ public static TimeseriesMetadata deserializeFrom( Set excludedMeasurements, boolean needChunkMetadataForDataTypeWithValuesInStatistics, boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) { + return deserializeFrom( + buffer, + measurementId -> !excludedMeasurements.contains(measurementId), + needChunkMetadataForDataTypeWithValuesInStatistics, + needChunkMetadataForDataTypeWithoutValuesInStatistics); + } + + /** + * Deserializes one TimeseriesMetadata and decodes its chunk metadata only when {@code + * decodeChunkMetadata} accepts the measurement. Statistics are always read; the chunk metadata + * region is skipped when it is not decoded. + */ + public static TimeseriesMetadata deserializeFrom( + ByteBuffer buffer, + Predicate decodeChunkMetadata, + boolean needChunkMetadataForDataTypeWithValuesInStatistics, + boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) { byte timeseriesType = ReadWriteIOUtils.readByte(buffer); String measurementID = ReadWriteIOUtils.readVarIntString(buffer); TSDataType tsDataType = ReadWriteIOUtils.readDataType(buffer); @@ -221,12 +277,11 @@ public static TimeseriesMetadata deserializeFrom( timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize); timeseriesMetaData.setStatistics(statistics); - if (!excludedMeasurements.contains(measurementID) + if (decodeChunkMetadata.test(measurementID) && ((!tsDataType.hasNoValueInStatistics() && needChunkMetadataForDataTypeWithValuesInStatistics) || (tsDataType.hasNoValueInStatistics() && needChunkMetadataForDataTypeWithoutValuesInStatistics))) { - // measurement is not in the excluded set and need chunk metadata ByteBuffer byteBuffer = buffer.slice(); byteBuffer.limit(chunkMetaDataListDataSize); timeseriesMetaData.chunkMetadataList = new ArrayList<>(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 8cddf4c46e..08e1a5a632 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1248,7 +1248,7 @@ private void readLeafMeasurementGroup( if (endOffset - startOffset < Integer.MAX_VALUE) { ByteBuffer buffer = readData(startOffset, endOffset, ioSizeRecorder); while (buffer.hasRemaining()) { - TimeseriesMetadata metadata = deserializeTimeseriesMetadata(buffer); + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(buffer, requested); if (requested.contains(metadata.getMeasurementId())) { result.put(metadata.getMeasurementId(), metadata); } @@ -1262,7 +1262,7 @@ private void readLeafMeasurementGroup( synchronized (this) { tsFileInput.position(startOffset); while (tsFileInput.position() < endOffset) { - TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput); + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput, requested); if (requested.contains(metadata.getMeasurementId())) { result.put(metadata.getMeasurementId(), metadata); } @@ -1270,18 +1270,21 @@ private void readLeafMeasurementGroup( } } - private TimeseriesMetadata deserializeTimeseriesMetadata(ByteBuffer buffer) throws IOException { + private TimeseriesMetadata deserializeTimeseriesMetadata(ByteBuffer buffer, Set requested) + throws IOException { try { - return TimeseriesMetadata.deserializeFrom(buffer, true); + // skip the chunk metadata of the series this call does not return + return TimeseriesMetadata.deserializeFrom(buffer, requested::contains, true, true); } catch (Exception e) { logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); throw e; } } - private TimeseriesMetadata deserializeTimeseriesMetadata(TsFileInput input) throws IOException { + private TimeseriesMetadata deserializeTimeseriesMetadata(TsFileInput input, Set requested) + throws IOException { try { - return TimeseriesMetadata.deserializeFrom(input, true); + return TimeseriesMetadata.deserializeFrom(input, requested::contains, true, true); } catch (StopReadTsFileByInterruptException e) { throw e; } catch (Exception e) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java index 736eed3746..1291040ab2 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java @@ -36,53 +36,54 @@ public class MetadataIndexNodeTest { @Test public void testBinarySearchInChildren() { List list = new ArrayList<>(); - list.add(new MeasurementMetadataIndexEntry("s0", -1L)); - list.add(new MeasurementMetadataIndexEntry("s5", -1L)); - list.add(new MeasurementMetadataIndexEntry("s10", -1L)); - list.add(new MeasurementMetadataIndexEntry("s15", -1L)); - list.add(new MeasurementMetadataIndexEntry("s20", -1L)); + list.add(new MeasurementMetadataIndexEntry("a", -1L)); + list.add(new MeasurementMetadataIndexEntry("c", -1L)); + list.add(new MeasurementMetadataIndexEntry("f", -1L)); + list.add(new MeasurementMetadataIndexEntry("m", -1L)); + list.add(new MeasurementMetadataIndexEntry("z", -1L)); MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L, MetadataIndexNodeType.LEAF_MEASUREMENT); - Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0", false)); - Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10", false)); - Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13", false)); - Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23", false)); - Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s13", true)); - Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s23", true)); + Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("a", false)); + Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("f", false)); + Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("g", false)); + Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("zz", false)); + Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("g", true)); + Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("zz", true)); } @Test public void testGetChildIndexEntries() { List list = new ArrayList<>(); - list.add(new MeasurementMetadataIndexEntry("s0", 0L)); - list.add(new MeasurementMetadataIndexEntry("s5", 10L)); - list.add(new MeasurementMetadataIndexEntry("s10", 20L)); - list.add(new MeasurementMetadataIndexEntry("s15", 30L)); - list.add(new MeasurementMetadataIndexEntry("s20", 40L)); + list.add(new MeasurementMetadataIndexEntry("a", 0L)); + list.add(new MeasurementMetadataIndexEntry("c", 10L)); + list.add(new MeasurementMetadataIndexEntry("f", 20L)); + list.add(new MeasurementMetadataIndexEntry("m", 30L)); + list.add(new MeasurementMetadataIndexEntry("z", 40L)); MetadataIndexNode node = new MetadataIndexNode(list, 50L, MetadataIndexNodeType.LEAF_MEASUREMENT); // few keys: monotonic binary search, each key gets the floor child and its end offset List> pairs = - node.getChildIndexEntries(Arrays.asList("s1", "s12"), false); + node.getChildIndexEntries(Arrays.asList("b", "d"), false); Assert.assertEquals(2, pairs.size()); Assert.assertEquals(0L, pairs.get(0).getLeft().getOffset()); Assert.assertEquals(10L, pairs.get(0).getRight().longValue()); - Assert.assertEquals(20L, pairs.get(1).getLeft().getOffset()); - Assert.assertEquals(30L, pairs.get(1).getRight().longValue()); + Assert.assertEquals(10L, pairs.get(1).getLeft().getOffset()); + Assert.assertEquals(20L, pairs.get(1).getRight().longValue()); // many keys: merge search, consecutive keys of one child share the same pair instance List> merged = - node.getChildIndexEntries(Arrays.asList("s0", "s1", "s2", "s3", "s4", "s5", "s6"), false); - Assert.assertEquals(7, merged.size()); - Assert.assertSame(merged.get(0), merged.get(4)); - Assert.assertNotSame(merged.get(0), merged.get(5)); - Assert.assertEquals(10L, merged.get(5).getLeft().getOffset()); - Assert.assertEquals(20L, merged.get(5).getRight().longValue()); + node.getChildIndexEntries(Arrays.asList("a", "b", "c", "d", "e"), false); + Assert.assertEquals(5, merged.size()); + Assert.assertSame(merged.get(0), merged.get(1)); + Assert.assertSame(merged.get(2), merged.get(4)); + Assert.assertNotSame(merged.get(1), merged.get(2)); + Assert.assertEquals(10L, merged.get(2).getLeft().getOffset()); + Assert.assertEquals(20L, merged.get(4).getRight().longValue()); // a key before the first child has no floor, and the last child ends at the node end offset - Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("a"), false).get(0)); + Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("0"), false).get(0)); Assert.assertEquals( 50L, node.getChildIndexEntries(Collections.singletonList("zz"), false) @@ -91,11 +92,45 @@ public void testGetChildIndexEntries() { .longValue()); // exact search returns null for a missing key - Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("s13"), true).get(0)); + Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("e"), true).get(0)); + + // duplicate keys stay eligible in both search strategies + int[] duplicates = + node.getChildIndexEntries(Arrays.asList("c", "c"), true).stream() + .mapToInt(pair -> node.getChildren().indexOf(pair.getLeft())) + .toArray(); + Assert.assertArrayEquals(new int[] {1, 1}, duplicates); + int[] mergedDuplicates = + node.getChildIndexEntries(Arrays.asList("c", "c", "f", "f", "m"), true).stream() + .mapToInt(pair -> node.getChildren().indexOf(pair.getLeft())) + .toArray(); + Assert.assertArrayEquals(new int[] {1, 1, 2, 2, 3}, mergedDuplicates); // an empty node has no floor entry MetadataIndexNode empty = new MetadataIndexNode(new ArrayList<>(), 0L, MetadataIndexNodeType.LEAF_MEASUREMENT); Assert.assertNull(empty.getChildIndexEntries(Collections.singletonList("s1"), false).get(0)); } + + @Test + public void testGetChildIndexEntriesComparesChildrenWithMixedDeviceIds() { + // plain device ids cannot be compared to a string-array child in this direction, so the + // batch search must compare the child to the key, like the binary searches do + List children = new ArrayList<>(); + children.add(new DeviceMetadataIndexEntry(new StringArrayDeviceID("d1"), 0L)); + children.add(new DeviceMetadataIndexEntry(new StringArrayDeviceID("d2"), 10L)); + children.add(new DeviceMetadataIndexEntry(new StringArrayDeviceID("d3"), 20L)); + MetadataIndexNode node = + new MetadataIndexNode(children, 30L, MetadataIndexNodeType.LEAF_MEASUREMENT); + + List keys = + Arrays.asList(new PlainDeviceID("d1"), new PlainDeviceID("d2"), new PlainDeviceID("d4")); + List> pairs = node.getChildIndexEntries(keys, false); + Assert.assertEquals(keys.size(), pairs.size()); + for (int i = 0; i < keys.size(); i++) { + Pair expected = node.getChildIndexEntry(keys.get(i), false); + Assert.assertEquals(expected.getLeft().getOffset(), pairs.get(i).getLeft().getOffset()); + Assert.assertEquals(expected.getRight(), pairs.get(i).getRight()); + } + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index f76962bc2e..b164b86508 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -21,10 +21,15 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.utils.FileGenerator; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -33,7 +38,9 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -176,4 +183,85 @@ public void testReadTimeseriesMetadataInBatch() throws IOException { } } } + + @Test + public void testReadTimeseriesMetadataInBatchThroughInternalNode() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("BatchInternalNode.tsfile")); + file.delete(); + IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d1"); + int measurementCount = 20; + List schemas = new ArrayList<>(measurementCount); + List present = new ArrayList<>(measurementCount); + for (int i = 0; i < measurementCount; i++) { + String measurement = String.format("m%02d", i); + present.add(measurement); + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + for (IMeasurementSchema schema : schemas) { + writer.registerTimeseries(device, schema); + } + for (int time = 0; time < 5; time++) { + TSRecord record = new TSRecord(device, time); + for (String measurement : present) { + record.addPoint(measurement, time); + } + writer.writeRecord(record); + } + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + long[] offset = null; + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + while (iterator.hasNext()) { + IDeviceID current = iterator.next(); + if (current.equals(device)) { + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + break; + } + } + Assert.assertNotNull(offset); + + List allReads = new ArrayList<>(); + Map all = + reader.readTimeseriesMetadata(device, offset, present, false, allReads::add); + Assert.assertEquals(measurementCount, all.size()); + // the batch read resolves several measurements per node/group read + Assert.assertFalse(allReads.isEmpty()); + Assert.assertTrue(allReads.size() < measurementCount); + for (String measurement : present) { + TimeseriesMetadata metadata = all.get(measurement); + Assert.assertNotNull(measurement, metadata); + Assert.assertEquals(measurement, metadata.getMeasurementId()); + Assert.assertNotNull(measurement, metadata.getChunkMetadataList()); + Assert.assertFalse(measurement, metadata.getChunkMetadataList().isEmpty()); + } + + // absent names must not add reads + Set withAbsent = new HashSet<>(present); + withAbsent.add("a"); + withAbsent.add("m99"); + withAbsent.add("zzz"); + List readsWithAbsent = new ArrayList<>(); + Map withAbsentResult = + reader.readTimeseriesMetadata(device, offset, withAbsent, false, readsWithAbsent::add); + Assert.assertEquals(measurementCount, withAbsentResult.size()); + Assert.assertEquals(allReads, readsWithAbsent); + + // a single searched measurement can warm the siblings that share its leaf group + List siblingReads = new ArrayList<>(); + Map siblings = + reader.readTimeseriesMetadata( + device, + offset, + Collections.singletonList("m00"), + new HashSet<>(present), + false, + siblingReads::add); + Assert.assertTrue(siblings.containsKey("m00")); + Assert.assertTrue(present.containsAll(siblings.keySet())); + Assert.assertTrue(siblingReads.size() < measurementCount); + } finally { + Assert.assertTrue(file.delete()); + } + } } From 8c00963d9c0cb51fa7ed4577a7e4230aed97932c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 09:58:08 +0800 Subject: [PATCH 3/8] Route batch reads by floor child and filter absent names up front Replace the global sort in the set-based reader with per-node floor routing: each name is bucketed by the child it sorts into, and the leaf level stops once every group is selected. Remove the now-unused batch index search. Let callers pass a mayExist predicate so a group that only absent names point to is not read, and document what a missing name means with and without the predicate. Rename the set-based reader to readTimeseriesMetadataForMeasurements so a literal null offset is no longer ambiguous with the single-measurement overloads. The non-predicate TsFileInput deserializer delegates to the predicate one instead of duplicating its body. Strengthen the tests: assert the exact node/group read count, add aligned (with the time column) and table-model fixtures, and cover what an absent name costs with and without the predicate. --- .../file/metadata/MetadataIndexNode.java | 99 +------- .../file/metadata/TimeseriesMetadata.java | 30 +-- .../tsfile/read/TsFileSequenceReader.java | 215 +++++++++++++----- .../file/metadata/MetadataIndexNodeTest.java | 85 ------- .../read/TimeSeriesMetadataReadTest.java | 144 ++++++++++-- 5 files changed, 300 insertions(+), 273 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 54797ad799..a46c383cf5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -33,12 +33,10 @@ import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; public class MetadataIndexNode { - private static final double LOG2 = Math.log(2); protected static final TSFileConfig config = TSFileDescriptor.getInstance().getConfig(); protected final List children; protected long endOffset; @@ -127,6 +125,12 @@ public static MetadataIndexNode deserializeFrom( return new MetadataIndexNode(children, offset, nodeType); } + /** + * Searches the floor child entry of the given key. + * + *

In floor mode a key that sorts before the first child returns child 0, so a caller that must + * not read a group which cannot hold the key should compare the key with the first child first. + */ public Pair getChildIndexEntry(Comparable key, boolean exactSearch) { return getChildIndexEntry(binarySearchInChildren(key, exactSearch)); } @@ -144,33 +148,6 @@ private Pair getChildIndexEntry(int index) { return new Pair<>(children.get(index), childEndOffset); } - /** - * Searches the floor child entry of every key in one pass. The same pair instance is returned for - * consecutive keys that belong to the same child, so callers can group keys by identity. - * - * @param keys keys sorted in ascending order - * @param exactSearch whether to return null instead of the floor child when the key is missing - */ - public List> getChildIndexEntries( - List keys, boolean exactSearch) { - int[] indexes = - keys.size() >= children.size() - || keys.size() * Math.log(children.size()) / LOG2 > keys.size() + children.size() - ? mergeSearchInChildren(keys, exactSearch) - : binarySearchInChildren(keys, exactSearch); - List> pairs = new ArrayList<>(indexes.length); - Pair previousPair = null; - int previousIndex = -1; - for (int index : indexes) { - if (index != previousIndex) { - previousPair = getChildIndexEntry(index); - previousIndex = index; - } - pairs.add(previousPair); - } - return pairs; - } - int binarySearchInChildren(Comparable key, boolean exactSearch) { int low = 0; int high = children.size() - 1; @@ -197,70 +174,6 @@ int binarySearchInChildren(Comparable key, boolean exactSearch) { } } - int[] binarySearchInChildren(List keys, boolean exactSearch) { - int[] results = new int[keys.size()]; - Arrays.fill(results, -1); - int currentLow = 0; - int high = children.size() - 1; - - for (int i = 0; i < keys.size(); i++) { - Comparable key = keys.get(i); - if (currentLow > high) { - Arrays.fill(results, i, keys.size(), exactSearch ? -1 : currentLow - 1); - return results; - } - - int foundIndex = -1; - int start = currentLow; - int end = high; - while (start <= end) { - int mid = (start + end) >>> 1; - int cmp = children.get(mid).getCompareKey().compareTo(key); - if (cmp < 0) { - start = mid + 1; - } else if (cmp > 0) { - end = mid - 1; - } else { - foundIndex = mid; - break; - } - } - - if (foundIndex >= 0) { - results[i] = foundIndex; - // keep the matched child eligible: the key list may contain duplicates - currentLow = foundIndex; - } else { - results[i] = exactSearch ? -1 : start - 1; - currentLow = start; - } - } - return results; - } - - int[] mergeSearchInChildren(List keys, boolean exactSearch) { - int[] results = new int[keys.size()]; - int i = 0; - int j = 0; - while (i < keys.size() && j < children.size()) { - // compare in the same direction as binarySearchInChildren: the key and the child compare - // key may be different implementations that only support child.compareTo(key) - int cmp = children.get(j).getCompareKey().compareTo(keys.get(i)); - if (cmp == 0) { - results[i] = j; - // keep the child eligible for the next key, which may duplicate this one - i++; - } else if (cmp < 0) { - j++; - } else { - results[i] = exactSearch ? -1 : j - 1; - i++; - } - } - Arrays.fill(results, i, keys.size(), exactSearch ? -1 : children.size() - 1); - return results; - } - public boolean isDeviceLevel() { return this.nodeType == MetadataIndexNodeType.INTERNAL_DEVICE || this.nodeType == MetadataIndexNodeType.LEAF_DEVICE; diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java index 5d4a64fcdb..56e71b1065 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java @@ -168,31 +168,11 @@ public static TimeseriesMetadata deserializeFrom( boolean needChunkMetadataForDataTypeWithValuesInStatistics, boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) throws IOException { - InputStream inputStream = tsFileInput.wrapAsInputStream(); - TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata(); - timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(inputStream)); - timeseriesMetaData.setMeasurementId(ReadWriteIOUtils.readVarIntString(inputStream)); - timeseriesMetaData.setTsDataType(ReadWriteIOUtils.readDataType(inputStream)); - int chunkMetaDataListDataSize = ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream); - timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize); - timeseriesMetaData.setStatistics( - Statistics.deserialize(inputStream, timeseriesMetaData.dataType)); - long startOffset = tsFileInput.position(); - if ((!timeseriesMetaData.getTsDataType().hasNoValueInStatistics() - && needChunkMetadataForDataTypeWithValuesInStatistics) - || (timeseriesMetaData.getTsDataType().hasNoValueInStatistics() - && needChunkMetadataForDataTypeWithoutValuesInStatistics)) { - timeseriesMetaData.chunkMetadataList = new ArrayList<>(); - while (tsFileInput.position() < startOffset + chunkMetaDataListDataSize) { - timeseriesMetaData.chunkMetadataList.add( - ChunkMetadata.deserializeFrom(inputStream, timeseriesMetaData)); - } - // minimize the storage of an ArrayList instance. - timeseriesMetaData.chunkMetadataList.trimToSize(); - } else { - tsFileInput.position(startOffset + chunkMetaDataListDataSize); - } - return timeseriesMetaData; + return deserializeFrom( + tsFileInput, + id -> true, + needChunkMetadataForDataTypeWithValuesInStatistics, + needChunkMetadataForDataTypeWithoutValuesInStatistics); } /** diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 08e1a5a632..3fdefc13c6 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -112,11 +112,11 @@ import java.util.Queue; import java.util.Set; import java.util.TreeMap; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongConsumer; +import java.util.function.Predicate; import java.util.stream.Collectors; public class TsFileSequenceReader implements AutoCloseable { @@ -1097,55 +1097,70 @@ public List readTimeseriesMetadata( /** * Reads the {@link TimeseriesMetadata} of the given measurements of a device, visiting each - * measurement leaf group that holds a requested measurement at most once. + * measurement leaf group that holds one of them at most once. * * @param deviceMetadataIndexNodeOffset measurement metadata index node offset of an external * TsFile; null to locate the device through the table metadata index * @param measurements measurements to read, in any order * @param ignoreNotExistDevice whether to return an empty map instead of throwing when the device * is not recorded in the file metadata - * @return the metadata of the measurements found in this file; absent measurements are missing - * from the map + * @return the metadata of the measurements found in the visited leaf groups; absent measurements + * are missing from the map */ - public Map readTimeseriesMetadata( + public Map readTimeseriesMetadataForMeasurements( IDeviceID device, long[] deviceMetadataIndexNodeOffset, Collection measurements, boolean ignoreNotExistDevice, LongConsumer ioSizeRecorder) throws IOException { - return readTimeseriesMetadata( + return readTimeseriesMetadataForMeasurements( device, deviceMetadataIndexNodeOffset, measurements, null, + null, ignoreNotExistDevice, ioSizeRecorder); } /** * Reads the {@link TimeseriesMetadata} of the measurements that share a leaf group with one of - * {@code measurements}. Leaf groups that no searched measurement maps into are not read, and each - * visited leaf group is read once. + * {@code measurements}. Every searched measurement selects the leaf group that holds its floor + * key, and each visited group is read once. + * + *

An absent measurement still selects its floor group, so a group that holds none of the + * requested series is read when an absent measurement sorts into it. Pass {@code mayExist} to + * avoid those reads: a measurement is tested before it selects a group, and a rejected + * measurement selects nothing. The predicate must have no false negatives, as a {@link + * BloomFilter} does not; do not pass one for an offset-addressed external TsFile whose + * BloomFilter may not cover the series. + * + *

A returned measurement that is missing from the result was rejected by {@code mayExist}, or + * its floor group was visited and does not contain it; a measurement before the first key is + * missing without a read. With {@code mayExist}, every searched measurement missing from the + * result is therefore absent from the file. * * @param measurements measurements that select the leaf groups to visit * @param returnedMeasurements measurements to deserialize from the visited groups; null means * {@code measurements}. It may contain measurements outside {@code measurements}, which is * how a caller warms the metadata cache with the siblings of a single requested measurement - * @return the metadata of the returned measurements found in this file; absent measurements are - * missing from the map + * @param mayExist optional filter tested on a measurement before it selects a group; null reads + * every floor group + * @return the metadata of the returned measurements found in the visited leaf groups */ - public Map readTimeseriesMetadata( + @SuppressWarnings("unchecked") + public Map readTimeseriesMetadataForMeasurements( IDeviceID device, long[] deviceMetadataIndexNodeOffset, Collection measurements, Set returnedMeasurements, + Predicate mayExist, boolean ignoreNotExistDevice, LongConsumer ioSizeRecorder) throws IOException { - if (measurements.isEmpty()) { - return Collections.emptyMap(); - } + // the device is looked up even for an empty request, so an absent device is reported the same + // way as for a non-empty one MetadataIndexNode measurementIndexNode = getMeasurementMetadataIndexNode(device, deviceMetadataIndexNodeOffset, ioSizeRecorder); if (measurementIndexNode == null) { @@ -1155,19 +1170,45 @@ public Map readTimeseriesMetadata( throw new IOException( Messages.format("error.read.device_not_in_metadata_file", device, file)); } - List sortedMeasurements = new ArrayList<>(new TreeSet<>(measurements)); + if (measurements.isEmpty()) { + return Collections.emptyMap(); + } Set returned = - returnedMeasurements == null ? new HashSet<>(sortedMeasurements) : returnedMeasurements; - Map result = new HashMap<>(); + returnedMeasurements != null + ? returnedMeasurements + : measurements instanceof Set + ? (Set) measurements + : new HashSet<>(measurements); + Map result = + new HashMap<>(Math.max(16, (int) (returned.size() / 0.75f) + 1)); readMeasurementsInIndexNode( - measurementIndexNode, sortedMeasurements, returned, result, ioSizeRecorder); + measurementIndexNode, measurements, returned, mayExist, result, ioSizeRecorder); return result; } - public Map readTimeseriesMetadata( + public Map readTimeseriesMetadataForMeasurements( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Collection measurements, + Set returnedMeasurements, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { + return readTimeseriesMetadataForMeasurements( + device, + deviceMetadataIndexNodeOffset, + measurements, + returnedMeasurements, + null, + ignoreNotExistDevice, + ioSizeRecorder); + } + + public Map readTimeseriesMetadataForMeasurements( IDeviceID device, Collection measurements, boolean ignoreNotExistDevice) throws IOException { - return readTimeseriesMetadata(device, null, measurements, null, ignoreNotExistDevice, null); + return readTimeseriesMetadataForMeasurements( + device, null, measurements, null, null, ignoreNotExistDevice, null); } private MetadataIndexNode getMeasurementMetadataIndexNode( @@ -1188,9 +1229,6 @@ private MetadataIndexNode getMeasurementMetadataIndexNode( return null; } ByteBuffer buffer = readData(deviceEntry.left.getOffset(), deviceEntry.right, ioSizeRecorder); - if (deviceIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { - return deviceIndexNode; - } return deserializeMeasurementMetadataIndexNode(buffer); } @@ -1207,49 +1245,112 @@ private MetadataIndexNode deserializeMeasurementMetadataIndexNode(ByteBuffer buf private void readMeasurementsInIndexNode( MetadataIndexNode node, - List sortedMeasurements, - Set requested, + Collection measurements, + Set returned, + Predicate mayExist, Map result, LongConsumer ioSizeRecorder) throws IOException { - List> entries = - node.getChildIndexEntries(sortedMeasurements, false); - int groupStart = 0; - for (int i = 1; i <= sortedMeasurements.size(); i++) { - if (i < sortedMeasurements.size() && entries.get(i) == entries.get(groupStart)) { + List children = node.getChildren(); + if (children.isEmpty()) { + return; + } + if (MetadataIndexNodeType.LEAF_MEASUREMENT.equals(node.getNodeType())) { + boolean[] selected = new boolean[children.size()]; + int remaining = children.size(); + for (String measurement : measurements) { + int index = floorChild(children, measurement); + if (index < 0 || selected[index]) { + continue; + } + if (mayExist != null && !mayExist.test(measurement)) { + continue; + } + selected[index] = true; + if (--remaining == 0) { + break; + } + } + for (int i = 0; i < selected.length; i++) { + if (selected[i]) { + readLeafMeasurementGroup(node, i, returned, result, ioSizeRecorder); + } + } + return; + } + // an internal node: route each measurement to its floor child and read each visited child once + List> buckets = new ArrayList<>(children.size()); + for (int i = 0; i < children.size(); i++) { + buckets.add(null); + } + for (String measurement : measurements) { + int index = floorChild(children, measurement); + if (index < 0) { continue; } - Pair entry = entries.get(groupStart); - if (entry != null) { - if (node.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) { - readLeafMeasurementGroup(entry, requested, result, ioSizeRecorder); - } else { - ByteBuffer buffer = readData(entry.left.getOffset(), entry.right, ioSizeRecorder); - readMeasurementsInIndexNode( - deserializeMeasurementMetadataIndexNode(buffer), - sortedMeasurements.subList(groupStart, i), - requested, - result, - ioSizeRecorder); + List bucket = buckets.get(index); + if (bucket == null) { + if (mayExist != null && !mayExist.test(measurement)) { + continue; } + bucket = new ArrayList<>(); + buckets.set(index, bucket); + } + bucket.add(measurement); + } + for (int i = 0; i < buckets.size(); i++) { + List bucket = buckets.get(i); + if (bucket != null) { + ByteBuffer buffer = + readData(children.get(i).getOffset(), childEndOffset(node, i), ioSizeRecorder); + readMeasurementsInIndexNode( + deserializeMeasurementMetadataIndexNode(buffer), + bucket, + returned, + mayExist, + result, + ioSizeRecorder); } - groupStart = i; } } + private static int floorChild(List children, String key) { + int low = 0; + int high = children.size() - 1; + int floor = -1; + while (low <= high) { + int mid = (low + high) >>> 1; + if (children.get(mid).getCompareKey().compareTo(key) <= 0) { + floor = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return floor; + } + + private static long childEndOffset(MetadataIndexNode node, int childIndex) { + return childIndex == node.getChildren().size() - 1 + ? node.getEndOffset() + : node.getChildren().get(childIndex + 1).getOffset(); + } + private void readLeafMeasurementGroup( - Pair entry, - Set requested, + MetadataIndexNode node, + int childIndex, + Set returned, Map result, LongConsumer ioSizeRecorder) throws IOException { - long startOffset = entry.left.getOffset(); - long endOffset = entry.right; + long startOffset = node.getChildren().get(childIndex).getOffset(); + long endOffset = childEndOffset(node, childIndex); + Predicate isReturned = returned::contains; if (endOffset - startOffset < Integer.MAX_VALUE) { ByteBuffer buffer = readData(startOffset, endOffset, ioSizeRecorder); while (buffer.hasRemaining()) { - TimeseriesMetadata metadata = deserializeTimeseriesMetadata(buffer, requested); - if (requested.contains(metadata.getMeasurementId())) { + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(buffer, isReturned); + if (isReturned.test(metadata.getMeasurementId())) { result.put(metadata.getMeasurementId(), metadata); } } @@ -1262,29 +1363,29 @@ private void readLeafMeasurementGroup( synchronized (this) { tsFileInput.position(startOffset); while (tsFileInput.position() < endOffset) { - TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput, requested); - if (requested.contains(metadata.getMeasurementId())) { + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput, isReturned); + if (isReturned.test(metadata.getMeasurementId())) { result.put(metadata.getMeasurementId(), metadata); } } } } - private TimeseriesMetadata deserializeTimeseriesMetadata(ByteBuffer buffer, Set requested) - throws IOException { + private TimeseriesMetadata deserializeTimeseriesMetadata( + ByteBuffer buffer, Predicate decodeChunkMetadata) throws IOException { try { // skip the chunk metadata of the series this call does not return - return TimeseriesMetadata.deserializeFrom(buffer, requested::contains, true, true); + return TimeseriesMetadata.deserializeFrom(buffer, decodeChunkMetadata, true, true); } catch (Exception e) { logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); throw e; } } - private TimeseriesMetadata deserializeTimeseriesMetadata(TsFileInput input, Set requested) - throws IOException { + private TimeseriesMetadata deserializeTimeseriesMetadata( + TsFileInput input, Predicate decodeChunkMetadata) throws IOException { try { - return TimeseriesMetadata.deserializeFrom(input, requested::contains, true, true); + return TimeseriesMetadata.deserializeFrom(input, decodeChunkMetadata, true, true); } catch (StopReadTsFileByInterruptException e) { throw e; } catch (Exception e) { diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java index 1291040ab2..e5afeb6ac2 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java @@ -21,14 +21,11 @@ import org.apache.tsfile.file.IMetadataIndexEntry; import org.apache.tsfile.file.metadata.enums.MetadataIndexNodeType; -import org.apache.tsfile.utils.Pair; import org.junit.Assert; import org.junit.Test; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; public class MetadataIndexNodeTest { @@ -51,86 +48,4 @@ public void testBinarySearchInChildren() { Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("g", true)); Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("zz", true)); } - - @Test - public void testGetChildIndexEntries() { - List list = new ArrayList<>(); - list.add(new MeasurementMetadataIndexEntry("a", 0L)); - list.add(new MeasurementMetadataIndexEntry("c", 10L)); - list.add(new MeasurementMetadataIndexEntry("f", 20L)); - list.add(new MeasurementMetadataIndexEntry("m", 30L)); - list.add(new MeasurementMetadataIndexEntry("z", 40L)); - MetadataIndexNode node = - new MetadataIndexNode(list, 50L, MetadataIndexNodeType.LEAF_MEASUREMENT); - - // few keys: monotonic binary search, each key gets the floor child and its end offset - List> pairs = - node.getChildIndexEntries(Arrays.asList("b", "d"), false); - Assert.assertEquals(2, pairs.size()); - Assert.assertEquals(0L, pairs.get(0).getLeft().getOffset()); - Assert.assertEquals(10L, pairs.get(0).getRight().longValue()); - Assert.assertEquals(10L, pairs.get(1).getLeft().getOffset()); - Assert.assertEquals(20L, pairs.get(1).getRight().longValue()); - - // many keys: merge search, consecutive keys of one child share the same pair instance - List> merged = - node.getChildIndexEntries(Arrays.asList("a", "b", "c", "d", "e"), false); - Assert.assertEquals(5, merged.size()); - Assert.assertSame(merged.get(0), merged.get(1)); - Assert.assertSame(merged.get(2), merged.get(4)); - Assert.assertNotSame(merged.get(1), merged.get(2)); - Assert.assertEquals(10L, merged.get(2).getLeft().getOffset()); - Assert.assertEquals(20L, merged.get(4).getRight().longValue()); - - // a key before the first child has no floor, and the last child ends at the node end offset - Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("0"), false).get(0)); - Assert.assertEquals( - 50L, - node.getChildIndexEntries(Collections.singletonList("zz"), false) - .get(0) - .getRight() - .longValue()); - - // exact search returns null for a missing key - Assert.assertNull(node.getChildIndexEntries(Collections.singletonList("e"), true).get(0)); - - // duplicate keys stay eligible in both search strategies - int[] duplicates = - node.getChildIndexEntries(Arrays.asList("c", "c"), true).stream() - .mapToInt(pair -> node.getChildren().indexOf(pair.getLeft())) - .toArray(); - Assert.assertArrayEquals(new int[] {1, 1}, duplicates); - int[] mergedDuplicates = - node.getChildIndexEntries(Arrays.asList("c", "c", "f", "f", "m"), true).stream() - .mapToInt(pair -> node.getChildren().indexOf(pair.getLeft())) - .toArray(); - Assert.assertArrayEquals(new int[] {1, 1, 2, 2, 3}, mergedDuplicates); - - // an empty node has no floor entry - MetadataIndexNode empty = - new MetadataIndexNode(new ArrayList<>(), 0L, MetadataIndexNodeType.LEAF_MEASUREMENT); - Assert.assertNull(empty.getChildIndexEntries(Collections.singletonList("s1"), false).get(0)); - } - - @Test - public void testGetChildIndexEntriesComparesChildrenWithMixedDeviceIds() { - // plain device ids cannot be compared to a string-array child in this direction, so the - // batch search must compare the child to the key, like the binary searches do - List children = new ArrayList<>(); - children.add(new DeviceMetadataIndexEntry(new StringArrayDeviceID("d1"), 0L)); - children.add(new DeviceMetadataIndexEntry(new StringArrayDeviceID("d2"), 10L)); - children.add(new DeviceMetadataIndexEntry(new StringArrayDeviceID("d3"), 20L)); - MetadataIndexNode node = - new MetadataIndexNode(children, 30L, MetadataIndexNodeType.LEAF_MEASUREMENT); - - List keys = - Arrays.asList(new PlainDeviceID("d1"), new PlainDeviceID("d2"), new PlainDeviceID("d4")); - List> pairs = node.getChildIndexEntries(keys, false); - Assert.assertEquals(keys.size(), pairs.size()); - for (int i = 0; i < keys.size(); i++) { - Pair expected = node.getChildIndexEntry(keys.get(i), false); - Assert.assertEquals(expected.getLeft().getOffset(), pairs.get(i).getLeft().getOffset()); - Assert.assertEquals(expected.getRight(), pairs.get(i).getRight()); - } - } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index b164b86508..29c5d15943 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -21,13 +21,16 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; import org.apache.tsfile.read.common.Path; import org.apache.tsfile.utils.FileGenerator; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.Tablet; import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; @@ -112,7 +115,7 @@ public void testReadTimeseriesMetadataInBatch() throws IOException { new HashSet<>(Arrays.asList("s1", "s2", "s3", "s4", "s5", "s6", "s8")); Map batch = - reader.readTimeseriesMetadata(device, requested, false); + reader.readTimeseriesMetadataForMeasurements(device, requested, false); // s8 does not exist in the file Assert.assertEquals(6, batch.size()); Assert.assertNull(batch.get("s8")); @@ -138,7 +141,7 @@ public void testReadTimeseriesMetadataInBatch() throws IOException { // a single searched measurement can warm the cache with its group siblings Map withSiblings = - reader.readTimeseriesMetadata( + reader.readTimeseriesMetadataForMeasurements( device, null, java.util.Collections.singletonList("s1"), @@ -155,9 +158,11 @@ public void testReadTimeseriesMetadataInBatch() throws IOException { // an absent device is tolerated only when ignoreNotExistDevice is set IDeviceID absentDevice = new Path("d_not_exist", "s1", true).getIDeviceID(); - Assert.assertTrue(reader.readTimeseriesMetadata(absentDevice, requested, true).isEmpty()); + Assert.assertTrue( + reader.readTimeseriesMetadataForMeasurements(absentDevice, requested, true).isEmpty()); Assert.assertThrows( - IOException.class, () -> reader.readTimeseriesMetadata(absentDevice, requested, false)); + IOException.class, + () -> reader.readTimeseriesMetadataForMeasurements(absentDevice, requested, false)); // the offset-addressed variant used by external tsfiles reads the same metadata LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); @@ -173,7 +178,7 @@ public void testReadTimeseriesMetadataInBatch() throws IOException { } Assert.assertNotNull(offset); Map viaOffset = - reader.readTimeseriesMetadata(current, offset, requested, false, null); + reader.readTimeseriesMetadataForMeasurements(current, offset, requested, false, null); Assert.assertEquals(batch.keySet(), viaOffset.keySet()); for (String measurement : batch.keySet()) { Assert.assertEquals( @@ -223,11 +228,11 @@ public void testReadTimeseriesMetadataInBatchThroughInternalNode() throws Except List allReads = new ArrayList<>(); Map all = - reader.readTimeseriesMetadata(device, offset, present, false, allReads::add); + reader.readTimeseriesMetadataForMeasurements( + device, offset, present, false, allReads::add); Assert.assertEquals(measurementCount, all.size()); - // the batch read resolves several measurements per node/group read - Assert.assertFalse(allReads.isEmpty()); - Assert.assertTrue(allReads.size() < measurementCount); + // 20 series at index degree 3: one root, three leaf index nodes and seven leaf groups + Assert.assertEquals(11, allReads.size()); for (String measurement : present) { TimeseriesMetadata metadata = all.get(measurement); Assert.assertNotNull(measurement, metadata); @@ -236,21 +241,56 @@ public void testReadTimeseriesMetadataInBatchThroughInternalNode() throws Except Assert.assertFalse(measurement, metadata.getChunkMetadataList().isEmpty()); } - // absent names must not add reads + // a name before the first key, or one whose floor group a present name already selects, + // adds no read Set withAbsent = new HashSet<>(present); withAbsent.add("a"); withAbsent.add("m99"); withAbsent.add("zzz"); List readsWithAbsent = new ArrayList<>(); Map withAbsentResult = - reader.readTimeseriesMetadata(device, offset, withAbsent, false, readsWithAbsent::add); + reader.readTimeseriesMetadataForMeasurements( + device, offset, withAbsent, false, readsWithAbsent::add); Assert.assertEquals(measurementCount, withAbsentResult.size()); Assert.assertEquals(allReads, readsWithAbsent); + // an absent name whose floor group no present name selects makes the reader visit that + // group; mayExist is what lets a caller avoid it + List subset = Collections.singletonList("m00"); + List subsetReads = new ArrayList<>(); + Map subsetResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, subset, false, subsetReads::add); + Assert.assertEquals(1, subsetResult.size()); + // the root, one leaf index node and one leaf group + Assert.assertEquals(3, subsetReads.size()); + + List subsetWithAbsent = Arrays.asList("m00", "m10x"); + List subsetWithAbsentReads = new ArrayList<>(); + Map subsetWithAbsentResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, subsetWithAbsent, false, subsetWithAbsentReads::add); + Assert.assertEquals(1, subsetWithAbsentResult.size()); + // the absent name sorts into a second leaf index node and leaf group + Assert.assertEquals(5, subsetWithAbsentReads.size()); + + List filteredReads = new ArrayList<>(); + Map filteredResult = + reader.readTimeseriesMetadataForMeasurements( + device, + offset, + subsetWithAbsent, + null, + name -> !name.equals("m10x"), + false, + filteredReads::add); + Assert.assertEquals(1, filteredResult.size()); + Assert.assertEquals(subsetReads, filteredReads); + // a single searched measurement can warm the siblings that share its leaf group List siblingReads = new ArrayList<>(); Map siblings = - reader.readTimeseriesMetadata( + reader.readTimeseriesMetadataForMeasurements( device, offset, Collections.singletonList("m00"), @@ -259,7 +299,85 @@ public void testReadTimeseriesMetadataInBatchThroughInternalNode() throws Except siblingReads::add); Assert.assertTrue(siblings.containsKey("m00")); Assert.assertTrue(present.containsAll(siblings.keySet())); - Assert.assertTrue(siblingReads.size() < measurementCount); + Assert.assertEquals(subsetReads, siblingReads); + } finally { + Assert.assertTrue(file.delete()); + } + } + + @Test + public void testReadAlignedMeasurementsWithTimeColumn() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("BatchAligned.tsfile")); + file.delete(); + IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d1"); + List schemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(device, schemas); + for (int time = 0; time < 5; time++) { + TSRecord record = new TSRecord(device, time); + record.addPoint("s1", time); + record.addPoint("s2", time); + writer.writeRecord(record); + } + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + List reads = new ArrayList<>(); + Map result = + reader.readTimeseriesMetadataForMeasurements( + device, null, Arrays.asList("", "s1", "s2", "s9"), false, reads::add); + // the time column travels with the value columns of an aligned device + Assert.assertTrue(result.containsKey("")); + Assert.assertEquals(5, result.get("").getStatistics().getCount()); + Assert.assertEquals(new HashSet<>(Arrays.asList("", "s1", "s2")), result.keySet()); + Assert.assertFalse(reads.isEmpty()); + } finally { + Assert.assertTrue(file.delete()); + } + } + + @Test + public void testReadTableModelMeasurements() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("BatchTable.tsfile")); + file.delete(); + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("device", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32)), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); + Tablet tablet = + new Tablet( + tableSchema.getTableName(), + Arrays.asList("device", "s1", "s2"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); + for (int i = 0; i < 5; i++) { + tablet.addTimestamp(i, i); + tablet.addValue("device", i, "d1"); + tablet.addValue("s1", i, i); + tablet.addValue("s2", i, i); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerTableSchema(tableSchema); + writer.writeTable(tablet); + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + TsFileDeviceIterator iterator = reader.getAllDevicesIteratorWithIsAligned(); + Assert.assertTrue(iterator.hasNext()); + IDeviceID device = iterator.next().getLeft(); + List reads = new ArrayList<>(); + Map result = + reader.readTimeseriesMetadataForMeasurements( + device, null, Arrays.asList("s1", "s2", "s9"), false, reads::add); + Assert.assertEquals(new HashSet<>(Arrays.asList("s1", "s2")), result.keySet()); + Assert.assertEquals(5, result.get("s1").getStatistics().getCount()); + Assert.assertFalse(reads.isEmpty()); } finally { Assert.assertTrue(file.delete()); } From c99f54f0d477a32840c38dbfb36e6877d06aa517 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 11:19:10 +0800 Subject: [PATCH 4/8] Clarify the mayExist false-negative wording --- .../java/org/apache/tsfile/read/TsFileSequenceReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 3fdefc13c6..da670c174a 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1132,8 +1132,8 @@ public Map readTimeseriesMetadataForMeasurements( *

An absent measurement still selects its floor group, so a group that holds none of the * requested series is read when an absent measurement sorts into it. Pass {@code mayExist} to * avoid those reads: a measurement is tested before it selects a group, and a rejected - * measurement selects nothing. The predicate must have no false negatives, as a {@link - * BloomFilter} does not; do not pass one for an offset-addressed external TsFile whose + * measurement selects nothing. The predicate must have no false negatives, because a {@link + * BloomFilter} has none; do not pass one for an offset-addressed external TsFile whose * BloomFilter may not cover the series. * *

A returned measurement that is missing from the result was rejected by {@code mayExist}, or From 030072582e5730130c93b4d5ef8d5b85781e90fd Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 16:33:55 +0800 Subject: [PATCH 5/8] Address the second review round on the selective reader - Size the result map from the searched set, not the return set: the return set may include many cache-warming siblings that live in groups this call does not visit. - For a streaming group above 2 GiB, record only the bytes actually read, excluding the chunk lists the selective deserializer seeks past. - Document that mayExist receives the bare measurement name, so a BloomFilter lookup has to build the full device.measurement path, and scope the absence guarantee to searched measurements: a missing warmed sibling is inconclusive. - Test the streaming predicate path on two chunk-bearing records: the rejected record's chunk list is skipped, the input lands on the next record, and the accepted record's chunk metadata matches. - Compare each chunk's offset, count, time bounds and numeric value statistics in the batch-vs-single assertion. --- .../tsfile/read/TsFileSequenceReader.java | 36 ++++++--- .../file/metadata/TimeseriesMetadataTest.java | 79 +++++++++++++++++++ .../read/TimeSeriesMetadataReadTest.java | 31 ++++++++ 3 files changed, 134 insertions(+), 12 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index da670c174a..f392d055b5 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1134,19 +1134,23 @@ public Map readTimeseriesMetadataForMeasurements( * avoid those reads: a measurement is tested before it selects a group, and a rejected * measurement selects nothing. The predicate must have no false negatives, because a {@link * BloomFilter} has none; do not pass one for an offset-addressed external TsFile whose - * BloomFilter may not cover the series. + * BloomFilter may not cover the series. It receives the bare measurement name, while a + * BloomFilter stores full {@code device.measurement} paths, so the caller has to build the path + * inside the predicate. * - *

A returned measurement that is missing from the result was rejected by {@code mayExist}, or - * its floor group was visited and does not contain it; a measurement before the first key is - * missing without a read. With {@code mayExist}, every searched measurement missing from the - * result is therefore absent from the file. + *

A searched measurement that is also requested for return is missing from the result only if + * {@code mayExist} rejected it (it then selects no group), or its floor group was visited and + * does not contain it (a measurement before the first key needs no read). A measurement in {@code + * returnedMeasurements} that is not in {@code measurements} is warmed only when it shares a + * visited group; a missing one is inconclusive, because its own group may not have been visited. * * @param measurements measurements that select the leaf groups to visit * @param returnedMeasurements measurements to deserialize from the visited groups; null means * {@code measurements}. It may contain measurements outside {@code measurements}, which is * how a caller warms the metadata cache with the siblings of a single requested measurement * @param mayExist optional filter tested on a measurement before it selects a group; null reads - * every floor group + * every floor group. It receives the bare measurement name; build the full path for a + * BloomFilter lookup. * @return the metadata of the returned measurements found in the visited leaf groups */ @SuppressWarnings("unchecked") @@ -1179,8 +1183,9 @@ public Map readTimeseriesMetadataForMeasurements( : measurements instanceof Set ? (Set) measurements : new HashSet<>(measurements); - Map result = - new HashMap<>(Math.max(16, (int) (returned.size() / 0.75f) + 1)); + // size from the searched set: the return set may include many cache-warming siblings that + // live in groups this call does not visit + Map result = new HashMap<>(Math.max(16, measurements.size())); readMeasurementsInIndexNode( measurementIndexNode, measurements, returned, mayExist, result, ioSizeRecorder); return result; @@ -1357,14 +1362,21 @@ private void readLeafMeasurementGroup( return; } // a group larger than Integer.MAX_VALUE cannot be held in one buffer - if (ioSizeRecorder != null) { - ioSizeRecorder.accept(endOffset - startOffset); - } synchronized (this) { tsFileInput.position(startOffset); while (tsFileInput.position() < endOffset) { + long recordStart = tsFileInput.position(); TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput, isReturned); - if (isReturned.test(metadata.getMeasurementId())) { + boolean returnedMeasurement = isReturned.test(metadata.getMeasurementId()); + if (ioSizeRecorder != null) { + // the selective deserializer seeks past the chunk list of a series it does not return + long consumed = tsFileInput.position() - recordStart; + if (!returnedMeasurement) { + consumed -= metadata.getDataSizeOfChunkMetaDataList(); + } + ioSizeRecorder.accept(consumed); + } + if (returnedMeasurement) { result.put(metadata.getMeasurementId(), metadata); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java index 3ac296afea..e550fa7ab6 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java @@ -19,6 +19,10 @@ package org.apache.tsfile.file.metadata; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.file.metadata.utils.TestHelper; import org.apache.tsfile.read.reader.LocalTsFileInput; import org.apache.tsfile.read.reader.TsFileInput; @@ -33,6 +37,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Paths; @@ -135,4 +140,78 @@ private void serialized(TimeseriesMetadata metaData) { } } } + + @Test + public void testStreamingDeserializerSkipsRejectedChunkList() throws IOException { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("streaming-predicate.tsfile")); + file.delete(); + try { + long firstChunkOffset = 1024L; + long secondChunkOffset = 2048L; + TimeseriesMetadata first = metadataWithOneChunk("s1", firstChunkOffset, 3); + TimeseriesMetadata second = metadataWithOneChunk("s2", secondChunkOffset, 5); + Assert.assertTrue(first.getDataSizeOfChunkMetaDataList() > 0); + int firstLength; + try (FileOutputStream out = new FileOutputStream(file)) { + firstLength = first.serializeTo(out); + second.serializeTo(out); + } + TsFileInput input = new LocalTsFileInput(Paths.get(file.getPath())); + try { + TimeseriesMetadata rejected = + TimeseriesMetadata.deserializeFrom(input, name -> false, true, true); + Assert.assertEquals("s1", rejected.getMeasurementId()); + // the rejected series' chunk list is skipped, not decoded + Assert.assertNull(rejected.getChunkMetadataList()); + // the input advanced exactly to the next record + Assert.assertEquals(firstLength, input.position()); + + TimeseriesMetadata accepted = + TimeseriesMetadata.deserializeFrom(input, name -> true, true, true); + Assert.assertEquals("s2", accepted.getMeasurementId()); + Assert.assertNotNull(accepted.getChunkMetadataList()); + Assert.assertEquals(1, accepted.getChunkMetadataList().size()); + ChunkMetadata chunk = (ChunkMetadata) accepted.getChunkMetadataList().get(0); + Assert.assertEquals(secondChunkOffset, chunk.getOffsetOfChunkHeader()); + Assert.assertEquals(5, chunk.getStatistics().getCount()); + Assert.assertEquals(0L, chunk.getStatistics().getStartTime()); + Assert.assertEquals(4L, chunk.getStatistics().getEndTime()); + } finally { + input.close(); + } + } finally { + file.delete(); + } + } + + private static TimeseriesMetadata metadataWithOneChunk( + String measurementId, long chunkOffset, int chunkCount) throws IOException { + Statistics seriesStatistics = + Statistics.getStatsByType(TSDataType.INT64); + seriesStatistics.setCount(chunkCount); + seriesStatistics.setStartTime(0); + seriesStatistics.setEndTime(chunkCount - 1); + Statistics chunkStatistics = + Statistics.getStatsByType(TSDataType.INT64); + chunkStatistics.setCount(chunkCount); + chunkStatistics.setStartTime(0); + chunkStatistics.setEndTime(chunkCount - 1); + ChunkMetadata chunk = + new ChunkMetadata( + measurementId, + TSDataType.INT64, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED, + chunkOffset, + chunkStatistics); + PublicBAOS chunkBuffer = new PublicBAOS(); + chunk.serializeTo(chunkBuffer, true); + return new TimeseriesMetadata( + (byte) 1, + chunkBuffer.size(), + measurementId, + TSDataType.INT64, + seriesStatistics, + chunkBuffer); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index 29c5d15943..80763726cb 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -23,6 +23,7 @@ import org.apache.tsfile.constant.TestConstant; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -137,6 +138,36 @@ public void testReadTimeseriesMetadataInBatch() throws IOException { measurement, single.getChunkMetadataList().size(), batched.getChunkMetadataList().size()); + for (int i = 0; i < single.getChunkMetadataList().size(); i++) { + ChunkMetadata singleChunk = (ChunkMetadata) single.getChunkMetadataList().get(i); + ChunkMetadata batchedChunk = (ChunkMetadata) batched.getChunkMetadataList().get(i); + Assert.assertEquals( + measurement, + singleChunk.getOffsetOfChunkHeader(), + batchedChunk.getOffsetOfChunkHeader()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getCount(), + batchedChunk.getStatistics().getCount()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getStartTime(), + batchedChunk.getStatistics().getStartTime()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getEndTime(), + batchedChunk.getStatistics().getEndTime()); + if (single.getTsDataType().isNumeric()) { + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getMinValue(), + batchedChunk.getStatistics().getMinValue()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getMaxValue(), + batchedChunk.getStatistics().getMaxValue()); + } + } } // a single searched measurement can warm the cache with its group siblings From 20e89a459b70be5737f9aae699fd0e0999343441 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 17:41:58 +0800 Subject: [PATCH 6/8] Route both index orderings and test the streaming byte accounting - Java indexes are sorted by UTF-16 code units while C++ indexes use UTF-8 bytes. The two disagree for supplementary characters mixed with the U+E000..U+FFFF range, so a binary search on a C++ index could miss an existing group. Compute the floor child under both orderings and visit both candidates when they differ. - Extract the streaming leaf-group loop so a reader-level test can drive it with a fake input: one accepted and one rejected chunk-bearing record, asserting the recorded bytes are the full record for the accepted one and the record minus its chunk list for the rejected one. - Add m00 + m03x to the leaf-filter test: they share an internal child but select different leaf groups, so the rejected-name read count now verifies the leaf-level filter. --- .../tsfile/read/TsFileSequenceReader.java | 115 ++++++++++++---- .../read/TimeSeriesMetadataReadTest.java | 124 ++++++++++++++++++ 2 files changed, 216 insertions(+), 23 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index f392d055b5..2e04a0629d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1264,15 +1264,26 @@ private void readMeasurementsInIndexNode( boolean[] selected = new boolean[children.size()]; int remaining = children.size(); for (String measurement : measurements) { - int index = floorChild(children, measurement); - if (index < 0 || selected[index]) { - continue; - } - if (mayExist != null && !mayExist.test(measurement)) { - continue; + int[] floors = floorChildren(children, measurement); + boolean tested = false; + boolean accepted = false; + for (int index : floors) { + if (index < 0 || selected[index]) { + continue; + } + if (!tested) { + tested = true; + accepted = mayExist == null || mayExist.test(measurement); + } + if (!accepted) { + break; + } + selected[index] = true; + if (--remaining == 0) { + break; + } } - selected[index] = true; - if (--remaining == 0) { + if (remaining == 0) { break; } } @@ -1289,19 +1300,29 @@ private void readMeasurementsInIndexNode( buckets.add(null); } for (String measurement : measurements) { - int index = floorChild(children, measurement); - if (index < 0) { + int[] floors = floorChildren(children, measurement); + // probe the filter only when this name could add a child that no accepted name selected yet + boolean selectsNewChild = false; + for (int index : floors) { + if (index >= 0 && buckets.get(index) == null) { + selectsNewChild = true; + break; + } + } + if (selectsNewChild && mayExist != null && !mayExist.test(measurement)) { continue; } - List bucket = buckets.get(index); - if (bucket == null) { - if (mayExist != null && !mayExist.test(measurement)) { + for (int index : floors) { + if (index < 0) { continue; } - bucket = new ArrayList<>(); - buckets.set(index, bucket); + List bucket = buckets.get(index); + if (bucket == null) { + bucket = new ArrayList<>(); + buckets.set(index, bucket); + } + bucket.add(measurement); } - bucket.add(measurement); } for (int i = 0; i < buckets.size(); i++) { List bucket = buckets.get(i); @@ -1319,13 +1340,28 @@ private void readMeasurementsInIndexNode( } } - private static int floorChild(List children, String key) { + /** + * Returns the floor child index under both orderings a TsFile may have been written with: Java + * indexes are sorted by UTF-16 code units, C++ indexes by UTF-8 bytes (code point order). The two + * disagree only for names that mix supplementary characters with the U+E000..U+FFFF range, so + * both candidates are returned when they differ and the caller visits both. + */ + static int[] floorChildren(List children, String key) { + int utf16Floor = floorChild(children, key, String::compareTo); + int codePointFloor = floorChild(children, key, TsFileSequenceReader::compareByCodePoint); + return utf16Floor == codePointFloor + ? new int[] {utf16Floor} + : new int[] {utf16Floor, codePointFloor}; + } + + private static int floorChild( + List children, String key, Comparator comparator) { int low = 0; int high = children.size() - 1; int floor = -1; while (low <= high) { int mid = (low + high) >>> 1; - if (children.get(mid).getCompareKey().compareTo(key) <= 0) { + if (comparator.compare((String) children.get(mid).getCompareKey(), key) <= 0) { floor = mid; low = mid + 1; } else { @@ -1335,6 +1371,22 @@ private static int floorChild(List children, String key) { return floor; } + /** Compares two measurement names by Unicode code point, the order C++ writers use. */ + private static int compareByCodePoint(String left, String right) { + int leftIndex = 0; + int rightIndex = 0; + while (leftIndex < left.length() && rightIndex < right.length()) { + int leftCodePoint = left.codePointAt(leftIndex); + int rightCodePoint = right.codePointAt(rightIndex); + if (leftCodePoint != rightCodePoint) { + return Integer.compare(leftCodePoint, rightCodePoint); + } + leftIndex += Character.charCount(leftCodePoint); + rightIndex += Character.charCount(rightCodePoint); + } + return Integer.compare(left.length() - leftIndex, right.length() - rightIndex); + } + private static long childEndOffset(MetadataIndexNode node, int childIndex) { return childIndex == node.getChildren().size() - 1 ? node.getEndOffset() @@ -1362,15 +1414,32 @@ private void readLeafMeasurementGroup( return; } // a group larger than Integer.MAX_VALUE cannot be held in one buffer + readLeafMeasurementGroupInStream( + tsFileInput, startOffset, endOffset, returned, result, ioSizeRecorder); + } + + /** + * Reads a leaf group through the given input and records only the bytes a selective read actually + * consumes: the chunk lists skipped for measurements the caller does not return are excluded. + */ + void readLeafMeasurementGroupInStream( + TsFileInput input, + long startOffset, + long endOffset, + Set returned, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + Predicate isReturned = returned::contains; synchronized (this) { - tsFileInput.position(startOffset); - while (tsFileInput.position() < endOffset) { - long recordStart = tsFileInput.position(); - TimeseriesMetadata metadata = deserializeTimeseriesMetadata(tsFileInput, isReturned); + input.position(startOffset); + while (input.position() < endOffset) { + long recordStart = input.position(); + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(input, isReturned); boolean returnedMeasurement = isReturned.test(metadata.getMeasurementId()); if (ioSizeRecorder != null) { // the selective deserializer seeks past the chunk list of a series it does not return - long consumed = tsFileInput.position() - recordStart; + long consumed = input.position() - recordStart; if (!returnedMeasurement) { consumed -= metadata.getDataSizeOfChunkMetaDataList(); } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index 80763726cb..01cf73ad02 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -23,12 +23,20 @@ import org.apache.tsfile.constant.TestConstant; import org.apache.tsfile.enums.ColumnCategory; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.IMetadataIndexEntry; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry; import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.reader.LocalTsFileInput; +import org.apache.tsfile.read.reader.TsFileInput; import org.apache.tsfile.utils.FileGenerator; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.Tablet; @@ -41,10 +49,14 @@ import org.junit.Test; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -318,6 +330,29 @@ public void testReadTimeseriesMetadataInBatchThroughInternalNode() throws Except Assert.assertEquals(1, filteredResult.size()); Assert.assertEquals(subsetReads, filteredReads); + // m03x shares m00's internal child but selects a different leaf group, so rejecting it must + // keep that leaf group from being read + List sameChildWithAbsent = Arrays.asList("m00", "m03x"); + List sameChildReads = new ArrayList<>(); + Map sameChildResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, sameChildWithAbsent, false, sameChildReads::add); + Assert.assertEquals(1, sameChildResult.size()); + Assert.assertEquals(4, sameChildReads.size()); + + List sameChildFilteredReads = new ArrayList<>(); + Map sameChildFilteredResult = + reader.readTimeseriesMetadataForMeasurements( + device, + offset, + sameChildWithAbsent, + null, + name -> !name.equals("m03x"), + false, + sameChildFilteredReads::add); + Assert.assertEquals(1, sameChildFilteredResult.size()); + Assert.assertEquals(subsetReads, sameChildFilteredReads); + // a single searched measurement can warm the siblings that share its leaf group List siblingReads = new ArrayList<>(); Map siblings = @@ -413,4 +448,93 @@ public void testReadTableModelMeasurements() throws Exception { Assert.assertTrue(file.delete()); } } + + @Test + public void testStreamingGroupReadRecordsOnlyConsumedBytes() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("StreamingGroupAccounting.tsfile")); + file.delete(); + try { + TimeseriesMetadata accepted = metadataWithOneChunk("s1", 1024L, 3); + TimeseriesMetadata rejected = metadataWithOneChunk("s2", 2048L, 5); + int acceptedLength; + int rejectedLength; + try (FileOutputStream out = new FileOutputStream(file)) { + acceptedLength = accepted.serializeTo(out); + rejectedLength = rejected.serializeTo(out); + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + TsFileInput input = new LocalTsFileInput(Paths.get(file.getPath())); + try { + List recorded = new ArrayList<>(); + Map result = new HashMap<>(); + reader.readLeafMeasurementGroupInStream( + input, + 0, + acceptedLength + rejectedLength, + new HashSet<>(Collections.singletonList("s1")), + result, + recorded::add); + Assert.assertEquals(Collections.singleton("s1"), result.keySet()); + Assert.assertEquals(2, recorded.size()); + // the accepted record is read in full; the rejected one only up to its chunk list + Assert.assertEquals(acceptedLength, recorded.get(0).longValue()); + Assert.assertEquals( + rejectedLength - rejected.getDataSizeOfChunkMetaDataList(), + recorded.get(1).longValue()); + } finally { + input.close(); + } + } + } finally { + file.delete(); + } + } + + @Test + public void testFloorRoutingHandlesBothIndexOrderings() { + // a C++ writer orders children by UTF-8 bytes (code points), which puts U+20000 after U+F900; + // Java's UTF-16 order puts it before. Both candidate groups must be reachable. + List children = new ArrayList<>(); + children.add(new MeasurementMetadataIndexEntry("豈", 0L)); + children.add(new MeasurementMetadataIndexEntry("𠀀", 10L)); + int[] floors = TsFileSequenceReader.floorChildren(children, "𠀀"); + Assert.assertEquals(2, floors.length); + Assert.assertEquals(-1, floors[0]); + Assert.assertEquals(1, floors[1]); + + int[] sameOrder = TsFileSequenceReader.floorChildren(children, "s1"); + Assert.assertEquals(1, sameOrder.length); + Assert.assertEquals(-1, sameOrder[0]); + } + + private static TimeseriesMetadata metadataWithOneChunk( + String measurementId, long chunkOffset, int chunkCount) throws IOException { + Statistics seriesStatistics = + Statistics.getStatsByType(TSDataType.INT64); + seriesStatistics.setCount(chunkCount); + seriesStatistics.setStartTime(0); + seriesStatistics.setEndTime(chunkCount - 1); + Statistics chunkStatistics = + Statistics.getStatsByType(TSDataType.INT64); + chunkStatistics.setCount(chunkCount); + chunkStatistics.setStartTime(0); + chunkStatistics.setEndTime(chunkCount - 1); + ChunkMetadata chunk = + new ChunkMetadata( + measurementId, + TSDataType.INT64, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED, + chunkOffset, + chunkStatistics); + PublicBAOS chunkBuffer = new PublicBAOS(); + chunk.serializeTo(chunkBuffer, true); + return new TimeseriesMetadata( + (byte) 1, + chunkBuffer.size(), + measurementId, + TSDataType.INT64, + seriesStatistics, + chunkBuffer); + } } From d6d7f52c1fc1d4cb7d9006691f02d3b0b9322824 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Nov 2026 19:12:57 +0800 Subject: [PATCH 7/8] Skip the second floor search when the name cannot reorder The two index orderings can disagree only where a surrogate code unit is compared with a unit in U+E000..U+FFFF. A name whose units are all below U+D800 (ASCII, Latin, common CJK) has the same relation to every child under both orders, so one binary search yields the same floor. Only names with a unit at or above U+D800 pay for the code-point search. --- .../tsfile/read/TsFileSequenceReader.java | 17 +++++++++++++++++ .../tsfile/read/TimeSeriesMetadataReadTest.java | 7 +++++++ 2 files changed, 24 insertions(+) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 2e04a0629d..525ff1db32 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1348,12 +1348,29 @@ private void readMeasurementsInIndexNode( */ static int[] floorChildren(List children, String key) { int utf16Floor = floorChild(children, key, String::compareTo); + if (!ordersMayDiffer(key)) { + return new int[] {utf16Floor}; + } int codePointFloor = floorChild(children, key, TsFileSequenceReader::compareByCodePoint); return utf16Floor == codePointFloor ? new int[] {utf16Floor} : new int[] {utf16Floor, codePointFloor}; } + /** + * Whether UTF-16 and code-point order can disagree for this name. They differ only between a + * surrogate code unit and a unit in U+E000..U+FFFF, so a name whose units are all below U+D800 + * (ASCII, Latin, common CJK) compares the same under both orders. + */ + static boolean ordersMayDiffer(String key) { + for (int i = 0; i < key.length(); i++) { + if (key.charAt(i) >= '\uD800') { + return true; + } + } + return false; + } + private static int floorChild( List children, String key, Comparator comparator) { int low = 0; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index 01cf73ad02..35ec31b59e 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -505,6 +505,13 @@ public void testFloorRoutingHandlesBothIndexOrderings() { int[] sameOrder = TsFileSequenceReader.floorChildren(children, "s1"); Assert.assertEquals(1, sameOrder.length); Assert.assertEquals(-1, sameOrder[0]); + + // only a unit at or above U+D800 can make the two orders disagree, so ASCII and common CJK + // names skip the second binary search + Assert.assertFalse(TsFileSequenceReader.ordersMayDiffer("s1")); + Assert.assertFalse(TsFileSequenceReader.ordersMayDiffer("温度")); + Assert.assertTrue(TsFileSequenceReader.ordersMayDiffer("豈")); + Assert.assertTrue(TsFileSequenceReader.ordersMayDiffer("𠀀")); } private static TimeseriesMetadata metadataWithOneChunk( From eb4f8fd036b35bf39a59c97cdaf0441eb2454417 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 20 Nov 2026 10:31:35 +0800 Subject: [PATCH 8/8] Revert the conditional second floor search Always compute the floor under both index orderings instead of relying on the code-unit threshold; the check couples the routing decision to a property of the name encoding that we do not want to depend on. --- .../tsfile/read/TsFileSequenceReader.java | 17 ----------------- .../tsfile/read/TimeSeriesMetadataReadTest.java | 7 ------- 2 files changed, 24 deletions(-) diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index 525ff1db32..2e04a0629d 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -1348,29 +1348,12 @@ private void readMeasurementsInIndexNode( */ static int[] floorChildren(List children, String key) { int utf16Floor = floorChild(children, key, String::compareTo); - if (!ordersMayDiffer(key)) { - return new int[] {utf16Floor}; - } int codePointFloor = floorChild(children, key, TsFileSequenceReader::compareByCodePoint); return utf16Floor == codePointFloor ? new int[] {utf16Floor} : new int[] {utf16Floor, codePointFloor}; } - /** - * Whether UTF-16 and code-point order can disagree for this name. They differ only between a - * surrogate code unit and a unit in U+E000..U+FFFF, so a name whose units are all below U+D800 - * (ASCII, Latin, common CJK) compares the same under both orders. - */ - static boolean ordersMayDiffer(String key) { - for (int i = 0; i < key.length(); i++) { - if (key.charAt(i) >= '\uD800') { - return true; - } - } - return false; - } - private static int floorChild( List children, String key, Comparator comparator) { int low = 0; diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index 35ec31b59e..01cf73ad02 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -505,13 +505,6 @@ public void testFloorRoutingHandlesBothIndexOrderings() { int[] sameOrder = TsFileSequenceReader.floorChildren(children, "s1"); Assert.assertEquals(1, sameOrder.length); Assert.assertEquals(-1, sameOrder[0]); - - // only a unit at or above U+D800 can make the two orders disagree, so ASCII and common CJK - // names skip the second binary search - Assert.assertFalse(TsFileSequenceReader.ordersMayDiffer("s1")); - Assert.assertFalse(TsFileSequenceReader.ordersMayDiffer("温度")); - Assert.assertTrue(TsFileSequenceReader.ordersMayDiffer("豈")); - Assert.assertTrue(TsFileSequenceReader.ordersMayDiffer("𠀀")); } private static TimeseriesMetadata metadataWithOneChunk(