diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java index 693b4ffb1..a46c383cf 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/MetadataIndexNode.java @@ -125,8 +125,17 @@ public static MetadataIndexNode deserializeFrom( return new MetadataIndexNode(children, offset, nodeType); } + /** + * Searches the floor child entry of the given key. + * + *

In floor mode a key that sorts before the first child returns child 0, so a caller that must + * not read a group which cannot hold the key should compare the key with the first child first. + */ public Pair getChildIndexEntry(Comparable key, boolean exactSearch) { - int index = binarySearchInChildren(key, exactSearch); + return getChildIndexEntry(binarySearchInChildren(key, exactSearch)); + } + + private Pair getChildIndexEntry(int index) { if (index == -1) { return null; } diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java index 790f457d1..56e71b106 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/TimeseriesMetadata.java @@ -44,6 +44,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.function.Predicate; import static org.apache.tsfile.utils.Preconditions.checkArgument; import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfCharArray; @@ -167,6 +168,23 @@ public static TimeseriesMetadata deserializeFrom( boolean needChunkMetadataForDataTypeWithValuesInStatistics, boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) throws IOException { + return deserializeFrom( + tsFileInput, + id -> true, + needChunkMetadataForDataTypeWithValuesInStatistics, + needChunkMetadataForDataTypeWithoutValuesInStatistics); + } + + /** + * Deserializes one TimeseriesMetadata from a file input and decodes its chunk metadata only when + * {@code decodeChunkMetadata} accepts the measurement. + */ + public static TimeseriesMetadata deserializeFrom( + TsFileInput tsFileInput, + Predicate decodeChunkMetadata, + boolean needChunkMetadataForDataTypeWithValuesInStatistics, + boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) + throws IOException { InputStream inputStream = tsFileInput.wrapAsInputStream(); TimeseriesMetadata timeseriesMetaData = new TimeseriesMetadata(); timeseriesMetaData.setTimeSeriesMetadataType(ReadWriteIOUtils.readByte(inputStream)); @@ -177,10 +195,11 @@ public static TimeseriesMetadata deserializeFrom( timeseriesMetaData.setStatistics( Statistics.deserialize(inputStream, timeseriesMetaData.dataType)); long startOffset = tsFileInput.position(); - if ((!timeseriesMetaData.getTsDataType().hasNoValueInStatistics() - && needChunkMetadataForDataTypeWithValuesInStatistics) - || (timeseriesMetaData.getTsDataType().hasNoValueInStatistics() - && needChunkMetadataForDataTypeWithoutValuesInStatistics)) { + if (decodeChunkMetadata.test(timeseriesMetaData.getMeasurementId()) + && ((!timeseriesMetaData.getTsDataType().hasNoValueInStatistics() + && needChunkMetadataForDataTypeWithValuesInStatistics) + || (timeseriesMetaData.getTsDataType().hasNoValueInStatistics() + && needChunkMetadataForDataTypeWithoutValuesInStatistics))) { timeseriesMetaData.chunkMetadataList = new ArrayList<>(); while (tsFileInput.position() < startOffset + chunkMetaDataListDataSize) { timeseriesMetaData.chunkMetadataList.add( @@ -208,6 +227,23 @@ public static TimeseriesMetadata deserializeFrom( Set excludedMeasurements, boolean needChunkMetadataForDataTypeWithValuesInStatistics, boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) { + return deserializeFrom( + buffer, + measurementId -> !excludedMeasurements.contains(measurementId), + needChunkMetadataForDataTypeWithValuesInStatistics, + needChunkMetadataForDataTypeWithoutValuesInStatistics); + } + + /** + * Deserializes one TimeseriesMetadata and decodes its chunk metadata only when {@code + * decodeChunkMetadata} accepts the measurement. Statistics are always read; the chunk metadata + * region is skipped when it is not decoded. + */ + public static TimeseriesMetadata deserializeFrom( + ByteBuffer buffer, + Predicate decodeChunkMetadata, + boolean needChunkMetadataForDataTypeWithValuesInStatistics, + boolean needChunkMetadataForDataTypeWithoutValuesInStatistics) { byte timeseriesType = ReadWriteIOUtils.readByte(buffer); String measurementID = ReadWriteIOUtils.readVarIntString(buffer); TSDataType tsDataType = ReadWriteIOUtils.readDataType(buffer); @@ -221,12 +257,11 @@ public static TimeseriesMetadata deserializeFrom( timeseriesMetaData.setDataSizeOfChunkMetaDataList(chunkMetaDataListDataSize); timeseriesMetaData.setStatistics(statistics); - if (!excludedMeasurements.contains(measurementID) + if (decodeChunkMetadata.test(measurementID) && ((!tsDataType.hasNoValueInStatistics() && needChunkMetadataForDataTypeWithValuesInStatistics) || (tsDataType.hasNoValueInStatistics() && needChunkMetadataForDataTypeWithoutValuesInStatistics))) { - // measurement is not in the excluded set and need chunk metadata ByteBuffer byteBuffer = buffer.slice(); byteBuffer.limit(chunkMetaDataListDataSize); timeseriesMetaData.chunkMetadataList = new ArrayList<>(); diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java index a3137149c..2e04a0629 100644 --- a/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java +++ b/java/tsfile/src/main/java/org/apache/tsfile/read/TsFileSequenceReader.java @@ -95,10 +95,12 @@ import java.nio.ByteBuffer; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Deque; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedList; @@ -114,6 +116,7 @@ import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.LongConsumer; +import java.util.function.Predicate; import java.util.stream.Collectors; public class TsFileSequenceReader implements AutoCloseable { @@ -1092,6 +1095,386 @@ public List readTimeseriesMetadata( return timeseriesMetadataList; } + /** + * Reads the {@link TimeseriesMetadata} of the given measurements of a device, visiting each + * measurement leaf group that holds one of them at most once. + * + * @param deviceMetadataIndexNodeOffset measurement metadata index node offset of an external + * TsFile; null to locate the device through the table metadata index + * @param measurements measurements to read, in any order + * @param ignoreNotExistDevice whether to return an empty map instead of throwing when the device + * is not recorded in the file metadata + * @return the metadata of the measurements found in the visited leaf groups; absent measurements + * are missing from the map + */ + public Map readTimeseriesMetadataForMeasurements( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Collection measurements, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { + return readTimeseriesMetadataForMeasurements( + device, + deviceMetadataIndexNodeOffset, + measurements, + null, + null, + ignoreNotExistDevice, + ioSizeRecorder); + } + + /** + * Reads the {@link TimeseriesMetadata} of the measurements that share a leaf group with one of + * {@code measurements}. Every searched measurement selects the leaf group that holds its floor + * key, and each visited group is read once. + * + *

An absent measurement still selects its floor group, so a group that holds none of the + * requested series is read when an absent measurement sorts into it. Pass {@code mayExist} to + * avoid those reads: a measurement is tested before it selects a group, and a rejected + * measurement selects nothing. The predicate must have no false negatives, because a {@link + * BloomFilter} has none; do not pass one for an offset-addressed external TsFile whose + * BloomFilter may not cover the series. It receives the bare measurement name, while a + * BloomFilter stores full {@code device.measurement} paths, so the caller has to build the path + * inside the predicate. + * + *

A searched measurement that is also requested for return is missing from the result only if + * {@code mayExist} rejected it (it then selects no group), or its floor group was visited and + * does not contain it (a measurement before the first key needs no read). A measurement in {@code + * returnedMeasurements} that is not in {@code measurements} is warmed only when it shares a + * visited group; a missing one is inconclusive, because its own group may not have been visited. + * + * @param measurements measurements that select the leaf groups to visit + * @param returnedMeasurements measurements to deserialize from the visited groups; null means + * {@code measurements}. It may contain measurements outside {@code measurements}, which is + * how a caller warms the metadata cache with the siblings of a single requested measurement + * @param mayExist optional filter tested on a measurement before it selects a group; null reads + * every floor group. It receives the bare measurement name; build the full path for a + * BloomFilter lookup. + * @return the metadata of the returned measurements found in the visited leaf groups + */ + @SuppressWarnings("unchecked") + public Map readTimeseriesMetadataForMeasurements( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Collection measurements, + Set returnedMeasurements, + Predicate mayExist, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { + // the device is looked up even for an empty request, so an absent device is reported the same + // way as for a non-empty one + MetadataIndexNode measurementIndexNode = + getMeasurementMetadataIndexNode(device, deviceMetadataIndexNodeOffset, ioSizeRecorder); + if (measurementIndexNode == null) { + if (ignoreNotExistDevice) { + return Collections.emptyMap(); + } + throw new IOException( + Messages.format("error.read.device_not_in_metadata_file", device, file)); + } + if (measurements.isEmpty()) { + return Collections.emptyMap(); + } + Set returned = + returnedMeasurements != null + ? returnedMeasurements + : measurements instanceof Set + ? (Set) measurements + : new HashSet<>(measurements); + // size from the searched set: the return set may include many cache-warming siblings that + // live in groups this call does not visit + Map result = new HashMap<>(Math.max(16, measurements.size())); + readMeasurementsInIndexNode( + measurementIndexNode, measurements, returned, mayExist, result, ioSizeRecorder); + return result; + } + + public Map readTimeseriesMetadataForMeasurements( + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Collection measurements, + Set returnedMeasurements, + boolean ignoreNotExistDevice, + LongConsumer ioSizeRecorder) + throws IOException { + return readTimeseriesMetadataForMeasurements( + device, + deviceMetadataIndexNodeOffset, + measurements, + returnedMeasurements, + null, + ignoreNotExistDevice, + ioSizeRecorder); + } + + public Map readTimeseriesMetadataForMeasurements( + IDeviceID device, Collection measurements, boolean ignoreNotExistDevice) + throws IOException { + return readTimeseriesMetadataForMeasurements( + device, null, measurements, null, null, ignoreNotExistDevice, null); + } + + private MetadataIndexNode getMeasurementMetadataIndexNode( + IDeviceID device, long[] deviceMetadataIndexNodeOffset, LongConsumer ioSizeRecorder) + throws IOException { + if (deviceMetadataIndexNodeOffset != null) { + ByteBuffer buffer = + readData( + deviceMetadataIndexNodeOffset[0], deviceMetadataIndexNodeOffset[1], ioSizeRecorder); + return deserializeMeasurementMetadataIndexNode(buffer); + } + readFileMetadata(ioSizeRecorder); + MetadataIndexNode deviceIndexNode = + tsFileMetaData.getTableMetadataIndexNode(device.getTableName()); + Pair deviceEntry = + getMetadataAndEndOffsetOfDeviceNode(deviceIndexNode, device, true, ioSizeRecorder); + if (deviceEntry == null) { + return null; + } + ByteBuffer buffer = readData(deviceEntry.left.getOffset(), deviceEntry.right, ioSizeRecorder); + return deserializeMeasurementMetadataIndexNode(buffer); + } + + private MetadataIndexNode deserializeMeasurementMetadataIndexNode(ByteBuffer buffer) + throws IOException { + try { + return deserializeConfig.measurementMetadataIndexNodeBufferDeserializer.deserialize( + buffer, deserializeConfig); + } catch (Exception e) { + logger.error(METADATA_INDEX_NODE_DESERIALIZE_ERROR, file); + throw e; + } + } + + private void readMeasurementsInIndexNode( + MetadataIndexNode node, + Collection measurements, + Set returned, + Predicate mayExist, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + List children = node.getChildren(); + if (children.isEmpty()) { + return; + } + if (MetadataIndexNodeType.LEAF_MEASUREMENT.equals(node.getNodeType())) { + boolean[] selected = new boolean[children.size()]; + int remaining = children.size(); + for (String measurement : measurements) { + int[] floors = floorChildren(children, measurement); + boolean tested = false; + boolean accepted = false; + for (int index : floors) { + if (index < 0 || selected[index]) { + continue; + } + if (!tested) { + tested = true; + accepted = mayExist == null || mayExist.test(measurement); + } + if (!accepted) { + break; + } + selected[index] = true; + if (--remaining == 0) { + break; + } + } + if (remaining == 0) { + break; + } + } + for (int i = 0; i < selected.length; i++) { + if (selected[i]) { + readLeafMeasurementGroup(node, i, returned, result, ioSizeRecorder); + } + } + return; + } + // an internal node: route each measurement to its floor child and read each visited child once + List> buckets = new ArrayList<>(children.size()); + for (int i = 0; i < children.size(); i++) { + buckets.add(null); + } + for (String measurement : measurements) { + int[] floors = floorChildren(children, measurement); + // probe the filter only when this name could add a child that no accepted name selected yet + boolean selectsNewChild = false; + for (int index : floors) { + if (index >= 0 && buckets.get(index) == null) { + selectsNewChild = true; + break; + } + } + if (selectsNewChild && mayExist != null && !mayExist.test(measurement)) { + continue; + } + for (int index : floors) { + if (index < 0) { + continue; + } + List bucket = buckets.get(index); + if (bucket == null) { + bucket = new ArrayList<>(); + buckets.set(index, bucket); + } + bucket.add(measurement); + } + } + for (int i = 0; i < buckets.size(); i++) { + List bucket = buckets.get(i); + if (bucket != null) { + ByteBuffer buffer = + readData(children.get(i).getOffset(), childEndOffset(node, i), ioSizeRecorder); + readMeasurementsInIndexNode( + deserializeMeasurementMetadataIndexNode(buffer), + bucket, + returned, + mayExist, + result, + ioSizeRecorder); + } + } + } + + /** + * Returns the floor child index under both orderings a TsFile may have been written with: Java + * indexes are sorted by UTF-16 code units, C++ indexes by UTF-8 bytes (code point order). The two + * disagree only for names that mix supplementary characters with the U+E000..U+FFFF range, so + * both candidates are returned when they differ and the caller visits both. + */ + static int[] floorChildren(List children, String key) { + int utf16Floor = floorChild(children, key, String::compareTo); + int codePointFloor = floorChild(children, key, TsFileSequenceReader::compareByCodePoint); + return utf16Floor == codePointFloor + ? new int[] {utf16Floor} + : new int[] {utf16Floor, codePointFloor}; + } + + private static int floorChild( + List children, String key, Comparator comparator) { + int low = 0; + int high = children.size() - 1; + int floor = -1; + while (low <= high) { + int mid = (low + high) >>> 1; + if (comparator.compare((String) children.get(mid).getCompareKey(), key) <= 0) { + floor = mid; + low = mid + 1; + } else { + high = mid - 1; + } + } + return floor; + } + + /** Compares two measurement names by Unicode code point, the order C++ writers use. */ + private static int compareByCodePoint(String left, String right) { + int leftIndex = 0; + int rightIndex = 0; + while (leftIndex < left.length() && rightIndex < right.length()) { + int leftCodePoint = left.codePointAt(leftIndex); + int rightCodePoint = right.codePointAt(rightIndex); + if (leftCodePoint != rightCodePoint) { + return Integer.compare(leftCodePoint, rightCodePoint); + } + leftIndex += Character.charCount(leftCodePoint); + rightIndex += Character.charCount(rightCodePoint); + } + return Integer.compare(left.length() - leftIndex, right.length() - rightIndex); + } + + private static long childEndOffset(MetadataIndexNode node, int childIndex) { + return childIndex == node.getChildren().size() - 1 + ? node.getEndOffset() + : node.getChildren().get(childIndex + 1).getOffset(); + } + + private void readLeafMeasurementGroup( + MetadataIndexNode node, + int childIndex, + Set returned, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + long startOffset = node.getChildren().get(childIndex).getOffset(); + long endOffset = childEndOffset(node, childIndex); + Predicate isReturned = returned::contains; + if (endOffset - startOffset < Integer.MAX_VALUE) { + ByteBuffer buffer = readData(startOffset, endOffset, ioSizeRecorder); + while (buffer.hasRemaining()) { + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(buffer, isReturned); + if (isReturned.test(metadata.getMeasurementId())) { + result.put(metadata.getMeasurementId(), metadata); + } + } + return; + } + // a group larger than Integer.MAX_VALUE cannot be held in one buffer + readLeafMeasurementGroupInStream( + tsFileInput, startOffset, endOffset, returned, result, ioSizeRecorder); + } + + /** + * Reads a leaf group through the given input and records only the bytes a selective read actually + * consumes: the chunk lists skipped for measurements the caller does not return are excluded. + */ + void readLeafMeasurementGroupInStream( + TsFileInput input, + long startOffset, + long endOffset, + Set returned, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + Predicate isReturned = returned::contains; + synchronized (this) { + input.position(startOffset); + while (input.position() < endOffset) { + long recordStart = input.position(); + TimeseriesMetadata metadata = deserializeTimeseriesMetadata(input, isReturned); + boolean returnedMeasurement = isReturned.test(metadata.getMeasurementId()); + if (ioSizeRecorder != null) { + // the selective deserializer seeks past the chunk list of a series it does not return + long consumed = input.position() - recordStart; + if (!returnedMeasurement) { + consumed -= metadata.getDataSizeOfChunkMetaDataList(); + } + ioSizeRecorder.accept(consumed); + } + if (returnedMeasurement) { + result.put(metadata.getMeasurementId(), metadata); + } + } + } + } + + private TimeseriesMetadata deserializeTimeseriesMetadata( + ByteBuffer buffer, Predicate decodeChunkMetadata) throws IOException { + try { + // skip the chunk metadata of the series this call does not return + return TimeseriesMetadata.deserializeFrom(buffer, decodeChunkMetadata, true, true); + } catch (Exception e) { + logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); + throw e; + } + } + + private TimeseriesMetadata deserializeTimeseriesMetadata( + TsFileInput input, Predicate decodeChunkMetadata) throws IOException { + try { + return TimeseriesMetadata.deserializeFrom(input, decodeChunkMetadata, true, true); + } catch (StopReadTsFileByInterruptException e) { + throw e; + } catch (Exception e) { + logger.error(Messages.get("log.read.sequence_reader_tsm_deserialize_error"), file); + throw e; + } + } + /* Get leaf MetadataIndexPair which contains path */ private Pair getLeafMetadataIndexPair( IDeviceID device, diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java index df18f6f39..e5afeb6ac 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/MetadataIndexNodeTest.java @@ -33,19 +33,19 @@ public class MetadataIndexNodeTest { @Test public void testBinarySearchInChildren() { List list = new ArrayList<>(); - list.add(new MeasurementMetadataIndexEntry("s0", -1L)); - list.add(new MeasurementMetadataIndexEntry("s5", -1L)); - list.add(new MeasurementMetadataIndexEntry("s10", -1L)); - list.add(new MeasurementMetadataIndexEntry("s15", -1L)); - list.add(new MeasurementMetadataIndexEntry("s20", -1L)); + list.add(new MeasurementMetadataIndexEntry("a", -1L)); + list.add(new MeasurementMetadataIndexEntry("c", -1L)); + list.add(new MeasurementMetadataIndexEntry("f", -1L)); + list.add(new MeasurementMetadataIndexEntry("m", -1L)); + list.add(new MeasurementMetadataIndexEntry("z", -1L)); MetadataIndexNode metadataIndexNode = new MetadataIndexNode(list, -1L, MetadataIndexNodeType.LEAF_MEASUREMENT); - Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("s0", false)); - Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s10", false)); - Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("s13", false)); - Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("s23", false)); - Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s13", true)); - Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("s23", true)); + Assert.assertEquals(0, metadataIndexNode.binarySearchInChildren("a", false)); + Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("f", false)); + Assert.assertEquals(2, metadataIndexNode.binarySearchInChildren("g", false)); + Assert.assertEquals(4, metadataIndexNode.binarySearchInChildren("zz", false)); + Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("g", true)); + Assert.assertEquals(-1, metadataIndexNode.binarySearchInChildren("zz", true)); } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java index 3ac296afe..e550fa7ab 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/TimeseriesMetadataTest.java @@ -19,6 +19,10 @@ package org.apache.tsfile.file.metadata; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.file.metadata.utils.TestHelper; import org.apache.tsfile.read.reader.LocalTsFileInput; import org.apache.tsfile.read.reader.TsFileInput; @@ -33,6 +37,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.io.Serializable; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.nio.file.Paths; @@ -135,4 +140,78 @@ private void serialized(TimeseriesMetadata metaData) { } } } + + @Test + public void testStreamingDeserializerSkipsRejectedChunkList() throws IOException { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("streaming-predicate.tsfile")); + file.delete(); + try { + long firstChunkOffset = 1024L; + long secondChunkOffset = 2048L; + TimeseriesMetadata first = metadataWithOneChunk("s1", firstChunkOffset, 3); + TimeseriesMetadata second = metadataWithOneChunk("s2", secondChunkOffset, 5); + Assert.assertTrue(first.getDataSizeOfChunkMetaDataList() > 0); + int firstLength; + try (FileOutputStream out = new FileOutputStream(file)) { + firstLength = first.serializeTo(out); + second.serializeTo(out); + } + TsFileInput input = new LocalTsFileInput(Paths.get(file.getPath())); + try { + TimeseriesMetadata rejected = + TimeseriesMetadata.deserializeFrom(input, name -> false, true, true); + Assert.assertEquals("s1", rejected.getMeasurementId()); + // the rejected series' chunk list is skipped, not decoded + Assert.assertNull(rejected.getChunkMetadataList()); + // the input advanced exactly to the next record + Assert.assertEquals(firstLength, input.position()); + + TimeseriesMetadata accepted = + TimeseriesMetadata.deserializeFrom(input, name -> true, true, true); + Assert.assertEquals("s2", accepted.getMeasurementId()); + Assert.assertNotNull(accepted.getChunkMetadataList()); + Assert.assertEquals(1, accepted.getChunkMetadataList().size()); + ChunkMetadata chunk = (ChunkMetadata) accepted.getChunkMetadataList().get(0); + Assert.assertEquals(secondChunkOffset, chunk.getOffsetOfChunkHeader()); + Assert.assertEquals(5, chunk.getStatistics().getCount()); + Assert.assertEquals(0L, chunk.getStatistics().getStartTime()); + Assert.assertEquals(4L, chunk.getStatistics().getEndTime()); + } finally { + input.close(); + } + } finally { + file.delete(); + } + } + + private static TimeseriesMetadata metadataWithOneChunk( + String measurementId, long chunkOffset, int chunkCount) throws IOException { + Statistics seriesStatistics = + Statistics.getStatsByType(TSDataType.INT64); + seriesStatistics.setCount(chunkCount); + seriesStatistics.setStartTime(0); + seriesStatistics.setEndTime(chunkCount - 1); + Statistics chunkStatistics = + Statistics.getStatsByType(TSDataType.INT64); + chunkStatistics.setCount(chunkCount); + chunkStatistics.setStartTime(0); + chunkStatistics.setEndTime(chunkCount - 1); + ChunkMetadata chunk = + new ChunkMetadata( + measurementId, + TSDataType.INT64, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED, + chunkOffset, + chunkStatistics); + PublicBAOS chunkBuffer = new PublicBAOS(); + chunk.serializeTo(chunkBuffer, true); + return new TimeseriesMetadata( + (byte) 1, + chunkBuffer.size(), + measurementId, + TSDataType.INT64, + seriesStatistics, + chunkBuffer); + } } diff --git a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java index e4bfbb44d..01cf73ad0 100644 --- a/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java +++ b/java/tsfile/src/test/java/org/apache/tsfile/read/TimeSeriesMetadataReadTest.java @@ -21,9 +21,27 @@ import org.apache.tsfile.common.conf.TSFileConfig; import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.constant.TestConstant; +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.IMetadataIndexEntry; +import org.apache.tsfile.file.metadata.ChunkMetadata; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.MeasurementMetadataIndexEntry; +import org.apache.tsfile.file.metadata.TableSchema; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.file.metadata.statistics.Statistics; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.read.reader.LocalTsFileInput; +import org.apache.tsfile.read.reader.TsFileInput; import org.apache.tsfile.utils.FileGenerator; +import org.apache.tsfile.utils.PublicBAOS; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.After; import org.junit.Assert; @@ -31,9 +49,17 @@ import org.junit.Test; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; public class TimeSeriesMetadataReadTest { @@ -93,4 +119,422 @@ public void testReadTimeseriesMetadata() throws IOException { Assert.assertEquals("s" + i, timeseriesMetadataList.get(i - 5).getMeasurementId()); } } + + @Test + public void testReadTimeseriesMetadataInBatch() throws IOException { + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + IDeviceID device = new Path("d1", "s1", true).getIDeviceID(); + Set requested = + new HashSet<>(Arrays.asList("s1", "s2", "s3", "s4", "s5", "s6", "s8")); + + Map batch = + reader.readTimeseriesMetadataForMeasurements(device, requested, false); + // s8 does not exist in the file + Assert.assertEquals(6, batch.size()); + Assert.assertNull(batch.get("s8")); + for (String measurement : Arrays.asList("s1", "s2", "s3", "s4", "s5", "s6")) { + TimeseriesMetadata single = reader.readTimeseriesMetadata(device, measurement, false, null); + TimeseriesMetadata batched = batch.get(measurement); + Assert.assertNotNull(measurement, single); + Assert.assertNotNull(measurement, batched); + Assert.assertEquals(measurement, single.getMeasurementId(), batched.getMeasurementId()); + Assert.assertEquals( + measurement, single.getStatistics().getCount(), batched.getStatistics().getCount()); + Assert.assertEquals( + measurement, + single.getStatistics().getStartTime(), + batched.getStatistics().getStartTime()); + Assert.assertEquals( + measurement, single.getStatistics().getEndTime(), batched.getStatistics().getEndTime()); + Assert.assertEquals( + measurement, + single.getChunkMetadataList().size(), + batched.getChunkMetadataList().size()); + for (int i = 0; i < single.getChunkMetadataList().size(); i++) { + ChunkMetadata singleChunk = (ChunkMetadata) single.getChunkMetadataList().get(i); + ChunkMetadata batchedChunk = (ChunkMetadata) batched.getChunkMetadataList().get(i); + Assert.assertEquals( + measurement, + singleChunk.getOffsetOfChunkHeader(), + batchedChunk.getOffsetOfChunkHeader()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getCount(), + batchedChunk.getStatistics().getCount()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getStartTime(), + batchedChunk.getStatistics().getStartTime()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getEndTime(), + batchedChunk.getStatistics().getEndTime()); + if (single.getTsDataType().isNumeric()) { + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getMinValue(), + batchedChunk.getStatistics().getMinValue()); + Assert.assertEquals( + measurement, + singleChunk.getStatistics().getMaxValue(), + batchedChunk.getStatistics().getMaxValue()); + } + } + } + + // a single searched measurement can warm the cache with its group siblings + Map withSiblings = + reader.readTimeseriesMetadataForMeasurements( + device, + null, + java.util.Collections.singletonList("s1"), + new HashSet<>(Arrays.asList("s1", "s2", "s3")), + false, + null); + Assert.assertEquals(new HashSet<>(Arrays.asList("s1", "s2", "s3")), withSiblings.keySet()); + for (Map.Entry entry : withSiblings.entrySet()) { + Assert.assertEquals( + entry.getKey(), + batch.get(entry.getKey()).getStatistics().getCount(), + entry.getValue().getStatistics().getCount()); + } + + // an absent device is tolerated only when ignoreNotExistDevice is set + IDeviceID absentDevice = new Path("d_not_exist", "s1", true).getIDeviceID(); + Assert.assertTrue( + reader.readTimeseriesMetadataForMeasurements(absentDevice, requested, true).isEmpty()); + Assert.assertThrows( + IOException.class, + () -> reader.readTimeseriesMetadataForMeasurements(absentDevice, requested, false)); + + // the offset-addressed variant used by external tsfiles reads the same metadata + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + IDeviceID current = null; + long[] offset = null; + while (iterator.hasNext()) { + IDeviceID next = iterator.next(); + if (next.equals(device)) { + current = next; + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + break; + } + } + Assert.assertNotNull(offset); + Map viaOffset = + reader.readTimeseriesMetadataForMeasurements(current, offset, requested, false, null); + Assert.assertEquals(batch.keySet(), viaOffset.keySet()); + for (String measurement : batch.keySet()) { + Assert.assertEquals( + measurement, + batch.get(measurement).getStatistics().getCount(), + viaOffset.get(measurement).getStatistics().getCount()); + } + } + } + + @Test + public void testReadTimeseriesMetadataInBatchThroughInternalNode() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("BatchInternalNode.tsfile")); + file.delete(); + IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d1"); + int measurementCount = 20; + List schemas = new ArrayList<>(measurementCount); + List present = new ArrayList<>(measurementCount); + for (int i = 0; i < measurementCount; i++) { + String measurement = String.format("m%02d", i); + present.add(measurement); + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + for (IMeasurementSchema schema : schemas) { + writer.registerTimeseries(device, schema); + } + for (int time = 0; time < 5; time++) { + TSRecord record = new TSRecord(device, time); + for (String measurement : present) { + record.addPoint(measurement, time); + } + writer.writeRecord(record); + } + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + long[] offset = null; + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + while (iterator.hasNext()) { + IDeviceID current = iterator.next(); + if (current.equals(device)) { + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + break; + } + } + Assert.assertNotNull(offset); + + List allReads = new ArrayList<>(); + Map all = + reader.readTimeseriesMetadataForMeasurements( + device, offset, present, false, allReads::add); + Assert.assertEquals(measurementCount, all.size()); + // 20 series at index degree 3: one root, three leaf index nodes and seven leaf groups + Assert.assertEquals(11, allReads.size()); + for (String measurement : present) { + TimeseriesMetadata metadata = all.get(measurement); + Assert.assertNotNull(measurement, metadata); + Assert.assertEquals(measurement, metadata.getMeasurementId()); + Assert.assertNotNull(measurement, metadata.getChunkMetadataList()); + Assert.assertFalse(measurement, metadata.getChunkMetadataList().isEmpty()); + } + + // a name before the first key, or one whose floor group a present name already selects, + // adds no read + Set withAbsent = new HashSet<>(present); + withAbsent.add("a"); + withAbsent.add("m99"); + withAbsent.add("zzz"); + List readsWithAbsent = new ArrayList<>(); + Map withAbsentResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, withAbsent, false, readsWithAbsent::add); + Assert.assertEquals(measurementCount, withAbsentResult.size()); + Assert.assertEquals(allReads, readsWithAbsent); + + // an absent name whose floor group no present name selects makes the reader visit that + // group; mayExist is what lets a caller avoid it + List subset = Collections.singletonList("m00"); + List subsetReads = new ArrayList<>(); + Map subsetResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, subset, false, subsetReads::add); + Assert.assertEquals(1, subsetResult.size()); + // the root, one leaf index node and one leaf group + Assert.assertEquals(3, subsetReads.size()); + + List subsetWithAbsent = Arrays.asList("m00", "m10x"); + List subsetWithAbsentReads = new ArrayList<>(); + Map subsetWithAbsentResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, subsetWithAbsent, false, subsetWithAbsentReads::add); + Assert.assertEquals(1, subsetWithAbsentResult.size()); + // the absent name sorts into a second leaf index node and leaf group + Assert.assertEquals(5, subsetWithAbsentReads.size()); + + List filteredReads = new ArrayList<>(); + Map filteredResult = + reader.readTimeseriesMetadataForMeasurements( + device, + offset, + subsetWithAbsent, + null, + name -> !name.equals("m10x"), + false, + filteredReads::add); + Assert.assertEquals(1, filteredResult.size()); + Assert.assertEquals(subsetReads, filteredReads); + + // m03x shares m00's internal child but selects a different leaf group, so rejecting it must + // keep that leaf group from being read + List sameChildWithAbsent = Arrays.asList("m00", "m03x"); + List sameChildReads = new ArrayList<>(); + Map sameChildResult = + reader.readTimeseriesMetadataForMeasurements( + device, offset, sameChildWithAbsent, false, sameChildReads::add); + Assert.assertEquals(1, sameChildResult.size()); + Assert.assertEquals(4, sameChildReads.size()); + + List sameChildFilteredReads = new ArrayList<>(); + Map sameChildFilteredResult = + reader.readTimeseriesMetadataForMeasurements( + device, + offset, + sameChildWithAbsent, + null, + name -> !name.equals("m03x"), + false, + sameChildFilteredReads::add); + Assert.assertEquals(1, sameChildFilteredResult.size()); + Assert.assertEquals(subsetReads, sameChildFilteredReads); + + // a single searched measurement can warm the siblings that share its leaf group + List siblingReads = new ArrayList<>(); + Map siblings = + reader.readTimeseriesMetadataForMeasurements( + device, + offset, + Collections.singletonList("m00"), + new HashSet<>(present), + false, + siblingReads::add); + Assert.assertTrue(siblings.containsKey("m00")); + Assert.assertTrue(present.containsAll(siblings.keySet())); + Assert.assertEquals(subsetReads, siblingReads); + } finally { + Assert.assertTrue(file.delete()); + } + } + + @Test + public void testReadAlignedMeasurementsWithTimeColumn() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("BatchAligned.tsfile")); + file.delete(); + IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.sg.d1"); + List schemas = + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32), + new MeasurementSchema("s3", TSDataType.INT32)); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(device, schemas); + for (int time = 0; time < 5; time++) { + TSRecord record = new TSRecord(device, time); + record.addPoint("s1", time); + record.addPoint("s2", time); + writer.writeRecord(record); + } + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + List reads = new ArrayList<>(); + Map result = + reader.readTimeseriesMetadataForMeasurements( + device, null, Arrays.asList("", "s1", "s2", "s9"), false, reads::add); + // the time column travels with the value columns of an aligned device + Assert.assertTrue(result.containsKey("")); + Assert.assertEquals(5, result.get("").getStatistics().getCount()); + Assert.assertEquals(new HashSet<>(Arrays.asList("", "s1", "s2")), result.keySet()); + Assert.assertFalse(reads.isEmpty()); + } finally { + Assert.assertTrue(file.delete()); + } + } + + @Test + public void testReadTableModelMeasurements() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("BatchTable.tsfile")); + file.delete(); + TableSchema tableSchema = + new TableSchema( + "t1", + Arrays.asList( + new MeasurementSchema("device", TSDataType.STRING), + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32)), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); + Tablet tablet = + new Tablet( + tableSchema.getTableName(), + Arrays.asList("device", "s1", "s2"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32, TSDataType.INT32), + Arrays.asList(ColumnCategory.TAG, ColumnCategory.FIELD, ColumnCategory.FIELD)); + for (int i = 0; i < 5; i++) { + tablet.addTimestamp(i, i); + tablet.addValue("device", i, "d1"); + tablet.addValue("s1", i, i); + tablet.addValue("s2", i, i); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerTableSchema(tableSchema); + writer.writeTable(tablet); + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + TsFileDeviceIterator iterator = reader.getAllDevicesIteratorWithIsAligned(); + Assert.assertTrue(iterator.hasNext()); + IDeviceID device = iterator.next().getLeft(); + List reads = new ArrayList<>(); + Map result = + reader.readTimeseriesMetadataForMeasurements( + device, null, Arrays.asList("s1", "s2", "s9"), false, reads::add); + Assert.assertEquals(new HashSet<>(Arrays.asList("s1", "s2")), result.keySet()); + Assert.assertEquals(5, result.get("s1").getStatistics().getCount()); + Assert.assertFalse(reads.isEmpty()); + } finally { + Assert.assertTrue(file.delete()); + } + } + + @Test + public void testStreamingGroupReadRecordsOnlyConsumedBytes() throws Exception { + File file = new File(TestConstant.BASE_OUTPUT_PATH.concat("StreamingGroupAccounting.tsfile")); + file.delete(); + try { + TimeseriesMetadata accepted = metadataWithOneChunk("s1", 1024L, 3); + TimeseriesMetadata rejected = metadataWithOneChunk("s2", 2048L, 5); + int acceptedLength; + int rejectedLength; + try (FileOutputStream out = new FileOutputStream(file)) { + acceptedLength = accepted.serializeTo(out); + rejectedLength = rejected.serializeTo(out); + } + try (TsFileSequenceReader reader = new TsFileSequenceReader(FILE_PATH)) { + TsFileInput input = new LocalTsFileInput(Paths.get(file.getPath())); + try { + List recorded = new ArrayList<>(); + Map result = new HashMap<>(); + reader.readLeafMeasurementGroupInStream( + input, + 0, + acceptedLength + rejectedLength, + new HashSet<>(Collections.singletonList("s1")), + result, + recorded::add); + Assert.assertEquals(Collections.singleton("s1"), result.keySet()); + Assert.assertEquals(2, recorded.size()); + // the accepted record is read in full; the rejected one only up to its chunk list + Assert.assertEquals(acceptedLength, recorded.get(0).longValue()); + Assert.assertEquals( + rejectedLength - rejected.getDataSizeOfChunkMetaDataList(), + recorded.get(1).longValue()); + } finally { + input.close(); + } + } + } finally { + file.delete(); + } + } + + @Test + public void testFloorRoutingHandlesBothIndexOrderings() { + // a C++ writer orders children by UTF-8 bytes (code points), which puts U+20000 after U+F900; + // Java's UTF-16 order puts it before. Both candidate groups must be reachable. + List children = new ArrayList<>(); + children.add(new MeasurementMetadataIndexEntry("豈", 0L)); + children.add(new MeasurementMetadataIndexEntry("𠀀", 10L)); + int[] floors = TsFileSequenceReader.floorChildren(children, "𠀀"); + Assert.assertEquals(2, floors.length); + Assert.assertEquals(-1, floors[0]); + Assert.assertEquals(1, floors[1]); + + int[] sameOrder = TsFileSequenceReader.floorChildren(children, "s1"); + Assert.assertEquals(1, sameOrder.length); + Assert.assertEquals(-1, sameOrder[0]); + } + + private static TimeseriesMetadata metadataWithOneChunk( + String measurementId, long chunkOffset, int chunkCount) throws IOException { + Statistics seriesStatistics = + Statistics.getStatsByType(TSDataType.INT64); + seriesStatistics.setCount(chunkCount); + seriesStatistics.setStartTime(0); + seriesStatistics.setEndTime(chunkCount - 1); + Statistics chunkStatistics = + Statistics.getStatsByType(TSDataType.INT64); + chunkStatistics.setCount(chunkCount); + chunkStatistics.setStartTime(0); + chunkStatistics.setEndTime(chunkCount - 1); + ChunkMetadata chunk = + new ChunkMetadata( + measurementId, + TSDataType.INT64, + TSEncoding.PLAIN, + CompressionType.UNCOMPRESSED, + chunkOffset, + chunkStatistics); + PublicBAOS chunkBuffer = new PublicBAOS(); + chunk.serializeTo(chunkBuffer, true); + return new TimeseriesMetadata( + (byte) 1, + chunkBuffer.size(), + measurementId, + TSDataType.INT64, + seriesStatistics, + chunkBuffer); + } }