From 7f90e2474e5c1a51326d6a42940f17d1728f522c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 20 Nov 2026 16:38:16 +0800 Subject: [PATCH] Batch scan TimeseriesMetadata for wide-table queries Resolve an aligned device's metadata with one set-based reader call per (device, file) instead of one call per column. The cache checks the time column first, filters absent series with the BloomFilter before taking the device lock, and remembers read-confirmed absence. The single-column entry keeps its own path with sibling warming. TsFile side: apache/tsfile#1123 (merged as 0d89263e), consumed through the dated 2.5.1-261120-SNAPSHOT. --- .../batch/PipeTabletEventPlainBatch.java | 17 +- .../request/PipeTransferTabletRawReq.java | 15 +- .../request/PipeTransferTabletRawReqV2.java | 18 +- .../fragment/FragmentInstanceContext.java | 14 +- .../operator/source/FileLoaderUtils.java | 49 +- .../buffer/TimeSeriesMetadataCache.java | 690 +++++++++++--- .../buffer/TimeSeriesMetadataCacheTest.java | 839 +++++++++++++++++- .../series/AlignedSeriesScanBatchTest.java | 359 ++++++++ pom.xml | 2 +- 9 files changed, 1806 insertions(+), 197 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java index 66baaed98bd1c..92efffcb4558c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/batch/PipeTabletEventPlainBatch.java @@ -32,10 +32,12 @@ import org.apache.tsfile.read.common.type.Type; import org.apache.tsfile.utils.BitMap; import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.utils.RamUsageEstimator; import org.apache.tsfile.utils.ReadWriteIOUtils; import org.apache.tsfile.write.record.Tablet; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -293,13 +295,14 @@ private static long calculateTabletSizeInBytes(final Tablet tablet) { } private static ByteBuffer serializeTabletWithAlignment( - final Tablet tablet, final boolean isAligned) { - final ByteBuffer buffer = - ByteBuffer.allocate(Math.addExact(tablet.serializedSize(), Byte.BYTES)); - tablet.serialize(buffer); - ReadWriteIOUtils.write(isAligned, buffer); - buffer.flip(); - return buffer; + final Tablet tablet, final boolean isAligned) throws IOException { + try (final PublicBAOS byteArrayOutputStream = + new PublicBAOS(Math.addExact(tablet.serializedSize(), Byte.BYTES)); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size()); + } } static boolean mayAppendTablet(final Tablet target, final Tablet source) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java index 7faf5449f1f77..00907ed8302f5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReq.java @@ -272,12 +272,15 @@ public byte[] toTPipeTransferBytes() throws IOException { throw new IOException(DataNodePipeMessages.CANNOT_SERIALIZE_BOTH_TABLET_AND_STATEMENT_ARE); } - final ByteBuffer buffer = ByteBuffer.allocate(calculateAirGapSerializedSize(tabletToSerialize)); - ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer); - ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), buffer); - tabletToSerialize.serialize(buffer); - ReadWriteIOUtils.write(isAlignedToSerialize, buffer); - return buffer.array(); + try (final PublicBAOS byteArrayOutputStream = + new PublicBAOS(calculateAirGapSerializedSize(tabletToSerialize)); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream); + tabletToSerialize.serialize(outputStream); + ReadWriteIOUtils.write(isAlignedToSerialize, outputStream); + return byteArrayOutputStream.toByteArray(); + } } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java index df83276fdf7a0..dd6941d7f7d8f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/sink/payload/evolvable/request/PipeTransferTabletRawReqV2.java @@ -174,14 +174,16 @@ public static PipeTransferTabletRawReqV2 fromTPipeTransferReq( public static byte[] toTPipeTransferBytes( final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException { - final ByteBuffer buffer = - ByteBuffer.allocate(calculateAirGapSerializedSize(tablet, dataBaseName)); - ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer); - ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), buffer); - tablet.serialize(buffer); - ReadWriteIOUtils.write(isAligned, buffer); - ReadWriteIOUtils.write(dataBaseName, buffer); - return buffer.array(); + try (final PublicBAOS byteArrayOutputStream = + new PublicBAOS(calculateAirGapSerializedSize(tablet, dataBaseName)); + final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) { + ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream); + ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream); + tablet.serialize(outputStream); + ReadWriteIOUtils.write(isAligned, outputStream); + ReadWriteIOUtils.write(dataBaseName, outputStream); + return byteArrayOutputStream.toByteArray(); + } } static int calculateSerializedSize(final Tablet tablet, final String dataBaseName) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java index 6d33ea28a6066..5ff5dca4c1e6d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java @@ -161,9 +161,10 @@ public class FragmentInstanceContext extends QueryContext { // empty for zero time partitions private List timePartitions; - // An optimization during restart changes the time index from FILE TIME INDEX - // to DEVICE TIME INDEX, which may cause a related validation false positive. - private boolean ignoreNotExistsDevice = false; + // Queries can start before the data regions finish recovering after a restart. During that + // window the time index may still change from FILE TIME INDEX to DEVICE TIME INDEX, so an + // absent device must not be treated as corruption or remembered as absent. + private boolean isStorageEngineRecovering = false; private QueryDataSourceType queryDataSourceType = QueryDataSourceType.SERIES_SCAN; @@ -456,7 +457,7 @@ protected FragmentInstanceContext( public void start() { long now = System.currentTimeMillis(); - ignoreNotExistsDevice = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions(); + isStorageEngineRecovering = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions(); executionStartTime.compareAndSet(null, now); startNanos.compareAndSet(0, System.nanoTime()); @@ -1642,8 +1643,9 @@ public long getUnclosedSeqFileNum() { return unclosedSeqFileNum.get(); } - public boolean ignoreNotExistsDevice() { - return ignoreNotExistsDevice; + /** Whether this query started while the storage engine was still recovering its data regions. */ + public boolean isStorageEngineRecovering() { + return isStorageEngineRecovering; } /** diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 4374e974db5bc..12b44da4a77fa 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -30,7 +30,6 @@ import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; -import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader; import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader; @@ -47,7 +46,6 @@ import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; -import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.ITimeSeriesMetadata; import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -62,6 +60,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -112,7 +111,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata( seriesPath.getDeviceId(), seriesPath.getMeasurement()), allSensors, - context.ignoreNotExistsDevice() + context.isStorageEngineRecovering() || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, context.isDebug(), context); @@ -341,23 +340,29 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr // the order of timeSeriesMetadata list is same as subSensorList's order TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); List valueMeasurementList = alignedPath.getMeasurementList(); + // resolves the time column and every selected sensor in one call: the cache reads each leaf + // group that holds one of them once and answers the whole group from that read, so a wide + // projection does not walk the device metadata index once per column Set allSensors = alignedPath.getAllSensors(); - boolean isDebug = context.isDebug(); - String filePath = resource.getTsFilePath(); - IDeviceID deviceId = alignedPath.getDeviceId(); // when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex // we should not ignore the non-exist of device in TsFileMetadata - TimeseriesMetadata timeColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""), + boolean ignoreNotExists = + context.isStorageEngineRecovering() + || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + + Map metadata = + cache.getDeviceTimeseriesMetadata( + resource.getTsFilePath(), + resource.getTsFileID(), + alignedPath.getDeviceId(), allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, - isDebug, + ignoreNotExists, + context.isDebug(), context, rootMeasurementMetadataIndexNodeOffset); + + TimeseriesMetadata timeColumn = metadata.get(""); if (timeColumn != null) { // only need time column, like count_time aggregation if (valueMeasurementList.isEmpty()) { @@ -375,21 +380,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr } else { List valueTimeSeriesMetadataList = new ArrayList<>(valueMeasurementList.size()); - // if all the queried aligned sensors does not exist, we will return null - boolean exist = false; for (String measurement : valueMeasurementList) { - TimeseriesMetadata valueColumn = - cache.get( - filePath, - new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement), - allSensors, - context.ignoreNotExistsDevice() - || resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE, - isDebug, - context, - rootMeasurementMetadataIndexNodeOffset); + valueTimeSeriesMetadataList.add(metadata.get(measurement)); + } + // if all the queried aligned sensors do not exist, skip the device in this file + boolean exist = false; + for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) { exist = (exist || (valueColumn != null)); - valueTimeSeriesMetadataList.add(valueColumn); } if (!ignoreAllNullRows || exist) { alignedTimeSeriesMetadata = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java index 68457df0ebbcd..ac7070f6aa80c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCache.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; +import org.apache.iotdb.db.queryengine.execution.fragment.QueryStatistics; import org.apache.iotdb.db.queryengine.metric.TimeSeriesMetadataCacheMetrics; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; @@ -35,7 +36,6 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Weigher; -import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.TimeseriesMetadata; @@ -48,13 +48,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.lang.ref.WeakReference; +import java.util.Collection; import java.util.Collections; -import java.util.List; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.WeakHashMap; import java.util.concurrent.atomic.AtomicLong; import java.util.function.LongConsumer; @@ -86,10 +87,17 @@ public class TimeSeriesMetadataCache { private final AtomicLong entryAverageSize = new AtomicLong(0); - private final Map> devices = - Collections.synchronizedMap(new WeakHashMap<>()); private static final String SEPARATOR = "$"; + private static final int DEVICE_LOCK_STRIPES = 1024; + private static final Object[] DEVICE_LOCKS = new Object[DEVICE_LOCK_STRIPES]; + + static { + for (int i = 0; i < DEVICE_LOCK_STRIPES; i++) { + DEVICE_LOCKS[i] = new Object(); + } + } + private final AtomicLong evictedExistingEntryCount = new AtomicLong(0); private final AtomicLong evictedNonExistingEntryCount = new AtomicLong(0); @@ -137,7 +145,6 @@ public static TimeSeriesMetadataCache getInstance() { return TimeSeriesMetadataCache.TimeSeriesMetadataCacheHolder.INSTANCE; } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -149,7 +156,6 @@ public TimeseriesMetadata get( return get(filePath, key, allSensors, ignoreNotExists, debug, queryContext, null); } - @SuppressWarnings({"squid:S1860", "squid:S6541", "squid:S3776"}) // Suppress synchronize warning public TimeseriesMetadata get( String filePath, TimeSeriesMetadataCacheKey key, @@ -160,162 +166,580 @@ public TimeseriesMetadata get( long[] deviceMetadataIndexNodeOffset) throws IOException { long startTime = System.nanoTime(); - long loadBloomFilterTime = 0; + boolean externalTsFile = queryContext.isExternalTsFileScan(); + boolean cacheEnabled = CACHE_ENABLE && !externalTsFile; LongConsumer timeSeriesMetadataIoSizeRecorder = queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; LongConsumer bloomFilterIoSizeRecorder = queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; - boolean cacheHit = true; - boolean externalTsFile = queryContext.isExternalTsFileScan(); + long loadBloomFilterTime = 0; + boolean cacheHit = false; try { - if (!CACHE_ENABLE || externalTsFile) { - String deviceStringFormat = key.device.toString(); - cacheHit = false; - - // bloom filter part - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, bloomFilterIoSizeRecorder, externalTsFile); - if (deviceMetadataIndexNodeOffset == null) { - BloomFilter bloomFilter = reader.readBloomFilter(bloomFilterIoSizeRecorder); - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount().incrementAndGet(); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { - loadBloomFilterTime = System.nanoTime() - startTime; - return null; + if (cacheEnabled) { + TimeseriesMetadata cached = lruCache.getIfPresent(key); + if (cached != null) { + cacheHit = true; + return copyOrNull(cached, debug, key, filePath); + } + } + String deviceName = key.device.toString(); + if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + BloomFilter bloomFilter = + loadBloomFilter( + filePath, + key.tsFileID, + queryContext, + debug, + bloomFilterIoSizeRecorder, + externalTsFile); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + if (bloomFilter != null + && !bloomFilter.contains(deviceName + IoTDBConstant.PATH_SEPARATOR + key.measurement)) { + if (debug) { + DEBUG_LOGGER.info(StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, key); } - loadBloomFilterTime = System.nanoTime() - startTime; + return null; } - - TimeseriesMetadata timeseriesMetadata = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - return (timeseriesMetadata == null || timeseriesMetadata.getStatistics().getCount() == 0) - ? null - : timeseriesMetadata; } - - TimeseriesMetadata timeseriesMetadata = lruCache.getIfPresent(key); - - if (timeseriesMetadata == null) { + if (!cacheEnabled) { + return readSingle( + filePath, + key, + allSensors, + deviceMetadataIndexNodeOffset, + ignoreNotExists, + debug, + externalTsFile, + false, + timeSeriesMetadataIoSizeRecorder); + } + synchronized (deviceLock(deviceName, filePath)) { + TimeseriesMetadata cached = lruCache.getIfPresent(key); + if (cached != null) { + cacheHit = true; + return copyOrNull(cached, debug, key, filePath); + } if (debug) { DEBUG_LOGGER.info( StorageEngineMessages.CACHE_MISS_IN_FILE, key.device, key.measurement, filePath); DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, key.device, allSensors); } - String deviceStringFormat = key.device.toString(); - // allow for the parallelism of different devices - synchronized ( - devices.computeIfAbsent( - deviceStringFormat + SEPARATOR + filePath, WeakReference::new)) { - // double check - timeseriesMetadata = lruCache.getIfPresent(key); - if (timeseriesMetadata == null) { - cacheHit = false; - - if (deviceMetadataIndexNodeOffset == null) { - long loadBloomFilterStartTime = System.nanoTime(); - // bloom filter part - BloomFilter bloomFilter = - BloomFilterCache.getInstance() - .get( - new BloomFilterCache.BloomFilterCacheKey(filePath, key.tsFileID), - debug, - bloomFilterIoSizeRecorder, - queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount() - ::addAndGet, - queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount() - ::addAndGet, - false); - if (bloomFilter != null - && !bloomFilter.contains( - deviceStringFormat + TsFileConstant.PATH_SEPARATOR + key.measurement)) { - if (debug) { - DEBUG_LOGGER.info( - StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, key); - } - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - return null; - } - - loadBloomFilterTime = System.nanoTime() - loadBloomFilterStartTime; - } - TsFileSequenceReader reader = - FileReaderManager.getInstance() - .get(filePath, key.tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); - List timeSeriesMetadataList = - reader.readTimeseriesMetadata( - key.device, - deviceMetadataIndexNodeOffset, - key.measurement, - allSensors, - ignoreNotExists, - timeSeriesMetadataIoSizeRecorder); - // put TimeSeriesMetadata of all sensors used in this read into cache - for (TimeseriesMetadata metadata : timeSeriesMetadataList) { - TimeSeriesMetadataCacheKey k = - new TimeSeriesMetadataCacheKey( - key.tsFileID, key.device, metadata.getMeasurementId()); - if (metadata.getStatistics().getCount() != 0) { - lruCache.put(k, metadata); - } - if (metadata.getMeasurementId().equals(key.measurement)) { - timeseriesMetadata = metadata.getStatistics().getCount() == 0 ? null : metadata; - } + return readSingle( + filePath, + key, + allSensors, + deviceMetadataIndexNodeOffset, + ignoreNotExists, + debug, + externalTsFile, + true, + timeSeriesMetadataIoSizeRecorder); + } + } finally { + long elapsed = System.nanoTime() - startTime; + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); + if (cacheHit) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(1); + statistics + .getLoadTimeSeriesMetadataFromCacheTime() + .getAndAdd(elapsed - loadBloomFilterTime); + } else { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(1); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(elapsed - loadBloomFilterTime); + } + } + } + + private static TimeseriesMetadata copyOrNull( + TimeseriesMetadata cached, boolean debug, TimeSeriesMetadataCacheKey key, String filePath) { + if (cached == NULL_EXISTS_CACHE_PLACE_HOLDER) { + if (debug) { + DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); + } + return null; + } + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, + key.device, + key.measurement, + filePath, + cached); + } + return new TimeseriesMetadata(cached); + } + + /** + * Reads one measurement and warms the cache with its group siblings: the reader deserializes + * every entry of {@code allSensors} in the visited group, and the cache keeps the non-empty ones + * so later single-column reads hit the LRU. + */ + private TimeseriesMetadata readSingle( + String filePath, + TimeSeriesMetadataCacheKey key, + Set allSensors, + long[] deviceMetadataIndexNodeOffset, + boolean ignoreNotExists, + boolean debug, + boolean externalTsFile, + boolean cacheEnabled, + LongConsumer ioSizeRecorder) + throws IOException { + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, key.tsFileID, true, ioSizeRecorder, externalTsFile); + Map loaded = + reader.readTimeseriesMetadataForMeasurements( + key.device, + deviceMetadataIndexNodeOffset, + Collections.singleton(key.measurement), + allSensors, + ignoreNotExists, + ioSizeRecorder); + if (cacheEnabled) { + cacheLoadedMetadata(key.tsFileID, key.device, loaded); + } + TimeseriesMetadata requestedMetadata = loaded.get(key.measurement); + if (requestedMetadata != null && requestedMetadata.getStatistics().getCount() == 0) { + requestedMetadata = null; + } + if (requestedMetadata == null) { + // the bloom filter already ruled out the measurements that cannot be in the file, so a + // missing or empty one is read-confirmed absent + cacheNonExistenceIfNeeded( + key.tsFileID, key.device, key.measurement, cacheEnabled, ignoreNotExists); + if (debug) { + DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); + } + return null; + } + return new TimeseriesMetadata(requestedMetadata); + } + + /** + * Resolve the metadata of {@code requested} measurements of a device in one file. + * + *

Cached measurements are returned directly. Each remaining measurement is first checked + * against the file's bloom filter: a measurement the filter rules out is answered with null and + * not remembered, because probing the filter again is cheaper than a cache entry. The + * measurements the filter cannot rule out are loaded together in one reader call, which visits + * every leaf group that holds one of them once. A measurement that call does not return, or + * returns with an empty statistics count, is absent from the file and is remembered with a + * negative cache entry, so a later query does not walk its leaf group again. + * + *

Offset-addressed external TsFiles may not have a bloom filter that covers their series, so + * bloom filtering is skipped for them and the reader settles every requested measurement. + * + *

{@code requested} must contain the time column {@code ""}; the aligned loader always adds + * it. The call resolves the full projected set in one reader call, so no sibling warming set is + * needed. + * + *

The returned map holds copies of the cached (or freshly read) metadata, so the caller may + * apply {@code setModified} and {@code setChunkMetadataLoader} without corrupting cached entries. + * Load statistics are counted per requested measurement. + * + * @param deviceMetadataIndexNodeOffset non-null for external tsfiles, which address the device + * metadata index by offset; bloom filtering is skipped in that case + * @return a map with an entry for every measurement in {@code requested}: a metadata copy when + * the series exists and is non-empty, null otherwise + */ + public Map getDeviceTimeseriesMetadata( + String filePath, + TsFileID tsFileID, + IDeviceID device, + Set requested, + boolean ignoreNotExists, + boolean debug, + QueryContext queryContext, + long[] deviceMetadataIndexNodeOffset) + throws IOException { + long startTime = System.nanoTime(); + long cachePhaseEndTime = 0; + long loadBloomFilterTime = 0; + long bloomFilterTimeBeforeCachePhase = 0; + long bloomFilterTimeAfterCachePhase = 0; + long cacheHitCount = 0; + long diskLookupCount = 0; + boolean externalTsFile = queryContext.isExternalTsFileScan(); + LongConsumer timeSeriesMetadataIoSizeRecorder = + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataActualIOSize()::addAndGet; + LongConsumer bloomFilterIoSizeRecorder = + queryContext.getQueryStatistics().getLoadBloomFilterActualIOSize()::addAndGet; + + Map result = new HashMap<>(requested.size()); + Set pending = new HashSet<>(requested.size()); + + try { + if (CACHE_ENABLE && !externalTsFile) { + String deviceName = device.toString(); + String devicePrefix = deviceName + IoTDBConstant.PATH_SEPARATOR; + BloomFilter bloomFilter = null; + // an aligned device always has its time column, so check it before the per-column cache + // pass: a file without data for this device then costs one lookup and at most one + // BloomFilter probe instead of one lookup per selected measurement + TimeseriesMetadata cachedTime = + lruCache.getIfPresent(new TimeSeriesMetadataCacheKey(tsFileID, device, "")); + if (cachedTime == NULL_EXISTS_CACHE_PLACE_HOLDER) { + cacheHitCount = 1; + cachePhaseEndTime = System.nanoTime(); + fillMissingWithNull(requested, result); + return result; + } + if (cachedTime != null) { + result.put("", new TimeseriesMetadata(cachedTime)); + } else if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, debug, bloomFilterIoSizeRecorder, false); + bloomFilterTimeBeforeCachePhase = System.nanoTime() - bloomFilterStartTime; + loadBloomFilterTime += bloomFilterTimeBeforeCachePhase; + if (bloomFilter != null && !bloomFilter.contains(devicePrefix)) { + fillMissingWithNull(requested, result); + return result; + } + } + collectCachedMetadata(tsFileID, device, requested, result, pending, filePath, debug); + cacheHitCount = result.size(); + if (pending.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; + } + cachePhaseEndTime = System.nanoTime(); + if (bloomFilter == null && deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, debug, bloomFilterIoSizeRecorder, false); + bloomFilterTimeAfterCachePhase = System.nanoTime() - bloomFilterStartTime; + loadBloomFilterTime += bloomFilterTimeAfterCachePhase; + } + // the filter and the cache are thread-safe, so a rejected measurement never enters the + // device lock; only the survivors need the locked recheck and the reader + bloomAbsentMeasurements( + tsFileID, device, pending, result, bloomFilter, devicePrefix, debug); + if (pending.isEmpty()) { + return result; + } + // serialize reads of the same device+file (same lock key as get) so threads don't each + // walk the device metadata index and race on the shared reader + synchronized (deviceLock(deviceName, filePath)) { + // another thread may have loaded some of the survivors while this one waited + Iterator pendingIterator = pending.iterator(); + while (pendingIterator.hasNext()) { + String measurement = pendingIterator.next(); + TimeseriesMetadata cached = + lruCache.getIfPresent( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + if (cached == null) { + continue; } - if (timeseriesMetadata == null - && !ignoreNotExists - && memoryConfig.isMayCacheNonExistSeries()) { - lruCache.put(key, NULL_EXISTS_CACHE_PLACE_HOLDER); + pendingIterator.remove(); + cacheHitCount++; + result.put( + measurement, + cached == NULL_EXISTS_CACHE_PLACE_HOLDER ? null : new TimeseriesMetadata(cached)); + } + if (pending.isEmpty()) { + cachePhaseEndTime = System.nanoTime(); + return result; + } + if (debug) { + for (String measurement : pending) { + DEBUG_LOGGER.info( + StorageEngineMessages.CACHE_MISS_IN_FILE, device, measurement, filePath); } + DEBUG_LOGGER.info(StorageEngineMessages.DEVICE_ALL_SENSORS, device, requested); } + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, false); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + ignoreNotExists, + true, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + return result; } } - if (timeseriesMetadata == null || timeseriesMetadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { + + // cache disabled or external tsfiles: nothing was resolved from the cache + pending.addAll(requested); + String devicePrefix = device.toString() + IoTDBConstant.PATH_SEPARATOR; + // offset-addressed external files may not have a bloom filter that covers their series + BloomFilter bloomFilter = null; + if (deviceMetadataIndexNodeOffset == null) { + long bloomFilterStartTime = System.nanoTime(); + bloomFilter = + loadBloomFilter( + filePath, tsFileID, queryContext, debug, bloomFilterIoSizeRecorder, externalTsFile); + loadBloomFilterTime = System.nanoTime() - bloomFilterStartTime; + } + if (deviceAbsent(bloomFilter, devicePrefix)) { + fillMissingWithNull(requested, result); + return result; + } + bloomAbsentMeasurements(tsFileID, device, pending, result, bloomFilter, devicePrefix, debug); + if (pending.isEmpty()) { + return result; + } + TsFileSequenceReader reader = + FileReaderManager.getInstance() + .get(filePath, tsFileID, true, timeSeriesMetadataIoSizeRecorder, externalTsFile); + readAndMerge( + reader, + tsFileID, + device, + deviceMetadataIndexNodeOffset, + pending, + ignoreNotExists, + false, + debug, + result, + timeSeriesMetadataIoSizeRecorder); + return result; + } finally { + long endTime = System.nanoTime(); + // every requested measurement that was not served by the cache is a lookup, including the + // ones an early return rejected with the BloomFilter + diskLookupCount = requested.size() - cacheHitCount; + long cacheLookupTime; + long diskLookupTime; + if (cacheHitCount == 0 || cachePhaseEndTime == 0) { + cacheLookupTime = 0; + diskLookupTime = endTime - startTime - loadBloomFilterTime; + } else if (diskLookupCount == 0) { + cacheLookupTime = endTime - startTime - loadBloomFilterTime; + diskLookupTime = 0; + } else { + // the pre-cache BloomFilter load belongs to the cache interval; only a load after the + // cache phase belongs to the disk interval + cacheLookupTime = cachePhaseEndTime - startTime - bloomFilterTimeBeforeCachePhase; + diskLookupTime = endTime - cachePhaseEndTime - bloomFilterTimeAfterCachePhase; + } + recordLoadStatistics( + queryContext, + cacheHitCount, + cacheLookupTime, + diskLookupCount, + diskLookupTime, + loadBloomFilterTime); + } + } + + private BloomFilter loadBloomFilter( + String filePath, + TsFileID tsFileID, + QueryContext queryContext, + boolean debug, + LongConsumer bloomFilterIoSizeRecorder, + boolean externalTsFile) + throws IOException { + return BloomFilterCache.getInstance() + .get( + new BloomFilterCache.BloomFilterCacheKey(filePath, tsFileID), + debug, + bloomFilterIoSizeRecorder, + queryContext.getQueryStatistics().getLoadBloomFilterFromCacheCount()::addAndGet, + queryContext.getQueryStatistics().getLoadBloomFilterFromDiskCount()::addAndGet, + externalTsFile); + } + + /** Whether the bloom filter proves that the aligned device has no data in this file. */ + private static boolean deviceAbsent(BloomFilter bloomFilter, String devicePrefix) { + return bloomFilter != null && !bloomFilter.contains(devicePrefix); + } + + /** + * Removes the pending measurements the bloom filter proves absent and answers them with null. + * They are not remembered: a later query probes the filter again instead of keeping a negative + * cache entry, which would add one entry per absent column per file. + */ + private static void bloomAbsentMeasurements( + TsFileID tsFileID, + IDeviceID device, + Set pending, + Map result, + BloomFilter bloomFilter, + String devicePrefix, + boolean debug) { + if (bloomFilter == null || pending.isEmpty()) { + return; + } + Iterator iterator = pending.iterator(); + while (iterator.hasNext()) { + String measurement = iterator.next(); + // the device gate already probed the time column + if (measurement.isEmpty() || bloomFilter.contains(devicePrefix + measurement)) { + continue; + } + iterator.remove(); + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.TS_METADATA_FILTERED_BY_BLOOM_FILTER, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + } + result.put(measurement, null); + } + } + + /** + * Reads the pending measurements from the reader and merges the result into {@code result}. + * + *

The pending set is exactly the aligned projection that was not resolved by the cache or the + * bloom filter, so the reader deserializes it as-is and every loaded entry belongs in the result. + * A pending measurement the reader does not return was routed to a group that was read, so it is + * absent from the file and is remembered with a negative cache entry. + */ + private void readAndMerge( + TsFileSequenceReader reader, + TsFileID tsFileID, + IDeviceID device, + long[] deviceMetadataIndexNodeOffset, + Set pending, + boolean ignoreNotExists, + boolean cacheEnabled, + boolean debug, + Map result, + LongConsumer ioSizeRecorder) + throws IOException { + Map loaded = + reader.readTimeseriesMetadataForMeasurements( + device, + deviceMetadataIndexNodeOffset, + pending, + pending, + ignoreNotExists, + ioSizeRecorder); + if (cacheEnabled) { + cacheLoadedMetadata(tsFileID, device, loaded); + } + for (Map.Entry entry : loaded.entrySet()) { + String measurement = entry.getKey(); + pending.remove(measurement); + TimeseriesMetadata metadata = entry.getValue(); + if (metadata.getStatistics().getCount() == 0) { + if (debug) { + DEBUG_LOGGER.info( + StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement)); + } + result.put(measurement, null); + cacheNonExistenceIfNeeded(tsFileID, device, measurement, cacheEnabled, ignoreNotExists); + } else { + result.put(measurement, new TimeseriesMetadata(metadata)); + } + } + // what is left in pending was routed to a leaf group that was read and not found there + for (String measurement : pending) { + result.put(measurement, null); + cacheNonExistenceIfNeeded(tsFileID, device, measurement, cacheEnabled, ignoreNotExists); + } + } + + /** Keeps every non-empty measurement of a group read so later single-column gets hit the LRU. */ + private void cacheLoadedMetadata( + TsFileID tsFileID, IDeviceID device, Map loaded) { + for (Map.Entry entry : loaded.entrySet()) { + if (entry.getValue().getStatistics().getCount() != 0) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, entry.getKey()), entry.getValue()); + } + } + } + + private static void fillMissingWithNull( + Set requested, Map result) { + for (String measurement : requested) { + result.putIfAbsent(measurement, null); + } + } + + private void cacheNonExistenceIfNeeded( + TsFileID tsFileID, + IDeviceID device, + String measurement, + boolean cacheEnabled, + boolean ignoreNotExists) { + if (cacheEnabled && !ignoreNotExists && memoryConfig.isMayCacheNonExistSeries()) { + lruCache.put( + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement), + NULL_EXISTS_CACHE_PLACE_HOLDER); + } + } + + private static void recordLoadStatistics( + QueryContext queryContext, + long cacheHitCount, + long cacheLookupTime, + long diskLookupCount, + long diskLookupTime, + long bloomFilterTime) { + QueryStatistics statistics = queryContext.getQueryStatistics(); + statistics.getLoadBloomFilterTime().getAndAdd(bloomFilterTime); + if (cacheHitCount > 0) { + statistics.getLoadTimeSeriesMetadataFromCacheCount().getAndAdd(cacheHitCount); + statistics.getLoadTimeSeriesMetadataFromCacheTime().getAndAdd(cacheLookupTime); + } + if (diskLookupCount > 0 || diskLookupTime > 0) { + statistics.getLoadTimeSeriesMetadataFromDiskCount().getAndAdd(diskLookupCount); + statistics.getLoadTimeSeriesMetadataFromDiskTime().getAndAdd(diskLookupTime); + } + } + + /** + * Resolve cached measurements into {@code resolved} and collect the rest into {@code missing}. + */ + private void collectCachedMetadata( + TsFileID tsFileID, + IDeviceID device, + Collection measurements, + Map resolved, + Collection missing, + String filePath, + boolean debug) { + for (String measurement : measurements) { + TimeSeriesMetadataCacheKey key = + new TimeSeriesMetadataCacheKey(tsFileID, device, measurement); + TimeseriesMetadata metadata = lruCache.getIfPresent(key); + if (metadata == null) { + missing.add(measurement); + } else if (metadata == NULL_EXISTS_CACHE_PLACE_HOLDER) { if (debug) { DEBUG_LOGGER.info(StorageEngineMessages.FILE_NO_SUCH_TIME_SERIES, key); } - return null; + resolved.put(measurement, null); } else { if (debug) { DEBUG_LOGGER.info( StorageEngineMessages.STORAGE_LOG_GET_TIMESERIES_METADATA_IN_FILE_FROM_CACHE_36652729, - key.device, - key.measurement, + device, + measurement, filePath, - timeseriesMetadata); + metadata); } - return new TimeseriesMetadata(timeseriesMetadata); - } - } finally { - queryContext.getQueryStatistics().getLoadBloomFilterTime().getAndAdd(loadBloomFilterTime); - if (cacheHit) { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromCacheTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); - } else { - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskCount() - .incrementAndGet(); - queryContext - .getQueryStatistics() - .getLoadTimeSeriesMetadataFromDiskTime() - .getAndAdd(System.nanoTime() - startTime - loadBloomFilterTime); + resolved.put(measurement, new TimeseriesMetadata(metadata)); } } } + /** + * Striped, strongly referenced monitor for one device+file. A WeakHashMap entry can be reclaimed + * by GC between calls, which would let two threads hold "the same" lock simultaneously. + */ + private static Object deviceLock(String deviceString, String filePath) { + return DEVICE_LOCKS[ + Math.floorMod((deviceString + SEPARATOR + filePath).hashCode(), DEVICE_LOCK_STRIPES)]; + } + public double calculateTimeSeriesMetadataHitRatio() { return lruCache.stats().hitRate(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java index 87526eb594b9f..6734011fe3318 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/buffer/TimeSeriesMetadataCacheTest.java @@ -22,17 +22,27 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType; import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.file.metadata.IDeviceID.Factory; import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.file.metadata.statistics.Statistics; +import org.apache.tsfile.read.LazyTsFileDeviceIterator; import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.utils.BloomFilter; +import org.apache.tsfile.utils.PublicBAOS; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; +import org.apache.tsfile.write.schema.IMeasurementSchema; import org.apache.tsfile.write.schema.MeasurementSchema; import org.junit.Ignore; import org.junit.Test; @@ -41,8 +51,12 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -51,9 +65,11 @@ import java.util.function.LongConsumer; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; public class TimeSeriesMetadataCacheTest { @@ -241,20 +257,20 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { fileReaderManager.setReaderForTest(tsFileID, true, reader); Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(null); Mockito.when( - reader.readTimeseriesMetadata( + reader.readTimeseriesMetadataForMeasurements( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(true), Mockito.any(LongConsumer.class))) - .thenReturn(Collections.emptyList()); + .thenReturn(Collections.emptyMap()); Mockito.when( - reader.readTimeseriesMetadata( + reader.readTimeseriesMetadataForMeasurements( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class))) .thenThrow(new IOException("missing device")); @@ -264,11 +280,11 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { IOException.class, () -> cache.get(filePath, key, allSensors, false, false, new QueryContext(false, false))); Mockito.verify(reader) - .readTimeseriesMetadata( + .readTimeseriesMetadataForMeasurements( Mockito.eq(deviceID), Mockito.isNull(), - Mockito.eq(measurement), - Mockito.eq(allSensors), + Mockito.anyCollection(), + Mockito.anySet(), Mockito.eq(false), Mockito.any(LongConsumer.class)); } finally { @@ -280,4 +296,807 @@ public void testIgnoreNotExistsResultIsNotCached() throws Exception { .setMayCacheNonExistSeries(mayCacheNonExistSeries); } } + + private static Map getDeviceTimeseriesMetadata( + TimeSeriesMetadataCache cache, + String filePath, + TsFileID tsFileID, + IDeviceID deviceID, + Set allSensors, + boolean ignoreNotExists) + throws IOException { + return cache.getDeviceTimeseriesMetadata( + filePath, + tsFileID, + deviceID, + allSensors, + ignoreNotExists, + false, + new QueryContext(false, false), + null); + } + + private static void stubGroupRead( + TsFileSequenceReader reader, + IDeviceID deviceID, + boolean ignoreNotExists, + List group) + throws IOException { + Map loaded = new HashMap<>(); + for (TimeseriesMetadata metadata : group) { + loaded.put(metadata.getMeasurementId(), metadata); + } + Mockito.when( + reader.readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(ignoreNotExists), + Mockito.any(LongConsumer.class))) + .thenReturn(loaded); + } + + @Test + public void testGetTimeseriesMetadata() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-device.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + TimeseriesMetadata timeColumn = + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS()); + + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + TimeseriesMetadata s1 = + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS()); + + // an empty series (count == 0) is mapped to null and cached as the null placeholder + TimeseriesMetadata emptyS2 = + new TimeseriesMetadata( + (byte) 0, + 0, + "s2", + TSDataType.INT32, + Statistics.getStatsByType(TSDataType.INT32), + new PublicBAOS()); + + List group = new ArrayList<>(); + group.add(timeColumn); + group.add(s1); + group.add(emptyS2); + // the group read must carry chunk metadata, otherwise the downstream chunk loader NPEs + s1.setChunkMetadataList(Collections.singletonList(new ChunkMetadata())); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map result = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNotNull(result.get("")); + assertEquals("", result.get("").getMeasurementId()); + assertNotNull(result.get("s1")); + assertEquals("s1", result.get("s1").getMeasurementId()); + // chunk metadata must flow through to the result (and the cached copy) unchanged + assertNotNull(result.get("s1").getChunkMetadataList()); + assertNull(result.get("s2")); + + // a second lookup of cached measurements must be served without touching the reader again + Set cachedSensors = new HashSet<>(Arrays.asList("", "s1")); + Map cachedResult = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, cachedSensors, false); + assertNotNull(cachedResult.get("s1")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + + // the group read also populated the per-measurement cache + TimeseriesMetadata cached = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cached); + assertEquals("s1", cached.getMeasurementId()); + + // the cache hands out copies: mutating a returned entry must not poison later reads + result.get("s1").setModified(true); + TimeseriesMetadata cachedAgain = + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s1"), + Collections.singleton("s1"), + true, + false, + new QueryContext(false, false)); + assertNotNull(cachedAgain); + assertFalse(cachedAgain.isModified()); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceSkippedWhenBloomFilterExcludesIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-bloom.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = Collections.singleton(""); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the bloom filter proves the device is missing, so the device must not be read at all + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenReturn(BloomFilter.getEmptyBloomFilter(0.05, 1)); + + Map result = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(result.get("")); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testAbsentDeviceThrowsWhenBloomFilterCannotExcludeIt() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-absent-device-throw.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + // ignoreNotExists keeps the missing device silent, but caches nothing + Map ignored = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + assertNull(ignored.get("s2")); + + // a normal query must surface the reader's failure instead of a silently empty device + assertThrows( + IOException.class, + () -> + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchMetadataMatchesPerColumn() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + int columnCount = 700; + File file = new File("target/batch-compare.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.compare.d1"); + List schemas = new ArrayList<>(columnCount); + for (int i = 0; i < columnCount; i++) { + schemas.add(new MeasurementSchema("s" + i, TSDataType.INT32)); + } + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries(deviceID, schemas); + for (long time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + for (int i = 0; i < columnCount; i++) { + record.addPoint("s" + i, (int) time); + } + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = file.getAbsolutePath(); + Set allSensors = new HashSet<>(); + for (int i = 0; i < columnCount; i++) { + allSensors.add("s" + i); + } + allSensors.add(""); + allSensors.add("absent"); + + Map perColumn = new HashMap<>(); + for (String measurement : allSensors) { + perColumn.put( + measurement, + cache.get( + filePath, + new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement), + allSensors, + false, + false, + new QueryContext(false, false))); + } + cache.clear(); + + Map batch = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertEquals(allSensors.size(), batch.size()); + for (String measurement : allSensors) { + TimeseriesMetadata expected = perColumn.get(measurement); + TimeseriesMetadata actual = batch.get(measurement); + if (expected == null) { + assertNull(measurement, actual); + continue; + } + assertNotNull(measurement, actual); + assertEquals(measurement, actual.getMeasurementId()); + assertEquals( + measurement, expected.getStatistics().getCount(), actual.getStatistics().getCount()); + assertEquals( + measurement, + expected.getStatistics().getStartTime(), + actual.getStatistics().getStartTime()); + assertEquals( + measurement, + expected.getStatistics().getEndTime(), + actual.getStatistics().getEndTime()); + assertEquals( + measurement, + expected.getChunkMetadataList().size(), + actual.getChunkMetadataList().size()); + for (int i = 0; i < expected.getChunkMetadataList().size(); i++) { + ChunkMetadata expectedChunk = (ChunkMetadata) expected.getChunkMetadataList().get(i); + ChunkMetadata actualChunk = (ChunkMetadata) actual.getChunkMetadataList().get(i); + assertEquals( + measurement, + expectedChunk.getOffsetOfChunkHeader(), + actualChunk.getOffsetOfChunkHeader()); + assertEquals( + measurement, + expectedChunk.getStatistics().getStartTime(), + actualChunk.getStatistics().getStartTime()); + assertEquals( + measurement, + expectedChunk.getStatistics().getEndTime(), + actualChunk.getStatistics().getEndTime()); + } + } + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + assertTrue(file.delete()); + } + } + + @Test + public void testBloomFilterExcludedMeasurementsSkipRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-time-column-filtered.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // both the time column and s1 are excluded, so no leaf group needs to be read + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Map result = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNull(result.get("")); + assertNull(result.get("s1")); + Mockito.verify(reader, Mockito.never()) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.any(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testEvictedMeasurementReloadsOnlyItsGroup() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-single-miss.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + Statistics statistics = Statistics.getStatsByType(TSDataType.INT32); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + TimeseriesMetadata s2 = + new TimeseriesMetadata((byte) 0, 0, "s2", TSDataType.INT32, statistics, new PublicBAOS()); + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + group.add(s2); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(first.get("s2")); + + // evict one measurement: the next call reads only the group that holds it, and that one + // read also resolves the other members of the group + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "s2")); + Map second = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + assertNotNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBatchIgnoreNotExistsDoesNotCachePlaceholders() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-ignore-not-exists.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.absent"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, true, Collections.emptyList()); + Mockito.when( + reader.readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class))) + .thenThrow(new IOException("device absent")); + + Map ignored = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, true); + assertNull(ignored.get("")); + assertNull(ignored.get("s1")); + + // ignoreNotExists must not write NULL placeholders: a normal query still has to read + assertThrows( + IOException.class, + () -> + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false)); + // the empty group read resolves the time column as absent, which ends the load for the + // device, plus the failing normal query + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testEvictedMeasurementsShareOneGroupRead() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-few-missing.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2", "s3", "s4", "s5")); + + List group = new ArrayList<>(); + for (String measurement : allSensors) { + TSDataType dataType = measurement.isEmpty() ? TSDataType.INT64 : TSDataType.INT32; + Statistics statistics = Statistics.getStatsByType(dataType); + statistics.setCount(1); + statistics.setStartTime(0); + statistics.setEndTime(10); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, measurement, dataType, statistics, new PublicBAOS())); + } + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(first.get("s5")); + + // evict three measurements: they share one leaf group, so a single read must resolve them + for (String measurement : Arrays.asList("s2", "s3", "s4")) { + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, measurement)); + } + + Map second = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNotNull(second.get("s2")); + assertNotNull(second.get("s3")); + assertNotNull(second.get("s4")); + Mockito.verify(reader, Mockito.times(2)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testReaderConfirmedAbsenceIsRemembered() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-confirmed-absent.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 3); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + bloomFilter.add(deviceID.toString() + ".s2"); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(first.get("s2")); + + // the reader confirmed s2 absent, so the second query is answered from the cache alone + Map second = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(second.get("s2")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBloomExcludedMeasurementIsNotRemembered() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-bloom-excluded.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + // the device exists, but s1 is not in the file + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 1); + bloomFilter.add(deviceID.toString() + "."); + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))).thenReturn(bloomFilter); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + stubGroupRead(reader, deviceID, false, group); + + Map first = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(first.get("s1")); + + // s1 was excluded by the bloom filter, not read, so it is probed again instead of being + // remembered; the second query still needs no reader call + Map second = + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + assertNull(second.get("s1")); + Mockito.verify(reader, Mockito.times(1)) + .readTimeseriesMetadataForMeasurements( + Mockito.eq(deviceID), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.eq(false), + Mockito.any(LongConsumer.class)); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testBloomLoadBeforeCachePhaseIsNotSubtractedFromDiskTime() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + FileReaderManager fileReaderManager = FileReaderManager.getInstance(); + TsFileID tsFileID = new TsFileID(); + cache.clear(); + bloomFilterCache.clear(); + try { + String filePath = "target/batch-bloom-phase.tsfile"; + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.device"); + Set allSensors = new HashSet<>(Arrays.asList("", "s1")); + + TsFileSequenceReader reader = Mockito.mock(TsFileSequenceReader.class); + fileReaderManager.setReaderForTest(tsFileID, true, reader); + BloomFilter bloomFilter = BloomFilter.getEmptyBloomFilter(0.05, 2); + bloomFilter.add(deviceID.toString() + "."); + bloomFilter.add(deviceID.toString() + ".s1"); + // a slow load makes the accounting visible: if the pre-cache load is subtracted from the + // disk interval, the recorded disk time turns negative + Mockito.when(reader.readBloomFilter(Mockito.any(LongConsumer.class))) + .thenAnswer( + invocation -> { + Thread.sleep(20); + return bloomFilter; + }); + + Statistics timeStatistics = Statistics.getStatsByType(TSDataType.INT64); + timeStatistics.setCount(1); + timeStatistics.setStartTime(0); + timeStatistics.setEndTime(10); + Statistics s1Statistics = Statistics.getStatsByType(TSDataType.INT32); + s1Statistics.setCount(1); + s1Statistics.setStartTime(0); + s1Statistics.setEndTime(10); + List group = new ArrayList<>(); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "", TSDataType.INT64, timeStatistics, new PublicBAOS())); + group.add( + new TimeseriesMetadata( + (byte) 0, 0, "s1", TSDataType.INT32, s1Statistics, new PublicBAOS())); + stubGroupRead(reader, deviceID, false, group); + + // warm both entries + getDeviceTimeseriesMetadata(cache, filePath, tsFileID, deviceID, allSensors, false); + + // evict only the time column, and drop the bloom cache so the next call reloads it + cache.remove(new TimeSeriesMetadataCacheKey(tsFileID, deviceID, "")); + bloomFilterCache.clear(); + + QueryContext queryContext = new QueryContext(false, false); + Map result = + cache.getDeviceTimeseriesMetadata( + filePath, tsFileID, deviceID, allSensors, false, false, queryContext, null); + assertNotNull(result.get("")); + assertNotNull(result.get("s1")); + assertTrue( + "cache time: " + + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromCacheTime().get(), + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromCacheTime().get() >= 0); + assertTrue( + "disk time: " + + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskTime().get(), + queryContext.getQueryStatistics().getLoadTimeSeriesMetadataFromDiskTime().get() >= 0); + } finally { + cache.clear(); + bloomFilterCache.clear(); + fileReaderManager.closeFileAndRemoveReader(tsFileID); + } + } + + @Test + public void testExternalTsFileReadsAllRequestedMeasurements() throws Exception { + TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + File file = new File("target/external-cache-disabled.tsfile"); + file.delete(); + IDeviceID deviceID = Factory.DEFAULT_FACTORY.create("root.external.d1"); + try (TsFileWriter writer = new TsFileWriter(file)) { + writer.registerAlignedTimeseries( + deviceID, + Arrays.asList( + new MeasurementSchema("s1", TSDataType.INT32), + new MeasurementSchema("s2", TSDataType.INT32))); + for (int time = 0; time < 3; time++) { + TSRecord record = new TSRecord(deviceID, time); + record.addPoint("s1", time); + record.addPoint("s2", time); + writer.writeRecord(record); + } + } + cache.clear(); + bloomFilterCache.clear(); + long[] offset; + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath())) { + LazyTsFileDeviceIterator iterator = new LazyTsFileDeviceIterator(reader); + assertTrue(iterator.hasNext()); + iterator.next(); + offset = iterator.getCurrentDeviceMeasurementNodeOffset(); + } + // external tsfiles bypass the metadata cache: every requested measurement must still be read + TsFileResource resource = new TsFileResource(file); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + String filePath = file.getAbsolutePath(); + // the external registry keys readers by path and closes one when its last reference is + // released, so hold one for the duration of the test + FileReaderManager.getInstance().increaseExternalFileReaderReference(filePath); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.EXTERNAL_TSFILE_SCAN); + try { + Set allSensors = new HashSet<>(Arrays.asList("", "s1", "s2")); + Map result = + cache.getDeviceTimeseriesMetadata( + filePath, + resource.getTsFileID(), + deviceID, + allSensors, + false, + false, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT, + offset); + assertEquals(allSensors.size(), result.size()); + for (String measurement : allSensors) { + assertNotNull(measurement, result.get(measurement)); + assertEquals(measurement, result.get(measurement).getMeasurementId()); + } + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setQueryDataSourceType( + QueryDataSourceType.SERIES_SCAN); + cache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().decreaseExternalFileReaderReference(filePath); + assertTrue(file.delete()); + } + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java new file mode 100644 index 0000000000000..91835b8b30f2d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/read/reader/series/AlignedSeriesScanBatchTest.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.storageengine.dataregion.read.reader.series; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.AlignedFullPath; +import org.apache.iotdb.commons.path.AlignedPath; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.queryengine.execution.operator.source.AlignedSeriesScanUtil; +import org.apache.iotdb.db.queryengine.plan.planner.plan.parameter.SeriesScanOptions; +import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import org.apache.iotdb.db.storageengine.buffer.BloomFilterCache; +import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; +import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource; +import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.apache.tsfile.block.column.Column; +import org.apache.tsfile.common.conf.TSFileDescriptor; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.read.common.TimeRange; +import org.apache.tsfile.read.common.block.TsBlock; +import org.apache.tsfile.write.chunk.AlignedChunkWriterImpl; +import org.apache.tsfile.write.chunk.IChunkWriter; +import org.apache.tsfile.write.schema.IMeasurementSchema; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.apache.tsfile.write.writer.TsFileIOWriter; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.function.LongConsumer; + +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter; +import static org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.writeAlignedChunk; + +/** + * Verifies metadata loading on a real device wider than one measurement leaf group: each selected + * column is resolved from the leaf group that holds it, and one group read serves every selected + * column of that group. + */ +public class AlignedSeriesScanBatchTest { + + private static final String DATABASE = "root.wide_pd"; + private static final IDeviceID DEVICE = + IDeviceID.Factory.DEFAULT_FACTORY.create(DATABASE + ".d1"); + private static final int COLUMN_COUNT = 300; + private static final int PRESENT_COLUMN_COUNT = 30; + private static final int ROW_COUNT = 10; + + @Test + public void testScanReadsWideAlignedDevice() throws Exception { + Assert.assertTrue( + "the fixture must span more than one measurement leaf group", + COLUMN_COUNT + 1 > TSFileDescriptor.getInstance().getConfig().getMaxDegreeOfIndexNode()); + File file = newFile(1); + TsFileResource resource = writeWideDevice(file, COLUMN_COUNT); + TsFileSequenceReader fileReader = new TsFileSequenceReader(file.getAbsolutePath()); + TsFileSequenceReader reader = Mockito.spy(fileReader); + FileReaderManager.getInstance().setReaderForTest(resource.getTsFileID(), true, reader); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + metadataCache.clear(); + bloomFilterCache.clear(); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + // the unified path is only reachable from table dialect, which forces this flag to false + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + long cacheHitsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(alignedPath(columnNames(COLUMN_COUNT)), resource); + + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertFalse("s" + column + " row " + row, valueColumn.isNull(row)); + Assert.assertEquals("s" + column + " row " + row, row, valueColumn.getInt(row)); + } + } + + // a cold cache: every selected series is a disk lookup, counted per series + Assert.assertEquals( + cacheHitsBefore, + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromCacheCount() + .get()); + // per-series counters: a cold scan counts one disk lookup per selected measurement + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); + // the whole projection is resolved with one reader call for this file, not one per column + Mockito.verify(reader) + .readTimeseriesMetadataForMeasurements( + Mockito.any(), + Mockito.isNull(), + Mockito.anyCollection(), + Mockito.anySet(), + Mockito.anyBoolean(), + Mockito.any(LongConsumer.class)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testSparseProjectionFiltersAbsentColumns() throws Exception { + File file = newFile(2); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + AlignedFullPath scanPath = alignedPath(columnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + try { + metadataCache.clear(); + bloomFilterCache.clear(); + long diskReadsBefore = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get(); + + TsBlock block = scan(scanPath, resource); + + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertEquals(row, block.getTimeByIndex(row)); + } + for (int column = 0; column < COLUMN_COUNT; column++) { + Column valueColumn = block.getColumn(column); + boolean present = column < PRESENT_COLUMN_COUNT; + for (int row = 0; row < ROW_COUNT; row++) { + if (present) { + Assert.assertFalse(valueColumn.isNull(row)); + Assert.assertEquals(row, valueColumn.getInt(row)); + } else { + Assert.assertTrue("s" + column + " row " + row, valueColumn.isNull(row)); + } + } + } + + // the bloom filter excludes the 270 absent columns; a few may survive as false positives, + // but the sparse projection must still look up far fewer series than it selects + long diskReads = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT + .getQueryStatistics() + .getLoadTimeSeriesMetadataFromDiskCount() + .get() + - diskReadsBefore; + // per-series counters: bloom-excluded measurements count as disk lookups, as they did on the + // per-column path + Assert.assertTrue("disk lookups: " + diskReads, diskReads >= 1); + Assert.assertTrue("disk lookups: " + diskReads, diskReads <= COLUMN_COUNT + 1); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + @Test + public void testAllNullProjection() throws Exception { + File file = newFile(3); + TsFileResource resource = writeWideDevice(file, PRESENT_COLUMN_COUNT); + TimeSeriesMetadataCache metadataCache = TimeSeriesMetadataCache.getInstance(); + BloomFilterCache bloomFilterCache = BloomFilterCache.getInstance(); + // every selected field is absent from the file + AlignedFullPath absentPath = alignedPath(absentColumnNames(COLUMN_COUNT)); + boolean previousIgnoreAllNullRows = + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.isIgnoreAllNullRows(); + try { + // table dialect: all-null rows are kept + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(false); + metadataCache.clear(); + bloomFilterCache.clear(); + TsBlock block = scan(absentPath, resource); + Assert.assertEquals(ROW_COUNT, block.getPositionCount()); + Assert.assertEquals(COLUMN_COUNT, block.getValueColumnCount()); + for (int column = 0; column < COLUMN_COUNT; column++) { + for (int row = 0; row < ROW_COUNT; row++) { + Assert.assertTrue("absent" + column + " row " + row, block.getColumn(column).isNull(row)); + } + } + + // tree dialect: an all-null projection skips the file + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(true); + metadataCache.clear(); + bloomFilterCache.clear(); + Assert.assertFalse(hasFile(absentPath, resource)); + } finally { + EnvironmentUtils.TEST_QUERY_FI_CONTEXT.setIgnoreAllNullRows(previousIgnoreAllNullRows); + cleanup(file, resource, metadataCache, bloomFilterCache); + } + } + + private static TsBlock scan(AlignedFullPath scanPath, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextFile, true); + SeriesReaderTestUtil.assertWithHasNext(seriesScanUtil::hasNextChunk, true); + Assert.assertTrue(seriesScanUtil.hasNextPage()); + return seriesScanUtil.nextPage(); + } + + private static boolean hasFile(AlignedFullPath scanPath, TsFileResource resource) + throws IOException { + AlignedSeriesScanUtil seriesScanUtil = newScanUtil(scanPath, resource); + for (int attempt = 0; attempt < 1000; attempt++) { + Optional hasNextFile = seriesScanUtil.hasNextFile(); + if (hasNextFile.isPresent()) { + return hasNextFile.get(); + } + } + throw new IllegalStateException("series scan did not advance after 1000 attempts"); + } + + private static AlignedSeriesScanUtil newScanUtil( + AlignedFullPath scanPath, TsFileResource resource) throws IOException { + SeriesScanOptions.Builder builder = new SeriesScanOptions.Builder(); + builder.withAllSensors(new HashSet<>(scanPath.getMeasurementList())); + SeriesScanOptions scanOptions = builder.build(); + AlignedSeriesScanUtil seriesScanUtil = + new AlignedSeriesScanUtil( + scanPath, Ordering.ASC, scanOptions, EnvironmentUtils.TEST_QUERY_FI_CONTEXT); + seriesScanUtil.initQueryDataSource( + new QueryDataSource(Collections.singletonList(resource), Collections.emptyList())); + return seriesScanUtil; + } + + private static AlignedFullPath alignedPath(List measurementNames) { + List schemas = new ArrayList<>(measurementNames.size()); + for (String measurement : measurementNames) { + schemas.add(new MeasurementSchema(measurement, TSDataType.INT32)); + } + return new AlignedFullPath(DEVICE, measurementNames, schemas); + } + + private static List columnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("s" + i); + } + return names; + } + + private static List absentColumnNames(int count) { + List names = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + names.add("absent" + i); + } + return names; + } + + private static File newFile(int version) { + File file = new File(TestConstant.getTestTsFilePath(DATABASE, 0, 0, version)); + if (!file.getParentFile().exists()) { + Assert.assertTrue(file.getParentFile().mkdirs()); + } + return file; + } + + private static TsFileResource writeWideDevice(File file, int presentColumnCount) + throws IOException, IllegalPathException { + TsFileResource resource = new TsFileResource(file); + List writtenPaths = new ArrayList<>(presentColumnCount); + List dataTypes = new ArrayList<>(presentColumnCount); + List encodings = new ArrayList<>(presentColumnCount); + List compressionTypes = new ArrayList<>(presentColumnCount); + for (int i = 0; i < presentColumnCount; i++) { + writtenPaths.add(new AlignedPath(DEVICE.toString(), "s" + i)); + dataTypes.add(TSDataType.INT32); + encodings.add(TSEncoding.PLAIN); + compressionTypes.add(CompressionType.UNCOMPRESSED); + } + try (TsFileIOWriter writer = new TsFileIOWriter(file)) { + writer.startChunkGroup(DEVICE); + for (IChunkWriter chunkWriter : + createChunkWriter(writtenPaths, dataTypes, encodings, compressionTypes, true)) { + writeAlignedChunk( + (AlignedChunkWriterImpl) chunkWriter, + writer, + Collections.singletonList(new TimeRange(0L, ROW_COUNT - 1L)), + true); + } + writer.endChunkGroup(); + writer.endFile(); + } + resource.updateStartTime(DEVICE, 0); + resource.updateEndTime(DEVICE, ROW_COUNT - 1L); + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + return resource; + } + + private static void cleanup( + File file, + TsFileResource resource, + TimeSeriesMetadataCache metadataCache, + BloomFilterCache bloomFilterCache) + throws IOException { + metadataCache.clear(); + bloomFilterCache.clear(); + FileReaderManager.getInstance().closeFileAndRemoveReader(resource.getTsFileID()); + Assert.assertTrue(file.delete()); + } +} diff --git a/pom.xml b/pom.xml index 425530778fe66..7458fec64e943 100644 --- a/pom.xml +++ b/pom.xml @@ -143,7 +143,7 @@ 0.25.0 1.9 1.5.6-3 - 2.5.1-261118-SNAPSHOT + 2.5.1-261120-SNAPSHOT en