Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,12 @@
import org.apache.tsfile.read.common.type.Type;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.PublicBAOS;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.record.Tablet;

import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
Expand Down Expand Up @@ -293,13 +295,14 @@ private static long calculateTabletSizeInBytes(final Tablet tablet) {
}

private static ByteBuffer serializeTabletWithAlignment(
final Tablet tablet, final boolean isAligned) {
final ByteBuffer buffer =
ByteBuffer.allocate(Math.addExact(tablet.serializedSize(), Byte.BYTES));
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
buffer.flip();
return buffer;
final Tablet tablet, final boolean isAligned) throws IOException {
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(Math.addExact(tablet.serializedSize(), Byte.BYTES));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
tablet.serialize(outputStream);
ReadWriteIOUtils.write(isAligned, outputStream);
return ByteBuffer.wrap(byteArrayOutputStream.getBuf(), 0, byteArrayOutputStream.size());
}
}

static boolean mayAppendTablet(final Tablet target, final Tablet source) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,12 +272,15 @@ public byte[] toTPipeTransferBytes() throws IOException {
throw new IOException(DataNodePipeMessages.CANNOT_SERIALIZE_BOTH_TABLET_AND_STATEMENT_ARE);
}

final ByteBuffer buffer = ByteBuffer.allocate(calculateAirGapSerializedSize(tabletToSerialize));
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), buffer);
tabletToSerialize.serialize(buffer);
ReadWriteIOUtils.write(isAlignedToSerialize, buffer);
return buffer.array();
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateAirGapSerializedSize(tabletToSerialize));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW.getType(), outputStream);
tabletToSerialize.serialize(outputStream);
ReadWriteIOUtils.write(isAlignedToSerialize, outputStream);
return byteArrayOutputStream.toByteArray();
}
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,14 +174,16 @@ public static PipeTransferTabletRawReqV2 fromTPipeTransferReq(

public static byte[] toTPipeTransferBytes(
final Tablet tablet, final boolean isAligned, final String dataBaseName) throws IOException {
final ByteBuffer buffer =
ByteBuffer.allocate(calculateAirGapSerializedSize(tablet, dataBaseName));
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), buffer);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), buffer);
tablet.serialize(buffer);
ReadWriteIOUtils.write(isAligned, buffer);
ReadWriteIOUtils.write(dataBaseName, buffer);
return buffer.array();
try (final PublicBAOS byteArrayOutputStream =
new PublicBAOS(calculateAirGapSerializedSize(tablet, dataBaseName));
final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream)) {
ReadWriteIOUtils.write(IoTDBSinkRequestVersion.VERSION_1.getVersion(), outputStream);
ReadWriteIOUtils.write(PipeRequestType.TRANSFER_TABLET_RAW_V2.getType(), outputStream);
tablet.serialize(outputStream);
ReadWriteIOUtils.write(isAligned, outputStream);
ReadWriteIOUtils.write(dataBaseName, outputStream);
return byteArrayOutputStream.toByteArray();
}
}

static int calculateSerializedSize(final Tablet tablet, final String dataBaseName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,10 @@ public class FragmentInstanceContext extends QueryContext {
// empty for zero time partitions
private List<Long> timePartitions;

// An optimization during restart changes the time index from FILE TIME INDEX
// to DEVICE TIME INDEX, which may cause a related validation false positive.
private boolean ignoreNotExistsDevice = false;
// Queries can start before the data regions finish recovering after a restart. During that
// window the time index may still change from FILE TIME INDEX to DEVICE TIME INDEX, so an
// absent device must not be treated as corruption or remembered as absent.
private boolean isStorageEngineRecovering = false;

private QueryDataSourceType queryDataSourceType = QueryDataSourceType.SERIES_SCAN;

Expand Down Expand Up @@ -456,7 +457,7 @@ protected FragmentInstanceContext(

public void start() {
long now = System.currentTimeMillis();
ignoreNotExistsDevice = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions();
isStorageEngineRecovering = !StorageEngine.getInstance().isReadyForNonReadWriteFunctions();
executionStartTime.compareAndSet(null, now);
startNanos.compareAndSet(0, System.nanoTime());

Expand Down Expand Up @@ -1642,8 +1643,9 @@ public long getUnclosedSeqFileNum() {
return unclosedSeqFileNum.get();
}

public boolean ignoreNotExistsDevice() {
return ignoreNotExistsDevice;
/** Whether this query started while the storage engine was still recovering its data regions. */
public boolean isStorageEngineRecovering() {
return isStorageEngineRecovering;
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskAlignedChunkLoader;
import org.apache.iotdb.db.storageengine.dataregion.read.reader.chunk.DiskChunkLoader;
Expand All @@ -47,7 +46,6 @@
import org.apache.tsfile.file.metadata.AbstractAlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.AlignedTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.IChunkMetadata;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.ITimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TableDeviceTimeSeriesMetadata;
import org.apache.tsfile.file.metadata.TimeseriesMetadata;
Expand All @@ -62,6 +60,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;

Expand Down Expand Up @@ -112,7 +111,7 @@ public static TimeseriesMetadata loadTimeSeriesMetadata(
seriesPath.getDeviceId(),
seriesPath.getMeasurement()),
allSensors,
context.ignoreNotExistsDevice()
context.isStorageEngineRecovering()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
context.isDebug(),
context);
Expand Down Expand Up @@ -341,23 +340,29 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
// the order of timeSeriesMetadata list is same as subSensorList's order
TimeSeriesMetadataCache cache = TimeSeriesMetadataCache.getInstance();
List<String> valueMeasurementList = alignedPath.getMeasurementList();
// resolves the time column and every selected sensor in one call: the cache reads each leaf
// group that holds one of them once and answers the whole group from that read, so a wide
// projection does not walk the device metadata index once per column
Set<String> allSensors = alignedPath.getAllSensors();
boolean isDebug = context.isDebug();
String filePath = resource.getTsFilePath();
IDeviceID deviceId = alignedPath.getDeviceId();

// when resource.getTimeIndexType() == 1, TsFileResource.timeIndexType is deviceTimeIndex
// we should not ignore the non-exist of device in TsFileMetadata
TimeseriesMetadata timeColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, ""),
boolean ignoreNotExists =
context.isStorageEngineRecovering()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE;

Map<String, TimeseriesMetadata> metadata =
cache.getDeviceTimeseriesMetadata(
resource.getTsFilePath(),
resource.getTsFileID(),
alignedPath.getDeviceId(),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
ignoreNotExists,
context.isDebug(),
context,
rootMeasurementMetadataIndexNodeOffset);

TimeseriesMetadata timeColumn = metadata.get("");
if (timeColumn != null) {
// only need time column, like count_time aggregation
if (valueMeasurementList.isEmpty()) {
Expand All @@ -375,21 +380,13 @@ private static AbstractAlignedTimeSeriesMetadata loadAlignedTimeSeriesMetadataFr
} else {
List<TimeseriesMetadata> valueTimeSeriesMetadataList =
new ArrayList<>(valueMeasurementList.size());
// if all the queried aligned sensors does not exist, we will return null
boolean exist = false;
for (String measurement : valueMeasurementList) {
TimeseriesMetadata valueColumn =
cache.get(
filePath,
new TimeSeriesMetadataCacheKey(resource.getTsFileID(), deviceId, measurement),
allSensors,
context.ignoreNotExistsDevice()
|| resource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE,
isDebug,
context,
rootMeasurementMetadataIndexNodeOffset);
valueTimeSeriesMetadataList.add(metadata.get(measurement));
}
// if all the queried aligned sensors do not exist, skip the device in this file
boolean exist = false;
for (TimeseriesMetadata valueColumn : valueTimeSeriesMetadataList) {
exist = (exist || (valueColumn != null));
valueTimeSeriesMetadataList.add(valueColumn);
}
if (!ignoreAllNullRows || exist) {
alignedTimeSeriesMetadata =
Expand Down
Loading