diff --git a/backend/pom.xml b/backend/pom.xml index 719be9e..8151717 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -38,7 +38,7 @@ 17 - 2.3.0 + 2.5.0 3.2.0 2.8.0 1.22.0 diff --git a/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java b/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java index 3afe24f..4ba7de7 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java @@ -78,11 +78,13 @@ public Cache scanTaskCache(TsFileProperties tsFileProperties) *

The health checker is a pure Java class with no Spring dependencies, so it is registered as * a bean here to allow injection into Spring-managed services. * + * @param tsFileProperties configuration supplying the chunk cap for the data-area traversal * @return a new TsFileHealthChecker instance */ @Bean - public TsFileHealthChecker tsFileHealthChecker() { - logger.info("Initializing TsFileHealthChecker bean"); - return new TsFileHealthChecker(); + public TsFileHealthChecker tsFileHealthChecker(TsFileProperties tsFileProperties) { + int maxChunks = tsFileProperties.getScan().getMaxChunks(); + logger.info("Initializing TsFileHealthChecker bean with maxChunks={}", maxChunks); + return new TsFileHealthChecker(maxChunks); } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java b/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java index c7023fd..f910354 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java @@ -235,6 +235,14 @@ public static class ScanProperties { /** Time-to-live in minutes for cached scan results. */ private int cacheTtlMinutes = 60; + /** + * 单个文件允许建立紧凑 chunk 索引的最大 chunk 数。 + * + *

超过该阈值时不报错、不 OOM,而是降级为只做结构与元数据检查,跳过数据区全量遍历,并在 {@code ScanResult} 的错误列表中以 {@link + * org.apache.tsfile.viewer.dto.ScanErrorType#DATA_VALIDATION_SKIPPED} 标注。默认 5,000,000。 + */ + private int maxChunks = 5_000_000; + public int getMaxConcurrent() { return maxConcurrent; } @@ -274,5 +282,13 @@ public int getCacheTtlMinutes() { public void setCacheTtlMinutes(int cacheTtlMinutes) { this.cacheTtlMinutes = cacheTtlMinutes; } + + public int getMaxChunks() { + return maxChunks; + } + + public void setMaxChunks(int maxChunks) { + this.maxChunks = maxChunks; + } } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java index 4724d72..12a511c 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java @@ -20,6 +20,7 @@ package org.apache.tsfile.viewer.controller; import java.io.IOException; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -27,8 +28,15 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.DeviceMeasurementsDTO; +import org.apache.tsfile.viewer.dto.DeviceRefDTO; +import org.apache.tsfile.viewer.dto.MeasurementDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; @@ -40,7 +48,9 @@ *

Provides endpoints for: * *

* *

Validates: Requirement 8.3 (Metadata endpoint) @@ -58,13 +68,14 @@ public MetadataController(MetadataService metadataService) { } /** - * Gets complete metadata for a TsFile. + * Gets the metadata summary for a TsFile. * - *

Returns metadata including version, time range, device/measurement counts, and detailed - * lists of measurements, RowGroups, and Chunks. + *

Returns version, time range, device/measurement counts and schema details. The {@code + * rowGroups} and {@code chunks} fields are kept for contract compatibility but are always empty; + * details come from the paging endpoints below. * * @param fileId the file identifier - * @return TsFileMetadataDTO with complete metadata + * @return TsFileMetadataDTO with summary metadata */ @GetMapping("/{fileId}") public ResponseEntity getMetadata(@PathVariable String fileId) @@ -74,4 +85,101 @@ public ResponseEntity getMetadata(@PathVariable String fileId TsFileMetadataDTO metadata = metadataService.getMetadata(fileId); return ResponseEntity.ok(metadata); } + + /** + * Gets one page of devices, optionally scoped to a table and filtered by a substring. + * + *

Cheap enough for pickers: it is derived from the cached device index, not from a table + * query. + * + * @param fileId the file identifier + * @param tableName only devices of this table + * @param query case-insensitive substring the device name must contain + * @param offset zero-based offset, negative values are treated as 0 + * @param limit page size, defaults to 50 and is capped at 200 + * @return page of devices + */ + @GetMapping("/{fileId}/devices") + public ResponseEntity> getDevices( + @PathVariable String fileId, + @RequestParam(required = false) String tableName, + @RequestParam(required = false) String query, + @RequestParam(required = false) Integer offset, + @RequestParam(required = false) Integer limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + int effectiveOffset = PageDTO.normalizeOffset(offset); + int effectiveLimit = PageDTO.normalizeLimit(limit); + + PageDTO page = + metadataService.getDevicesPage(fileId, tableName, query, effectiveOffset, effectiveLimit); + return ResponseEntity.ok(page); + } + + /** + * Gets the measurements carried by one device. + * + *

Tree model files nest measurements under devices, so a client showing a device picker needs + * the measurement choices of the selected device instead of the whole file's union. + * + * @param fileId the file identifier + * @param device device name as returned by the RowGroup paging endpoint + * @return measurements of that device, empty when the device is not part of the file + */ + @GetMapping("/{fileId}/devices/{device}/measurements") + public ResponseEntity getDeviceMeasurements( + @PathVariable String fileId, @PathVariable String device) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting measurements of device={} for fileId={}", device, fileId); + + List measurements = metadataService.getDeviceMeasurements(fileId, device); + return ResponseEntity.ok(new DeviceMeasurementsDTO(device, measurements)); + } + + /** + * Gets one page of RowGroups of a TsFile. + * + * @param fileId the file identifier + * @param offset zero-based offset, negative values are treated as 0 + * @param limit page size, defaults to 50 and is capped at 200 + * @return page of RowGroups + */ + @GetMapping("/{fileId}/row-groups") + public ResponseEntity> getRowGroups( + @PathVariable String fileId, + @RequestParam(name = "offset", required = false) Integer offset, + @RequestParam(name = "limit", required = false) Integer limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + int pageOffset = PageDTO.normalizeOffset(offset); + int pageLimit = PageDTO.normalizeLimit(limit); + + logger.debug( + "Getting RowGroup page for fileId={}, offset={}, limit={}", fileId, pageOffset, pageLimit); + + PageDTO page = metadataService.getRowGroupsPage(fileId, pageOffset, pageLimit); + return ResponseEntity.ok(page); + } + + /** + * Gets one page of Chunks of a TsFile. + * + * @param fileId the file identifier + * @param offset zero-based offset, negative values are treated as 0 + * @param limit page size, defaults to 50 and is capped at 200 + * @return page of Chunks + */ + @GetMapping("/{fileId}/chunks") + public ResponseEntity> getChunks( + @PathVariable String fileId, + @RequestParam(name = "offset", required = false) Integer offset, + @RequestParam(name = "limit", required = false) Integer limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + int pageOffset = PageDTO.normalizeOffset(offset); + int pageLimit = PageDTO.normalizeLimit(limit); + + logger.debug( + "Getting chunk page for fileId={}, offset={}, limit={}", fileId, pageOffset, pageLimit); + + PageDTO page = metadataService.getChunksPage(fileId, pageOffset, pageLimit); + return ResponseEntity.ok(page); + } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java new file mode 100644 index 0000000..219ac3e --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import java.util.List; + +/** + * Measurements carried by a single device. + * + *

Tree model files nest measurements under devices, so the measurement options a client offers + * must be scoped to the selected device rather than to the whole file. + */ +public class DeviceMeasurementsDTO { + + private String device; + private List measurements; + + public DeviceMeasurementsDTO() {} + + public DeviceMeasurementsDTO(String device, List measurements) { + this.device = device; + this.measurements = measurements; + } + + public String getDevice() { + return device; + } + + public void setDevice(String device) { + this.device = device; + } + + public List getMeasurements() { + return measurements; + } + + public void setMeasurements(List measurements) { + this.measurements = measurements; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java new file mode 100644 index 0000000..156738b --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import java.util.List; + +/** + * A device reference for pickers: its identifier, owning table and TAG values. + * + *

Derived from the device identifier itself, which is {@code tableName.tag1.tag2...}, so it + * needs no row scan. + */ +public class DeviceRefDTO { + + private String device; + private String tableName; + private List tagValues; + + public DeviceRefDTO() {} + + public DeviceRefDTO(String device, String tableName, List tagValues) { + this.device = device; + this.tableName = tableName; + this.tagValues = tagValues; + } + + public String getDevice() { + return device; + } + + public void setDevice(String device) { + this.device = device; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public List getTagValues() { + return tagValues; + } + + public void setTagValues(List tagValues) { + this.tagValues = tagValues; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java new file mode 100644 index 0000000..771de55 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import java.util.List; + +/** + * Generic page envelope used by the paginated metadata endpoints. + * + *

Serializes as {@code {"items": [...], "total": n, "offset": n, "limit": n, "hasMore": bool}}. + * + * @param element type of one page + */ +public class PageDTO { + + /** Page size applied when the client does not ask for one. */ + public static final int DEFAULT_LIMIT = 50; + + /** Hard cap of a single request, keeping one page's memory footprint bounded. */ + public static final int MAX_LIMIT = 200; + + private List items; + private int total; + private int offset; + private int limit; + private boolean hasMore; + + /** Default constructor for JSON deserialization. */ + public PageDTO() {} + + /** + * Creates a page envelope. + * + * @param items page elements + * @param total total number of elements across all pages + * @param offset offset of the first element of this page + * @param limit page size requested for this page + * @param hasMore whether elements beyond this page exist + */ + public PageDTO(List items, int total, int offset, int limit, boolean hasMore) { + this.items = items; + this.total = total; + this.offset = offset; + this.limit = limit; + this.hasMore = hasMore; + } + + /** + * Builds a page envelope, deriving {@code hasMore} from the payload. + * + * @param items page elements (null is treated as an empty page) + * @param total total number of elements across all pages + * @param offset offset of the first element of this page + * @param limit page size requested for this page + * @param element type + * @return page envelope + */ + public static PageDTO of(List items, int total, int offset, int limit) { + List safeItems = items == null ? List.of() : items; + int safeTotal = Math.max(total, 0); + boolean hasMore = (long) offset + safeItems.size() < safeTotal; + return new PageDTO<>(safeItems, safeTotal, offset, limit, hasMore); + } + + /** + * Normalizes a requested offset: missing or negative values become 0. + * + * @param offset raw request value + * @return non-negative offset + */ + public static int normalizeOffset(Integer offset) { + return offset == null || offset < 0 ? 0 : offset; + } + + /** + * Normalizes a requested limit: missing or non-positive values fall back to {@link + * #DEFAULT_LIMIT}, values above {@link #MAX_LIMIT} are capped to it. + * + * @param limit raw request value + * @return effective page size in {@code [1, MAX_LIMIT]} + */ + public static int normalizeLimit(Integer limit) { + if (limit == null || limit <= 0) { + return DEFAULT_LIMIT; + } + return Math.min(limit, MAX_LIMIT); + } + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + + public int getTotal() { + return total; + } + + public void setTotal(int total) { + this.total = total; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public boolean isHasMore() { + return hasMore; + } + + public void setHasMore(boolean hasMore) { + this.hasMore = hasMore; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java index a9e535d..18a5b4f 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java @@ -38,5 +38,12 @@ public enum ScanErrorType { TIMESERIES_METADATA_MISMATCH, /** An IOException occurred while reading the file. */ - DATA_READ_ERROR + DATA_READ_ERROR, + + /** + * The file holds more chunks than {@code tsfile.scan.max-chunks} allows, so the data-area + * traversal was skipped. Structure and metadata checks still ran — this is a degraded, not a + * failed, result. + */ + DATA_VALIDATION_SKIPPED } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java index 509cfbc..2951bc2 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java @@ -131,7 +131,9 @@ public DataPreviewResponse previewData(DataPreviewRequest request) filters.getTimeRange() != null ? filters.getTimeRange().getStartTime() : null, filters.getTimeRange() != null ? filters.getTimeRange().getEndTime() : null, readLimit, - 0); + 0, + filters.getDevices(), + filters.getMeasurements()); // Check timeout checkTimeout(startTime, timeoutSeconds); @@ -206,7 +208,9 @@ public ChartDataResponse queryChartData(ChartDataRequest request) request.getStartTime(), request.getEndTime(), tsFileProperties.getQuery().getMaxResultSize(), - 0); + 0, + request.getDevices(), + request.getMeasurements()); // Check timeout checkTimeout(startTime, timeoutSeconds); diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java index b2965cb..2d4c61f 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java @@ -21,7 +21,11 @@ import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; @@ -30,12 +34,20 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.tsfile.file.metadata.IDeviceID; +import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.viewer.config.TsFileProperties; -import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.DeviceRefDTO; +import org.apache.tsfile.viewer.dto.MeasurementDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.dto.TimeRange; +import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.EmptyFileException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.tsfile.DeviceIndex; import org.apache.tsfile.viewer.tsfile.TsFileParser; import jakarta.annotation.PostConstruct; @@ -46,9 +58,9 @@ *

Provides functionality for: * *

* *

Validates: Requirement 2.1-2.5, 5.1 (Metadata extraction, caching) @@ -58,12 +70,22 @@ public class MetadataService { private static final Logger logger = LoggerFactory.getLogger(MetadataService.class); + /** + * Device indexes are small (a device name list plus one int per device) but each one pins data + * for a whole file, so this cache stays deliberately tiny instead of tracking file count. + */ + private static final int DEVICE_INDEX_CACHE_SIZE = 8; + private final TsFileProperties tsFileProperties; private final FileService fileService; private final TsFileParser tsFileParser; + /** Cached summaries only: they carry no RowGroup or Chunk lists, so entries stay small. */ private Cache metadataCache; + /** Cached device indexes, validated against the file's mtime and size on every use. */ + private Cache deviceIndexCache; + public MetadataService( TsFileProperties tsFileProperties, FileService fileService, TsFileParser tsFileParser) { this.tsFileProperties = tsFileProperties; @@ -78,7 +100,10 @@ public void init() { int ttlMinutes = tsFileProperties.getCache().getMetadata().getTtlMinutes(); logger.info( - "Initializing MetadataService cache with maxSize={}, ttlMinutes={}", maxSize, ttlMinutes); + "Initializing MetadataService cache with maxSize={}, ttlMinutes={}, deviceIndexMaxSize={}", + maxSize, + ttlMinutes, + DEVICE_INDEX_CACHE_SIZE); this.metadataCache = Caffeine.newBuilder() @@ -86,15 +111,22 @@ public void init() { .expireAfterWrite(ttlMinutes, TimeUnit.MINUTES) .recordStats() .build(); + this.deviceIndexCache = + Caffeine.newBuilder() + .maximumSize(DEVICE_INDEX_CACHE_SIZE) + .expireAfterWrite(ttlMinutes, TimeUnit.MINUTES) + .build(); } /** - * Gets metadata for a TsFile by fileId. + * Gets the metadata summary for a TsFile by fileId. * - *

First checks the cache, then parses the file if not cached. + *

The returned {@code rowGroups} and {@code chunks} lists are always empty: details are served + * by {@link #getRowGroupsPage} and {@link #getChunksPage} instead, so building a summary never + * materializes the whole file's chunk metadata. * * @param fileId the file identifier - * @return TsFileMetadataDTO with complete metadata + * @return TsFileMetadataDTO with summary metadata * @throws TsFileNotFoundException if the file is not found * @throws AccessDeniedException if the path is not allowed * @throws IOException if the file cannot be read @@ -103,21 +135,210 @@ public TsFileMetadataDTO getMetadata(String fileId) throws TsFileNotFoundException, AccessDeniedException, IOException { logger.debug("Getting metadata for fileId={}", fileId); - // Check cache first TsFileMetadataDTO cached = metadataCache.getIfPresent(fileId); if (cached != null) { logger.debug("Returning cached metadata for fileId={}", fileId); return cached; } - // Get file path and parse metadata + String filePath = resolveFilePath(fileId); + logger.debug("Parsing metadata summary for file: {}", filePath); + + TsFileParser.SummaryMetadata summary = tsFileParser.parseSummary(filePath); + TsFileMetadataDTO metadata = toSummaryDTO(fileId, summary); + + metadataCache.put(fileId, metadata); + deviceIndexCache.put(fileId, summary.deviceIndex()); + logger.debug("Cached metadata for fileId={}", fileId); + + return metadata; + } + + /** + * Gets one page of RowGroups (one RowGroup per device). + * + *

Only the devices inside the page window are read from the file. + * + * @param fileId the file identifier + * @param offset zero-based offset of the first RowGroup + * @param limit maximum number of RowGroups to return + * @return page of RowGroups with the file's total RowGroup count + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public PageDTO getRowGroupsPage(String fileId, int offset, int limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting RowGroup page for fileId={}, offset={}, limit={}", fileId, offset, limit); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + int total = deviceIndex.deviceCount(); + + List items = List.of(); + if (offset < total && limit > 0) { + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + items = tsFileParser.readRowGroupPage(reader, deviceIndex, offset, limit); + } + } + + return PageDTO.of(items, total, offset, limit); + } + + /** + * Gets one page of devices, optionally scoped to a table and filtered by a substring. + * + *

Derived from the cached device index rather than a row scan: a device identifier already + * carries its table name and TAG values ({@code tableName.tag1.tag2}), so listing devices of a + * table costs a pass over cached names instead of a full table query. + * + * @param fileId the file identifier + * @param tableName only devices of this table, or null for all devices + * @param query case-insensitive substring the device name must contain, or null + * @param offset zero-based offset of the first device + * @param limit maximum number of devices to return + * @return page of devices with the number of matching devices as total + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public PageDTO getDevicesPage( + String fileId, String tableName, String query, int offset, int limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug( + "Getting device page for fileId={}, tableName={}, query={}, offset={}, limit={}", + fileId, + tableName, + query, + offset, + limit); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + + String tablePrefix = tableName == null ? "" : tableName.trim(); + String loweredQuery = query == null ? "" : query.trim().toLowerCase(Locale.ROOT); + boolean hasTable = !tablePrefix.isEmpty(); + boolean hasQuery = !loweredQuery.isEmpty(); + + List page = new ArrayList<>(); + int matched = 0; + int total = 0; + // Two passes so only the requested page is materialised + for (IDeviceID device : deviceIndex.devices()) { + String name = device.toString(); + if (hasTable && !isInTable(name, tablePrefix)) { + continue; + } + if (hasQuery && !name.toLowerCase(Locale.ROOT).contains(loweredQuery)) { + continue; + } + if (matched >= offset && page.size() < limit) { + page.add(toDeviceRef(name, hasTable ? tablePrefix : null)); + } + matched++; + total = matched; + } + + return PageDTO.of(page, total, offset, limit); + } + + private static boolean isInTable(String deviceName, String tableName) { + return deviceName.equals(tableName) || deviceName.startsWith(tableName + "."); + } + + private static DeviceRefDTO toDeviceRef(String deviceName, String tableName) { + if (tableName == null) { + return new DeviceRefDTO(deviceName, null, List.of()); + } + String remainder = deviceName.length() > tableName.length() + 1 + ? deviceName.substring(tableName.length() + 1) + : ""; + List tagValues = remainder.isEmpty() ? List.of() : List.of(remainder.split("\\.")); + return new DeviceRefDTO(deviceName, tableName, tagValues); + } + + /** + * Gets the measurements carried by one device. + * + *

Only that device's metadata is read; the device is resolved through the cached device index, + * so this does not walk the file. + * + * @param fileId the file identifier + * @param deviceName device name as returned by the RowGroup paging endpoint + * @return measurements of that device, empty when the device is not part of the file + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public List getDeviceMeasurements(String fileId, String deviceName) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting measurements of device={} for fileId={}", deviceName, fileId); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + + IDeviceID match = null; + for (IDeviceID device : deviceIndex.devices()) { + if (device.toString().equals(deviceName)) { + match = device; + break; + } + } + if (match == null) { + logger.debug("Device {} is not part of file {}", deviceName, filePath); + return List.of(); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + return tsFileParser.readDeviceMeasurements(reader, match); + } + } + + /** + * Gets one page of Chunks, crossing device boundaries when a page needs to. + * + *

Only the devices the page touches are read from the file. + * + * @param fileId the file identifier + * @param offset zero-based offset of the first chunk + * @param limit maximum number of chunks to return + * @return page of Chunks with the file's total chunk count + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public PageDTO getChunksPage(String fileId, int offset, int limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting chunk page for fileId={}, offset={}, limit={}", fileId, offset, limit); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + int total = deviceIndex.chunkCount(); + + List items = List.of(); + if (offset < total && limit > 0) { + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + items = tsFileParser.readChunkPage(reader, deviceIndex, offset, limit); + } + } + + return PageDTO.of(items, total, offset, limit); + } + + /** + * Resolves a fileId to an existing, non-empty file path. + * + * @param fileId the file identifier + * @return absolute file path + */ + private String resolveFilePath(String fileId) + throws TsFileNotFoundException, AccessDeniedException { String filePath = fileService.getFilePath(fileId); - logger.debug("Parsing metadata for file: {}", filePath); // Pre-check: reject empty files before attempting to parse try { - long fileSize = Files.size(Paths.get(filePath)); - if (fileSize == 0) { + if (Files.size(Paths.get(filePath)) == 0) { throw new EmptyFileException( "The file is empty (0 bytes). It may be a placeholder or incomplete write."); } @@ -125,49 +346,75 @@ public TsFileMetadataDTO getMetadata(String fileId) logger.warn("Cannot determine file size for {}: {}", filePath, e.getMessage()); } - TsFileMetadataDTO metadata = parseMetadata(fileId, filePath); + return filePath; + } - // Cache the result - metadataCache.put(fileId, metadata); - logger.debug("Cached metadata for fileId={}", fileId); + /** + * Returns the cached device index of a file, rebuilding it when the file changed on disk. + * + * @param fileId the file identifier + * @param filePath the resolved file path + * @return device index of the file's current version + */ + private DeviceIndex getDeviceIndex(String fileId, String filePath) throws IOException { + DeviceIndex cached = deviceIndexCache.getIfPresent(fileId); + if (cached != null && !isStale(cached, filePath)) { + return cached; + } - return metadata; + DeviceIndex deviceIndex = tsFileParser.parseDeviceIndex(filePath); + deviceIndexCache.put(fileId, deviceIndex); + return deviceIndex; + } + + /** + * Tells whether a cached index no longer matches the file on disk. + * + * @param deviceIndex cached index + * @param filePath file path + * @return true when the file's mtime or size differs from the indexed version + */ + private static boolean isStale(DeviceIndex deviceIndex, String filePath) { + try { + Path path = Paths.get(filePath); + return !deviceIndex.matches(Files.getLastModifiedTime(path).toMillis(), Files.size(path)); + } catch (IOException e) { + logger.debug("Cannot stat {}: {}", filePath, e.getMessage()); + return true; + } } /** - * Parses metadata from a TsFile. + * Converts parsed summary metadata to the API DTO. * * @param fileId the file identifier - * @param filePath the file path - * @return TsFileMetadataDTO with parsed metadata - * @throws IOException if the file cannot be read + * @param summary parsed summary metadata + * @return DTO with empty RowGroup and Chunk lists */ - private TsFileMetadataDTO parseMetadata(String fileId, String filePath) throws IOException { - // Parse all metadata in a single pass (opens the file only once instead of 5-7 times) - TsFileParser.AllMetadata all = tsFileParser.parseAll(filePath); + private TsFileMetadataDTO toSummaryDTO(String fileId, TsFileParser.SummaryMetadata summary) { + TsFileParser.BasicMetadata basic = summary.basic(); // Check if the file is empty (valid structure but no data) - if (all.basic().deviceCount() == 0 - && all.measurements().isEmpty() - && all.rowGroups().isEmpty() - && all.chunks().isEmpty() - && all.tables().isEmpty()) { + if (basic.deviceCount() == 0 + && summary.measurements().isEmpty() + && summary.tables().isEmpty() + && summary.deviceIndex().chunkCount() == 0) { throw new EmptyFileException( "The TsFile is valid but contains no data. It may be a newly created or empty file."); } return TsFileMetadataDTO.builder() .fileId(fileId) - .version(all.basic().version()) - .timeRange(new TimeRange(all.basic().startTime(), all.basic().endTime())) - .deviceCount(all.basic().deviceCount()) - .measurementCount(all.basic().measurementCount()) - .rowGroupCount(all.rowGroups().size()) - .chunkCount(all.chunks().size()) - .measurements(all.measurements()) - .rowGroups(all.rowGroups()) - .chunks(all.chunks()) - .tables(all.tables()) + .version(basic.version()) + .timeRange(new TimeRange(basic.startTime(), basic.endTime())) + .deviceCount(basic.deviceCount()) + .measurementCount(basic.measurementCount()) + .rowGroupCount(basic.rowGroupCount()) + .chunkCount(basic.chunkCount()) + .measurements(summary.measurements()) + .rowGroups(List.of()) + .chunks(List.of()) + .tables(summary.tables()) .build(); } @@ -179,12 +426,14 @@ private TsFileMetadataDTO parseMetadata(String fileId, String filePath) throws I public void invalidateCache(String fileId) { logger.debug("Invalidating cache for fileId={}", fileId); metadataCache.invalidate(fileId); + deviceIndexCache.invalidate(fileId); } /** Invalidates all cached metadata. */ public void invalidateAllCache() { logger.info("Invalidating all metadata cache"); metadataCache.invalidateAll(); + deviceIndexCache.invalidateAll(); } /** diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java index 71b5476..d63ac27 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java @@ -48,6 +48,7 @@ import org.apache.tsfile.viewer.dto.TableListResponse.TableInfo; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.tsfile.TableSchemaSupport; /** * Service for table-level data operations. @@ -89,7 +90,7 @@ public TableListResponse getTableList(String fileId) List tableInfos = new ArrayList<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(filePath)).build()) { - List schemas = reader.getAllTableSchema(); + List schemas = TableSchemaSupport.readAll(reader); if (schemas == null || schemas.isEmpty()) { logger.debug("No tables found in file: {}", filePath); @@ -161,7 +162,7 @@ public DeviceListResponse getDeviceList(String fileId, String tableName) Map deviceMap = new HashMap<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(filePath)).build()) { - List schemas = reader.getAllTableSchema(); + List schemas = TableSchemaSupport.readAll(reader); if (schemas == null || schemas.isEmpty()) { return DeviceListResponse.builder().devices(new ArrayList<>()).totalCount(0).build(); diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java new file mode 100644 index 0000000..afadc31 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Compact, primitive-array index of chunk metadata keyed by chunk-header offset. + * + *

Replaces the former {@code Map}, whose per-chunk cost is far larger than + * the three fields the health check actually reads (a boxed {@code Long} key, a {@code HashMap} + * node and a full {@code IChunkMetadata} carrying its statistics). This index packs those three + * fields into primitive arrays: {@code 8 (offset) + 4 (measurement id) + 8 (start time) + 8 (end + * time) = 28 bytes} per chunk, so the 1,838,571 chunks of a large real-world file need about 49 MiB + * instead of several hundred MiB. + * + *

The marker-based traversal that consumes this index only ever moves forward through the file, + * so a {@link Cursor} advances monotonically and the whole lookup pass costs O(n) rather than O(n + * log n). A backward query — which traversal cannot produce — falls back to a binary search, so + * correctness never depends on that ordering assumption. + */ +final class ChunkMetadataIndex { + + /** Signals that the configured chunk cap was exceeded while building the index. */ + static final class ChunkCountExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final int limit; + + ChunkCountExceededException(int limit) { + super("Chunk count exceeds the configured limit of " + limit); + this.limit = limit; + } + + int getLimit() { + return limit; + } + } + + private static final int INITIAL_CAPACITY = 1024; + + /** Cap on the number of entries; adding one more throws {@link ChunkCountExceededException}. */ + private final int maxChunks; + + /** Distinct measurement names; chunk entries store an {@code int} index into this list. */ + private final List measurementPool = new ArrayList<>(); + + private final Map measurementIdByMeasurement = new HashMap<>(); + + private long[] offsets = new long[INITIAL_CAPACITY]; + private int[] measurementIds = new int[INITIAL_CAPACITY]; + private long[] startTimes = new long[INITIAL_CAPACITY]; + private long[] endTimes = new long[INITIAL_CAPACITY]; + private int size; + private boolean sealed; + + ChunkMetadataIndex(int maxChunks) { + this.maxChunks = maxChunks; + } + + /** + * Appends one chunk entry. + * + * @param offset offset of the chunk header (including its marker byte) + * @param measurement measurement name owning the chunk + * @param startTime first timestamp of the chunk + * @param endTime last timestamp of the chunk + * @throws ChunkCountExceededException if the index already holds {@code maxChunks} entries + */ + void add(long offset, String measurement, long startTime, long endTime) { + if (sealed) { + throw new IllegalStateException("Chunk metadata index is already sealed"); + } + if (size >= maxChunks) { + throw new ChunkCountExceededException(maxChunks); + } + if (size == offsets.length) { + grow(); + } + offsets[size] = offset; + measurementIds[size] = measurementId(measurement); + startTimes[size] = startTime; + endTimes[size] = endTime; + size++; + } + + int size() { + return size; + } + + boolean isEmpty() { + return size == 0; + } + + long offsetAt(int index) { + return offsets[index]; + } + + String measurementAt(int index) { + return measurementPool.get(measurementIds[index]); + } + + long startTimeAt(int index) { + return startTimes[index]; + } + + long endTimeAt(int index) { + return endTimes[index]; + } + + /** + * Sorts the entries by chunk offset and trims the backing arrays to the exact size. + * + *

Lookups require a sealed index. + */ + void seal() { + if (sealed) { + return; + } + sortByOffset(0, size - 1); + offsets = Arrays.copyOf(offsets, size); + measurementIds = Arrays.copyOf(measurementIds, size); + startTimes = Arrays.copyOf(startTimes, size); + endTimes = Arrays.copyOf(endTimes, size); + sealed = true; + } + + /** Creates a forward-only lookup cursor; the returned cursor is cheap and single-pass. */ + Cursor cursor() { + if (!sealed) { + throw new IllegalStateException("seal() must be called before creating a cursor"); + } + return new Cursor(this); + } + + private int measurementId(String measurement) { + Integer id = measurementIdByMeasurement.get(measurement); + if (id == null) { + id = measurementPool.size(); + measurementPool.add(measurement); + measurementIdByMeasurement.put(measurement, id); + } + return id; + } + + private void grow() { + int newCapacity = offsets.length + (offsets.length >> 1) + 1; + offsets = Arrays.copyOf(offsets, newCapacity); + measurementIds = Arrays.copyOf(measurementIds, newCapacity); + startTimes = Arrays.copyOf(startTimes, newCapacity); + endTimes = Arrays.copyOf(endTimes, newCapacity); + } + + private void sortByOffset(int low, int high) { + int lo = low; + int hi = high; + while (lo < hi) { + if (hi - lo < 16) { + insertionSortByOffset(lo, hi); + return; + } + long pivot = medianOfThree(lo, hi); + int less = lo; + int greater = hi; + int current = lo; + // Three-way partition: offsets are unique in practice, but equal keys must not loop forever. + while (current <= greater) { + if (offsets[current] < pivot) { + swap(current, less); + current++; + less++; + } else if (offsets[current] > pivot) { + swap(current, greater); + greater--; + } else { + current++; + } + } + // Recurse into the smaller side only, keeping the stack depth logarithmic. + if (less - lo < hi - greater) { + sortByOffset(lo, less - 1); + lo = greater + 1; + } else { + sortByOffset(greater + 1, hi); + hi = less - 1; + } + } + } + + private void insertionSortByOffset(int lo, int hi) { + for (int i = lo + 1; i <= hi; i++) { + int j = i; + while (j > lo && offsets[j - 1] > offsets[j]) { + swap(j - 1, j); + j--; + } + } + } + + private long medianOfThree(int lo, int hi) { + int mid = lo + ((hi - lo) >>> 1); + long a = offsets[lo]; + long b = offsets[mid]; + long c = offsets[hi]; + if (a < b) { + return b < c ? b : Math.max(a, c); + } + return a < c ? a : Math.max(b, c); + } + + private void swap(int i, int j) { + if (i == j) { + return; + } + long offset = offsets[i]; + offsets[i] = offsets[j]; + offsets[j] = offset; + + int measurementId = measurementIds[i]; + measurementIds[i] = measurementIds[j]; + measurementIds[j] = measurementId; + + long startTime = startTimes[i]; + startTimes[i] = startTimes[j]; + startTimes[j] = startTime; + + long endTime = endTimes[i]; + endTimes[i] = endTimes[j]; + endTimes[j] = endTime; + } + + private int binarySearch(long offset) { + int low = 0; + int high = size - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + long midOffset = offsets[mid]; + if (midOffset < offset) { + low = mid + 1; + } else if (midOffset > offset) { + high = mid - 1; + } else { + return mid; + } + } + return -1; + } + + /** Forward-only cursor over a sealed {@link ChunkMetadataIndex}. Not thread-safe. */ + static final class Cursor { + + private final ChunkMetadataIndex index; + private int position; + + private Cursor(ChunkMetadataIndex index) { + this.index = index; + } + + /** + * Returns the entry index whose chunk-header offset equals {@code offset}, or {@code -1} when + * the chunk is not present in the index. + * + *

Offsets requested in ascending order cost O(1) amortized; a request that moves backwards + * (impossible for file traversal) falls back to a binary search. + */ + int find(long offset) { + if (position > 0 && index.offsets[position - 1] >= offset) { + return index.binarySearch(offset); + } + while (position < index.size && index.offsets[position] < offset) { + position++; + } + if (position < index.size && index.offsets[position] == offset) { + return position; + } + return -1; + } + + String measurement(int index) { + return this.index.measurementAt(index); + } + + long startTime(int index) { + return this.index.startTimeAt(index); + } + + long endTime(int index) { + return this.index.endTimeAt(index); + } + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java new file mode 100644 index 0000000..03bfeb2 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import java.util.List; + +import org.apache.tsfile.file.metadata.IDeviceID; + +/** + * Immutable per-file index of devices and their chunk counts. + * + *

Holds only device names and an int prefix-sum array, so it stays small (a few MB even for + * ~100k devices) and never contains chunk or timeseries metadata. Every page request is served from + * this index plus per-device reads, which is what keeps paging incremental. + * + *

The {@code lastModified}/{@code size} pair identifies the file version the index was built + * from; {@link #matches(long, long)} is used to detect a rewritten file. + */ +public final class DeviceIndex { + + private final long lastModified; + private final long size; + private final List devices; + private final int[] chunkPrefix; + + private DeviceIndex( + long lastModified, long size, List devices, int[] chunkPrefix) { + this.lastModified = lastModified; + this.size = size; + this.devices = devices; + this.chunkPrefix = chunkPrefix; + } + + /** + * Creates an index from a device list and the chunk count of each device. + * + * @param lastModified file modification time the index was built from + * @param size file size the index was built from + * @param devices devices in file order + * @param chunkCounts chunk count per device, index-aligned with {@code devices} + * @return device index + */ + public static DeviceIndex of( + long lastModified, long size, List devices, int[] chunkCounts) { + if (devices.size() != chunkCounts.length) { + throw new IllegalArgumentException( + "devices and chunkCounts must be index-aligned: " + + devices.size() + + " vs " + + chunkCounts.length); + } + int[] prefix = new int[devices.size() + 1]; + for (int i = 0; i < chunkCounts.length; i++) { + prefix[i + 1] = prefix[i] + Math.max(chunkCounts[i], 0); + } + return new DeviceIndex(lastModified, size, List.copyOf(devices), prefix); + } + + /** + * Tells whether this index still describes the file's current version. + * + * @param lastModified current file modification time + * @param size current file size + * @return true when the index was built from that exact version + */ + public boolean matches(long lastModified, long size) { + return this.lastModified == lastModified && this.size == size; + } + + /** Number of devices, which is also the number of RowGroups of this file. */ + public int deviceCount() { + return devices.size(); + } + + /** Devices in file order. */ + public List devices() { + return devices; + } + + /** Total number of chunks in the file. */ + public int chunkCount() { + return chunkPrefix[devices.size()]; + } + + /** + * Number of chunks that precede the given device in file order. + * + * @param devicePosition position of the device in {@link #devices()} + * @return absolute index of that device's first chunk + */ + public int chunksBeforeDevice(int devicePosition) { + return chunkPrefix[devicePosition]; + } + + /** + * Number of chunks of the given device. + * + * @param devicePosition position of the device in {@link #devices()} + * @return chunk count of that device + */ + public int chunkCountOf(int devicePosition) { + return chunkPrefix[devicePosition + 1] - chunkPrefix[devicePosition]; + } + + /** + * Finds the device that owns the chunk at the given absolute position, so a chunk page can start + * reading at exactly the device it needs instead of from the first device. + * + * @param chunkOffset absolute chunk position + * @return position of the owning device, or {@link #deviceCount()} when the offset is past the + * last chunk + */ + public int deviceIndexForChunk(int chunkOffset) { + int low = 0; + int high = devices.size() - 1; + int result = devices.size(); + while (low <= high) { + int mid = (low + high) >>> 1; + if (chunkPrefix[mid + 1] > chunkOffset) { + result = mid; + high = mid - 1; + } else { + low = mid + 1; + } + } + return result; + } + + /** File modification time this index was built from. */ + public long lastModified() { + return lastModified; + } + + /** File size this index was built from. */ + public long size() { + return size; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java new file mode 100644 index 0000000..8006bd9 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import java.io.IOException; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.v4.ITsFileReader; + +/** + * Reads the table schemas of a V4 TsFile without failing on tree model files. + * + *

Callers use an empty result to decide that the file is a tree model file and to fall back to + * {@link org.apache.tsfile.read.TsFileSequenceReader}. {@code ITsFileReader.getAllTableSchema()} + * does not honour that contract: on a tree model file the underlying {@code + * DeviceTableModelReader} dereferences a null table schema map and throws {@link + * NullPointerException} instead of returning an empty list. + */ +public final class TableSchemaSupport { + + private static final Logger logger = LoggerFactory.getLogger(TableSchemaSupport.class); + + private TableSchemaSupport() {} + + /** + * Returns the table schemas of the file, or an empty list when the file carries no table schema. + * + * @param reader open V4 TsFile reader + * @return table schemas, never null + * @throws IOException if the metadata cannot be read + */ + public static List readAll(ITsFileReader reader) throws IOException { + try { + List schemas = reader.getAllTableSchema(); + return schemas == null ? List.of() : schemas; + } catch (NullPointerException e) { + // tsfile 2.x throws NPE here for tree model files instead of returning null or an empty list. + logger.debug("File carries no table schema (tree model): {}", e.getMessage()); + return List.of(); + } + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java index 5625cbb..4baa939 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -217,6 +218,35 @@ public ReadFilter setMaxValue(Double maxValue) { /** Reads data from a TsFile with time range filtering and pagination. */ public DataReadResult readDataByTimeRange( File filePath, Long startTime, Long endTime, int limit, int offset) throws IOException { + return readDataByTimeRange(filePath, startTime, endTime, limit, offset, null, null); + } + + /** + * Reads data from a TsFile, optionally restricted to the given devices and measurements. + * + *

The restriction is pushed down into the tree model read path. Without it a tree model file + * is read device by device in storage order, so the row limit is exhausted by the first device and + * a later device (or its measurements) can never be matched by the post-read filters. + * + * @param filePath file to read + * @param startTime inclusive lower time bound, or null + * @param endTime inclusive upper time bound, or null + * @param limit maximum number of rows to return + * @param offset number of rows to skip + * @param devices device names to restrict to, or null for all devices + * @param measurements measurement names to restrict to, or null for all measurements + * @return rows read, plus column metadata and warnings + * @throws IOException if the file cannot be read + */ + public DataReadResult readDataByTimeRange( + File filePath, + Long startTime, + Long endTime, + int limit, + int offset, + List devices, + List measurements) + throws IOException { long effStart = startTime != null ? startTime : Long.MIN_VALUE; long effEnd = endTime != null ? endTime : Long.MAX_VALUE; List results = new ArrayList<>(); @@ -228,13 +258,13 @@ public DataReadResult readDataByTimeRange( try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { List schemas = new ArrayList<>(); - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { schemas.add(ts); } // If no TableSchema found, this is a tree model file - use TsFileSequenceReader if (schemas.isEmpty()) { - return readTreeModelData(filePath, effStart, effEnd, limit, offset); + return readTreeModelData(filePath, effStart, effEnd, limit, offset, devices, measurements); } for (TableSchema ts : schemas) { @@ -349,7 +379,7 @@ public DataReadResult readDataByTimeRange( int skipped = 0, collected = 0; boolean hasMore = false; - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { if (collected >= limit) { hasMore = true; break; @@ -450,7 +480,7 @@ public DataReadResult readDataWithFilter(File filePath, ReadFilter filter, int l boolean hasMore = false; try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { if (collected >= limit) { hasMore = true; break; @@ -571,7 +601,7 @@ public int streamData(File filePath, ReadFilter filter, Consumer consum int processed = 0; try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { String tblName = ts.getTableName(); if (fTbls != null && !fTbls.isEmpty() && !fTbls.contains(tblName)) continue; List cols = ts.getColumnSchemas(); @@ -661,7 +691,7 @@ public DataReadResult readDataByTimestamps(File filePath, List timestamps) List colTypes = new ArrayList<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { String tblName = ts.getTableName(); List cols = ts.getColumnSchemas(); List categories = ts.getColumnTypes(); @@ -737,7 +767,7 @@ public DataReadResult readDataByTimestamps(File filePath, List timestamps) public List getTableNames(File filePath) throws IOException { List names = new ArrayList<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) names.add(ts.getTableName()); + for (TableSchema ts : TableSchemaSupport.readAll(reader)) names.add(ts.getTableName()); } return names; } @@ -766,7 +796,7 @@ public long countRows(File filePath, Long startTime, Long endTime) throws IOExce long effEnd = endTime != null ? endTime : Long.MAX_VALUE; long total = 0; try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { List qCols = new ArrayList<>(); for (IMeasurementSchema c : ts.getColumnSchemas()) qCols.add(c.getMeasurementName()); if (qCols.isEmpty()) continue; @@ -793,20 +823,33 @@ public long countRows(File filePath, Long startTime, Long endTime) throws IOExce * to read device/measurement structure and data. */ private DataReadResult readTreeModelData( - File filePath, long effStart, long effEnd, int limit, int offset) throws IOException { + File filePath, + long effStart, + long effEnd, + int limit, + int offset, + List selectedDevices, + List selectedMeasurements) + throws IOException { List results = new ArrayList<>(); List colNames = new ArrayList<>(); List colTypes = new ArrayList<>(); int skipped = 0, collected = 0; boolean hasMore = false; + List deviceFilter = selectedDevices != null ? selectedDevices : List.of(); + List measurementFilter = + selectedMeasurements != null ? selectedMeasurements : List.of(); try (TsFileSequenceReader seqReader = new TsFileSequenceReader(filePath.getAbsolutePath()); TsFileReader tsFileReader = new TsFileReader(seqReader)) { - // Get all devices and their measurements - Map> deviceMeasurements = new HashMap<>(); + // Get all devices and their measurements, keeping storage order so paging is stable + Map> deviceMeasurements = new LinkedHashMap<>(); for (IDeviceID device : seqReader.getAllDevices()) { + if (!deviceFilter.isEmpty() && !deviceFilter.contains(device.toString())) { + continue; + } Map> seriesMetaData = seqReader.readChunkMetadataInDevice(device); Map measurements = new HashMap<>(); @@ -820,7 +863,12 @@ private DataReadResult readTreeModelData( } } } - deviceMeasurements.put(device, measurements); + if (!measurementFilter.isEmpty()) { + measurements.keySet().retainAll(measurementFilter); + } + if (!measurements.isEmpty()) { + deviceMeasurements.put(device, measurements); + } } // Build column names from all unique measurements diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java index b3d4304..5df161d 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java @@ -72,6 +72,30 @@ public class TsFileHealthChecker { private static final Logger logger = LoggerFactory.getLogger(TsFileHealthChecker.class); + /** Default cap on the number of chunks a single file may contribute to the chunk index. */ + public static final int DEFAULT_MAX_CHUNKS = 5_000_000; + + /** Number of devices processed between two device-metadata cache evictions. */ + private static final int DEVICE_CACHE_CLEAR_INTERVAL = 1000; + + private final int maxChunks; + + /** Creates a checker using {@link #DEFAULT_MAX_CHUNKS}. */ + public TsFileHealthChecker() { + this(DEFAULT_MAX_CHUNKS); + } + + /** + * Creates a checker with an explicit chunk cap. + * + * @param maxChunks maximum number of chunks indexed for a single file; a file above the cap has + * its data-area traversal skipped instead of blowing up the heap. Non-positive values fall + * back to {@link #DEFAULT_MAX_CHUNKS}. + */ + public TsFileHealthChecker(int maxChunks) { + this.maxChunks = maxChunks > 0 ? maxChunks : DEFAULT_MAX_CHUNKS; + } + /** * Performs a full health check on a single TsFile following IoTDB's validation approach. * @@ -79,7 +103,8 @@ public class TsFileHealthChecker { * *

    *
  1. Validate file completeness (isComplete) - *
  2. Build chunk metadata offset map for cross-referencing + *
  3. Build a compact chunk metadata index for cross-referencing (skipped, together with the + * data-area traversal below, when the file exceeds {@code tsfile.scan.max-chunks}) *
  4. Sequential marker-based traversal validating: *
      *
    • Chunk header offset matches metadata @@ -121,18 +146,41 @@ public ScanResult check(Path filePath) { "File is not complete (missing tail magic string or truncated)", ErrorSeverity.CRITICAL)); } else { - // Step 2: Build chunk metadata map - Map chunkMetadataMap = getChunkMetadataMap(reader); - if (chunkMetadataMap.isEmpty()) { + // Step 2: Build the compact chunk metadata index + ChunkMetadataIndex chunkMetadataIndex = null; + boolean chunkLimitExceeded = false; + try { + chunkMetadataIndex = buildChunkMetadataIndex(reader); + } catch (ChunkMetadataIndex.ChunkCountExceededException e) { + // Degrade instead of failing or exhausting the heap: structure and metadata were still + // checked, only the full data-area traversal is skipped. + chunkLimitExceeded = true; + logger.warn( + "Chunk count of {} exceeds tsfile.scan.max-chunks={}, skipping data validation", + filePath, + maxChunks); errors.add( new FileError( - ScanErrorType.STRUCTURE_CORRUPT, - "File content", - "No data found in the file (empty chunk metadata)", - ErrorSeverity.ERROR)); - } else { - // Step 3: Marker-based traversal with data validation - validateDataCorrectness(reader, chunkMetadataMap, errors); + ScanErrorType.DATA_VALIDATION_SKIPPED, + "Data area validation", + "Data area validation skipped: chunk count exceeds tsfile.scan.max-chunks (" + + e.getLimit() + + "). Structure and metadata checks still ran.", + ErrorSeverity.WARNING)); + } + + if (!chunkLimitExceeded) { + if (chunkMetadataIndex.isEmpty()) { + errors.add( + new FileError( + ScanErrorType.STRUCTURE_CORRUPT, + "File content", + "No data found in the file (empty chunk metadata)", + ErrorSeverity.ERROR)); + } else { + // Step 3: Marker-based traversal with data validation + validateDataCorrectness(reader, chunkMetadataIndex, errors); + } } } } catch (IOException e) { @@ -180,8 +228,9 @@ public ScanResult quickCheck(Path filePath) { "File is not complete (missing tail magic string or truncated)", ErrorSeverity.CRITICAL)); } else { - Map chunkMetadataMap = getChunkMetadataMap(reader); - if (chunkMetadataMap.isEmpty()) { + // Only the "does the file hold any chunk at all" question matters here, so stop at the + // first chunk instead of indexing every chunk in the file. + if (!hasAnyChunk(reader)) { errors.add( new FileError( ScanErrorType.STRUCTURE_CORRUPT, @@ -219,12 +268,11 @@ public ScanResult quickCheck(Path filePath) { * validateTsFileDataCorrectness} approach. */ private void validateDataCorrectness( - TsFileSequenceReader reader, - Map chunkMetadataMap, - List errors) { + TsFileSequenceReader reader, ChunkMetadataIndex chunkMetadataIndex, List errors) { List> alignedTimeBatches = new ArrayList<>(); Map valueColumn2TimeBatchIndex = new HashMap<>(); + ChunkMetadataIndex.Cursor chunkCursor = chunkMetadataIndex.cursor(); try { reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); @@ -241,17 +289,17 @@ private void validateDataCorrectness( case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: long chunkOffset = reader.position(); ChunkHeader header = reader.readChunkHeader(marker); - IChunkMetadata chunkMetadata = chunkMetadataMap.get(chunkOffset - Byte.BYTES); + int chunkIndex = chunkCursor.find(chunkOffset - Byte.BYTES); + String metadataMeasurement = chunkIndex >= 0 ? chunkCursor.measurement(chunkIndex) : null; // Validate chunk offset matches metadata - if (chunkMetadata != null - && !chunkMetadata.getMeasurementUid().equals(header.getMeasurementID())) { + if (chunkIndex >= 0 && !metadataMeasurement.equals(header.getMeasurementID())) { errors.add( new FileError( ScanErrorType.STRUCTURE_CORRUPT, "Chunk at offset " + (chunkOffset - Byte.BYTES), "Chunk measurement ID mismatch: metadata says '" - + chunkMetadata.getMeasurementUid() + + metadataMeasurement + "' but chunk header says '" + header.getMeasurementID() + "'", @@ -302,11 +350,11 @@ private void validateDataCorrectness( long pageHeaderStartTime = isHasStatistic ? pageHeader.getStartTime() - : (chunkMetadata != null ? chunkMetadata.getStartTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.startTime(chunkIndex) : 0); long pageHeaderEndTime = isHasStatistic ? pageHeader.getEndTime() - : (chunkMetadata != null ? chunkMetadata.getEndTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.endTime(chunkIndex) : 0); validateTimeFrame( alignedTimeBatch, @@ -347,11 +395,11 @@ private void validateDataCorrectness( long pageHeaderStartTime = isHasStatistic ? pageHeader.getStartTime() - : (chunkMetadata != null ? chunkMetadata.getStartTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.startTime(chunkIndex) : 0); long pageHeaderEndTime = isHasStatistic ? pageHeader.getEndTime() - : (chunkMetadata != null ? chunkMetadata.getEndTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.endTime(chunkIndex) : 0); long pageStartTime = Long.MAX_VALUE; long previousTime = Long.MIN_VALUE; @@ -532,21 +580,62 @@ private void validateTimeFrame( // --------------------------------------------------------------------------- /** - * Builds a map of chunk header offset to IChunkMetadata for cross-referencing during traversal. + * Builds the compact chunk metadata index used for cross-referencing during traversal. + * + *

      Devices are read one at a time and dropped immediately, so the transient peak stays bounded + * by a single device instead of the whole file's {@link TimeseriesMetadata} tree. The reader's + * device-metadata cache is evicted periodically; nothing in this method holds a reference into + * that cache, so eviction is safe at any point. + * + * @throws ChunkMetadataIndex.ChunkCountExceededException if the file holds more than {@code + * maxChunks} chunks */ - private Map getChunkMetadataMap(TsFileSequenceReader reader) - throws IOException { - Map offset2ChunkMetadata = new HashMap<>(); - Map> device2Metadata = - reader.getAllTimeseriesMetadata(true); - for (Map.Entry> entry : device2Metadata.entrySet()) { - for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) { - for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) { - offset2ChunkMetadata.put(chunkMetadata.getOffsetOfChunkHeader(), chunkMetadata); + ChunkMetadataIndex buildChunkMetadataIndex(TsFileSequenceReader reader) throws IOException { + ChunkMetadataIndex index = new ChunkMetadataIndex(maxChunks); + int devicesSinceCacheClear = 0; + for (IDeviceID device : reader.getAllDevices()) { + for (TimeseriesMetadata timeseriesMetadata : reader.getDeviceTimeseriesMetadata(device)) { + List chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); + if (chunkMetadataList == null || chunkMetadataList.isEmpty()) { + continue; } + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + index.add( + chunkMetadata.getOffsetOfChunkHeader(), + chunkMetadata.getMeasurementUid(), + chunkMetadata.getStartTime(), + chunkMetadata.getEndTime()); + } + } + if (++devicesSinceCacheClear >= DEVICE_CACHE_CLEAR_INTERVAL) { + devicesSinceCacheClear = 0; + reader.clearCachedDeviceMetadata(); + } + } + index.seal(); + return index; + } + + /** + * Returns whether the file holds at least one chunk, stopping at the first one found. + * + *

      Used by {@link #quickCheck(Path)}, which needs the "non-empty" verdict only. + */ + private boolean hasAnyChunk(TsFileSequenceReader reader) throws IOException { + int devicesSinceCacheClear = 0; + for (IDeviceID device : reader.getAllDevices()) { + for (TimeseriesMetadata timeseriesMetadata : reader.getDeviceTimeseriesMetadata(device)) { + List chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); + if (chunkMetadataList != null && !chunkMetadataList.isEmpty()) { + return true; + } + } + if (++devicesSinceCacheClear >= DEVICE_CACHE_CLEAR_INTERVAL) { + devicesSinceCacheClear = 0; + reader.clearCachedDeviceMetadata(); } } - return offset2ChunkMetadata; + return false; } /** diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java index 19cdeb8..d22fa39 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java @@ -21,9 +21,11 @@ import java.io.IOException; import java.io.Serializable; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -32,6 +34,7 @@ import java.util.Set; import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -385,77 +388,29 @@ public BasicMetadata parseMetadata(Path filePath) throws IOException { * @throws IOException if the file cannot be read */ private ParsedMetadata parseMetadataInternal(TsFileSequenceReader reader) throws IOException { - // Get version byte versionNumber = reader.readVersionNumber(); + ScanResult scan = scan(reader, List.of(), false); String version = "V" + versionNumber; - // Get all devices - List devices = reader.getAllDevices(); - int deviceCount = devices.size(); - - // Get all timeseries metadata to calculate measurements, time range, and counts - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - - // Calculate unique measurements and time range - Set uniqueMeasurements = new HashSet<>(); - long minStartTime = Long.MAX_VALUE; - long maxEndTime = Long.MIN_VALUE; - int totalChunkCount = 0; - int rowGroupCount = 0; - - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - List timeseriesMetadataList = entry.getValue(); - rowGroupCount++; // Each device entry represents a RowGroup - - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { - uniqueMeasurements.add(tsMetadata.getMeasurementId()); - - // Get time range from statistics - Statistics stats = tsMetadata.getStatistics(); - if (stats != null) { - if (stats.getStartTime() < minStartTime) { - minStartTime = stats.getStartTime(); - } - if (stats.getEndTime() > maxEndTime) { - maxEndTime = stats.getEndTime(); - } - } - - // Count chunks - List chunkMetadataList = tsMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - totalChunkCount += chunkMetadataList.size(); - } - } - } - - // Handle case where no data exists - if (minStartTime == Long.MAX_VALUE) { - minStartTime = 0; - } - if (maxEndTime == Long.MIN_VALUE) { - maxEndTime = 0; - } - logger.debug( "Parsed metadata: version={}, devices={}, measurements={}, rowGroups={}, chunks={}", version, - deviceCount, - uniqueMeasurements.size(), - rowGroupCount, - totalChunkCount); + scan.devices().size(), + scan.measurementCount(), + scan.devices().size(), + totalChunks(scan)); return new ParsedMetadata( version, - minStartTime, - maxEndTime, - deviceCount, - uniqueMeasurements.size(), - rowGroupCount, - totalChunkCount); + scan.startTime(), + scan.endTime(), + scan.devices().size(), + scan.measurementCount(), + scan.devices().size(), + totalChunks(scan)); } + /** * Parses measurement schema details from a TsFileSequenceReader. * @@ -478,157 +433,12 @@ private ParsedMetadata parseMetadataInternal(TsFileSequenceReader reader) throws */ public List parseMeasurements(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing measurements from TsFile"); - - Map measurementMap = new LinkedHashMap<>(); - - // Try to read as V4 table model first to get column categories and schema info - Map columnCategories = new HashMap<>(); - Map columnSchemas = new HashMap<>(); - boolean isTableModel = false; - try { - String tsfilePath = reader.getFileName(); - Path path = Paths.get(tsfilePath); - logger.debug("Attempting to read TableSchema from: {}", tsfilePath); - try (ITsFileReader v4Reader = new TsFileReaderBuilder().file(path.toFile()).build()) { - List schemas = v4Reader.getAllTableSchema(); - if (schemas != null && !schemas.isEmpty()) { - isTableModel = true; - logger.debug("Found {} table schemas in V4 format", schemas.size()); - for (TableSchema schema : schemas) { - logger.debug("Processing table schema: {}", schema.getTableName()); - List schemaList = schema.getColumnSchemas(); - List categories = schema.getColumnTypes(); - - // Column schemas and categories lists are parallel - same index corresponds to - // same column - if (schemaList != null && categories != null) { - for (int i = 0; i < schemaList.size() && i < categories.size(); i++) { - IMeasurementSchema columnSchema = schemaList.get(i); - ColumnCategory category = categories.get(i); - String columnName = columnSchema.getMeasurementName(); - - if (category != null) { - columnCategories.put(columnName, category); - columnSchemas.put(columnName, columnSchema); - logger.debug(" Column {} has category {}", columnName, category); - - // For table model, add TAG columns directly since they won't appear in - // TimeseriesMetadata - if (category == ColumnCategory.TAG) { - String dataType = - columnSchema.getType() != null - ? columnSchema.getType().toString() - : "STRING"; - String encoding = - columnSchema.getEncodingType() != null - ? columnSchema.getEncodingType().toString() - : "PLAIN"; - String compression = - columnSchema.getCompressor() != null - ? columnSchema.getCompressor().toString() - : "UNCOMPRESSED"; - - measurementMap.put( - columnName, - new MeasurementInfo(columnName, dataType, encoding, compression, "TAG")); - } - } - } - } - } - } - } - } catch (Exception e) { - logger.debug("Not a V4 table model file or failed to read TableSchema: {}", e.getMessage()); - } - - // Parse measurements from TimeseriesMetadata - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - List timeseriesMetadataList = entry.getValue(); - - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { - String measurementId = tsMetadata.getMeasurementId(); - - // Skip if already processed - if (measurementMap.containsKey(measurementId)) { - continue; - } - - // Get data type from TimeseriesMetadata - String dataType = tsMetadata.getTsDataType().toString(); - - // Get encoding and compression - prioritize TableSchema for V4 table model - String encoding = "UNKNOWN"; - String compression = "UNKNOWN"; - - // For V4 table model, try to get from TableSchema first - IMeasurementSchema tableSchema = columnSchemas.get(measurementId); - if (tableSchema != null) { - if (tableSchema.getEncodingType() != null) { - encoding = tableSchema.getEncodingType().toString(); - } - if (tableSchema.getCompressor() != null) { - compression = tableSchema.getCompressor().toString(); - } - logger.debug( - "Got encoding/compression from TableSchema for {}: {}/{}", - measurementId, - encoding, - compression); - } else { - // Fallback to ChunkHeader for tree model files - // Read ChunkHeader from file to get encoding and compression - List chunkMetadataList = tsMetadata.getChunkMetadataList(); - if (chunkMetadataList != null && !chunkMetadataList.isEmpty()) { - IChunkMetadata firstChunk = chunkMetadataList.get(0); - if (firstChunk instanceof ChunkMetadata chunkMeta) { - try { - // Read the chunk which includes the header with encoding/compression info - org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); - if (chunk != null && chunk.getHeader() != null) { - org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); - if (chunkHeader.getEncodingType() != null) { - encoding = chunkHeader.getEncodingType().toString(); - } - if (chunkHeader.getCompressionType() != null) { - compression = chunkHeader.getCompressionType().toString(); - } - logger.debug( - "Read encoding/compression from ChunkHeader for {}: {}/{}", - measurementId, - encoding, - compression); - } - } catch (Exception e) { - logger.debug( - "Could not read ChunkHeader for measurement {}: {}", - measurementId, - e.getMessage()); - } - } - } - } - - // Get column category from V4 TableSchema if available - String columnCategory = "FIELD"; // Default to FIELD - ColumnCategory category = columnCategories.get(measurementId); - if (category != null) { - columnCategory = category.toString(); - } - - measurementMap.put( - measurementId, - new MeasurementInfo(measurementId, dataType, encoding, compression, columnCategory)); - } - } - - logger.debug("Parsed {} unique measurements", measurementMap.size()); - return new ArrayList<>(measurementMap.values()); + ScanResult scan = scan(reader, readTableSchemas(reader), true); + logger.debug("Parsed {} unique measurements", scan.measurements().size()); + return scan.measurements(); } + /** * Parses table schemas from a V4 table model TsFile. * @@ -641,84 +451,12 @@ public List parseMeasurements(TsFileSequenceReader reader) thro */ public List parseTables(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing table schemas from TsFile"); - - List tableInfos = new ArrayList<>(); - - try { - String tsfilePath = reader.getFileName(); - Path path = Paths.get(tsfilePath); - logger.debug("Attempting to read TableSchema from: {}", tsfilePath); - - try (ITsFileReader v4Reader = new TsFileReaderBuilder().file(path.toFile()).build()) { - List schemas = v4Reader.getAllTableSchema(); - - if (schemas == null || schemas.isEmpty()) { - logger.debug("No table schemas found - this is a tree model file"); - return tableInfos; - } - - logger.debug("Found {} table schemas in V4 format", schemas.size()); - - for (TableSchema schema : schemas) { - String tableName = schema.getTableName(); - logger.debug("Processing table: {}", tableName); - - List tagColumns = new ArrayList<>(); - List fieldColumns = new ArrayList<>(); - - List columnSchemas = schema.getColumnSchemas(); - List categories = schema.getColumnTypes(); - - if (columnSchemas != null && categories != null) { - for (int i = 0; i < columnSchemas.size() && i < categories.size(); i++) { - IMeasurementSchema columnSchema = columnSchemas.get(i); - ColumnCategory category = categories.get(i); - String columnName = columnSchema.getMeasurementName(); - - String dataType = - columnSchema.getType() != null ? columnSchema.getType().toString() : "UNKNOWN"; - String encoding = - columnSchema.getEncodingType() != null - ? columnSchema.getEncodingType().toString() - : "PLAIN"; - String compression = - columnSchema.getCompressor() != null - ? columnSchema.getCompressor().toString() - : "UNCOMPRESSED"; - - MeasurementInfo columnInfo = - new MeasurementInfo( - columnName, dataType, encoding, compression, category.toString()); - - if (category == ColumnCategory.TAG) { - tagColumns.add(columnInfo); - logger.debug(" TAG column: {}", columnName); - } else if (category == ColumnCategory.FIELD) { - fieldColumns.add(columnInfo); - logger.debug(" FIELD column: {}", columnName); - } - } - } - - TableInfo tableInfo = new TableInfo(tableName, tagColumns, fieldColumns); - tableInfos.add(tableInfo); - - logger.debug( - "Table '{}': {} TAG columns, {} FIELD columns, {} total", - tableName, - tagColumns.size(), - fieldColumns.size(), - tableInfo.getTotalColumns()); - } - } - } catch (Exception e) { - logger.debug("Not a V4 table model file or failed to read tables: {}", e.getMessage()); - } - + List tableInfos = toTableInfos(readTableSchemas(reader)); logger.debug("Parsed {} tables total", tableInfos.size()); return tableInfos; } + /** * Parses RowGroup metadata from a TsFileSequenceReader. * @@ -731,53 +469,54 @@ public List parseTables(TsFileSequenceReader reader) throws IOExcepti */ public List parseRowGroups(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing RowGroups from TsFile"); + List devices = reader.getAllDevices(); + List rowGroups = new ArrayList<>(devices.size()); + for (int i = 0; i < devices.size(); i++) { + rowGroups.add(readRowGroupInfo(reader, devices.get(i), i)); + } + logger.debug("Parsed {} RowGroups", rowGroups.size()); + return rowGroups; + } - List rowGroups = new ArrayList<>(); - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - - int index = 0; - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - IDeviceID device = entry.getKey(); - List timeseriesMetadataList = entry.getValue(); - - long minStartTime = Long.MAX_VALUE; - long maxEndTime = Long.MIN_VALUE; - int chunkCount = 0; + /** + * Reads the RowGroup metadata of a single device. + * + * @param reader TsFile reader + * @param device device to read + * @param index RowGroup index of that device + * @return RowGroup metadata of that single device + * @throws IOException if reading fails + */ + private RowGroupInfo readRowGroupInfo(TsFileSequenceReader reader, IDeviceID device, int index) + throws IOException { + long minStartTime = Long.MAX_VALUE; + long maxEndTime = Long.MIN_VALUE; + int chunkCount = 0; - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { + List deviceMetadata = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadata != null) { + for (TimeseriesMetadata tsMetadata : deviceMetadata) { Statistics stats = tsMetadata.getStatistics(); if (stats != null) { - if (stats.getStartTime() < minStartTime) { - minStartTime = stats.getStartTime(); - } - if (stats.getEndTime() > maxEndTime) { - maxEndTime = stats.getEndTime(); - } - } - - List chunkMetadataList = tsMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - chunkCount += chunkMetadataList.size(); + minStartTime = Math.min(minStartTime, stats.getStartTime()); + maxEndTime = Math.max(maxEndTime, stats.getEndTime()); } - } - // Handle case where no data exists - if (minStartTime == Long.MAX_VALUE) { - minStartTime = 0; - } - if (maxEndTime == Long.MIN_VALUE) { - maxEndTime = 0; + chunkCount += countChunks(tsMetadata.getChunkMetadataList()); } + } - rowGroups.add( - new RowGroupInfo(index++, device.toString(), minStartTime, maxEndTime, chunkCount)); + if (minStartTime == Long.MAX_VALUE) { + minStartTime = 0; + } + if (maxEndTime == Long.MIN_VALUE) { + maxEndTime = 0; } - logger.debug("Parsed {} RowGroups", rowGroups.size()); - return rowGroups; + return new RowGroupInfo(index, device.toString(), minStartTime, maxEndTime, chunkCount); } + /** * Parses Chunk metadata from a TsFileSequenceReader. * @@ -790,101 +529,111 @@ public List parseRowGroups(TsFileSequenceReader reader) throws IOE */ public List parseChunks(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing Chunks from TsFile"); + List chunks = new ArrayList<>(); + for (IDeviceID device : reader.getAllDevices()) { + chunks.addAll(readChunkInfos(reader, device)); + } + logger.debug("Parsed {} Chunks", chunks.size()); + return chunks; + } + /** + * Reads the chunk metadata of a single device. + * + *

      Bounded by one device instead of the whole file, which is what keeps a chunk page + * incremental. + * + * @param reader TsFile reader + * @param device device to read + * @return chunk metadata of that single device, in file order + * @throws IOException if reading fails + */ + private List readChunkInfos(TsFileSequenceReader reader, IDeviceID device) + throws IOException { List chunks = new ArrayList<>(); - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); + List deviceMetadata = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadata == null) { + return chunks; + } - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - IDeviceID device = entry.getKey(); - List timeseriesMetadataList = entry.getValue(); + for (TimeseriesMetadata tsMetadata : deviceMetadata) { + String measurementId = tsMetadata.getMeasurementId(); + List chunkMetadataList = tsMetadata.getChunkMetadataList(); - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { - String measurementId = tsMetadata.getMeasurementId(); - List chunkMetadataList = tsMetadata.getChunkMetadataList(); + if (chunkMetadataList == null) { + continue; + } - if (chunkMetadataList == null) { + for (IChunkMetadata iChunkMeta : chunkMetadataList) { + if (!(iChunkMeta instanceof ChunkMetadata chunkMeta)) { continue; } - for (IChunkMetadata iChunkMeta : chunkMetadataList) { - if (!(iChunkMeta instanceof ChunkMetadata chunkMeta)) { - continue; - } - - long offset = chunkMeta.getOffsetOfChunkHeader(); - Statistics stats = chunkMeta.getStatistics(); + long offset = chunkMeta.getOffsetOfChunkHeader(); + Statistics stats = chunkMeta.getStatistics(); - long startTime = stats != null ? stats.getStartTime() : 0; - long endTime = stats != null ? stats.getEndTime() : 0; - long numOfPoints = chunkMeta.getNumOfPoints(); + long startTime = stats != null ? stats.getStartTime() : 0; + long endTime = stats != null ? stats.getEndTime() : 0; + long numOfPoints = chunkMeta.getNumOfPoints(); - // Get data type - String dataType = - chunkMeta.getDataType() != null ? chunkMeta.getDataType().toString() : "UNKNOWN"; + String dataType = + chunkMeta.getDataType() != null ? chunkMeta.getDataType().toString() : "UNKNOWN"; - // Get encoding and compression from ChunkHeader - String encoding = "UNKNOWN"; - String compression = "UNKNOWN"; - try { - org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); - if (chunk != null && chunk.getHeader() != null) { - org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); - if (chunkHeader.getEncodingType() != null) { - encoding = chunkHeader.getEncodingType().toString(); - } - if (chunkHeader.getCompressionType() != null) { - compression = chunkHeader.getCompressionType().toString(); - } + String encoding = "UNKNOWN"; + String compression = "UNKNOWN"; + try { + org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); + if (chunk != null && chunk.getHeader() != null) { + org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); + if (chunkHeader.getEncodingType() != null) { + encoding = chunkHeader.getEncodingType().toString(); } - } catch (Exception e) { - logger.debug( - "Could not read ChunkHeader for chunk at offset {}: {}", offset, e.getMessage()); - } - - // Get min/max values from statistics - String minValue = null; - String maxValue = null; - if (stats != null) { - try { - Object min = stats.getMinValue(); - Object max = stats.getMaxValue(); - minValue = min != null ? min.toString() : null; - maxValue = max != null ? max.toString() : null; - } catch (Exception e) { - logger.debug("Could not get min/max values from statistics: {}", e.getMessage()); + if (chunkHeader.getCompressionType() != null) { + compression = chunkHeader.getCompressionType().toString(); } } + } catch (Exception e) { + logger.debug( + "Could not read ChunkHeader for chunk at offset {}: {}", offset, e.getMessage()); + } - // Calculate compression ratio - double compressionRatio = calculateCompressionRatio(chunkMeta, numOfPoints); - - // Size is estimated from the chunk metadata - long size = estimateChunkSize(chunkMeta); - - chunks.add( - new ChunkInfo( - measurementId, - offset, - size, - compressionRatio, - device.toString(), - startTime, - endTime, - numOfPoints, - dataType, - encoding, - compression, - minValue, - maxValue)); + String minValue = null; + String maxValue = null; + if (stats != null) { + try { + Object min = stats.getMinValue(); + Object max = stats.getMaxValue(); + minValue = min != null ? min.toString() : null; + maxValue = max != null ? max.toString() : null; + } catch (Exception e) { + logger.debug("Could not get min/max values from statistics: {}", e.getMessage()); + } } + + double compressionRatio = calculateCompressionRatio(chunkMeta, numOfPoints); + long size = estimateChunkSize(chunkMeta); + + chunks.add( + new ChunkInfo( + measurementId, + offset, + size, + compressionRatio, + device.toString(), + startTime, + endTime, + numOfPoints, + dataType, + encoding, + compression, + minValue, + maxValue)); } } - - logger.debug("Parsed {} Chunks", chunks.size()); return chunks; } + /** * Calculates an estimated compression ratio for a chunk. * @@ -939,133 +688,492 @@ private long estimateChunkSize(ChunkMetadata chunkMeta) { return chunkMeta.serializedSize(true); } - // ============== Unified Parse Method ============== + // ============== Incremental Summary & Paging ============== - /** - * Result of parsing all metadata from a TsFile in a single pass. Opens the file only once, - * avoiding the overhead of 5+ separate file opens that the individual parse methods incur. - */ - public record AllMetadata( + /** Summary metadata of a TsFile plus the device index used by the paging endpoints. */ + public record SummaryMetadata( BasicMetadata basic, List measurements, - List rowGroups, - List chunks, - List tables) {} + List tables, + DeviceIndex deviceIndex) {} /** - * Parses all metadata from a TsFile in a single pass, opening the file only once. + * Parses summary metadata from a TsFile in one streaming pass. * - *

      This is significantly more efficient than calling {@link #parseMetadata(String)}, {@link - * #parseMeasurements(String)}, {@link #parseRowGroups(String)}, {@link #parseChunks(String)}, and - * {@link #parseTables(String)} separately, which each open their own reader (5-7 file opens - * total). + *

      Never materializes the file's whole timeseries metadata: each device's metadata is read, + * folded into counters, and dropped before the next device is visited. * * @param filePath path to the TsFile - * @return AllMetadata containing all parsed metadata + * @return summary metadata plus the device index of the same file version * @throws IOException if the file cannot be read */ - public AllMetadata parseAll(String filePath) throws IOException { - logger.debug("Parsing all metadata from TsFile in single pass: {}", filePath); + public SummaryMetadata parseSummary(String filePath) throws IOException { + logger.debug("Parsing summary metadata from TsFile: {}", filePath); + Path path = Paths.get(filePath); + long lastModified = Files.getLastModifiedTime(path).toMillis(); + long size = Files.size(path); + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { - ParsedMetadata parsed = parseMetadataInternal(reader); + String version = "V" + reader.readVersionNumber(); + List schemas = readTableSchemas(reader); + ScanResult scan = scan(reader, schemas, true); + BasicMetadata basic = new BasicMetadata( - parsed.getVersion(), - parsed.getStartTime(), - parsed.getEndTime(), - parsed.getDeviceCount(), - parsed.getMeasurementCount(), - parsed.getRowGroupCount(), - parsed.getChunkCount()); + version, + scan.startTime(), + scan.endTime(), + scan.devices().size(), + scan.measurementCount(), + scan.devices().size(), + totalChunks(scan)); List measurements = - parseMeasurements(reader).stream() - .map( - info -> - MeasurementDTO.builder() - .name(info.getName()) - .dataType(info.getDataType()) - .encoding(info.getEncoding()) - .compression(info.getCompression()) - .columnCategory(info.getColumnCategory()) - .build()) - .toList(); - - List rowGroups = - parseRowGroups(reader).stream() - .map( - info -> - RowGroupDTO.builder() - .index(info.getIndex()) - .device(info.getDevice()) - .startTime(info.getStartTime()) - .endTime(info.getEndTime()) - .chunkCount(info.getChunkCount()) - .build()) - .toList(); - - List chunks = - parseChunks(reader).stream() - .map( - info -> - ChunkDTO.builder() - .measurement(info.getMeasurement()) - .offset(info.getOffset()) - .size(info.getSize()) - .compressionRatio(info.getCompressionRatio()) - .device(info.getDevice()) - .dataType(info.getDataType()) - .encoding(info.getEncoding()) - .compression(info.getCompression()) - .startTime(info.getStartTime()) - .endTime(info.getEndTime()) - .numOfPoints(info.getNumOfPoints()) - .minValue(info.getMinValue()) - .maxValue(info.getMaxValue()) - .build()) - .toList(); - + scan.measurements().stream().map(TsFileParser::toMeasurementDTO).toList(); List tables = - parseTables(reader).stream() - .map( - info -> { - List tagDTOs = - info.getTagColumns().stream() - .map( - col -> - MeasurementDTO.builder() - .name(col.getName()) - .dataType(col.getDataType()) - .encoding(col.getEncoding()) - .compression(col.getCompression()) - .columnCategory(col.getColumnCategory()) - .build()) - .toList(); - List fieldDTOs = - info.getFieldColumns().stream() - .map( - col -> - MeasurementDTO.builder() - .name(col.getName()) - .dataType(col.getDataType()) - .encoding(col.getEncoding()) - .compression(col.getCompression()) - .columnCategory(col.getColumnCategory()) - .build()) - .toList(); - return org.apache.tsfile.viewer.dto.TableDTO.builder() - .tableName(info.getTableName()) - .tagColumns(tagDTOs) - .fieldColumns(fieldDTOs) - .totalColumns(info.getTotalColumns()) - .build(); - }) - .toList(); - - return new AllMetadata(basic, measurements, rowGroups, chunks, tables); + toTableInfos(schemas).stream().map(TsFileParser::toTableDTO).toList(); + + return new SummaryMetadata( + basic, + measurements, + tables, + DeviceIndex.of(lastModified, size, scan.devices(), scan.chunkCounts())); + } + } + + /** + * Builds the device index of a TsFile without parsing measurements. + * + * @param filePath path to the TsFile + * @return device index stamped with the file version it was read from + * @throws IOException if the file cannot be read + */ + public DeviceIndex parseDeviceIndex(String filePath) throws IOException { + Path path = Paths.get(filePath); + long lastModified = Files.getLastModifiedTime(path).toMillis(); + long size = Files.size(path); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + ScanResult scan = scan(reader, List.of(), false); + return DeviceIndex.of(lastModified, size, scan.devices(), scan.chunkCounts()); + } + } + + /** + * Reads the measurements carried by one device. + * + *

      Tree model files nest measurements under devices, so the measurement choices a client + * offers have to come from the selected device rather than from the whole file. Only that + * device's metadata is read. + * + * @param reader TsFile reader + * @param device device to read + * @return measurements of that device, sorted by name, excluding the VECTOR time column + * @throws IOException if reading fails + */ + public List readDeviceMeasurements( + TsFileSequenceReader reader, IDeviceID device) throws IOException { + Map> seriesMetadata = reader.readChunkMetadataInDevice(device); + List measurements = new ArrayList<>(); + for (Map.Entry> entry : seriesMetadata.entrySet()) { + List chunkMetadata = entry.getValue(); + if (chunkMetadata == null || chunkMetadata.isEmpty()) { + continue; + } + TSDataType dataType = chunkMetadata.get(0).getDataType(); + // VECTOR is the aligned time column, not a user measurement (same rule as the read path) + if (dataType == TSDataType.VECTOR) { + continue; + } + measurements.add( + MeasurementDTO.builder().name(entry.getKey()).dataType(dataType.toString()).build()); + } + measurements.sort(Comparator.comparing(MeasurementDTO::getName)); + return measurements; + } + + /** + * Reads one page of RowGroups. + * + *

      Only the devices inside the page window are read, so a page's memory is bounded by the page + * itself instead of the file's device count. + * + * @param reader TsFile reader + * @param deviceIndex index of the same file version + * @param offset absolute index of the first RowGroup to return + * @param limit maximum number of RowGroups to return + * @return RowGroups in file order; empty when the offset is past the last RowGroup + * @throws IOException if reading fails + */ + public List readRowGroupPage( + TsFileSequenceReader reader, DeviceIndex deviceIndex, int offset, int limit) + throws IOException { + int total = deviceIndex.deviceCount(); + if (limit <= 0 || offset < 0 || offset >= total) { + return List.of(); + } + + int end = Math.min(total, offset + limit); + List page = new ArrayList<>(end - offset); + for (int index = offset; index < end; index++) { + page.add(toRowGroupDTO(readRowGroupInfo(reader, deviceIndex.devices().get(index), index))); + } + return page; + } + + /** + * Reads one page of Chunks, stitching devices together when the page crosses a device boundary. + * + *

      Starts at the device that owns {@code offset} and stops once the page is full, so only the + * devices the page touches are ever read. + * + * @param reader TsFile reader + * @param deviceIndex index of the same file version + * @param offset absolute index of the first chunk to return + * @param limit maximum number of chunks to return + * @return chunks in file order; empty when the offset is past the last chunk + * @throws IOException if reading fails + */ + public List readChunkPage( + TsFileSequenceReader reader, DeviceIndex deviceIndex, int offset, int limit) + throws IOException { + int total = deviceIndex.chunkCount(); + if (limit <= 0 || offset < 0 || offset >= total) { + return List.of(); + } + + int end = Math.min(total, offset + limit); + List page = new ArrayList<>(end - offset); + + for (int position = deviceIndex.deviceIndexForChunk(offset); + position < deviceIndex.deviceCount() && page.size() < end - offset; + position++) { + int base = deviceIndex.chunksBeforeDevice(position); + List deviceChunks = readChunkInfos(reader, deviceIndex.devices().get(position)); + + for (int i = 0; i < deviceChunks.size(); i++) { + int absolute = base + i; + if (absolute < offset) { + continue; + } + if (absolute >= end) { + break; + } + page.add(toChunkDTO(deviceChunks.get(i))); + } } + return page; } + /** Counters of one streaming pass; no chunk or timeseries metadata outlives the scan. */ + private record ScanResult( + List devices, + int[] chunkCounts, + long startTime, + long endTime, + int measurementCount, + List measurements) {} + + /** + * Streams the file one device at a time and accumulates the summary counters. + * + * @param reader TsFile reader + * @param schemas V4 table schemas, already read, used for TAG columns and encoding/compression + * @param includeMeasurements whether measurement schema details are needed + * @return counters of the file plus its device list and per-device chunk counts + * @throws IOException if reading fails + */ + private ScanResult scan( + TsFileSequenceReader reader, List schemas, boolean includeMeasurements) + throws IOException { + List devices = reader.getAllDevices(); + int[] chunkCounts = new int[devices.size()]; + + Map columnCategories = new HashMap<>(); + Map columnSchemas = new HashMap<>(); + Map measurementMap = new LinkedHashMap<>(); + Set measurementNames = new HashSet<>(); + if (includeMeasurements) { + collectSchemaColumns(schemas, columnCategories, columnSchemas, measurementMap); + } + + long minStartTime = Long.MAX_VALUE; + long maxEndTime = Long.MIN_VALUE; + + for (int devicePosition = 0; devicePosition < devices.size(); devicePosition++) { + List deviceMetadata = + reader.getDeviceTimeseriesMetadata(devices.get(devicePosition)); + int deviceChunks = 0; + + if (deviceMetadata != null) { + for (TimeseriesMetadata tsMetadata : deviceMetadata) { + Statistics stats = tsMetadata.getStatistics(); + if (stats != null) { + minStartTime = Math.min(minStartTime, stats.getStartTime()); + maxEndTime = Math.max(maxEndTime, stats.getEndTime()); + } + + deviceChunks += countChunks(tsMetadata.getChunkMetadataList()); + + String measurementId = tsMetadata.getMeasurementId(); + measurementNames.add(measurementId); + if (includeMeasurements && !measurementMap.containsKey(measurementId)) { + measurementMap.put( + measurementId, + toMeasurementInfo(reader, tsMetadata, columnCategories, columnSchemas)); + } + } + } + chunkCounts[devicePosition] = deviceChunks; + } + + if (minStartTime == Long.MAX_VALUE) { + minStartTime = 0; + } + if (maxEndTime == Long.MIN_VALUE) { + maxEndTime = 0; + } + + return new ScanResult( + devices, + chunkCounts, + minStartTime, + maxEndTime, + measurementNames.size(), + includeMeasurements ? new ArrayList<>(measurementMap.values()) : List.of()); + } + + /** + * Counts the chunks that the paging endpoints address. + * + *

      Aligned files wrap some entries in non-{@link ChunkMetadata} objects that the chunk listing + * skips; counting them here keeps the reported total equal to the number of addressable chunks. + * + * @param chunkMetadataList chunk metadata of one timeseries, may be null + * @return number of addressable chunks + */ + private static int countChunks(List chunkMetadataList) { + if (chunkMetadataList == null) { + return 0; + } + int count = 0; + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + if (chunkMetadata instanceof ChunkMetadata) { + count++; + } + } + return count; + } + + private static int totalChunks(ScanResult scan) { + int total = 0; + for (int count : scan.chunkCounts()) { + total += count; + } + return total; + } + + /** + * Reads the V4 table schemas, returning an empty list for tree model files. + * + * @param reader TsFile reader + * @return table schemas of the file + */ + private List readTableSchemas(TsFileSequenceReader reader) { + try { + Path path = Paths.get(reader.getFileName()); + try (ITsFileReader v4Reader = new TsFileReaderBuilder().file(path.toFile()).build()) { + List schemas = v4Reader.getAllTableSchema(); + return schemas == null ? List.of() : schemas; + } + } catch (Exception e) { + logger.debug("Not a V4 table model file or failed to read TableSchema: {}", e.getMessage()); + return List.of(); + } + } + + private void collectSchemaColumns( + List schemas, + Map columnCategories, + Map columnSchemas, + Map measurementMap) { + for (TableSchema schema : schemas) { + List schemaList = schema.getColumnSchemas(); + List categories = schema.getColumnTypes(); + if (schemaList == null || categories == null) { + continue; + } + for (int i = 0; i < schemaList.size() && i < categories.size(); i++) { + IMeasurementSchema columnSchema = schemaList.get(i); + ColumnCategory category = categories.get(i); + if (category == null) { + continue; + } + String columnName = columnSchema.getMeasurementName(); + columnCategories.put(columnName, category); + columnSchemas.put(columnName, columnSchema); + + if (category == ColumnCategory.TAG) { + measurementMap.put( + columnName, + new MeasurementInfo( + columnName, + schemaDataType(columnSchema, "STRING"), + schemaEncoding(columnSchema), + schemaCompression(columnSchema), + "TAG")); + } + } + } + } + + private MeasurementInfo toMeasurementInfo( + TsFileSequenceReader reader, + TimeseriesMetadata tsMetadata, + Map columnCategories, + Map columnSchemas) { + String measurementId = tsMetadata.getMeasurementId(); + String dataType = tsMetadata.getTsDataType().toString(); + String encoding = "UNKNOWN"; + String compression = "UNKNOWN"; + + IMeasurementSchema tableSchema = columnSchemas.get(measurementId); + if (tableSchema != null) { + if (tableSchema.getEncodingType() != null) { + encoding = tableSchema.getEncodingType().toString(); + } + if (tableSchema.getCompressor() != null) { + compression = tableSchema.getCompressor().toString(); + } + } else { + List chunkMetadataList = tsMetadata.getChunkMetadataList(); + if (chunkMetadataList != null && !chunkMetadataList.isEmpty()) { + IChunkMetadata firstChunk = chunkMetadataList.get(0); + if (firstChunk instanceof ChunkMetadata chunkMeta) { + try { + org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); + if (chunk != null && chunk.getHeader() != null) { + org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); + if (chunkHeader.getEncodingType() != null) { + encoding = chunkHeader.getEncodingType().toString(); + } + if (chunkHeader.getCompressionType() != null) { + compression = chunkHeader.getCompressionType().toString(); + } + } + } catch (Exception e) { + logger.debug( + "Could not read ChunkHeader for measurement {}: {}", measurementId, e.getMessage()); + } + } + } + } + + ColumnCategory category = columnCategories.get(measurementId); + return new MeasurementInfo( + measurementId, + dataType, + encoding, + compression, + category != null ? category.toString() : "FIELD"); + } + + private List toTableInfos(List schemas) { + List tableInfos = new ArrayList<>(); + for (TableSchema schema : schemas) { + List tagColumns = new ArrayList<>(); + List fieldColumns = new ArrayList<>(); + + List columnSchemas = schema.getColumnSchemas(); + List categories = schema.getColumnTypes(); + if (columnSchemas != null && categories != null) { + for (int i = 0; i < columnSchemas.size() && i < categories.size(); i++) { + IMeasurementSchema columnSchema = columnSchemas.get(i); + ColumnCategory category = categories.get(i); + MeasurementInfo columnInfo = + new MeasurementInfo( + columnSchema.getMeasurementName(), + schemaDataType(columnSchema, "UNKNOWN"), + schemaEncoding(columnSchema), + schemaCompression(columnSchema), + category.toString()); + + if (category == ColumnCategory.TAG) { + tagColumns.add(columnInfo); + } else if (category == ColumnCategory.FIELD) { + fieldColumns.add(columnInfo); + } + } + } + tableInfos.add(new TableInfo(schema.getTableName(), tagColumns, fieldColumns)); + } + return tableInfos; + } + + private static String schemaDataType(IMeasurementSchema schema, String defaultValue) { + return schema.getType() != null ? schema.getType().toString() : defaultValue; + } + + private static String schemaEncoding(IMeasurementSchema schema) { + return schema.getEncodingType() != null ? schema.getEncodingType().toString() : "PLAIN"; + } + + private static String schemaCompression(IMeasurementSchema schema) { + return schema.getCompressor() != null ? schema.getCompressor().toString() : "UNCOMPRESSED"; + } + + private static MeasurementDTO toMeasurementDTO(MeasurementInfo info) { + return MeasurementDTO.builder() + .name(info.getName()) + .dataType(info.getDataType()) + .encoding(info.getEncoding()) + .compression(info.getCompression()) + .columnCategory(info.getColumnCategory()) + .build(); + } + + private static org.apache.tsfile.viewer.dto.TableDTO toTableDTO(TableInfo info) { + List tagDTOs = + info.getTagColumns().stream().map(TsFileParser::toMeasurementDTO).toList(); + List fieldDTOs = + info.getFieldColumns().stream().map(TsFileParser::toMeasurementDTO).toList(); + return org.apache.tsfile.viewer.dto.TableDTO.builder() + .tableName(info.getTableName()) + .tagColumns(tagDTOs) + .fieldColumns(fieldDTOs) + .totalColumns(info.getTotalColumns()) + .build(); + } + + private static RowGroupDTO toRowGroupDTO(RowGroupInfo info) { + return RowGroupDTO.builder() + .index(info.getIndex()) + .device(info.getDevice()) + .startTime(info.getStartTime()) + .endTime(info.getEndTime()) + .chunkCount(info.getChunkCount()) + .build(); + } + + private static ChunkDTO toChunkDTO(ChunkInfo info) { + return ChunkDTO.builder() + .measurement(info.getMeasurement()) + .offset(info.getOffset()) + .size(info.getSize()) + .compressionRatio(info.getCompressionRatio()) + .device(info.getDevice()) + .dataType(info.getDataType()) + .encoding(info.getEncoding()) + .compression(info.getCompression()) + .startTime(info.getStartTime()) + .endTime(info.getEndTime()) + .numOfPoints(info.getNumOfPoints()) + .minValue(info.getMinValue()) + .maxValue(info.getMaxValue()) + .build(); + } + + // ============== DTO Conversion Methods ============== /** Record containing basic TsFile metadata. */ diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index ebf37fa..85755fe 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -63,6 +63,9 @@ tsfile: task-timeout-minutes: 30 cache-max-size: 100 cache-ttl-minutes: 60 + # 单个文件允许建立紧凑 chunk 索引的最大 chunk 数。超过该阈值时不报错、不 OOM, + # 而是降级为只做结构与元数据检查、跳过数据区全量遍历,并在扫描结果中明确标注。 + max-chunks: 5000000 # SpringDoc OpenAPI Configuration springdoc: @@ -75,6 +78,6 @@ springdoc: # Logging Configuration logging: level: - org.apache.tsfile.viewer: DEBUG + org.apache.tsfile.viewer: INFO org.apache.tsfile: INFO org.springframework.web: INFO diff --git a/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java new file mode 100644 index 0000000..3f5fe52 --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.controller; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.DeviceRefDTO; +import org.apache.tsfile.viewer.dto.MeasurementDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; +import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.service.MetadataService; + +/** + * Tests for the paginated metadata endpoints: parameter defaults, clamping and response shape. + * + *

      Validates: offset default, offset clamping, limit cap, empty page past the end, JSON contract + * and the 404 semantics of an unknown fileId. + */ +class MetadataControllerPaginationTest { + + private static final int TOTAL_ROW_GROUPS = 87_488; + private static final int TOTAL_CHUNKS = 2_000_000; + + private MetadataService metadataService; + private MetadataController controller; + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + metadataService = mock(MetadataService.class); + controller = new MetadataController(metadataService); + mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + @DisplayName("row-groups defaults to offset=0 and limit=50") + void shouldApplyRowGroupDefaults() throws Exception { + when(metadataService.getRowGroupsPage("file-1", 0, 50)) + .thenReturn(PageDTO.of(List.of(rowGroup(0, "device_0")), TOTAL_ROW_GROUPS, 0, 50)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/row-groups")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(50)) + .andExpect(jsonPath("$.total").value(TOTAL_ROW_GROUPS)) + .andExpect(jsonPath("$.hasMore").value(true)) + .andExpect(jsonPath("$.items.length()").value(1)) + .andExpect(jsonPath("$.items[0].device").value("device_0")) + .andExpect(jsonPath("$.items[0].index").value(0)) + .andReturn(); + + System.out.println("[row-groups] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("negative offset becomes 0 and limit above 200 is capped") + void shouldClampRowGroupParameters() throws Exception { + when(metadataService.getRowGroupsPage("file-1", 0, 200)) + .thenReturn(PageDTO.of(List.of(rowGroup(0, "device_0")), TOTAL_ROW_GROUPS, 0, 200)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/row-groups").param("offset", "-7").param("limit", "100000")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(200)) + .andReturn(); + + verify(metadataService).getRowGroupsPage("file-1", 0, 200); + System.out.println("[row-groups clamped] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("offset beyond total returns an empty page with the real total") + void shouldReturnEmptyPagePastTheEnd() throws Exception { + when(metadataService.getRowGroupsPage("file-1", 100_000, 50)) + .thenReturn(PageDTO.of(List.of(), TOTAL_ROW_GROUPS, 100_000, 50)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/row-groups").param("offset", "100000")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(100_000)) + .andExpect(jsonPath("$.total").value(TOTAL_ROW_GROUPS)) + .andExpect(jsonPath("$.hasMore").value(false)) + .andExpect(jsonPath("$.items.length()").value(0)) + .andReturn(); + + System.out.println("[row-groups past end] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("chunks defaults to offset=0 and limit=50 and returns the element contract") + void shouldApplyChunkDefaults() throws Exception { + when(metadataService.getChunksPage("file-1", 0, 50)) + .thenReturn(PageDTO.of(List.of(chunk("s0", 0, "device_0")), TOTAL_CHUNKS, 0, 50)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/chunks")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(50)) + .andExpect(jsonPath("$.total").value(TOTAL_CHUNKS)) + .andExpect(jsonPath("$.hasMore").value(true)) + .andExpect(jsonPath("$.items[0].measurement").value("s0")) + .andExpect(jsonPath("$.items[0].device").value("device_0")) + .andExpect(jsonPath("$.items[0].numOfPoints").value(10)) + .andReturn(); + + System.out.println("[chunks] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("chunks honours offset and limit inside the page") + void shouldForwardChunkPageWindow() throws Exception { + when(metadataService.getChunksPage("file-1", 100, 3)) + .thenReturn(PageDTO.of(List.of(chunk("s1", 40, "device_1")), TOTAL_CHUNKS, 100, 3)); + + mockMvc + .perform(get("/api/meta/file-1/chunks").param("offset", "100").param("limit", "3")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(100)) + .andExpect(jsonPath("$.limit").value(3)) + .andExpect(jsonPath("$.hasMore").value(true)); + + verify(metadataService).getChunksPage("file-1", 100, 3); + } + + @Test + @DisplayName("unknown fileId keeps the existing 404 exception semantics") + void shouldPropagateNotFound() throws Exception { + when(metadataService.getRowGroupsPage(eq("missing"), anyInt(), anyInt())) + .thenThrow(new TsFileNotFoundException("File not found: missing", "missing")); + when(metadataService.getChunksPage(eq("missing"), anyInt(), anyInt())) + .thenThrow(new TsFileNotFoundException("File not found: missing", "missing")); + + assertThatThrownBy(() -> controller.getRowGroups("missing", null, null)) + .isInstanceOf(TsFileNotFoundException.class) + .hasMessageContaining("missing"); + assertThatThrownBy(() -> controller.getChunks("missing", null, null)) + .isInstanceOf(TsFileNotFoundException.class) + .hasMessageContaining("missing"); + } + + @Test + @DisplayName("summary response keeps rowGroups and chunks as empty arrays") + void shouldKeepSummaryContractWithEmptyDetailLists() throws Exception { + org.apache.tsfile.viewer.dto.TsFileMetadataDTO summary = + org.apache.tsfile.viewer.dto.TsFileMetadataDTO.builder() + .fileId("file-1") + .version("V3") + .deviceCount(87_488) + .measurementCount(12) + .rowGroupCount(87_488) + .chunkCount(2_000_000) + .measurements(List.of()) + .rowGroups(List.of()) + .chunks(List.of()) + .tables(List.of()) + .build(); + when(metadataService.getMetadata("file-1")).thenReturn(summary); + + mockMvc + .perform(get("/api/meta/file-1")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.rowGroups").isArray()) + .andExpect(jsonPath("$.rowGroups.length()").value(0)) + .andExpect(jsonPath("$.chunks").isArray()) + .andExpect(jsonPath("$.chunks.length()").value(0)) + .andExpect(jsonPath("$.rowGroupCount").value(87_488)) + .andExpect(jsonPath("$.chunkCount").value(2_000_000)); + + assertThat(summary.getRowGroups()).isEmpty(); + assertThat(summary.getChunks()).isEmpty(); + } + + private static RowGroupDTO rowGroup(int index, String device) { + return RowGroupDTO.builder() + .index(index) + .device(device) + .startTime(1_700_000_000_000L) + .endTime(1_700_000_001_000L) + .chunkCount(3) + .build(); + } + + private static ChunkDTO chunk(String measurement, long offset, String device) { + return ChunkDTO.builder() + .measurement(measurement) + .offset(offset) + .size(128) + .compressionRatio(2.5) + .device(device) + .dataType("INT64") + .encoding("RLE") + .compression("SNAPPY") + .startTime(1_700_000_000_000L) + .endTime(1_700_000_001_000L) + .numOfPoints(10L) + .minValue("1") + .maxValue("10") + .build(); + } + @Test + @DisplayName("device measurements are returned for a known device") + void shouldReturnDeviceMeasurements() throws Exception { + when(metadataService.getDeviceMeasurements("file-1", "root.a.b")) + .thenReturn( + List.of( + MeasurementDTO.builder().name("temperature").dataType("FLOAT").build(), + MeasurementDTO.builder().name("humidity").dataType("INT32").build())); + + mockMvc + .perform(get("/api/meta/file-1/devices/root.a.b/measurements")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.device").value("root.a.b")) + .andExpect(jsonPath("$.measurements.length()").value(2)) + .andExpect(jsonPath("$.measurements[0].name").value("temperature")); + } + + @Test + @DisplayName("device measurements are empty for an unknown device") + void shouldReturnEmptyMeasurementsForUnknownDevice() throws Exception { + when(metadataService.getDeviceMeasurements("file-1", "root.unknown")).thenReturn(List.of()); + + mockMvc + .perform(get("/api/meta/file-1/devices/root.unknown/measurements")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.device").value("root.unknown")) + .andExpect(jsonPath("$.measurements.length()").value(0)); + } + + @Test + @DisplayName("devices default to offset=0 and limit=50") + void shouldApplyDeviceDefaults() throws Exception { + when(metadataService.getDevicesPage("file-1", "t1", null, 0, 50)) + .thenReturn( + PageDTO.of( + List.of(new DeviceRefDTO("t1.d1", "t1", List.of("d1"))), 3, 0, 50)); + + mockMvc + .perform(get("/api/meta/file-1/devices").param("tableName", "t1")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(50)) + .andExpect(jsonPath("$.total").value(3)) + .andExpect(jsonPath("$.hasMore").value(true)) + .andExpect(jsonPath("$.items[0].device").value("t1.d1")) + .andExpect(jsonPath("$.items[0].tableName").value("t1")); + } + + @Test + @DisplayName("devices clamp limit to 200 and normalise a negative offset") + void shouldClampDeviceParameters() throws Exception { + when(metadataService.getDevicesPage("file-1", null, "abc", 0, 200)) + .thenReturn(PageDTO.of(List.of(), 0, 0, 200)); + + mockMvc + .perform( + get("/api/meta/file-1/devices") + .param("query", "abc") + .param("offset", "-5") + .param("limit", "500")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(200)); + } +} diff --git a/backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java b/backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java new file mode 100644 index 0000000..247015c --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Boundary tests for the pagination envelope and its parameter rules. + * + *

      Validates: offset default/negative/overflow, limit default/cap, hasMore correctness. + */ +class PageDTOTest { + + @Test + @DisplayName("offset is 0 when missing or negative") + void shouldNormalizeOffset() { + assertThat(PageDTO.normalizeOffset(null)).isZero(); + assertThat(PageDTO.normalizeOffset(0)).isZero(); + assertThat(PageDTO.normalizeOffset(-1)).isZero(); + assertThat(PageDTO.normalizeOffset(-1000)).isZero(); + assertThat(PageDTO.normalizeOffset(7)).isEqualTo(7); + } + + @Test + @DisplayName("limit defaults to 50 and is capped at 200") + void shouldNormalizeLimit() { + assertThat(PageDTO.normalizeLimit(null)).isEqualTo(PageDTO.DEFAULT_LIMIT).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(0)).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(-5)).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(1)).isEqualTo(1); + assertThat(PageDTO.normalizeLimit(50)).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(200)).isEqualTo(200); + assertThat(PageDTO.normalizeLimit(201)).isEqualTo(PageDTO.MAX_LIMIT).isEqualTo(200); + assertThat(PageDTO.normalizeLimit(Integer.MAX_VALUE)).isEqualTo(200); + } + + @Test + @DisplayName("first page reports hasMore when elements remain") + void shouldReportHasMoreOnFirstPage() { + PageDTO page = PageDTO.of(items(50), 100, 0, 50); + + assertThat(page.getOffset()).isZero(); + assertThat(page.getLimit()).isEqualTo(50); + assertThat(page.getTotal()).isEqualTo(100); + assertThat(page.getItems()).hasSize(50); + assertThat(page.isHasMore()).isTrue(); + } + + @Test + @DisplayName("last page reports no more elements when it ends exactly at total") + void shouldReportNoMoreAtExactBoundary() { + PageDTO page = PageDTO.of(items(50), 100, 50, 50); + + assertThat(page.isHasMore()).isFalse(); + } + + @Test + @DisplayName("partial page before the end still reports hasMore") + void shouldReportHasMoreOnPartialPage() { + PageDTO page = PageDTO.of(items(3), 10, 7, 50); + + assertThat(page.getItems()).hasSize(3); + assertThat(page.isHasMore()).isFalse(); + + PageDTO shorterPage = PageDTO.of(items(2), 10, 6, 2); + assertThat(shorterPage.isHasMore()).isTrue(); + } + + @Test + @DisplayName("offset beyond total yields an empty page with the real total") + void shouldReturnEmptyPageBeyondTotal() { + PageDTO page = PageDTO.of(List.of(), 87_488, 87_488, 50); + + assertThat(page.getItems()).isEmpty(); + assertThat(page.getTotal()).isEqualTo(87_488); + assertThat(page.getOffset()).isEqualTo(87_488); + assertThat(page.getLimit()).isEqualTo(50); + assertThat(page.isHasMore()).isFalse(); + } + + @Test + @DisplayName("far offset never overflows into hasMore=true") + void shouldNotOverflowOnHugeOffset() { + PageDTO page = PageDTO.of(List.of(), 10, Integer.MAX_VALUE, 200); + + assertThat(page.isHasMore()).isFalse(); + } + + @Test + @DisplayName("null items are treated as an empty page") + void shouldTreatNullItemsAsEmpty() { + PageDTO page = PageDTO.of(null, 0, 0, 50); + + assertThat(page.getItems()).isEmpty(); + assertThat(page.getTotal()).isZero(); + assertThat(page.isHasMore()).isFalse(); + } + + private static List items(int count) { + return java.util.stream.IntStream.range(0, count).mapToObj(i -> "item-" + i).toList(); + } +} diff --git a/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java index d4e3e1f..0e39dbc 100644 --- a/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java +++ b/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java @@ -412,4 +412,42 @@ void shouldStreamFilteredTreeModelData() throws Exception { } } } + + @Nested + @DisplayName("Filter Pushdown Tests") + class FilterPushdownTests { + + private File testFile; + + @BeforeEach + void createTestFile() throws Exception { + // Note: TsFileTestUtils writes a single table carrying a device_name FIELD, so this fixture + // cannot express a genuinely multi device tree model file. The device name pushdown + // (reading only the requested devices) is therefore verified against a real multi device + // tree model file rather than here; what is covered below is that a measurement filter is + // pushed into the read path instead of being applied only after reading. + testFile = + TsFileTestUtils.createTreeModelMultipleDevices( + tempDir.resolve("multi-device-tree.tsfile"), + Arrays.asList("root.device1", "root.device2"), + 50); + } + + @Test + @DisplayName("Should return rows when filtering by measurement") + void shouldReturnRowsForMeasurementFilter() throws Exception { + DataReadResult all = dataReader.readDataByTimeRange(testFile, null, null, 1000, 0); + assertThat(all.getData()).isNotEmpty(); + String measurement = all.getData().get(0).getValues().keySet().iterator().next(); + + DataReadResult filtered = + dataReader.readDataByTimeRange( + testFile, null, null, 1000, 0, null, List.of(measurement)); + + assertThat(filtered.getData()).isNotEmpty(); + for (DataRow row : filtered.getData()) { + assertThat(row.getValues()).containsKey(measurement); + } + } + } } diff --git a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java new file mode 100644 index 0000000..070f7d6 --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.read.TsFileSequenceReader; + +import org.apache.tsfile.viewer.dto.ErrorSeverity; +import org.apache.tsfile.viewer.dto.HealthStatus; +import org.apache.tsfile.viewer.dto.ScanErrorType; +import org.apache.tsfile.viewer.service.FileError; +import org.apache.tsfile.viewer.service.ScanResult; + +/** + * Tests for the compact chunk metadata index that replaced the per-chunk {@code Map} in {@link TsFileHealthChecker}. + * + *

      Covers: structure construction and lookup correctness, {@code quickCheck} early exit, the + * over-limit degradation path, and equality of the scan conclusions against the pre-change + * metadata map. + */ +class TsFileHealthCheckerCompactIndexTest { + + private static void assertEntry( + ChunkMetadataIndex index, + int position, + long offset, + String measurement, + long startTime, + long endTime) { + assertThat(index.offsetAt(position)).isEqualTo(offset); + assertThat(index.measurementAt(position)).isEqualTo(measurement); + assertThat(index.startTimeAt(position)).isEqualTo(startTime); + assertThat(index.endTimeAt(position)).isEqualTo(endTime); + } + + // ------------------------------------------------------------------------- + // Compact structure: construction and lookups + // ------------------------------------------------------------------------- + + @Nested + class CompactIndexTests { + + @Test + void seal_sortsByOffsetAndKeepsPayloadAttached() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + index.add(300L, "s3", 30L, 39L); + index.add(100L, "s1", 10L, 19L); + index.add(200L, "s2", 20L, 29L); + + index.seal(); + + assertThat(index.size()).isEqualTo(3); + assertEntry(index, 0, 100L, "s1", 10L, 19L); + assertEntry(index, 1, 200L, "s2", 20L, 29L); + assertEntry(index, 2, 300L, "s3", 30L, 39L); + } + + @Test + void cursor_forwardLookupsMatchEveryEntry() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + index.add(20L, "b", 200L, 209L); + index.add(10L, "a", 100L, 109L); + index.add(30L, "a", 300L, 309L); + index.seal(); + + ChunkMetadataIndex.Cursor cursor = index.cursor(); + + for (int position = 0; position < index.size(); position++) { + int found = cursor.find(index.offsetAt(position)); + assertThat(found).isEqualTo(position); + assertThat(cursor.measurement(found)).isEqualTo(index.measurementAt(position)); + assertThat(cursor.startTime(found)).isEqualTo(index.startTimeAt(position)); + assertThat(cursor.endTime(found)).isEqualTo(index.endTimeAt(position)); + } + } + + @Test + void cursor_returnsMinusOneForAbsentOffsetsAndKeepsAdvancing() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + index.add(100L, "a", 1L, 2L); + index.add(200L, "a", 3L, 4L); + index.add(300L, "a", 5L, 6L); + index.seal(); + + ChunkMetadataIndex.Cursor cursor = index.cursor(); + + assertThat(cursor.find(50L)).isEqualTo(-1); + assertThat(cursor.find(150L)).isEqualTo(-1); + assertThat(cursor.find(200L)).isEqualTo(1); + assertThat(cursor.find(250L)).isEqualTo(-1); + assertThat(cursor.find(350L)).isEqualTo(-1); + assertThat(cursor.find(300L)).isEqualTo(2); + } + + @Test + void cursor_backwardLookupFallsBackToBinarySearch() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + for (int i = 0; i < 40; i++) { + index.add(i * 10L, "m" + (i % 3), i, i); + } + index.seal(); + + ChunkMetadataIndex.Cursor cursor = index.cursor(); + + assertThat(cursor.find(390L)).isEqualTo(39); + // Traversal never asks for a smaller offset, but the answer must still be correct. + assertThat(cursor.find(10L)).isEqualTo(1); + assertThat(cursor.find(390L)).isEqualTo(39); + } + + @Test + void seal_sortsLargeOutOfOrderInputBeyondInitialCapacity() { + int entryCount = 5_000; + ChunkMetadataIndex index = new ChunkMetadataIndex(entryCount); + Map sourceIndexByOffset = new HashMap<>(); + // Deterministic shuffle so entries arrive far out of offset order. + for (int i = 0; i < entryCount; i++) { + int offset = (i * 7919) % entryCount; + // Payload is derived from the source index, so it must follow the entry through the sort. + index.add(offset, "s" + (i % 7), i * 1000L, i * 1000L + 999L); + sourceIndexByOffset.put(offset, i); + } + + index.seal(); + + assertThat(index.size()).isEqualTo(entryCount); + ChunkMetadataIndex.Cursor cursor = index.cursor(); + for (int offset = 0; offset < entryCount; offset++) { + int found = cursor.find(offset); + assertThat(found).as("offset %s must be indexed", offset).isGreaterThanOrEqualTo(0); + assertThat(index.offsetAt(found)).isEqualTo(offset); + + int sourceIndex = sourceIndexByOffset.get(offset); + assertThat(index.measurementAt(found)).isEqualTo("s" + (sourceIndex % 7)); + assertThat(index.startTimeAt(found)).isEqualTo(sourceIndex * 1000L); + assertThat(index.endTimeAt(found)).isEqualTo(sourceIndex * 1000L + 999L); + } + } + + @Test + void seal_toleratesDuplicateOffsets() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + for (int i = 0; i < 40; i++) { + index.add(7L, "dup" + i, i, i); + } + + index.seal(); + + assertThat(index.size()).isEqualTo(40); + assertThat(index.cursor().find(7L)).isGreaterThanOrEqualTo(0); + } + + @Test + void add_beyondMaxChunksThrowsWithLimit() { + ChunkMetadataIndex index = new ChunkMetadataIndex(2); + index.add(1L, "a", 0L, 0L); + index.add(2L, "a", 0L, 0L); + + assertThatThrownBy(() -> index.add(3L, "a", 0L, 0L)) + .isInstanceOf(ChunkMetadataIndex.ChunkCountExceededException.class) + .hasMessageContaining("2"); + } + + @Test + void emptyIndexHasNoEntriesAndFindsNothing() { + ChunkMetadataIndex index = new ChunkMetadataIndex(10); + index.seal(); + + assertThat(index.isEmpty()).isTrue(); + assertThat(index.cursor().find(0L)).isEqualTo(-1); + } + + @Test + void add_afterSealAndCursorBeforeSealAreRejected() { + ChunkMetadataIndex index = new ChunkMetadataIndex(10); + index.add(1L, "a", 0L, 5L); + + assertThatThrownBy(index::cursor).isInstanceOf(IllegalStateException.class); + + index.seal(); + assertThatThrownBy(() -> index.add(2L, "b", 6L, 9L)) + .isInstanceOf(IllegalStateException.class); + } + } + + // ------------------------------------------------------------------------- + // Checker integration: equivalence, degradation, early exit + // ------------------------------------------------------------------------- + + @Nested + class CheckerIntegrationTests { + + private static final List DEVICE_IDS = List.of("d1", "d2", "d3"); + + private Path createSampleFile(Path tempDir) throws Exception { + return TsFileTestUtils.createTableModelMultipleDevices( + tempDir.resolve("compact-index-sample.tsfile"), "wind", DEVICE_IDS, 20) + .toPath(); + } + + /** The pre-change implementation: one map entry per chunk, built from all metadata at once. */ + private Map legacyChunkMetadataMap(Path file) throws IOException { + Map offsetToChunkMetadata = new HashMap<>(); + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.toString())) { + for (List timeseriesMetadataList : + reader.getAllTimeseriesMetadata(true).values()) { + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { + for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) { + offsetToChunkMetadata.put(chunkMetadata.getOffsetOfChunkHeader(), chunkMetadata); + } + } + } + } + return offsetToChunkMetadata; + } + + @Test + void compactIndex_matchesLegacyMetadataMap_sameFileSameConclusion(@TempDir Path tempDir) + throws Exception { + Path file = createSampleFile(tempDir); + + Map legacyMap = legacyChunkMetadataMap(file); + assertThat(legacyMap).as("the generated file must contain chunks").isNotEmpty(); + + TsFileHealthChecker checker = new TsFileHealthChecker(); + List ascendingOffsets = new ArrayList<>(legacyMap.keySet()); + ascendingOffsets.sort(null); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.toString())) { + ChunkMetadataIndex compactIndex = checker.buildChunkMetadataIndex(reader); + + assertThat(compactIndex.isEmpty()).isEqualTo(legacyMap.isEmpty()); + assertThat(compactIndex.size()).isEqualTo(legacyMap.size()); + + ChunkMetadataIndex.Cursor cursor = compactIndex.cursor(); + for (Long offset : ascendingOffsets) { + int found = cursor.find(offset); + assertThat(found).as("offset %s must be indexed", offset).isGreaterThanOrEqualTo(0); + IChunkMetadata expected = legacyMap.get(offset); + assertThat(cursor.measurement(found)).isEqualTo(expected.getMeasurementUid()); + assertThat(cursor.startTime(found)).isEqualTo(expected.getStartTime()); + assertThat(cursor.endTime(found)).isEqualTo(expected.getEndTime()); + } + } + + // Below the cap the conclusion is unchanged: data-area traversal runs, no skip marker. + ScanResult result = checker.check(file); + assertThat(result.getHealthStatus()).isEqualTo(HealthStatus.HEALTHY); + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getFilePath()).isEqualTo(file.toString()); + assertThat(result.getFileSize()).isGreaterThan(0); + } + + @Test + void check_overChunkLimit_degradesAndReportsSkip(@TempDir Path tempDir) throws Exception { + Path file = createSampleFile(tempDir); + int chunkCount = legacyChunkMetadataMap(file).size(); + assertThat(chunkCount).isGreaterThan(1); + + ScanResult result = new TsFileHealthChecker(1).check(file); + + assertThat(result.getHealthStatus()).isEqualTo(HealthStatus.WARNING); + assertThat(result.getErrors()).hasSize(1); + FileError skipError = result.getErrors().get(0); + assertThat(skipError.getErrorType()).isEqualTo(ScanErrorType.DATA_VALIDATION_SKIPPED); + assertThat(skipError.getSeverity()).isEqualTo(ErrorSeverity.WARNING); + assertThat(skipError.getDescription()).contains("max-chunks").contains("skipped"); + assertThat(result.getFilePath()).isEqualTo(file.toString()); + assertThat(result.getFileSize()).isGreaterThan(0); + assertThat(result.getScanDurationMs()).isGreaterThanOrEqualTo(0); + } + + @Test + void quickCheck_stopsAtFirstChunk(@TempDir Path tempDir) throws Exception { + Path file = createSampleFile(tempDir); + int chunkCount = legacyChunkMetadataMap(file).size(); + assertThat(chunkCount).isGreaterThan(1); + + TsFileHealthChecker oneChunkChecker = new TsFileHealthChecker(1); + ScanResult quickResult = oneChunkChecker.quickCheck(file); + assertThat(quickResult.getHealthStatus()).isEqualTo(HealthStatus.HEALTHY); + assertThat(quickResult.getErrors()).isEmpty(); + + // The same cap makes the full check degrade, which proves the file holds more than one chunk + // and that quickCheck answered without indexing all of them. + ScanResult fullResult = oneChunkChecker.check(file); + assertThat(fullResult.getErrors()) + .extracting(FileError::getErrorType) + .containsExactly(ScanErrorType.DATA_VALIDATION_SKIPPED); + } + } +} diff --git a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java index 0252d80..afd7b13 100644 --- a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java +++ b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java @@ -50,6 +50,7 @@ class TsFileHealthCheckerPropertyTest { *

    • CHUNK_STATISTICS_MISMATCH → ERROR *
    • TIMESERIES_METADATA_MISMATCH → ERROR *
    • DATA_READ_ERROR → WARNING + *
    • DATA_VALIDATION_SKIPPED → WARNING *
    */ private static final Map EXPECTED_SEVERITY_MAP = @@ -58,7 +59,8 @@ class TsFileHealthCheckerPropertyTest { ScanErrorType.STRUCTURE_CORRUPT, ErrorSeverity.ERROR, ScanErrorType.CHUNK_STATISTICS_MISMATCH, ErrorSeverity.ERROR, ScanErrorType.TIMESERIES_METADATA_MISMATCH, ErrorSeverity.ERROR, - ScanErrorType.DATA_READ_ERROR, ErrorSeverity.WARNING); + ScanErrorType.DATA_READ_ERROR, ErrorSeverity.WARNING, + ScanErrorType.DATA_VALIDATION_SKIPPED, ErrorSeverity.WARNING); // --------------------------------------------------------------------------- // Property 5: ScanResult 结构完整性与严重程度映射 diff --git a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java new file mode 100644 index 0000000..d2106c7 --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; + +/** + * Tests for incremental RowGroup/Chunk paging against a real TsFile. + * + *

    Checks that device-based pages tile the full metadata exactly, that a chunk page crossing a + * device boundary is stitched correctly, and that offsets past the end return an empty page. + * + *

    Validates: pagination boundaries plus the "no full materialization" paging contract. + */ +class TsFilePaginationTest { + + private static final int DEVICES = 4; + private static final int MEASUREMENTS = 3; + private static final int ROWS = 10; + + @TempDir Path tempDir; + + private final TsFileParser parser = new TsFileParser(); + + private Path tsFile; + + @BeforeEach + void setUp() throws Exception { + tsFile = createTableModelTsFile(tempDir.resolve("pagination.tsfile"), DEVICES, MEASUREMENTS, ROWS); + } + + @Test + @DisplayName("Device index totals match the full enumeration of the file") + void shouldMatchFullEnumeration() throws Exception { + DeviceIndex index = parser.parseDeviceIndex(tsFile.toString()); + List allRowGroups = parser.parseRowGroups(tsFile.toString()); + List allChunks = parser.parseChunks(tsFile.toString()); + + System.out.printf( + "[index] devices=%d chunks=%d chunksPerDevice=%d enumeratedRowGroups=%d enumeratedChunks=%d%n", + index.deviceCount(), + index.chunkCount(), + index.chunkCountOf(0), + allRowGroups.size(), + allChunks.size()); + + assertThat(index.deviceCount()).isEqualTo(DEVICES).isEqualTo(allRowGroups.size()); + assertThat(index.chunkCount()).isEqualTo(allChunks.size()); + for (int position = 0; position < index.deviceCount(); position++) { + assertThat(index.chunkCountOf(position)).isEqualTo(allRowGroups.get(position).getChunkCount()); + } + } + + @Test + @DisplayName("Device index is stamped with the file version it was built from") + void shouldStampDeviceIndexWithFileVersion() throws Exception { + DeviceIndex index = parser.parseDeviceIndex(tsFile.toString()); + + long lastModified = Files.getLastModifiedTime(tsFile).toMillis(); + long size = Files.size(tsFile); + + assertThat(index.matches(lastModified, size)).isTrue(); + assertThat(index.matches(lastModified + 1, size)).isFalse(); + assertThat(index.matches(lastModified, size + 1)).isFalse(); + } + + @Test + @DisplayName("offset=0 returns the first page") + void shouldReturnFirstRowGroupPage() throws Exception { + List all = parser.parseRowGroups(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + List page = parser.readRowGroupPage(reader, index(), 0, 2); + + assertThat(page).hasSize(2); + assertThat(page.get(0).getIndex()).isZero(); + assertThat(page.get(0).getDevice()).isEqualTo(all.get(0).getDevice()); + assertThat(page.get(1).getDevice()).isEqualTo(all.get(1).getDevice()); + } + } + + @Test + @DisplayName("RowGroup pages tile the whole list without gaps or duplicates") + void shouldTileEveryRowGroup() throws Exception { + List all = parser.parseRowGroups(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + List collected = new ArrayList<>(); + + int offset = 0; + List> pages = new ArrayList<>(); + while (offset < index.deviceCount()) { + List page = parser.readRowGroupPage(reader, index, offset, 2); + assertThat(page).isNotEmpty(); + pages.add(PageDTO.of(page, index.deviceCount(), offset, 2)); + collected.addAll(page); + offset += page.size(); + } + + assertThat(collected) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all); + assertThat(pages).isNotEmpty(); + assertThat(pages.get(pages.size() - 1).isHasMore()).isFalse(); + if (pages.size() > 1) { + assertThat(pages.get(0).isHasMore()).isTrue(); + } + } + } + + @Test + @DisplayName("RowGroup offset beyond total returns an empty list") + void shouldReturnEmptyRowGroupPageBeyondTotal() throws Exception { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + + assertThat(parser.readRowGroupPage(reader, index, index.deviceCount(), 50)).isEmpty(); + assertThat(parser.readRowGroupPage(reader, index, 10_000, 50)).isEmpty(); + assertThat(parser.readRowGroupPage(reader, index, Integer.MAX_VALUE, 200)).isEmpty(); + } + } + + @Test + @DisplayName("Chunk pages tile the whole list and cross device boundaries correctly") + void shouldTileEveryChunkIncludingDeviceBoundaries() throws Exception { + List all = parser.parseChunks(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + assertThat(index.chunkCount()).isEqualTo(all.size()); + + int chunksPerDevice = index.chunkCountOf(0); + assertThat(chunksPerDevice).isPositive(); + + // A page starting on the last chunk of a device and ending inside the next one. + int crossing = chunksPerDevice - 1; + List crossingPage = parser.readChunkPage(reader, index, crossing, 3); + assertThat(crossingPage).hasSize(3); + assertThat(crossingPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(crossing, crossing + 3)); + assertThat(crossingPage.get(0).getDevice()).isNotEqualTo(crossingPage.get(2).getDevice()); + + // A page starting exactly on a device boundary. + List boundaryPage = parser.readChunkPage(reader, index, chunksPerDevice, 4); + assertThat(boundaryPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(chunksPerDevice, chunksPerDevice + 4)); + + // A page aligned inside a single device. + List innerPage = parser.readChunkPage(reader, index, 1, 2); + assertThat(innerPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(1, 3)); + + // A trailing partial page. + List tailPage = parser.readChunkPage(reader, index, all.size() - 2, 50); + assertThat(tailPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(all.size() - 2, all.size())); + } + } + + @Test + @DisplayName("Chunk pages tile the whole list with a non-dividing page size") + void shouldTileAllChunksWithSmallPages() throws Exception { + List all = parser.parseChunks(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + List collected = new ArrayList<>(); + + int offset = 0; + while (offset < index.chunkCount()) { + List page = parser.readChunkPage(reader, index, offset, 2); + assertThat(page).isNotEmpty(); + collected.addAll(page); + offset += page.size(); + } + + assertThat(collected) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all); + } + } + + @Test + @DisplayName("Chunk offset beyond total returns an empty list") + void shouldReturnEmptyChunkPageBeyondTotal() throws Exception { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + + assertThat(parser.readChunkPage(reader, index, index.chunkCount(), 50)).isEmpty(); + assertThat(parser.readChunkPage(reader, index, 10_000, 50)).isEmpty(); + assertThat(parser.readChunkPage(reader, index, Integer.MAX_VALUE, 200)).isEmpty(); + } + } + + @Test + @DisplayName("Summary reports counts but never carries RowGroup or Chunk lists") + void shouldNotBuildDetailListsInSummary() throws Exception { + TsFileParser.SummaryMetadata summary = parser.parseSummary(tsFile.toString()); + + int enumeratedChunks = parser.parseChunks(tsFile.toString()).size(); + + System.out.printf( + "[summary] devices=%d measurements=%d measurementCount=%d chunks=%d%n", + summary.basic().deviceCount(), + summary.measurements().size(), + summary.basic().measurementCount(), + summary.basic().chunkCount()); + + assertThat(summary.basic().deviceCount()).isEqualTo(DEVICES); + assertThat(summary.basic().rowGroupCount()).isEqualTo(summary.deviceIndex().deviceCount()); + assertThat(summary.basic().chunkCount()).isEqualTo(summary.deviceIndex().chunkCount()); + assertThat(summary.basic().chunkCount()).isEqualTo(enumeratedChunks); + assertThat(summary.deviceIndex().deviceCount()).isEqualTo(DEVICES); + assertThat(summary.measurements()).isNotEmpty(); + assertThat(summary.basic().measurementCount()).isPositive(); + } + + private DeviceIndex index() throws IOException { + return parser.parseDeviceIndex(tsFile.toString()); + } + + private static Path createTableModelTsFile( + Path filePath, int numDevices, int numMeasurements, int numRows) throws Exception { + Files.deleteIfExists(filePath); + File file = filePath.toFile(); + + List columnNames = new ArrayList<>(); + List dataTypes = new ArrayList<>(); + List columnSchemas = new ArrayList<>(); + + columnNames.add("device_id"); + dataTypes.add(TSDataType.STRING); + columnSchemas.add( + new ColumnSchemaBuilder() + .name("device_id") + .dataType(TSDataType.STRING) + .category(ColumnCategory.TAG) + .build()); + + for (int m = 0; m < numMeasurements; m++) { + columnNames.add("s" + m); + dataTypes.add(TSDataType.INT64); + columnSchemas.add( + new ColumnSchemaBuilder() + .name("s" + m) + .dataType(TSDataType.INT64) + .category(ColumnCategory.FIELD) + .build()); + } + + TableSchema tableSchema = new TableSchema("test_table", columnSchemas); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { + for (int d = 0; d < numDevices; d++) { + Tablet tablet = new Tablet(columnNames, dataTypes); + for (int row = 0; row < numRows; row++) { + tablet.addTimestamp(row, d * 1_000_000L + row); + tablet.addValue(row, "device_id", "device_" + d); + for (int m = 0; m < numMeasurements; m++) { + tablet.addValue(row, "s" + m, (long) (d * 1000 + m * 100 + row)); + } + } + writer.write(tablet); + } + } + + return filePath; + } +} diff --git a/frontend/src/api/tsfile/meta.ts b/frontend/src/api/tsfile/meta.ts index 7ce0f5f..69692f2 100644 --- a/frontend/src/api/tsfile/meta.ts +++ b/frontend/src/api/tsfile/meta.ts @@ -17,9 +17,60 @@ * under the License. */ -import type { TsFileMetadata } from "./types"; +import type { + Chunk, + DeviceRef, + DeviceMeasurements, + PagedResponse, + RowGroup, + TsFileMetadata, +} from "./types"; import { apiClient } from "../request"; +/** + * 元数据摘要。 + * + * 注意:响应中的 rowGroups / chunks 恒为空数组(大文件的明细会撑爆后端内存), + * 行数看 rowGroupCount / chunkCount,明细走 getRowGroupsPage / getChunksPage。 + */ export function getMetadata(fileId: string) { return apiClient.get(`/meta/${fileId}`); } + +/** RowGroup 明细分页 */ +export function getRowGroupsPage(fileId: string, offset = 0, limit = 50) { + return apiClient.get>(`/meta/${fileId}/row-groups`, { + params: { offset, limit }, + }); +} + +/** Chunk 明细分页 */ +export function getChunksPage(fileId: string, offset = 0, limit = 50) { + return apiClient.get>(`/meta/${fileId}/chunks`, { + params: { offset, limit }, + }); +} + +/** + * 单个设备的测点清单。 + * + * 树模型的层级是「设备 → 测点」,因此测点下拉要跟随所选设备;设备不存在时返回空列表。 + */ +export function getDeviceMeasurements(fileId: string, device: string) { + return apiClient.get( + `/meta/${fileId}/devices/${encodeURIComponent(device)}/measurements`, + ); +} + +/** + * 设备分页(按表过滤 + 子串搜索)。 + * + * 表模型的层级是「表 → 设备 → 字段」,设备下拉必须按所选表取。该端点由已缓存的设备索引 + * 推导(设备名本身即 tableName.tag1.tag2),因此支持分页与搜索而不做全表行扫描。 + */ +export function getDevicesPage( + fileId: string, + params: { tableName?: string; query?: string; offset?: number; limit?: number } = {}, +) { + return apiClient.get>(`/meta/${fileId}/devices`, { params }); +} diff --git a/frontend/src/api/tsfile/scan-types.ts b/frontend/src/api/tsfile/scan-types.ts index 166f84d..a097385 100644 --- a/frontend/src/api/tsfile/scan-types.ts +++ b/frontend/src/api/tsfile/scan-types.ts @@ -35,7 +35,8 @@ export type ScanErrorType = | "DATA_READ_ERROR" | "FORMAT_INCOMPATIBLE" | "STRUCTURE_CORRUPT" - | "TIMESERIES_METADATA_MISMATCH"; + | "TIMESERIES_METADATA_MISMATCH" + | "DATA_VALIDATION_SKIPPED"; export type ErrorSeverity = "CRITICAL" | "ERROR" | "WARNING"; diff --git a/frontend/src/api/tsfile/types.ts b/frontend/src/api/tsfile/types.ts index c660cd8..8189887 100644 --- a/frontend/src/api/tsfile/types.ts +++ b/frontend/src/api/tsfile/types.ts @@ -100,6 +100,37 @@ export interface Chunk { maxValue?: string; } +/** + * 分页响应,由 /api/meta/{fileId}/row-groups 与 /api/meta/{fileId}/chunks 共用。 + * + * items 的元素类型由调用方指定;offset/limit 回显请求参数(limit 可能被服务端收窄), + * hasMore 是“是否还有下一页”的唯一判据,不要用 offset + items.length < total 自行推算。 + */ +export interface PagedResponse { + items: T[]; + total: number; + offset: number; + limit: number; + hasMore: boolean; +} + +/** 设备引用:设备名、所属表、TAG 值(都由设备标识本身推导,不需要行扫描) */ +export interface DeviceRef { + device: string; + tableName: string | null; + tagValues: string[]; +} + +/** + * 单个设备携带的测点。 + * + * 树模型文件的测点挂在设备下,测点选项应跟随所选设备,而不是取整文件并集。 + */ +export interface DeviceMeasurements { + device: string; + measurements: Measurement[]; +} + export interface TsFileMetadata { fileId: string; version: string; @@ -109,7 +140,9 @@ export interface TsFileMetadata { rowGroupCount: number; chunkCount: number; measurements: Measurement[]; + /** 摘要接口恒为空数组,明细请用 metaApi.getRowGroupsPage() 分页拉取 */ rowGroups: RowGroup[]; + /** 摘要接口恒为空数组,明细请用 metaApi.getChunksPage() 分页拉取 */ chunks: Chunk[]; tables?: Table[]; // V4 table model tables } diff --git a/frontend/src/components/tsfile/ChunksTable.vue b/frontend/src/components/tsfile/ChunksTable.vue new file mode 100644 index 0000000..0f89eeb --- /dev/null +++ b/frontend/src/components/tsfile/ChunksTable.vue @@ -0,0 +1,331 @@ + + + + + diff --git a/frontend/src/components/tsfile/RowGroupsTable.vue b/frontend/src/components/tsfile/RowGroupsTable.vue index d5fdb03..4f1a945 100644 --- a/frontend/src/components/tsfile/RowGroupsTable.vue +++ b/frontend/src/components/tsfile/RowGroupsTable.vue @@ -19,42 +19,149 @@