From b483c002cf4498d9c6328d9ecc5c47e262307207 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 12:56:29 +0800 Subject: [PATCH 1/9] =?UTF-8?q?feat(backend):=20=E5=85=83=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E6=98=8E=E7=BB=86=E6=94=B9=E7=9C=9F=E5=88=86=E9=A1=B5?= =?UTF-8?q?=EF=BC=8C=E6=91=98=E8=A6=81=E4=B8=8D=E5=86=8D=E5=85=A8=E9=87=8F?= =?UTF-8?q?=E7=89=A9=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 解析含 87,488 个 RowGroup 的 TsFile 时,后端把整文件的每个 chunk 逐条建成 DTO(约 200 万条)塞进 /api/meta 响应,堆被打爆;OOM 又打死了 Tomcat 的 Poller 线程,Acceptor 仍在 accept,导致服务"端口通但永无响应"假死。 - /api/meta/{fileId} 只返回摘要,rowGroups/chunks 恒为空数组(字段保留以兼容契约) - 新增 /api/meta/{fileId}/row-groups 与 /chunks 分页端点, offset/limit 归一化,limit 上限 200,hasMore 用 long 计算防溢出 - 摘要与分页都改为按设备增量读取,不再全量物化 TimeseriesMetadata - 摘要缓存只存摘要;新增设备索引缓存,按 mtime+size 校验文件版本 - tsfile 依赖 2.3.0 -> 2.5.0 - 应用日志级别 DEBUG -> INFO(原包名写错导致线上一直是 DEBUG) Tests: 77 run, 0 failures, 13 skipped(既有 @EnabledIf 集成用例因样本目录缺失而跳过) --- backend/pom.xml | 2 +- .../viewer/controller/MetadataController.java | 65 +- .../org/apache/tsfile/viewer/dto/PageDTO.java | 145 +++ .../viewer/service/MetadataService.java | 214 +++- .../tsfile/viewer/tsfile/DeviceIndex.java | 155 +++ .../tsfile/viewer/tsfile/TsFileParser.java | 1073 +++++++++-------- backend/src/main/resources/application.yml | 2 +- .../MetadataControllerPaginationTest.java | 242 ++++ .../apache/tsfile/viewer/dto/PageDTOTest.java | 124 ++ .../viewer/tsfile/TsFilePaginationTest.java | 317 +++++ 10 files changed, 1792 insertions(+), 547 deletions(-) create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java create mode 100644 backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java create mode 100644 backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java create mode 100644 backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java diff --git a/backend/pom.xml b/backend/pom.xml index 719be9e..8151717 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -38,7 +38,7 @@ 17 - 2.3.0 + 2.5.0 3.2.0 2.8.0 1.22.0 diff --git a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java index 4724d72..0913892 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java @@ -27,8 +27,12 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; @@ -40,7 +44,9 @@ *

Provides endpoints for: * *

* *

Validates: Requirement 8.3 (Metadata endpoint) @@ -58,13 +64,14 @@ public MetadataController(MetadataService metadataService) { } /** - * Gets complete metadata for a TsFile. + * Gets the metadata summary for a TsFile. * - *

Returns metadata including version, time range, device/measurement counts, and detailed - * lists of measurements, RowGroups, and Chunks. + *

Returns version, time range, device/measurement counts and schema details. The {@code + * rowGroups} and {@code chunks} fields are kept for contract compatibility but are always empty; + * details come from the paging endpoints below. * * @param fileId the file identifier - * @return TsFileMetadataDTO with complete metadata + * @return TsFileMetadataDTO with summary metadata */ @GetMapping("/{fileId}") public ResponseEntity getMetadata(@PathVariable String fileId) @@ -74,4 +81,52 @@ public ResponseEntity getMetadata(@PathVariable String fileId TsFileMetadataDTO metadata = metadataService.getMetadata(fileId); return ResponseEntity.ok(metadata); } + + /** + * Gets one page of RowGroups of a TsFile. + * + * @param fileId the file identifier + * @param offset zero-based offset, negative values are treated as 0 + * @param limit page size, defaults to 50 and is capped at 200 + * @return page of RowGroups + */ + @GetMapping("/{fileId}/row-groups") + public ResponseEntity> getRowGroups( + @PathVariable String fileId, + @RequestParam(name = "offset", required = false) Integer offset, + @RequestParam(name = "limit", required = false) Integer limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + int pageOffset = PageDTO.normalizeOffset(offset); + int pageLimit = PageDTO.normalizeLimit(limit); + + logger.debug( + "Getting RowGroup page for fileId={}, offset={}, limit={}", fileId, pageOffset, pageLimit); + + PageDTO page = metadataService.getRowGroupsPage(fileId, pageOffset, pageLimit); + return ResponseEntity.ok(page); + } + + /** + * Gets one page of Chunks of a TsFile. + * + * @param fileId the file identifier + * @param offset zero-based offset, negative values are treated as 0 + * @param limit page size, defaults to 50 and is capped at 200 + * @return page of Chunks + */ + @GetMapping("/{fileId}/chunks") + public ResponseEntity> getChunks( + @PathVariable String fileId, + @RequestParam(name = "offset", required = false) Integer offset, + @RequestParam(name = "limit", required = false) Integer limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + int pageOffset = PageDTO.normalizeOffset(offset); + int pageLimit = PageDTO.normalizeLimit(limit); + + logger.debug( + "Getting chunk page for fileId={}, offset={}, limit={}", fileId, pageOffset, pageLimit); + + PageDTO page = metadataService.getChunksPage(fileId, pageOffset, pageLimit); + return ResponseEntity.ok(page); + } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java new file mode 100644 index 0000000..771de55 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/PageDTO.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import java.util.List; + +/** + * Generic page envelope used by the paginated metadata endpoints. + * + *

Serializes as {@code {"items": [...], "total": n, "offset": n, "limit": n, "hasMore": bool}}. + * + * @param element type of one page + */ +public class PageDTO { + + /** Page size applied when the client does not ask for one. */ + public static final int DEFAULT_LIMIT = 50; + + /** Hard cap of a single request, keeping one page's memory footprint bounded. */ + public static final int MAX_LIMIT = 200; + + private List items; + private int total; + private int offset; + private int limit; + private boolean hasMore; + + /** Default constructor for JSON deserialization. */ + public PageDTO() {} + + /** + * Creates a page envelope. + * + * @param items page elements + * @param total total number of elements across all pages + * @param offset offset of the first element of this page + * @param limit page size requested for this page + * @param hasMore whether elements beyond this page exist + */ + public PageDTO(List items, int total, int offset, int limit, boolean hasMore) { + this.items = items; + this.total = total; + this.offset = offset; + this.limit = limit; + this.hasMore = hasMore; + } + + /** + * Builds a page envelope, deriving {@code hasMore} from the payload. + * + * @param items page elements (null is treated as an empty page) + * @param total total number of elements across all pages + * @param offset offset of the first element of this page + * @param limit page size requested for this page + * @param element type + * @return page envelope + */ + public static PageDTO of(List items, int total, int offset, int limit) { + List safeItems = items == null ? List.of() : items; + int safeTotal = Math.max(total, 0); + boolean hasMore = (long) offset + safeItems.size() < safeTotal; + return new PageDTO<>(safeItems, safeTotal, offset, limit, hasMore); + } + + /** + * Normalizes a requested offset: missing or negative values become 0. + * + * @param offset raw request value + * @return non-negative offset + */ + public static int normalizeOffset(Integer offset) { + return offset == null || offset < 0 ? 0 : offset; + } + + /** + * Normalizes a requested limit: missing or non-positive values fall back to {@link + * #DEFAULT_LIMIT}, values above {@link #MAX_LIMIT} are capped to it. + * + * @param limit raw request value + * @return effective page size in {@code [1, MAX_LIMIT]} + */ + public static int normalizeLimit(Integer limit) { + if (limit == null || limit <= 0) { + return DEFAULT_LIMIT; + } + return Math.min(limit, MAX_LIMIT); + } + + public List getItems() { + return items; + } + + public void setItems(List items) { + this.items = items; + } + + public int getTotal() { + return total; + } + + public void setTotal(int total) { + this.total = total; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public boolean isHasMore() { + return hasMore; + } + + public void setHasMore(boolean hasMore) { + this.hasMore = hasMore; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java index b2965cb..fec33b5 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java @@ -21,7 +21,9 @@ import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.List; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; @@ -30,12 +32,17 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.viewer.config.TsFileProperties; -import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.dto.TimeRange; +import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.EmptyFileException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.tsfile.DeviceIndex; import org.apache.tsfile.viewer.tsfile.TsFileParser; import jakarta.annotation.PostConstruct; @@ -46,9 +53,9 @@ *

Provides functionality for: * *

* *

Validates: Requirement 2.1-2.5, 5.1 (Metadata extraction, caching) @@ -58,12 +65,22 @@ public class MetadataService { private static final Logger logger = LoggerFactory.getLogger(MetadataService.class); + /** + * Device indexes are small (a device name list plus one int per device) but each one pins data + * for a whole file, so this cache stays deliberately tiny instead of tracking file count. + */ + private static final int DEVICE_INDEX_CACHE_SIZE = 8; + private final TsFileProperties tsFileProperties; private final FileService fileService; private final TsFileParser tsFileParser; + /** Cached summaries only: they carry no RowGroup or Chunk lists, so entries stay small. */ private Cache metadataCache; + /** Cached device indexes, validated against the file's mtime and size on every use. */ + private Cache deviceIndexCache; + public MetadataService( TsFileProperties tsFileProperties, FileService fileService, TsFileParser tsFileParser) { this.tsFileProperties = tsFileProperties; @@ -78,7 +95,10 @@ public void init() { int ttlMinutes = tsFileProperties.getCache().getMetadata().getTtlMinutes(); logger.info( - "Initializing MetadataService cache with maxSize={}, ttlMinutes={}", maxSize, ttlMinutes); + "Initializing MetadataService cache with maxSize={}, ttlMinutes={}, deviceIndexMaxSize={}", + maxSize, + ttlMinutes, + DEVICE_INDEX_CACHE_SIZE); this.metadataCache = Caffeine.newBuilder() @@ -86,15 +106,22 @@ public void init() { .expireAfterWrite(ttlMinutes, TimeUnit.MINUTES) .recordStats() .build(); + this.deviceIndexCache = + Caffeine.newBuilder() + .maximumSize(DEVICE_INDEX_CACHE_SIZE) + .expireAfterWrite(ttlMinutes, TimeUnit.MINUTES) + .build(); } /** - * Gets metadata for a TsFile by fileId. + * Gets the metadata summary for a TsFile by fileId. * - *

First checks the cache, then parses the file if not cached. + *

The returned {@code rowGroups} and {@code chunks} lists are always empty: details are served + * by {@link #getRowGroupsPage} and {@link #getChunksPage} instead, so building a summary never + * materializes the whole file's chunk metadata. * * @param fileId the file identifier - * @return TsFileMetadataDTO with complete metadata + * @return TsFileMetadataDTO with summary metadata * @throws TsFileNotFoundException if the file is not found * @throws AccessDeniedException if the path is not allowed * @throws IOException if the file cannot be read @@ -103,21 +130,100 @@ public TsFileMetadataDTO getMetadata(String fileId) throws TsFileNotFoundException, AccessDeniedException, IOException { logger.debug("Getting metadata for fileId={}", fileId); - // Check cache first TsFileMetadataDTO cached = metadataCache.getIfPresent(fileId); if (cached != null) { logger.debug("Returning cached metadata for fileId={}", fileId); return cached; } - // Get file path and parse metadata + String filePath = resolveFilePath(fileId); + logger.debug("Parsing metadata summary for file: {}", filePath); + + TsFileParser.SummaryMetadata summary = tsFileParser.parseSummary(filePath); + TsFileMetadataDTO metadata = toSummaryDTO(fileId, summary); + + metadataCache.put(fileId, metadata); + deviceIndexCache.put(fileId, summary.deviceIndex()); + logger.debug("Cached metadata for fileId={}", fileId); + + return metadata; + } + + /** + * Gets one page of RowGroups (one RowGroup per device). + * + *

Only the devices inside the page window are read from the file. + * + * @param fileId the file identifier + * @param offset zero-based offset of the first RowGroup + * @param limit maximum number of RowGroups to return + * @return page of RowGroups with the file's total RowGroup count + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public PageDTO getRowGroupsPage(String fileId, int offset, int limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting RowGroup page for fileId={}, offset={}, limit={}", fileId, offset, limit); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + int total = deviceIndex.deviceCount(); + + List items = List.of(); + if (offset < total && limit > 0) { + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + items = tsFileParser.readRowGroupPage(reader, deviceIndex, offset, limit); + } + } + + return PageDTO.of(items, total, offset, limit); + } + + /** + * Gets one page of Chunks, crossing device boundaries when a page needs to. + * + *

Only the devices the page touches are read from the file. + * + * @param fileId the file identifier + * @param offset zero-based offset of the first chunk + * @param limit maximum number of chunks to return + * @return page of Chunks with the file's total chunk count + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public PageDTO getChunksPage(String fileId, int offset, int limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting chunk page for fileId={}, offset={}, limit={}", fileId, offset, limit); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + int total = deviceIndex.chunkCount(); + + List items = List.of(); + if (offset < total && limit > 0) { + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + items = tsFileParser.readChunkPage(reader, deviceIndex, offset, limit); + } + } + + return PageDTO.of(items, total, offset, limit); + } + + /** + * Resolves a fileId to an existing, non-empty file path. + * + * @param fileId the file identifier + * @return absolute file path + */ + private String resolveFilePath(String fileId) + throws TsFileNotFoundException, AccessDeniedException { String filePath = fileService.getFilePath(fileId); - logger.debug("Parsing metadata for file: {}", filePath); // Pre-check: reject empty files before attempting to parse try { - long fileSize = Files.size(Paths.get(filePath)); - if (fileSize == 0) { + if (Files.size(Paths.get(filePath)) == 0) { throw new EmptyFileException( "The file is empty (0 bytes). It may be a placeholder or incomplete write."); } @@ -125,49 +231,75 @@ public TsFileMetadataDTO getMetadata(String fileId) logger.warn("Cannot determine file size for {}: {}", filePath, e.getMessage()); } - TsFileMetadataDTO metadata = parseMetadata(fileId, filePath); + return filePath; + } + + /** + * Returns the cached device index of a file, rebuilding it when the file changed on disk. + * + * @param fileId the file identifier + * @param filePath the resolved file path + * @return device index of the file's current version + */ + private DeviceIndex getDeviceIndex(String fileId, String filePath) throws IOException { + DeviceIndex cached = deviceIndexCache.getIfPresent(fileId); + if (cached != null && !isStale(cached, filePath)) { + return cached; + } - // Cache the result - metadataCache.put(fileId, metadata); - logger.debug("Cached metadata for fileId={}", fileId); + DeviceIndex deviceIndex = tsFileParser.parseDeviceIndex(filePath); + deviceIndexCache.put(fileId, deviceIndex); + return deviceIndex; + } - return metadata; + /** + * Tells whether a cached index no longer matches the file on disk. + * + * @param deviceIndex cached index + * @param filePath file path + * @return true when the file's mtime or size differs from the indexed version + */ + private static boolean isStale(DeviceIndex deviceIndex, String filePath) { + try { + Path path = Paths.get(filePath); + return !deviceIndex.matches(Files.getLastModifiedTime(path).toMillis(), Files.size(path)); + } catch (IOException e) { + logger.debug("Cannot stat {}: {}", filePath, e.getMessage()); + return true; + } } /** - * Parses metadata from a TsFile. + * Converts parsed summary metadata to the API DTO. * * @param fileId the file identifier - * @param filePath the file path - * @return TsFileMetadataDTO with parsed metadata - * @throws IOException if the file cannot be read + * @param summary parsed summary metadata + * @return DTO with empty RowGroup and Chunk lists */ - private TsFileMetadataDTO parseMetadata(String fileId, String filePath) throws IOException { - // Parse all metadata in a single pass (opens the file only once instead of 5-7 times) - TsFileParser.AllMetadata all = tsFileParser.parseAll(filePath); + private TsFileMetadataDTO toSummaryDTO(String fileId, TsFileParser.SummaryMetadata summary) { + TsFileParser.BasicMetadata basic = summary.basic(); // Check if the file is empty (valid structure but no data) - if (all.basic().deviceCount() == 0 - && all.measurements().isEmpty() - && all.rowGroups().isEmpty() - && all.chunks().isEmpty() - && all.tables().isEmpty()) { + if (basic.deviceCount() == 0 + && summary.measurements().isEmpty() + && summary.tables().isEmpty() + && summary.deviceIndex().chunkCount() == 0) { throw new EmptyFileException( "The TsFile is valid but contains no data. It may be a newly created or empty file."); } return TsFileMetadataDTO.builder() .fileId(fileId) - .version(all.basic().version()) - .timeRange(new TimeRange(all.basic().startTime(), all.basic().endTime())) - .deviceCount(all.basic().deviceCount()) - .measurementCount(all.basic().measurementCount()) - .rowGroupCount(all.rowGroups().size()) - .chunkCount(all.chunks().size()) - .measurements(all.measurements()) - .rowGroups(all.rowGroups()) - .chunks(all.chunks()) - .tables(all.tables()) + .version(basic.version()) + .timeRange(new TimeRange(basic.startTime(), basic.endTime())) + .deviceCount(basic.deviceCount()) + .measurementCount(basic.measurementCount()) + .rowGroupCount(basic.rowGroupCount()) + .chunkCount(basic.chunkCount()) + .measurements(summary.measurements()) + .rowGroups(List.of()) + .chunks(List.of()) + .tables(summary.tables()) .build(); } @@ -179,12 +311,14 @@ private TsFileMetadataDTO parseMetadata(String fileId, String filePath) throws I public void invalidateCache(String fileId) { logger.debug("Invalidating cache for fileId={}", fileId); metadataCache.invalidate(fileId); + deviceIndexCache.invalidate(fileId); } /** Invalidates all cached metadata. */ public void invalidateAllCache() { logger.info("Invalidating all metadata cache"); metadataCache.invalidateAll(); + deviceIndexCache.invalidateAll(); } /** diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java new file mode 100644 index 0000000..03bfeb2 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/DeviceIndex.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import java.util.List; + +import org.apache.tsfile.file.metadata.IDeviceID; + +/** + * Immutable per-file index of devices and their chunk counts. + * + *

Holds only device names and an int prefix-sum array, so it stays small (a few MB even for + * ~100k devices) and never contains chunk or timeseries metadata. Every page request is served from + * this index plus per-device reads, which is what keeps paging incremental. + * + *

The {@code lastModified}/{@code size} pair identifies the file version the index was built + * from; {@link #matches(long, long)} is used to detect a rewritten file. + */ +public final class DeviceIndex { + + private final long lastModified; + private final long size; + private final List devices; + private final int[] chunkPrefix; + + private DeviceIndex( + long lastModified, long size, List devices, int[] chunkPrefix) { + this.lastModified = lastModified; + this.size = size; + this.devices = devices; + this.chunkPrefix = chunkPrefix; + } + + /** + * Creates an index from a device list and the chunk count of each device. + * + * @param lastModified file modification time the index was built from + * @param size file size the index was built from + * @param devices devices in file order + * @param chunkCounts chunk count per device, index-aligned with {@code devices} + * @return device index + */ + public static DeviceIndex of( + long lastModified, long size, List devices, int[] chunkCounts) { + if (devices.size() != chunkCounts.length) { + throw new IllegalArgumentException( + "devices and chunkCounts must be index-aligned: " + + devices.size() + + " vs " + + chunkCounts.length); + } + int[] prefix = new int[devices.size() + 1]; + for (int i = 0; i < chunkCounts.length; i++) { + prefix[i + 1] = prefix[i] + Math.max(chunkCounts[i], 0); + } + return new DeviceIndex(lastModified, size, List.copyOf(devices), prefix); + } + + /** + * Tells whether this index still describes the file's current version. + * + * @param lastModified current file modification time + * @param size current file size + * @return true when the index was built from that exact version + */ + public boolean matches(long lastModified, long size) { + return this.lastModified == lastModified && this.size == size; + } + + /** Number of devices, which is also the number of RowGroups of this file. */ + public int deviceCount() { + return devices.size(); + } + + /** Devices in file order. */ + public List devices() { + return devices; + } + + /** Total number of chunks in the file. */ + public int chunkCount() { + return chunkPrefix[devices.size()]; + } + + /** + * Number of chunks that precede the given device in file order. + * + * @param devicePosition position of the device in {@link #devices()} + * @return absolute index of that device's first chunk + */ + public int chunksBeforeDevice(int devicePosition) { + return chunkPrefix[devicePosition]; + } + + /** + * Number of chunks of the given device. + * + * @param devicePosition position of the device in {@link #devices()} + * @return chunk count of that device + */ + public int chunkCountOf(int devicePosition) { + return chunkPrefix[devicePosition + 1] - chunkPrefix[devicePosition]; + } + + /** + * Finds the device that owns the chunk at the given absolute position, so a chunk page can start + * reading at exactly the device it needs instead of from the first device. + * + * @param chunkOffset absolute chunk position + * @return position of the owning device, or {@link #deviceCount()} when the offset is past the + * last chunk + */ + public int deviceIndexForChunk(int chunkOffset) { + int low = 0; + int high = devices.size() - 1; + int result = devices.size(); + while (low <= high) { + int mid = (low + high) >>> 1; + if (chunkPrefix[mid + 1] > chunkOffset) { + result = mid; + high = mid - 1; + } else { + low = mid + 1; + } + } + return result; + } + + /** File modification time this index was built from. */ + public long lastModified() { + return lastModified; + } + + /** File size this index was built from. */ + public long size() { + return size; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java index 19cdeb8..c6d1108 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.Serializable; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -385,77 +386,29 @@ public BasicMetadata parseMetadata(Path filePath) throws IOException { * @throws IOException if the file cannot be read */ private ParsedMetadata parseMetadataInternal(TsFileSequenceReader reader) throws IOException { - // Get version byte versionNumber = reader.readVersionNumber(); + ScanResult scan = scan(reader, List.of(), false); String version = "V" + versionNumber; - // Get all devices - List devices = reader.getAllDevices(); - int deviceCount = devices.size(); - - // Get all timeseries metadata to calculate measurements, time range, and counts - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - - // Calculate unique measurements and time range - Set uniqueMeasurements = new HashSet<>(); - long minStartTime = Long.MAX_VALUE; - long maxEndTime = Long.MIN_VALUE; - int totalChunkCount = 0; - int rowGroupCount = 0; - - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - List timeseriesMetadataList = entry.getValue(); - rowGroupCount++; // Each device entry represents a RowGroup - - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { - uniqueMeasurements.add(tsMetadata.getMeasurementId()); - - // Get time range from statistics - Statistics stats = tsMetadata.getStatistics(); - if (stats != null) { - if (stats.getStartTime() < minStartTime) { - minStartTime = stats.getStartTime(); - } - if (stats.getEndTime() > maxEndTime) { - maxEndTime = stats.getEndTime(); - } - } - - // Count chunks - List chunkMetadataList = tsMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - totalChunkCount += chunkMetadataList.size(); - } - } - } - - // Handle case where no data exists - if (minStartTime == Long.MAX_VALUE) { - minStartTime = 0; - } - if (maxEndTime == Long.MIN_VALUE) { - maxEndTime = 0; - } - logger.debug( "Parsed metadata: version={}, devices={}, measurements={}, rowGroups={}, chunks={}", version, - deviceCount, - uniqueMeasurements.size(), - rowGroupCount, - totalChunkCount); + scan.devices().size(), + scan.measurementCount(), + scan.devices().size(), + totalChunks(scan)); return new ParsedMetadata( version, - minStartTime, - maxEndTime, - deviceCount, - uniqueMeasurements.size(), - rowGroupCount, - totalChunkCount); + scan.startTime(), + scan.endTime(), + scan.devices().size(), + scan.measurementCount(), + scan.devices().size(), + totalChunks(scan)); } + /** * Parses measurement schema details from a TsFileSequenceReader. * @@ -478,157 +431,12 @@ private ParsedMetadata parseMetadataInternal(TsFileSequenceReader reader) throws */ public List parseMeasurements(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing measurements from TsFile"); - - Map measurementMap = new LinkedHashMap<>(); - - // Try to read as V4 table model first to get column categories and schema info - Map columnCategories = new HashMap<>(); - Map columnSchemas = new HashMap<>(); - boolean isTableModel = false; - try { - String tsfilePath = reader.getFileName(); - Path path = Paths.get(tsfilePath); - logger.debug("Attempting to read TableSchema from: {}", tsfilePath); - try (ITsFileReader v4Reader = new TsFileReaderBuilder().file(path.toFile()).build()) { - List schemas = v4Reader.getAllTableSchema(); - if (schemas != null && !schemas.isEmpty()) { - isTableModel = true; - logger.debug("Found {} table schemas in V4 format", schemas.size()); - for (TableSchema schema : schemas) { - logger.debug("Processing table schema: {}", schema.getTableName()); - List schemaList = schema.getColumnSchemas(); - List categories = schema.getColumnTypes(); - - // Column schemas and categories lists are parallel - same index corresponds to - // same column - if (schemaList != null && categories != null) { - for (int i = 0; i < schemaList.size() && i < categories.size(); i++) { - IMeasurementSchema columnSchema = schemaList.get(i); - ColumnCategory category = categories.get(i); - String columnName = columnSchema.getMeasurementName(); - - if (category != null) { - columnCategories.put(columnName, category); - columnSchemas.put(columnName, columnSchema); - logger.debug(" Column {} has category {}", columnName, category); - - // For table model, add TAG columns directly since they won't appear in - // TimeseriesMetadata - if (category == ColumnCategory.TAG) { - String dataType = - columnSchema.getType() != null - ? columnSchema.getType().toString() - : "STRING"; - String encoding = - columnSchema.getEncodingType() != null - ? columnSchema.getEncodingType().toString() - : "PLAIN"; - String compression = - columnSchema.getCompressor() != null - ? columnSchema.getCompressor().toString() - : "UNCOMPRESSED"; - - measurementMap.put( - columnName, - new MeasurementInfo(columnName, dataType, encoding, compression, "TAG")); - } - } - } - } - } - } - } - } catch (Exception e) { - logger.debug("Not a V4 table model file or failed to read TableSchema: {}", e.getMessage()); - } - - // Parse measurements from TimeseriesMetadata - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - List timeseriesMetadataList = entry.getValue(); - - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { - String measurementId = tsMetadata.getMeasurementId(); - - // Skip if already processed - if (measurementMap.containsKey(measurementId)) { - continue; - } - - // Get data type from TimeseriesMetadata - String dataType = tsMetadata.getTsDataType().toString(); - - // Get encoding and compression - prioritize TableSchema for V4 table model - String encoding = "UNKNOWN"; - String compression = "UNKNOWN"; - - // For V4 table model, try to get from TableSchema first - IMeasurementSchema tableSchema = columnSchemas.get(measurementId); - if (tableSchema != null) { - if (tableSchema.getEncodingType() != null) { - encoding = tableSchema.getEncodingType().toString(); - } - if (tableSchema.getCompressor() != null) { - compression = tableSchema.getCompressor().toString(); - } - logger.debug( - "Got encoding/compression from TableSchema for {}: {}/{}", - measurementId, - encoding, - compression); - } else { - // Fallback to ChunkHeader for tree model files - // Read ChunkHeader from file to get encoding and compression - List chunkMetadataList = tsMetadata.getChunkMetadataList(); - if (chunkMetadataList != null && !chunkMetadataList.isEmpty()) { - IChunkMetadata firstChunk = chunkMetadataList.get(0); - if (firstChunk instanceof ChunkMetadata chunkMeta) { - try { - // Read the chunk which includes the header with encoding/compression info - org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); - if (chunk != null && chunk.getHeader() != null) { - org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); - if (chunkHeader.getEncodingType() != null) { - encoding = chunkHeader.getEncodingType().toString(); - } - if (chunkHeader.getCompressionType() != null) { - compression = chunkHeader.getCompressionType().toString(); - } - logger.debug( - "Read encoding/compression from ChunkHeader for {}: {}/{}", - measurementId, - encoding, - compression); - } - } catch (Exception e) { - logger.debug( - "Could not read ChunkHeader for measurement {}: {}", - measurementId, - e.getMessage()); - } - } - } - } - - // Get column category from V4 TableSchema if available - String columnCategory = "FIELD"; // Default to FIELD - ColumnCategory category = columnCategories.get(measurementId); - if (category != null) { - columnCategory = category.toString(); - } - - measurementMap.put( - measurementId, - new MeasurementInfo(measurementId, dataType, encoding, compression, columnCategory)); - } - } - - logger.debug("Parsed {} unique measurements", measurementMap.size()); - return new ArrayList<>(measurementMap.values()); + ScanResult scan = scan(reader, readTableSchemas(reader), true); + logger.debug("Parsed {} unique measurements", scan.measurements().size()); + return scan.measurements(); } + /** * Parses table schemas from a V4 table model TsFile. * @@ -641,84 +449,12 @@ public List parseMeasurements(TsFileSequenceReader reader) thro */ public List parseTables(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing table schemas from TsFile"); - - List tableInfos = new ArrayList<>(); - - try { - String tsfilePath = reader.getFileName(); - Path path = Paths.get(tsfilePath); - logger.debug("Attempting to read TableSchema from: {}", tsfilePath); - - try (ITsFileReader v4Reader = new TsFileReaderBuilder().file(path.toFile()).build()) { - List schemas = v4Reader.getAllTableSchema(); - - if (schemas == null || schemas.isEmpty()) { - logger.debug("No table schemas found - this is a tree model file"); - return tableInfos; - } - - logger.debug("Found {} table schemas in V4 format", schemas.size()); - - for (TableSchema schema : schemas) { - String tableName = schema.getTableName(); - logger.debug("Processing table: {}", tableName); - - List tagColumns = new ArrayList<>(); - List fieldColumns = new ArrayList<>(); - - List columnSchemas = schema.getColumnSchemas(); - List categories = schema.getColumnTypes(); - - if (columnSchemas != null && categories != null) { - for (int i = 0; i < columnSchemas.size() && i < categories.size(); i++) { - IMeasurementSchema columnSchema = columnSchemas.get(i); - ColumnCategory category = categories.get(i); - String columnName = columnSchema.getMeasurementName(); - - String dataType = - columnSchema.getType() != null ? columnSchema.getType().toString() : "UNKNOWN"; - String encoding = - columnSchema.getEncodingType() != null - ? columnSchema.getEncodingType().toString() - : "PLAIN"; - String compression = - columnSchema.getCompressor() != null - ? columnSchema.getCompressor().toString() - : "UNCOMPRESSED"; - - MeasurementInfo columnInfo = - new MeasurementInfo( - columnName, dataType, encoding, compression, category.toString()); - - if (category == ColumnCategory.TAG) { - tagColumns.add(columnInfo); - logger.debug(" TAG column: {}", columnName); - } else if (category == ColumnCategory.FIELD) { - fieldColumns.add(columnInfo); - logger.debug(" FIELD column: {}", columnName); - } - } - } - - TableInfo tableInfo = new TableInfo(tableName, tagColumns, fieldColumns); - tableInfos.add(tableInfo); - - logger.debug( - "Table '{}': {} TAG columns, {} FIELD columns, {} total", - tableName, - tagColumns.size(), - fieldColumns.size(), - tableInfo.getTotalColumns()); - } - } - } catch (Exception e) { - logger.debug("Not a V4 table model file or failed to read tables: {}", e.getMessage()); - } - + List tableInfos = toTableInfos(readTableSchemas(reader)); logger.debug("Parsed {} tables total", tableInfos.size()); return tableInfos; } + /** * Parses RowGroup metadata from a TsFileSequenceReader. * @@ -731,53 +467,54 @@ public List parseTables(TsFileSequenceReader reader) throws IOExcepti */ public List parseRowGroups(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing RowGroups from TsFile"); + List devices = reader.getAllDevices(); + List rowGroups = new ArrayList<>(devices.size()); + for (int i = 0; i < devices.size(); i++) { + rowGroups.add(readRowGroupInfo(reader, devices.get(i), i)); + } + logger.debug("Parsed {} RowGroups", rowGroups.size()); + return rowGroups; + } - List rowGroups = new ArrayList<>(); - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); - - int index = 0; - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - IDeviceID device = entry.getKey(); - List timeseriesMetadataList = entry.getValue(); - - long minStartTime = Long.MAX_VALUE; - long maxEndTime = Long.MIN_VALUE; - int chunkCount = 0; + /** + * Reads the RowGroup metadata of a single device. + * + * @param reader TsFile reader + * @param device device to read + * @param index RowGroup index of that device + * @return RowGroup metadata of that single device + * @throws IOException if reading fails + */ + private RowGroupInfo readRowGroupInfo(TsFileSequenceReader reader, IDeviceID device, int index) + throws IOException { + long minStartTime = Long.MAX_VALUE; + long maxEndTime = Long.MIN_VALUE; + int chunkCount = 0; - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { + List deviceMetadata = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadata != null) { + for (TimeseriesMetadata tsMetadata : deviceMetadata) { Statistics stats = tsMetadata.getStatistics(); if (stats != null) { - if (stats.getStartTime() < minStartTime) { - minStartTime = stats.getStartTime(); - } - if (stats.getEndTime() > maxEndTime) { - maxEndTime = stats.getEndTime(); - } - } - - List chunkMetadataList = tsMetadata.getChunkMetadataList(); - if (chunkMetadataList != null) { - chunkCount += chunkMetadataList.size(); + minStartTime = Math.min(minStartTime, stats.getStartTime()); + maxEndTime = Math.max(maxEndTime, stats.getEndTime()); } - } - // Handle case where no data exists - if (minStartTime == Long.MAX_VALUE) { - minStartTime = 0; - } - if (maxEndTime == Long.MIN_VALUE) { - maxEndTime = 0; + chunkCount += countChunks(tsMetadata.getChunkMetadataList()); } + } - rowGroups.add( - new RowGroupInfo(index++, device.toString(), minStartTime, maxEndTime, chunkCount)); + if (minStartTime == Long.MAX_VALUE) { + minStartTime = 0; + } + if (maxEndTime == Long.MIN_VALUE) { + maxEndTime = 0; } - logger.debug("Parsed {} RowGroups", rowGroups.size()); - return rowGroups; + return new RowGroupInfo(index, device.toString(), minStartTime, maxEndTime, chunkCount); } + /** * Parses Chunk metadata from a TsFileSequenceReader. * @@ -790,101 +527,111 @@ public List parseRowGroups(TsFileSequenceReader reader) throws IOE */ public List parseChunks(TsFileSequenceReader reader) throws IOException { logger.debug("Parsing Chunks from TsFile"); + List chunks = new ArrayList<>(); + for (IDeviceID device : reader.getAllDevices()) { + chunks.addAll(readChunkInfos(reader, device)); + } + logger.debug("Parsed {} Chunks", chunks.size()); + return chunks; + } + /** + * Reads the chunk metadata of a single device. + * + *

Bounded by one device instead of the whole file, which is what keeps a chunk page + * incremental. + * + * @param reader TsFile reader + * @param device device to read + * @return chunk metadata of that single device, in file order + * @throws IOException if reading fails + */ + private List readChunkInfos(TsFileSequenceReader reader, IDeviceID device) + throws IOException { List chunks = new ArrayList<>(); - Map> allTimeseriesMetadata = - reader.getAllTimeseriesMetadata(true); + List deviceMetadata = reader.getDeviceTimeseriesMetadata(device); + if (deviceMetadata == null) { + return chunks; + } - for (Map.Entry> entry : allTimeseriesMetadata.entrySet()) { - IDeviceID device = entry.getKey(); - List timeseriesMetadataList = entry.getValue(); + for (TimeseriesMetadata tsMetadata : deviceMetadata) { + String measurementId = tsMetadata.getMeasurementId(); + List chunkMetadataList = tsMetadata.getChunkMetadataList(); - for (TimeseriesMetadata tsMetadata : timeseriesMetadataList) { - String measurementId = tsMetadata.getMeasurementId(); - List chunkMetadataList = tsMetadata.getChunkMetadataList(); + if (chunkMetadataList == null) { + continue; + } - if (chunkMetadataList == null) { + for (IChunkMetadata iChunkMeta : chunkMetadataList) { + if (!(iChunkMeta instanceof ChunkMetadata chunkMeta)) { continue; } - for (IChunkMetadata iChunkMeta : chunkMetadataList) { - if (!(iChunkMeta instanceof ChunkMetadata chunkMeta)) { - continue; - } - - long offset = chunkMeta.getOffsetOfChunkHeader(); - Statistics stats = chunkMeta.getStatistics(); + long offset = chunkMeta.getOffsetOfChunkHeader(); + Statistics stats = chunkMeta.getStatistics(); - long startTime = stats != null ? stats.getStartTime() : 0; - long endTime = stats != null ? stats.getEndTime() : 0; - long numOfPoints = chunkMeta.getNumOfPoints(); + long startTime = stats != null ? stats.getStartTime() : 0; + long endTime = stats != null ? stats.getEndTime() : 0; + long numOfPoints = chunkMeta.getNumOfPoints(); - // Get data type - String dataType = - chunkMeta.getDataType() != null ? chunkMeta.getDataType().toString() : "UNKNOWN"; + String dataType = + chunkMeta.getDataType() != null ? chunkMeta.getDataType().toString() : "UNKNOWN"; - // Get encoding and compression from ChunkHeader - String encoding = "UNKNOWN"; - String compression = "UNKNOWN"; - try { - org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); - if (chunk != null && chunk.getHeader() != null) { - org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); - if (chunkHeader.getEncodingType() != null) { - encoding = chunkHeader.getEncodingType().toString(); - } - if (chunkHeader.getCompressionType() != null) { - compression = chunkHeader.getCompressionType().toString(); - } + String encoding = "UNKNOWN"; + String compression = "UNKNOWN"; + try { + org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); + if (chunk != null && chunk.getHeader() != null) { + org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); + if (chunkHeader.getEncodingType() != null) { + encoding = chunkHeader.getEncodingType().toString(); } - } catch (Exception e) { - logger.debug( - "Could not read ChunkHeader for chunk at offset {}: {}", offset, e.getMessage()); - } - - // Get min/max values from statistics - String minValue = null; - String maxValue = null; - if (stats != null) { - try { - Object min = stats.getMinValue(); - Object max = stats.getMaxValue(); - minValue = min != null ? min.toString() : null; - maxValue = max != null ? max.toString() : null; - } catch (Exception e) { - logger.debug("Could not get min/max values from statistics: {}", e.getMessage()); + if (chunkHeader.getCompressionType() != null) { + compression = chunkHeader.getCompressionType().toString(); } } + } catch (Exception e) { + logger.debug( + "Could not read ChunkHeader for chunk at offset {}: {}", offset, e.getMessage()); + } - // Calculate compression ratio - double compressionRatio = calculateCompressionRatio(chunkMeta, numOfPoints); - - // Size is estimated from the chunk metadata - long size = estimateChunkSize(chunkMeta); - - chunks.add( - new ChunkInfo( - measurementId, - offset, - size, - compressionRatio, - device.toString(), - startTime, - endTime, - numOfPoints, - dataType, - encoding, - compression, - minValue, - maxValue)); + String minValue = null; + String maxValue = null; + if (stats != null) { + try { + Object min = stats.getMinValue(); + Object max = stats.getMaxValue(); + minValue = min != null ? min.toString() : null; + maxValue = max != null ? max.toString() : null; + } catch (Exception e) { + logger.debug("Could not get min/max values from statistics: {}", e.getMessage()); + } } + + double compressionRatio = calculateCompressionRatio(chunkMeta, numOfPoints); + long size = estimateChunkSize(chunkMeta); + + chunks.add( + new ChunkInfo( + measurementId, + offset, + size, + compressionRatio, + device.toString(), + startTime, + endTime, + numOfPoints, + dataType, + encoding, + compression, + minValue, + maxValue)); } } - - logger.debug("Parsed {} Chunks", chunks.size()); return chunks; } + /** * Calculates an estimated compression ratio for a chunk. * @@ -939,133 +686,459 @@ private long estimateChunkSize(ChunkMetadata chunkMeta) { return chunkMeta.serializedSize(true); } - // ============== Unified Parse Method ============== + // ============== Incremental Summary & Paging ============== - /** - * Result of parsing all metadata from a TsFile in a single pass. Opens the file only once, - * avoiding the overhead of 5+ separate file opens that the individual parse methods incur. - */ - public record AllMetadata( + /** Summary metadata of a TsFile plus the device index used by the paging endpoints. */ + public record SummaryMetadata( BasicMetadata basic, List measurements, - List rowGroups, - List chunks, - List tables) {} + List tables, + DeviceIndex deviceIndex) {} /** - * Parses all metadata from a TsFile in a single pass, opening the file only once. + * Parses summary metadata from a TsFile in one streaming pass. * - *

This is significantly more efficient than calling {@link #parseMetadata(String)}, {@link - * #parseMeasurements(String)}, {@link #parseRowGroups(String)}, {@link #parseChunks(String)}, and - * {@link #parseTables(String)} separately, which each open their own reader (5-7 file opens - * total). + *

Never materializes the file's whole timeseries metadata: each device's metadata is read, + * folded into counters, and dropped before the next device is visited. * * @param filePath path to the TsFile - * @return AllMetadata containing all parsed metadata + * @return summary metadata plus the device index of the same file version * @throws IOException if the file cannot be read */ - public AllMetadata parseAll(String filePath) throws IOException { - logger.debug("Parsing all metadata from TsFile in single pass: {}", filePath); + public SummaryMetadata parseSummary(String filePath) throws IOException { + logger.debug("Parsing summary metadata from TsFile: {}", filePath); + Path path = Paths.get(filePath); + long lastModified = Files.getLastModifiedTime(path).toMillis(); + long size = Files.size(path); + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { - ParsedMetadata parsed = parseMetadataInternal(reader); + String version = "V" + reader.readVersionNumber(); + List schemas = readTableSchemas(reader); + ScanResult scan = scan(reader, schemas, true); + BasicMetadata basic = new BasicMetadata( - parsed.getVersion(), - parsed.getStartTime(), - parsed.getEndTime(), - parsed.getDeviceCount(), - parsed.getMeasurementCount(), - parsed.getRowGroupCount(), - parsed.getChunkCount()); + version, + scan.startTime(), + scan.endTime(), + scan.devices().size(), + scan.measurementCount(), + scan.devices().size(), + totalChunks(scan)); List measurements = - parseMeasurements(reader).stream() - .map( - info -> - MeasurementDTO.builder() - .name(info.getName()) - .dataType(info.getDataType()) - .encoding(info.getEncoding()) - .compression(info.getCompression()) - .columnCategory(info.getColumnCategory()) - .build()) - .toList(); - - List rowGroups = - parseRowGroups(reader).stream() - .map( - info -> - RowGroupDTO.builder() - .index(info.getIndex()) - .device(info.getDevice()) - .startTime(info.getStartTime()) - .endTime(info.getEndTime()) - .chunkCount(info.getChunkCount()) - .build()) - .toList(); - - List chunks = - parseChunks(reader).stream() - .map( - info -> - ChunkDTO.builder() - .measurement(info.getMeasurement()) - .offset(info.getOffset()) - .size(info.getSize()) - .compressionRatio(info.getCompressionRatio()) - .device(info.getDevice()) - .dataType(info.getDataType()) - .encoding(info.getEncoding()) - .compression(info.getCompression()) - .startTime(info.getStartTime()) - .endTime(info.getEndTime()) - .numOfPoints(info.getNumOfPoints()) - .minValue(info.getMinValue()) - .maxValue(info.getMaxValue()) - .build()) - .toList(); - + scan.measurements().stream().map(TsFileParser::toMeasurementDTO).toList(); List tables = - parseTables(reader).stream() - .map( - info -> { - List tagDTOs = - info.getTagColumns().stream() - .map( - col -> - MeasurementDTO.builder() - .name(col.getName()) - .dataType(col.getDataType()) - .encoding(col.getEncoding()) - .compression(col.getCompression()) - .columnCategory(col.getColumnCategory()) - .build()) - .toList(); - List fieldDTOs = - info.getFieldColumns().stream() - .map( - col -> - MeasurementDTO.builder() - .name(col.getName()) - .dataType(col.getDataType()) - .encoding(col.getEncoding()) - .compression(col.getCompression()) - .columnCategory(col.getColumnCategory()) - .build()) - .toList(); - return org.apache.tsfile.viewer.dto.TableDTO.builder() - .tableName(info.getTableName()) - .tagColumns(tagDTOs) - .fieldColumns(fieldDTOs) - .totalColumns(info.getTotalColumns()) - .build(); - }) - .toList(); - - return new AllMetadata(basic, measurements, rowGroups, chunks, tables); + toTableInfos(schemas).stream().map(TsFileParser::toTableDTO).toList(); + + return new SummaryMetadata( + basic, + measurements, + tables, + DeviceIndex.of(lastModified, size, scan.devices(), scan.chunkCounts())); + } + } + + /** + * Builds the device index of a TsFile without parsing measurements. + * + * @param filePath path to the TsFile + * @return device index stamped with the file version it was read from + * @throws IOException if the file cannot be read + */ + public DeviceIndex parseDeviceIndex(String filePath) throws IOException { + Path path = Paths.get(filePath); + long lastModified = Files.getLastModifiedTime(path).toMillis(); + long size = Files.size(path); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + ScanResult scan = scan(reader, List.of(), false); + return DeviceIndex.of(lastModified, size, scan.devices(), scan.chunkCounts()); + } + } + + /** + * Reads one page of RowGroups. + * + *

Only the devices inside the page window are read, so a page's memory is bounded by the page + * itself instead of the file's device count. + * + * @param reader TsFile reader + * @param deviceIndex index of the same file version + * @param offset absolute index of the first RowGroup to return + * @param limit maximum number of RowGroups to return + * @return RowGroups in file order; empty when the offset is past the last RowGroup + * @throws IOException if reading fails + */ + public List readRowGroupPage( + TsFileSequenceReader reader, DeviceIndex deviceIndex, int offset, int limit) + throws IOException { + int total = deviceIndex.deviceCount(); + if (limit <= 0 || offset < 0 || offset >= total) { + return List.of(); + } + + int end = Math.min(total, offset + limit); + List page = new ArrayList<>(end - offset); + for (int index = offset; index < end; index++) { + page.add(toRowGroupDTO(readRowGroupInfo(reader, deviceIndex.devices().get(index), index))); + } + return page; + } + + /** + * Reads one page of Chunks, stitching devices together when the page crosses a device boundary. + * + *

Starts at the device that owns {@code offset} and stops once the page is full, so only the + * devices the page touches are ever read. + * + * @param reader TsFile reader + * @param deviceIndex index of the same file version + * @param offset absolute index of the first chunk to return + * @param limit maximum number of chunks to return + * @return chunks in file order; empty when the offset is past the last chunk + * @throws IOException if reading fails + */ + public List readChunkPage( + TsFileSequenceReader reader, DeviceIndex deviceIndex, int offset, int limit) + throws IOException { + int total = deviceIndex.chunkCount(); + if (limit <= 0 || offset < 0 || offset >= total) { + return List.of(); + } + + int end = Math.min(total, offset + limit); + List page = new ArrayList<>(end - offset); + + for (int position = deviceIndex.deviceIndexForChunk(offset); + position < deviceIndex.deviceCount() && page.size() < end - offset; + position++) { + int base = deviceIndex.chunksBeforeDevice(position); + List deviceChunks = readChunkInfos(reader, deviceIndex.devices().get(position)); + + for (int i = 0; i < deviceChunks.size(); i++) { + int absolute = base + i; + if (absolute < offset) { + continue; + } + if (absolute >= end) { + break; + } + page.add(toChunkDTO(deviceChunks.get(i))); + } + } + return page; + } + + /** Counters of one streaming pass; no chunk or timeseries metadata outlives the scan. */ + private record ScanResult( + List devices, + int[] chunkCounts, + long startTime, + long endTime, + int measurementCount, + List measurements) {} + + /** + * Streams the file one device at a time and accumulates the summary counters. + * + * @param reader TsFile reader + * @param schemas V4 table schemas, already read, used for TAG columns and encoding/compression + * @param includeMeasurements whether measurement schema details are needed + * @return counters of the file plus its device list and per-device chunk counts + * @throws IOException if reading fails + */ + private ScanResult scan( + TsFileSequenceReader reader, List schemas, boolean includeMeasurements) + throws IOException { + List devices = reader.getAllDevices(); + int[] chunkCounts = new int[devices.size()]; + + Map columnCategories = new HashMap<>(); + Map columnSchemas = new HashMap<>(); + Map measurementMap = new LinkedHashMap<>(); + Set measurementNames = new HashSet<>(); + if (includeMeasurements) { + collectSchemaColumns(schemas, columnCategories, columnSchemas, measurementMap); + } + + long minStartTime = Long.MAX_VALUE; + long maxEndTime = Long.MIN_VALUE; + + for (int devicePosition = 0; devicePosition < devices.size(); devicePosition++) { + List deviceMetadata = + reader.getDeviceTimeseriesMetadata(devices.get(devicePosition)); + int deviceChunks = 0; + + if (deviceMetadata != null) { + for (TimeseriesMetadata tsMetadata : deviceMetadata) { + Statistics stats = tsMetadata.getStatistics(); + if (stats != null) { + minStartTime = Math.min(minStartTime, stats.getStartTime()); + maxEndTime = Math.max(maxEndTime, stats.getEndTime()); + } + + deviceChunks += countChunks(tsMetadata.getChunkMetadataList()); + + String measurementId = tsMetadata.getMeasurementId(); + measurementNames.add(measurementId); + if (includeMeasurements && !measurementMap.containsKey(measurementId)) { + measurementMap.put( + measurementId, + toMeasurementInfo(reader, tsMetadata, columnCategories, columnSchemas)); + } + } + } + chunkCounts[devicePosition] = deviceChunks; + } + + if (minStartTime == Long.MAX_VALUE) { + minStartTime = 0; + } + if (maxEndTime == Long.MIN_VALUE) { + maxEndTime = 0; } + + return new ScanResult( + devices, + chunkCounts, + minStartTime, + maxEndTime, + measurementNames.size(), + includeMeasurements ? new ArrayList<>(measurementMap.values()) : List.of()); } + /** + * Counts the chunks that the paging endpoints address. + * + *

Aligned files wrap some entries in non-{@link ChunkMetadata} objects that the chunk listing + * skips; counting them here keeps the reported total equal to the number of addressable chunks. + * + * @param chunkMetadataList chunk metadata of one timeseries, may be null + * @return number of addressable chunks + */ + private static int countChunks(List chunkMetadataList) { + if (chunkMetadataList == null) { + return 0; + } + int count = 0; + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + if (chunkMetadata instanceof ChunkMetadata) { + count++; + } + } + return count; + } + + private static int totalChunks(ScanResult scan) { + int total = 0; + for (int count : scan.chunkCounts()) { + total += count; + } + return total; + } + + /** + * Reads the V4 table schemas, returning an empty list for tree model files. + * + * @param reader TsFile reader + * @return table schemas of the file + */ + private List readTableSchemas(TsFileSequenceReader reader) { + try { + Path path = Paths.get(reader.getFileName()); + try (ITsFileReader v4Reader = new TsFileReaderBuilder().file(path.toFile()).build()) { + List schemas = v4Reader.getAllTableSchema(); + return schemas == null ? List.of() : schemas; + } + } catch (Exception e) { + logger.debug("Not a V4 table model file or failed to read TableSchema: {}", e.getMessage()); + return List.of(); + } + } + + private void collectSchemaColumns( + List schemas, + Map columnCategories, + Map columnSchemas, + Map measurementMap) { + for (TableSchema schema : schemas) { + List schemaList = schema.getColumnSchemas(); + List categories = schema.getColumnTypes(); + if (schemaList == null || categories == null) { + continue; + } + for (int i = 0; i < schemaList.size() && i < categories.size(); i++) { + IMeasurementSchema columnSchema = schemaList.get(i); + ColumnCategory category = categories.get(i); + if (category == null) { + continue; + } + String columnName = columnSchema.getMeasurementName(); + columnCategories.put(columnName, category); + columnSchemas.put(columnName, columnSchema); + + if (category == ColumnCategory.TAG) { + measurementMap.put( + columnName, + new MeasurementInfo( + columnName, + schemaDataType(columnSchema, "STRING"), + schemaEncoding(columnSchema), + schemaCompression(columnSchema), + "TAG")); + } + } + } + } + + private MeasurementInfo toMeasurementInfo( + TsFileSequenceReader reader, + TimeseriesMetadata tsMetadata, + Map columnCategories, + Map columnSchemas) { + String measurementId = tsMetadata.getMeasurementId(); + String dataType = tsMetadata.getTsDataType().toString(); + String encoding = "UNKNOWN"; + String compression = "UNKNOWN"; + + IMeasurementSchema tableSchema = columnSchemas.get(measurementId); + if (tableSchema != null) { + if (tableSchema.getEncodingType() != null) { + encoding = tableSchema.getEncodingType().toString(); + } + if (tableSchema.getCompressor() != null) { + compression = tableSchema.getCompressor().toString(); + } + } else { + List chunkMetadataList = tsMetadata.getChunkMetadataList(); + if (chunkMetadataList != null && !chunkMetadataList.isEmpty()) { + IChunkMetadata firstChunk = chunkMetadataList.get(0); + if (firstChunk instanceof ChunkMetadata chunkMeta) { + try { + org.apache.tsfile.read.common.Chunk chunk = reader.readMemChunk(chunkMeta); + if (chunk != null && chunk.getHeader() != null) { + org.apache.tsfile.file.header.ChunkHeader chunkHeader = chunk.getHeader(); + if (chunkHeader.getEncodingType() != null) { + encoding = chunkHeader.getEncodingType().toString(); + } + if (chunkHeader.getCompressionType() != null) { + compression = chunkHeader.getCompressionType().toString(); + } + } + } catch (Exception e) { + logger.debug( + "Could not read ChunkHeader for measurement {}: {}", measurementId, e.getMessage()); + } + } + } + } + + ColumnCategory category = columnCategories.get(measurementId); + return new MeasurementInfo( + measurementId, + dataType, + encoding, + compression, + category != null ? category.toString() : "FIELD"); + } + + private List toTableInfos(List schemas) { + List tableInfos = new ArrayList<>(); + for (TableSchema schema : schemas) { + List tagColumns = new ArrayList<>(); + List fieldColumns = new ArrayList<>(); + + List columnSchemas = schema.getColumnSchemas(); + List categories = schema.getColumnTypes(); + if (columnSchemas != null && categories != null) { + for (int i = 0; i < columnSchemas.size() && i < categories.size(); i++) { + IMeasurementSchema columnSchema = columnSchemas.get(i); + ColumnCategory category = categories.get(i); + MeasurementInfo columnInfo = + new MeasurementInfo( + columnSchema.getMeasurementName(), + schemaDataType(columnSchema, "UNKNOWN"), + schemaEncoding(columnSchema), + schemaCompression(columnSchema), + category.toString()); + + if (category == ColumnCategory.TAG) { + tagColumns.add(columnInfo); + } else if (category == ColumnCategory.FIELD) { + fieldColumns.add(columnInfo); + } + } + } + tableInfos.add(new TableInfo(schema.getTableName(), tagColumns, fieldColumns)); + } + return tableInfos; + } + + private static String schemaDataType(IMeasurementSchema schema, String defaultValue) { + return schema.getType() != null ? schema.getType().toString() : defaultValue; + } + + private static String schemaEncoding(IMeasurementSchema schema) { + return schema.getEncodingType() != null ? schema.getEncodingType().toString() : "PLAIN"; + } + + private static String schemaCompression(IMeasurementSchema schema) { + return schema.getCompressor() != null ? schema.getCompressor().toString() : "UNCOMPRESSED"; + } + + private static MeasurementDTO toMeasurementDTO(MeasurementInfo info) { + return MeasurementDTO.builder() + .name(info.getName()) + .dataType(info.getDataType()) + .encoding(info.getEncoding()) + .compression(info.getCompression()) + .columnCategory(info.getColumnCategory()) + .build(); + } + + private static org.apache.tsfile.viewer.dto.TableDTO toTableDTO(TableInfo info) { + List tagDTOs = + info.getTagColumns().stream().map(TsFileParser::toMeasurementDTO).toList(); + List fieldDTOs = + info.getFieldColumns().stream().map(TsFileParser::toMeasurementDTO).toList(); + return org.apache.tsfile.viewer.dto.TableDTO.builder() + .tableName(info.getTableName()) + .tagColumns(tagDTOs) + .fieldColumns(fieldDTOs) + .totalColumns(info.getTotalColumns()) + .build(); + } + + private static RowGroupDTO toRowGroupDTO(RowGroupInfo info) { + return RowGroupDTO.builder() + .index(info.getIndex()) + .device(info.getDevice()) + .startTime(info.getStartTime()) + .endTime(info.getEndTime()) + .chunkCount(info.getChunkCount()) + .build(); + } + + private static ChunkDTO toChunkDTO(ChunkInfo info) { + return ChunkDTO.builder() + .measurement(info.getMeasurement()) + .offset(info.getOffset()) + .size(info.getSize()) + .compressionRatio(info.getCompressionRatio()) + .device(info.getDevice()) + .dataType(info.getDataType()) + .encoding(info.getEncoding()) + .compression(info.getCompression()) + .startTime(info.getStartTime()) + .endTime(info.getEndTime()) + .numOfPoints(info.getNumOfPoints()) + .minValue(info.getMinValue()) + .maxValue(info.getMaxValue()) + .build(); + } + + // ============== DTO Conversion Methods ============== /** Record containing basic TsFile metadata. */ diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index ebf37fa..03e0860 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -75,6 +75,6 @@ springdoc: # Logging Configuration logging: level: - org.apache.tsfile.viewer: DEBUG + org.apache.tsfile.viewer: INFO org.apache.tsfile: INFO org.springframework.web: INFO diff --git a/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java new file mode 100644 index 0000000..0be1e36 --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java @@ -0,0 +1,242 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.controller; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.MvcResult; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; +import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.service.MetadataService; + +/** + * Tests for the paginated metadata endpoints: parameter defaults, clamping and response shape. + * + *

Validates: offset default, offset clamping, limit cap, empty page past the end, JSON contract + * and the 404 semantics of an unknown fileId. + */ +class MetadataControllerPaginationTest { + + private static final int TOTAL_ROW_GROUPS = 87_488; + private static final int TOTAL_CHUNKS = 2_000_000; + + private MetadataService metadataService; + private MetadataController controller; + private MockMvc mockMvc; + + @BeforeEach + void setUp() { + metadataService = mock(MetadataService.class); + controller = new MetadataController(metadataService); + mockMvc = MockMvcBuilders.standaloneSetup(controller).build(); + } + + @Test + @DisplayName("row-groups defaults to offset=0 and limit=50") + void shouldApplyRowGroupDefaults() throws Exception { + when(metadataService.getRowGroupsPage("file-1", 0, 50)) + .thenReturn(PageDTO.of(List.of(rowGroup(0, "device_0")), TOTAL_ROW_GROUPS, 0, 50)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/row-groups")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(50)) + .andExpect(jsonPath("$.total").value(TOTAL_ROW_GROUPS)) + .andExpect(jsonPath("$.hasMore").value(true)) + .andExpect(jsonPath("$.items.length()").value(1)) + .andExpect(jsonPath("$.items[0].device").value("device_0")) + .andExpect(jsonPath("$.items[0].index").value(0)) + .andReturn(); + + System.out.println("[row-groups] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("negative offset becomes 0 and limit above 200 is capped") + void shouldClampRowGroupParameters() throws Exception { + when(metadataService.getRowGroupsPage("file-1", 0, 200)) + .thenReturn(PageDTO.of(List.of(rowGroup(0, "device_0")), TOTAL_ROW_GROUPS, 0, 200)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/row-groups").param("offset", "-7").param("limit", "100000")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(200)) + .andReturn(); + + verify(metadataService).getRowGroupsPage("file-1", 0, 200); + System.out.println("[row-groups clamped] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("offset beyond total returns an empty page with the real total") + void shouldReturnEmptyPagePastTheEnd() throws Exception { + when(metadataService.getRowGroupsPage("file-1", 100_000, 50)) + .thenReturn(PageDTO.of(List.of(), TOTAL_ROW_GROUPS, 100_000, 50)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/row-groups").param("offset", "100000")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(100_000)) + .andExpect(jsonPath("$.total").value(TOTAL_ROW_GROUPS)) + .andExpect(jsonPath("$.hasMore").value(false)) + .andExpect(jsonPath("$.items.length()").value(0)) + .andReturn(); + + System.out.println("[row-groups past end] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("chunks defaults to offset=0 and limit=50 and returns the element contract") + void shouldApplyChunkDefaults() throws Exception { + when(metadataService.getChunksPage("file-1", 0, 50)) + .thenReturn(PageDTO.of(List.of(chunk("s0", 0, "device_0")), TOTAL_CHUNKS, 0, 50)); + + MvcResult result = + mockMvc + .perform(get("/api/meta/file-1/chunks")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(50)) + .andExpect(jsonPath("$.total").value(TOTAL_CHUNKS)) + .andExpect(jsonPath("$.hasMore").value(true)) + .andExpect(jsonPath("$.items[0].measurement").value("s0")) + .andExpect(jsonPath("$.items[0].device").value("device_0")) + .andExpect(jsonPath("$.items[0].numOfPoints").value(10)) + .andReturn(); + + System.out.println("[chunks] " + result.getResponse().getContentAsString()); + } + + @Test + @DisplayName("chunks honours offset and limit inside the page") + void shouldForwardChunkPageWindow() throws Exception { + when(metadataService.getChunksPage("file-1", 100, 3)) + .thenReturn(PageDTO.of(List.of(chunk("s1", 40, "device_1")), TOTAL_CHUNKS, 100, 3)); + + mockMvc + .perform(get("/api/meta/file-1/chunks").param("offset", "100").param("limit", "3")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(100)) + .andExpect(jsonPath("$.limit").value(3)) + .andExpect(jsonPath("$.hasMore").value(true)); + + verify(metadataService).getChunksPage("file-1", 100, 3); + } + + @Test + @DisplayName("unknown fileId keeps the existing 404 exception semantics") + void shouldPropagateNotFound() throws Exception { + when(metadataService.getRowGroupsPage(eq("missing"), anyInt(), anyInt())) + .thenThrow(new TsFileNotFoundException("File not found: missing", "missing")); + when(metadataService.getChunksPage(eq("missing"), anyInt(), anyInt())) + .thenThrow(new TsFileNotFoundException("File not found: missing", "missing")); + + assertThatThrownBy(() -> controller.getRowGroups("missing", null, null)) + .isInstanceOf(TsFileNotFoundException.class) + .hasMessageContaining("missing"); + assertThatThrownBy(() -> controller.getChunks("missing", null, null)) + .isInstanceOf(TsFileNotFoundException.class) + .hasMessageContaining("missing"); + } + + @Test + @DisplayName("summary response keeps rowGroups and chunks as empty arrays") + void shouldKeepSummaryContractWithEmptyDetailLists() throws Exception { + org.apache.tsfile.viewer.dto.TsFileMetadataDTO summary = + org.apache.tsfile.viewer.dto.TsFileMetadataDTO.builder() + .fileId("file-1") + .version("V3") + .deviceCount(87_488) + .measurementCount(12) + .rowGroupCount(87_488) + .chunkCount(2_000_000) + .measurements(List.of()) + .rowGroups(List.of()) + .chunks(List.of()) + .tables(List.of()) + .build(); + when(metadataService.getMetadata("file-1")).thenReturn(summary); + + mockMvc + .perform(get("/api/meta/file-1")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.rowGroups").isArray()) + .andExpect(jsonPath("$.rowGroups.length()").value(0)) + .andExpect(jsonPath("$.chunks").isArray()) + .andExpect(jsonPath("$.chunks.length()").value(0)) + .andExpect(jsonPath("$.rowGroupCount").value(87_488)) + .andExpect(jsonPath("$.chunkCount").value(2_000_000)); + + assertThat(summary.getRowGroups()).isEmpty(); + assertThat(summary.getChunks()).isEmpty(); + } + + private static RowGroupDTO rowGroup(int index, String device) { + return RowGroupDTO.builder() + .index(index) + .device(device) + .startTime(1_700_000_000_000L) + .endTime(1_700_000_001_000L) + .chunkCount(3) + .build(); + } + + private static ChunkDTO chunk(String measurement, long offset, String device) { + return ChunkDTO.builder() + .measurement(measurement) + .offset(offset) + .size(128) + .compressionRatio(2.5) + .device(device) + .dataType("INT64") + .encoding("RLE") + .compression("SNAPPY") + .startTime(1_700_000_000_000L) + .endTime(1_700_000_001_000L) + .numOfPoints(10L) + .minValue("1") + .maxValue("10") + .build(); + } +} diff --git a/backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java b/backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java new file mode 100644 index 0000000..247015c --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/dto/PageDTOTest.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/** + * Boundary tests for the pagination envelope and its parameter rules. + * + *

Validates: offset default/negative/overflow, limit default/cap, hasMore correctness. + */ +class PageDTOTest { + + @Test + @DisplayName("offset is 0 when missing or negative") + void shouldNormalizeOffset() { + assertThat(PageDTO.normalizeOffset(null)).isZero(); + assertThat(PageDTO.normalizeOffset(0)).isZero(); + assertThat(PageDTO.normalizeOffset(-1)).isZero(); + assertThat(PageDTO.normalizeOffset(-1000)).isZero(); + assertThat(PageDTO.normalizeOffset(7)).isEqualTo(7); + } + + @Test + @DisplayName("limit defaults to 50 and is capped at 200") + void shouldNormalizeLimit() { + assertThat(PageDTO.normalizeLimit(null)).isEqualTo(PageDTO.DEFAULT_LIMIT).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(0)).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(-5)).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(1)).isEqualTo(1); + assertThat(PageDTO.normalizeLimit(50)).isEqualTo(50); + assertThat(PageDTO.normalizeLimit(200)).isEqualTo(200); + assertThat(PageDTO.normalizeLimit(201)).isEqualTo(PageDTO.MAX_LIMIT).isEqualTo(200); + assertThat(PageDTO.normalizeLimit(Integer.MAX_VALUE)).isEqualTo(200); + } + + @Test + @DisplayName("first page reports hasMore when elements remain") + void shouldReportHasMoreOnFirstPage() { + PageDTO page = PageDTO.of(items(50), 100, 0, 50); + + assertThat(page.getOffset()).isZero(); + assertThat(page.getLimit()).isEqualTo(50); + assertThat(page.getTotal()).isEqualTo(100); + assertThat(page.getItems()).hasSize(50); + assertThat(page.isHasMore()).isTrue(); + } + + @Test + @DisplayName("last page reports no more elements when it ends exactly at total") + void shouldReportNoMoreAtExactBoundary() { + PageDTO page = PageDTO.of(items(50), 100, 50, 50); + + assertThat(page.isHasMore()).isFalse(); + } + + @Test + @DisplayName("partial page before the end still reports hasMore") + void shouldReportHasMoreOnPartialPage() { + PageDTO page = PageDTO.of(items(3), 10, 7, 50); + + assertThat(page.getItems()).hasSize(3); + assertThat(page.isHasMore()).isFalse(); + + PageDTO shorterPage = PageDTO.of(items(2), 10, 6, 2); + assertThat(shorterPage.isHasMore()).isTrue(); + } + + @Test + @DisplayName("offset beyond total yields an empty page with the real total") + void shouldReturnEmptyPageBeyondTotal() { + PageDTO page = PageDTO.of(List.of(), 87_488, 87_488, 50); + + assertThat(page.getItems()).isEmpty(); + assertThat(page.getTotal()).isEqualTo(87_488); + assertThat(page.getOffset()).isEqualTo(87_488); + assertThat(page.getLimit()).isEqualTo(50); + assertThat(page.isHasMore()).isFalse(); + } + + @Test + @DisplayName("far offset never overflows into hasMore=true") + void shouldNotOverflowOnHugeOffset() { + PageDTO page = PageDTO.of(List.of(), 10, Integer.MAX_VALUE, 200); + + assertThat(page.isHasMore()).isFalse(); + } + + @Test + @DisplayName("null items are treated as an empty page") + void shouldTreatNullItemsAsEmpty() { + PageDTO page = PageDTO.of(null, 0, 0, 50); + + assertThat(page.getItems()).isEmpty(); + assertThat(page.getTotal()).isZero(); + assertThat(page.isHasMore()).isFalse(); + } + + private static List items(int count) { + return java.util.stream.IntStream.range(0, count).mapToObj(i -> "item-" + i).toList(); + } +} diff --git a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java new file mode 100644 index 0000000..d2106c7 --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFilePaginationTest.java @@ -0,0 +1,317 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ColumnSchema; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.TsFileSequenceReader; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.PageDTO; +import org.apache.tsfile.viewer.dto.RowGroupDTO; + +/** + * Tests for incremental RowGroup/Chunk paging against a real TsFile. + * + *

Checks that device-based pages tile the full metadata exactly, that a chunk page crossing a + * device boundary is stitched correctly, and that offsets past the end return an empty page. + * + *

Validates: pagination boundaries plus the "no full materialization" paging contract. + */ +class TsFilePaginationTest { + + private static final int DEVICES = 4; + private static final int MEASUREMENTS = 3; + private static final int ROWS = 10; + + @TempDir Path tempDir; + + private final TsFileParser parser = new TsFileParser(); + + private Path tsFile; + + @BeforeEach + void setUp() throws Exception { + tsFile = createTableModelTsFile(tempDir.resolve("pagination.tsfile"), DEVICES, MEASUREMENTS, ROWS); + } + + @Test + @DisplayName("Device index totals match the full enumeration of the file") + void shouldMatchFullEnumeration() throws Exception { + DeviceIndex index = parser.parseDeviceIndex(tsFile.toString()); + List allRowGroups = parser.parseRowGroups(tsFile.toString()); + List allChunks = parser.parseChunks(tsFile.toString()); + + System.out.printf( + "[index] devices=%d chunks=%d chunksPerDevice=%d enumeratedRowGroups=%d enumeratedChunks=%d%n", + index.deviceCount(), + index.chunkCount(), + index.chunkCountOf(0), + allRowGroups.size(), + allChunks.size()); + + assertThat(index.deviceCount()).isEqualTo(DEVICES).isEqualTo(allRowGroups.size()); + assertThat(index.chunkCount()).isEqualTo(allChunks.size()); + for (int position = 0; position < index.deviceCount(); position++) { + assertThat(index.chunkCountOf(position)).isEqualTo(allRowGroups.get(position).getChunkCount()); + } + } + + @Test + @DisplayName("Device index is stamped with the file version it was built from") + void shouldStampDeviceIndexWithFileVersion() throws Exception { + DeviceIndex index = parser.parseDeviceIndex(tsFile.toString()); + + long lastModified = Files.getLastModifiedTime(tsFile).toMillis(); + long size = Files.size(tsFile); + + assertThat(index.matches(lastModified, size)).isTrue(); + assertThat(index.matches(lastModified + 1, size)).isFalse(); + assertThat(index.matches(lastModified, size + 1)).isFalse(); + } + + @Test + @DisplayName("offset=0 returns the first page") + void shouldReturnFirstRowGroupPage() throws Exception { + List all = parser.parseRowGroups(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + List page = parser.readRowGroupPage(reader, index(), 0, 2); + + assertThat(page).hasSize(2); + assertThat(page.get(0).getIndex()).isZero(); + assertThat(page.get(0).getDevice()).isEqualTo(all.get(0).getDevice()); + assertThat(page.get(1).getDevice()).isEqualTo(all.get(1).getDevice()); + } + } + + @Test + @DisplayName("RowGroup pages tile the whole list without gaps or duplicates") + void shouldTileEveryRowGroup() throws Exception { + List all = parser.parseRowGroups(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + List collected = new ArrayList<>(); + + int offset = 0; + List> pages = new ArrayList<>(); + while (offset < index.deviceCount()) { + List page = parser.readRowGroupPage(reader, index, offset, 2); + assertThat(page).isNotEmpty(); + pages.add(PageDTO.of(page, index.deviceCount(), offset, 2)); + collected.addAll(page); + offset += page.size(); + } + + assertThat(collected) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all); + assertThat(pages).isNotEmpty(); + assertThat(pages.get(pages.size() - 1).isHasMore()).isFalse(); + if (pages.size() > 1) { + assertThat(pages.get(0).isHasMore()).isTrue(); + } + } + } + + @Test + @DisplayName("RowGroup offset beyond total returns an empty list") + void shouldReturnEmptyRowGroupPageBeyondTotal() throws Exception { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + + assertThat(parser.readRowGroupPage(reader, index, index.deviceCount(), 50)).isEmpty(); + assertThat(parser.readRowGroupPage(reader, index, 10_000, 50)).isEmpty(); + assertThat(parser.readRowGroupPage(reader, index, Integer.MAX_VALUE, 200)).isEmpty(); + } + } + + @Test + @DisplayName("Chunk pages tile the whole list and cross device boundaries correctly") + void shouldTileEveryChunkIncludingDeviceBoundaries() throws Exception { + List all = parser.parseChunks(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + assertThat(index.chunkCount()).isEqualTo(all.size()); + + int chunksPerDevice = index.chunkCountOf(0); + assertThat(chunksPerDevice).isPositive(); + + // A page starting on the last chunk of a device and ending inside the next one. + int crossing = chunksPerDevice - 1; + List crossingPage = parser.readChunkPage(reader, index, crossing, 3); + assertThat(crossingPage).hasSize(3); + assertThat(crossingPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(crossing, crossing + 3)); + assertThat(crossingPage.get(0).getDevice()).isNotEqualTo(crossingPage.get(2).getDevice()); + + // A page starting exactly on a device boundary. + List boundaryPage = parser.readChunkPage(reader, index, chunksPerDevice, 4); + assertThat(boundaryPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(chunksPerDevice, chunksPerDevice + 4)); + + // A page aligned inside a single device. + List innerPage = parser.readChunkPage(reader, index, 1, 2); + assertThat(innerPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(1, 3)); + + // A trailing partial page. + List tailPage = parser.readChunkPage(reader, index, all.size() - 2, 50); + assertThat(tailPage) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all.subList(all.size() - 2, all.size())); + } + } + + @Test + @DisplayName("Chunk pages tile the whole list with a non-dividing page size") + void shouldTileAllChunksWithSmallPages() throws Exception { + List all = parser.parseChunks(tsFile.toString()); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + List collected = new ArrayList<>(); + + int offset = 0; + while (offset < index.chunkCount()) { + List page = parser.readChunkPage(reader, index, offset, 2); + assertThat(page).isNotEmpty(); + collected.addAll(page); + offset += page.size(); + } + + assertThat(collected) + .usingRecursiveFieldByFieldElementComparator() + .containsExactlyElementsOf(all); + } + } + + @Test + @DisplayName("Chunk offset beyond total returns an empty list") + void shouldReturnEmptyChunkPageBeyondTotal() throws Exception { + try (TsFileSequenceReader reader = new TsFileSequenceReader(tsFile.toString())) { + DeviceIndex index = index(); + + assertThat(parser.readChunkPage(reader, index, index.chunkCount(), 50)).isEmpty(); + assertThat(parser.readChunkPage(reader, index, 10_000, 50)).isEmpty(); + assertThat(parser.readChunkPage(reader, index, Integer.MAX_VALUE, 200)).isEmpty(); + } + } + + @Test + @DisplayName("Summary reports counts but never carries RowGroup or Chunk lists") + void shouldNotBuildDetailListsInSummary() throws Exception { + TsFileParser.SummaryMetadata summary = parser.parseSummary(tsFile.toString()); + + int enumeratedChunks = parser.parseChunks(tsFile.toString()).size(); + + System.out.printf( + "[summary] devices=%d measurements=%d measurementCount=%d chunks=%d%n", + summary.basic().deviceCount(), + summary.measurements().size(), + summary.basic().measurementCount(), + summary.basic().chunkCount()); + + assertThat(summary.basic().deviceCount()).isEqualTo(DEVICES); + assertThat(summary.basic().rowGroupCount()).isEqualTo(summary.deviceIndex().deviceCount()); + assertThat(summary.basic().chunkCount()).isEqualTo(summary.deviceIndex().chunkCount()); + assertThat(summary.basic().chunkCount()).isEqualTo(enumeratedChunks); + assertThat(summary.deviceIndex().deviceCount()).isEqualTo(DEVICES); + assertThat(summary.measurements()).isNotEmpty(); + assertThat(summary.basic().measurementCount()).isPositive(); + } + + private DeviceIndex index() throws IOException { + return parser.parseDeviceIndex(tsFile.toString()); + } + + private static Path createTableModelTsFile( + Path filePath, int numDevices, int numMeasurements, int numRows) throws Exception { + Files.deleteIfExists(filePath); + File file = filePath.toFile(); + + List columnNames = new ArrayList<>(); + List dataTypes = new ArrayList<>(); + List columnSchemas = new ArrayList<>(); + + columnNames.add("device_id"); + dataTypes.add(TSDataType.STRING); + columnSchemas.add( + new ColumnSchemaBuilder() + .name("device_id") + .dataType(TSDataType.STRING) + .category(ColumnCategory.TAG) + .build()); + + for (int m = 0; m < numMeasurements; m++) { + columnNames.add("s" + m); + dataTypes.add(TSDataType.INT64); + columnSchemas.add( + new ColumnSchemaBuilder() + .name("s" + m) + .dataType(TSDataType.INT64) + .category(ColumnCategory.FIELD) + .build()); + } + + TableSchema tableSchema = new TableSchema("test_table", columnSchemas); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { + for (int d = 0; d < numDevices; d++) { + Tablet tablet = new Tablet(columnNames, dataTypes); + for (int row = 0; row < numRows; row++) { + tablet.addTimestamp(row, d * 1_000_000L + row); + tablet.addValue(row, "device_id", "device_" + d); + for (int m = 0; m < numMeasurements; m++) { + tablet.addValue(row, "s" + m, (long) (d * 1000 + m * 100 + row)); + } + } + writer.write(tablet); + } + } + + return filePath; + } +} From bda3d5bcac57dd22a3fd86758bf54b2a8364fe35 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 12:56:34 +0800 Subject: [PATCH 2/9] =?UTF-8?q?feat(frontend):=20RowGroup/Chunk=20?= =?UTF-8?q?=E6=98=8E=E7=BB=86=E6=94=B9=E8=99=9A=E6=8B=9F=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=20+=20=E5=88=86=E9=A1=B5=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 配合后端分页契约,明细不再一次性拉全量。 - 新增 usePagedList:追加式分页状态机(hasMore 闸门、切文件 generation 防旧响应回写、空页即停防无限重试、失败停止自动加载并可重试) - RowGroupsTable 改用 el-table-v2 虚拟行,滚动到底自动加载下一页; 按设备搜索只过滤已加载行,计数如实显示"已加载 / 总数" - 新增 ChunksTable(此前无此组件),同款虚拟滚动与分页 - 元数据页接入两个新表格,chunks 明细首次可见 - TableFilterPanel/TreeFilterPanel 的设备选项改由分页 RowGroup 有界扫描, 扫描不完整时标出 (已扫 / 总数),不再依赖恒空的 metadata.rowGroups - 显式引入 table-v2 样式:ElTableV2/ElAutoResizer 在 script setup 中手动 import,unplugin-vue-components 不会为其注入 CSS,缺失时表头会落到数据 下方、行不裁剪互相重叠 验证:pnpm build 通过;Playwright 实测 87,488 RowGroup / 1,838,571 Chunk 的文件,两表滚动加载正常、表头正确、行不重叠、零 console 错误 --- frontend/src/api/tsfile/meta.ts | 22 +- frontend/src/api/tsfile/types.ts | 16 + .../src/components/tsfile/ChunksTable.vue | 323 ++++++++++++++++++ .../src/components/tsfile/RowGroupsTable.vue | 231 +++++++++---- .../components/tsfile/TableFilterPanel.vue | 55 ++- .../src/components/tsfile/TreeFilterPanel.vue | 55 ++- frontend/src/composables/usePagedList.ts | 125 +++++++ frontend/src/views/tsfile/metadata/index.vue | 50 +-- 8 files changed, 779 insertions(+), 98 deletions(-) create mode 100644 frontend/src/components/tsfile/ChunksTable.vue create mode 100644 frontend/src/composables/usePagedList.ts diff --git a/frontend/src/api/tsfile/meta.ts b/frontend/src/api/tsfile/meta.ts index 7ce0f5f..2a1ceac 100644 --- a/frontend/src/api/tsfile/meta.ts +++ b/frontend/src/api/tsfile/meta.ts @@ -17,9 +17,29 @@ * under the License. */ -import type { TsFileMetadata } from "./types"; +import type { Chunk, PagedResponse, RowGroup, TsFileMetadata } from "./types"; import { apiClient } from "../request"; +/** + * 元数据摘要。 + * + * 注意:响应中的 rowGroups / chunks 恒为空数组(大文件的明细会撑爆后端内存), + * 行数看 rowGroupCount / chunkCount,明细走 getRowGroupsPage / getChunksPage。 + */ export function getMetadata(fileId: string) { return apiClient.get(`/meta/${fileId}`); } + +/** RowGroup 明细分页 */ +export function getRowGroupsPage(fileId: string, offset = 0, limit = 50) { + return apiClient.get>(`/meta/${fileId}/row-groups`, { + params: { offset, limit }, + }); +} + +/** Chunk 明细分页 */ +export function getChunksPage(fileId: string, offset = 0, limit = 50) { + return apiClient.get>(`/meta/${fileId}/chunks`, { + params: { offset, limit }, + }); +} diff --git a/frontend/src/api/tsfile/types.ts b/frontend/src/api/tsfile/types.ts index c660cd8..45ef593 100644 --- a/frontend/src/api/tsfile/types.ts +++ b/frontend/src/api/tsfile/types.ts @@ -100,6 +100,20 @@ export interface Chunk { maxValue?: string; } +/** + * 分页响应,由 /api/meta/{fileId}/row-groups 与 /api/meta/{fileId}/chunks 共用。 + * + * items 的元素类型由调用方指定;offset/limit 回显请求参数(limit 可能被服务端收窄), + * hasMore 是“是否还有下一页”的唯一判据,不要用 offset + items.length < total 自行推算。 + */ +export interface PagedResponse { + items: T[]; + total: number; + offset: number; + limit: number; + hasMore: boolean; +} + export interface TsFileMetadata { fileId: string; version: string; @@ -109,7 +123,9 @@ export interface TsFileMetadata { rowGroupCount: number; chunkCount: number; measurements: Measurement[]; + /** 摘要接口恒为空数组,明细请用 metaApi.getRowGroupsPage() 分页拉取 */ rowGroups: RowGroup[]; + /** 摘要接口恒为空数组,明细请用 metaApi.getChunksPage() 分页拉取 */ chunks: Chunk[]; tables?: Table[]; // V4 table model tables } diff --git a/frontend/src/components/tsfile/ChunksTable.vue b/frontend/src/components/tsfile/ChunksTable.vue new file mode 100644 index 0000000..be8c1e4 --- /dev/null +++ b/frontend/src/components/tsfile/ChunksTable.vue @@ -0,0 +1,323 @@ + + + + + diff --git a/frontend/src/components/tsfile/RowGroupsTable.vue b/frontend/src/components/tsfile/RowGroupsTable.vue index d5fdb03..4f1a945 100644 --- a/frontend/src/components/tsfile/RowGroupsTable.vue +++ b/frontend/src/components/tsfile/RowGroupsTable.vue @@ -19,42 +19,149 @@ From 129c278b7d9bb7222d0b06658c2dcd46c669fb7b Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 13:44:55 +0800 Subject: [PATCH 3/9] =?UTF-8?q?fix(backend):=20=E6=96=87=E4=BB=B6=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=E6=94=B9=E7=94=A8=E7=B4=A7=E5=87=91=20chunk=20?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=EF=BC=8C=E9=81=BF=E5=85=8D=E5=A4=A7=E6=96=87?= =?UTF-8?q?=E4=BB=B6=20OOM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TsFileHealthChecker 为文件里每个 chunk 建一条 HashMap 记录,1,838,571 个 chunk 的文件要上百万个元数据对象,扫描时撑爆堆。 - 新增 ChunkMetadataIndex:四组平行基本类型数组(offset/measurementId/ startTime/endTime),单 chunk 28B,183 万 chunk 约 49MiB(原结构约 350MiB 起) - 构建时逐设备读取并周期性 clearCachedDeviceMetadata,不全量物化 TimeseriesMetadata - 查找用单向前进游标(遍历本身从文件头顺序前进,O(n)),倒退请求回退二分 - quickCheck 只需"有没有 chunk",改为命中第一个即返回 - 新增 tsfile.scan.max-chunks(默认 5,000,000):超阈值不报错不 OOM,降级为 结构与元数据检查并在扫描结果里给出 DATA_VALIDATION_SKIPPED(WARNING 级) 实测:729MB / 183 万 chunk 的文件在 -Xmx512m 的一次性实例下 30s 扫完 (HEALTHY,无 OOM);闸门设为 1000 时 138ms 降级返回 WARNING。 Tests: 322 run, 0 failures, 17 skipped --- .../tsfile/viewer/config/ScanCacheConfig.java | 8 +- .../viewer/config/TsFileProperties.java | 16 + .../tsfile/viewer/dto/ScanErrorType.java | 9 +- .../viewer/tsfile/ChunkMetadataIndex.java | 313 +++++++++++++++++ .../viewer/tsfile/TsFileHealthChecker.java | 159 +++++++-- backend/src/main/resources/application.yml | 3 + .../TsFileHealthCheckerCompactIndexTest.java | 325 ++++++++++++++++++ .../TsFileHealthCheckerPropertyTest.java | 4 +- 8 files changed, 797 insertions(+), 40 deletions(-) create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java create mode 100644 backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java diff --git a/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java b/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java index 3afe24f..4ba7de7 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/config/ScanCacheConfig.java @@ -78,11 +78,13 @@ public Cache scanTaskCache(TsFileProperties tsFileProperties) *

The health checker is a pure Java class with no Spring dependencies, so it is registered as * a bean here to allow injection into Spring-managed services. * + * @param tsFileProperties configuration supplying the chunk cap for the data-area traversal * @return a new TsFileHealthChecker instance */ @Bean - public TsFileHealthChecker tsFileHealthChecker() { - logger.info("Initializing TsFileHealthChecker bean"); - return new TsFileHealthChecker(); + public TsFileHealthChecker tsFileHealthChecker(TsFileProperties tsFileProperties) { + int maxChunks = tsFileProperties.getScan().getMaxChunks(); + logger.info("Initializing TsFileHealthChecker bean with maxChunks={}", maxChunks); + return new TsFileHealthChecker(maxChunks); } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java b/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java index c7023fd..f910354 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/config/TsFileProperties.java @@ -235,6 +235,14 @@ public static class ScanProperties { /** Time-to-live in minutes for cached scan results. */ private int cacheTtlMinutes = 60; + /** + * 单个文件允许建立紧凑 chunk 索引的最大 chunk 数。 + * + *

超过该阈值时不报错、不 OOM,而是降级为只做结构与元数据检查,跳过数据区全量遍历,并在 {@code ScanResult} 的错误列表中以 {@link + * org.apache.tsfile.viewer.dto.ScanErrorType#DATA_VALIDATION_SKIPPED} 标注。默认 5,000,000。 + */ + private int maxChunks = 5_000_000; + public int getMaxConcurrent() { return maxConcurrent; } @@ -274,5 +282,13 @@ public int getCacheTtlMinutes() { public void setCacheTtlMinutes(int cacheTtlMinutes) { this.cacheTtlMinutes = cacheTtlMinutes; } + + public int getMaxChunks() { + return maxChunks; + } + + public void setMaxChunks(int maxChunks) { + this.maxChunks = maxChunks; + } } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java index a9e535d..18a5b4f 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/ScanErrorType.java @@ -38,5 +38,12 @@ public enum ScanErrorType { TIMESERIES_METADATA_MISMATCH, /** An IOException occurred while reading the file. */ - DATA_READ_ERROR + DATA_READ_ERROR, + + /** + * The file holds more chunks than {@code tsfile.scan.max-chunks} allows, so the data-area + * traversal was skipped. Structure and metadata checks still ran — this is a degraded, not a + * failed, result. + */ + DATA_VALIDATION_SKIPPED } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java new file mode 100644 index 0000000..afadc31 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/ChunkMetadataIndex.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Compact, primitive-array index of chunk metadata keyed by chunk-header offset. + * + *

Replaces the former {@code Map}, whose per-chunk cost is far larger than + * the three fields the health check actually reads (a boxed {@code Long} key, a {@code HashMap} + * node and a full {@code IChunkMetadata} carrying its statistics). This index packs those three + * fields into primitive arrays: {@code 8 (offset) + 4 (measurement id) + 8 (start time) + 8 (end + * time) = 28 bytes} per chunk, so the 1,838,571 chunks of a large real-world file need about 49 MiB + * instead of several hundred MiB. + * + *

The marker-based traversal that consumes this index only ever moves forward through the file, + * so a {@link Cursor} advances monotonically and the whole lookup pass costs O(n) rather than O(n + * log n). A backward query — which traversal cannot produce — falls back to a binary search, so + * correctness never depends on that ordering assumption. + */ +final class ChunkMetadataIndex { + + /** Signals that the configured chunk cap was exceeded while building the index. */ + static final class ChunkCountExceededException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + private final int limit; + + ChunkCountExceededException(int limit) { + super("Chunk count exceeds the configured limit of " + limit); + this.limit = limit; + } + + int getLimit() { + return limit; + } + } + + private static final int INITIAL_CAPACITY = 1024; + + /** Cap on the number of entries; adding one more throws {@link ChunkCountExceededException}. */ + private final int maxChunks; + + /** Distinct measurement names; chunk entries store an {@code int} index into this list. */ + private final List measurementPool = new ArrayList<>(); + + private final Map measurementIdByMeasurement = new HashMap<>(); + + private long[] offsets = new long[INITIAL_CAPACITY]; + private int[] measurementIds = new int[INITIAL_CAPACITY]; + private long[] startTimes = new long[INITIAL_CAPACITY]; + private long[] endTimes = new long[INITIAL_CAPACITY]; + private int size; + private boolean sealed; + + ChunkMetadataIndex(int maxChunks) { + this.maxChunks = maxChunks; + } + + /** + * Appends one chunk entry. + * + * @param offset offset of the chunk header (including its marker byte) + * @param measurement measurement name owning the chunk + * @param startTime first timestamp of the chunk + * @param endTime last timestamp of the chunk + * @throws ChunkCountExceededException if the index already holds {@code maxChunks} entries + */ + void add(long offset, String measurement, long startTime, long endTime) { + if (sealed) { + throw new IllegalStateException("Chunk metadata index is already sealed"); + } + if (size >= maxChunks) { + throw new ChunkCountExceededException(maxChunks); + } + if (size == offsets.length) { + grow(); + } + offsets[size] = offset; + measurementIds[size] = measurementId(measurement); + startTimes[size] = startTime; + endTimes[size] = endTime; + size++; + } + + int size() { + return size; + } + + boolean isEmpty() { + return size == 0; + } + + long offsetAt(int index) { + return offsets[index]; + } + + String measurementAt(int index) { + return measurementPool.get(measurementIds[index]); + } + + long startTimeAt(int index) { + return startTimes[index]; + } + + long endTimeAt(int index) { + return endTimes[index]; + } + + /** + * Sorts the entries by chunk offset and trims the backing arrays to the exact size. + * + *

Lookups require a sealed index. + */ + void seal() { + if (sealed) { + return; + } + sortByOffset(0, size - 1); + offsets = Arrays.copyOf(offsets, size); + measurementIds = Arrays.copyOf(measurementIds, size); + startTimes = Arrays.copyOf(startTimes, size); + endTimes = Arrays.copyOf(endTimes, size); + sealed = true; + } + + /** Creates a forward-only lookup cursor; the returned cursor is cheap and single-pass. */ + Cursor cursor() { + if (!sealed) { + throw new IllegalStateException("seal() must be called before creating a cursor"); + } + return new Cursor(this); + } + + private int measurementId(String measurement) { + Integer id = measurementIdByMeasurement.get(measurement); + if (id == null) { + id = measurementPool.size(); + measurementPool.add(measurement); + measurementIdByMeasurement.put(measurement, id); + } + return id; + } + + private void grow() { + int newCapacity = offsets.length + (offsets.length >> 1) + 1; + offsets = Arrays.copyOf(offsets, newCapacity); + measurementIds = Arrays.copyOf(measurementIds, newCapacity); + startTimes = Arrays.copyOf(startTimes, newCapacity); + endTimes = Arrays.copyOf(endTimes, newCapacity); + } + + private void sortByOffset(int low, int high) { + int lo = low; + int hi = high; + while (lo < hi) { + if (hi - lo < 16) { + insertionSortByOffset(lo, hi); + return; + } + long pivot = medianOfThree(lo, hi); + int less = lo; + int greater = hi; + int current = lo; + // Three-way partition: offsets are unique in practice, but equal keys must not loop forever. + while (current <= greater) { + if (offsets[current] < pivot) { + swap(current, less); + current++; + less++; + } else if (offsets[current] > pivot) { + swap(current, greater); + greater--; + } else { + current++; + } + } + // Recurse into the smaller side only, keeping the stack depth logarithmic. + if (less - lo < hi - greater) { + sortByOffset(lo, less - 1); + lo = greater + 1; + } else { + sortByOffset(greater + 1, hi); + hi = less - 1; + } + } + } + + private void insertionSortByOffset(int lo, int hi) { + for (int i = lo + 1; i <= hi; i++) { + int j = i; + while (j > lo && offsets[j - 1] > offsets[j]) { + swap(j - 1, j); + j--; + } + } + } + + private long medianOfThree(int lo, int hi) { + int mid = lo + ((hi - lo) >>> 1); + long a = offsets[lo]; + long b = offsets[mid]; + long c = offsets[hi]; + if (a < b) { + return b < c ? b : Math.max(a, c); + } + return a < c ? a : Math.max(b, c); + } + + private void swap(int i, int j) { + if (i == j) { + return; + } + long offset = offsets[i]; + offsets[i] = offsets[j]; + offsets[j] = offset; + + int measurementId = measurementIds[i]; + measurementIds[i] = measurementIds[j]; + measurementIds[j] = measurementId; + + long startTime = startTimes[i]; + startTimes[i] = startTimes[j]; + startTimes[j] = startTime; + + long endTime = endTimes[i]; + endTimes[i] = endTimes[j]; + endTimes[j] = endTime; + } + + private int binarySearch(long offset) { + int low = 0; + int high = size - 1; + while (low <= high) { + int mid = (low + high) >>> 1; + long midOffset = offsets[mid]; + if (midOffset < offset) { + low = mid + 1; + } else if (midOffset > offset) { + high = mid - 1; + } else { + return mid; + } + } + return -1; + } + + /** Forward-only cursor over a sealed {@link ChunkMetadataIndex}. Not thread-safe. */ + static final class Cursor { + + private final ChunkMetadataIndex index; + private int position; + + private Cursor(ChunkMetadataIndex index) { + this.index = index; + } + + /** + * Returns the entry index whose chunk-header offset equals {@code offset}, or {@code -1} when + * the chunk is not present in the index. + * + *

Offsets requested in ascending order cost O(1) amortized; a request that moves backwards + * (impossible for file traversal) falls back to a binary search. + */ + int find(long offset) { + if (position > 0 && index.offsets[position - 1] >= offset) { + return index.binarySearch(offset); + } + while (position < index.size && index.offsets[position] < offset) { + position++; + } + if (position < index.size && index.offsets[position] == offset) { + return position; + } + return -1; + } + + String measurement(int index) { + return this.index.measurementAt(index); + } + + long startTime(int index) { + return this.index.startTimeAt(index); + } + + long endTime(int index) { + return this.index.endTimeAt(index); + } + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java index b3d4304..5df161d 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileHealthChecker.java @@ -72,6 +72,30 @@ public class TsFileHealthChecker { private static final Logger logger = LoggerFactory.getLogger(TsFileHealthChecker.class); + /** Default cap on the number of chunks a single file may contribute to the chunk index. */ + public static final int DEFAULT_MAX_CHUNKS = 5_000_000; + + /** Number of devices processed between two device-metadata cache evictions. */ + private static final int DEVICE_CACHE_CLEAR_INTERVAL = 1000; + + private final int maxChunks; + + /** Creates a checker using {@link #DEFAULT_MAX_CHUNKS}. */ + public TsFileHealthChecker() { + this(DEFAULT_MAX_CHUNKS); + } + + /** + * Creates a checker with an explicit chunk cap. + * + * @param maxChunks maximum number of chunks indexed for a single file; a file above the cap has + * its data-area traversal skipped instead of blowing up the heap. Non-positive values fall + * back to {@link #DEFAULT_MAX_CHUNKS}. + */ + public TsFileHealthChecker(int maxChunks) { + this.maxChunks = maxChunks > 0 ? maxChunks : DEFAULT_MAX_CHUNKS; + } + /** * Performs a full health check on a single TsFile following IoTDB's validation approach. * @@ -79,7 +103,8 @@ public class TsFileHealthChecker { * *

    *
  1. Validate file completeness (isComplete) - *
  2. Build chunk metadata offset map for cross-referencing + *
  3. Build a compact chunk metadata index for cross-referencing (skipped, together with the + * data-area traversal below, when the file exceeds {@code tsfile.scan.max-chunks}) *
  4. Sequential marker-based traversal validating: *
      *
    • Chunk header offset matches metadata @@ -121,18 +146,41 @@ public ScanResult check(Path filePath) { "File is not complete (missing tail magic string or truncated)", ErrorSeverity.CRITICAL)); } else { - // Step 2: Build chunk metadata map - Map chunkMetadataMap = getChunkMetadataMap(reader); - if (chunkMetadataMap.isEmpty()) { + // Step 2: Build the compact chunk metadata index + ChunkMetadataIndex chunkMetadataIndex = null; + boolean chunkLimitExceeded = false; + try { + chunkMetadataIndex = buildChunkMetadataIndex(reader); + } catch (ChunkMetadataIndex.ChunkCountExceededException e) { + // Degrade instead of failing or exhausting the heap: structure and metadata were still + // checked, only the full data-area traversal is skipped. + chunkLimitExceeded = true; + logger.warn( + "Chunk count of {} exceeds tsfile.scan.max-chunks={}, skipping data validation", + filePath, + maxChunks); errors.add( new FileError( - ScanErrorType.STRUCTURE_CORRUPT, - "File content", - "No data found in the file (empty chunk metadata)", - ErrorSeverity.ERROR)); - } else { - // Step 3: Marker-based traversal with data validation - validateDataCorrectness(reader, chunkMetadataMap, errors); + ScanErrorType.DATA_VALIDATION_SKIPPED, + "Data area validation", + "Data area validation skipped: chunk count exceeds tsfile.scan.max-chunks (" + + e.getLimit() + + "). Structure and metadata checks still ran.", + ErrorSeverity.WARNING)); + } + + if (!chunkLimitExceeded) { + if (chunkMetadataIndex.isEmpty()) { + errors.add( + new FileError( + ScanErrorType.STRUCTURE_CORRUPT, + "File content", + "No data found in the file (empty chunk metadata)", + ErrorSeverity.ERROR)); + } else { + // Step 3: Marker-based traversal with data validation + validateDataCorrectness(reader, chunkMetadataIndex, errors); + } } } } catch (IOException e) { @@ -180,8 +228,9 @@ public ScanResult quickCheck(Path filePath) { "File is not complete (missing tail magic string or truncated)", ErrorSeverity.CRITICAL)); } else { - Map chunkMetadataMap = getChunkMetadataMap(reader); - if (chunkMetadataMap.isEmpty()) { + // Only the "does the file hold any chunk at all" question matters here, so stop at the + // first chunk instead of indexing every chunk in the file. + if (!hasAnyChunk(reader)) { errors.add( new FileError( ScanErrorType.STRUCTURE_CORRUPT, @@ -219,12 +268,11 @@ public ScanResult quickCheck(Path filePath) { * validateTsFileDataCorrectness} approach. */ private void validateDataCorrectness( - TsFileSequenceReader reader, - Map chunkMetadataMap, - List errors) { + TsFileSequenceReader reader, ChunkMetadataIndex chunkMetadataIndex, List errors) { List> alignedTimeBatches = new ArrayList<>(); Map valueColumn2TimeBatchIndex = new HashMap<>(); + ChunkMetadataIndex.Cursor chunkCursor = chunkMetadataIndex.cursor(); try { reader.position((long) TSFileConfig.MAGIC_STRING.getBytes().length + 1); @@ -241,17 +289,17 @@ private void validateDataCorrectness( case MetaMarker.ONLY_ONE_PAGE_VALUE_CHUNK_HEADER: long chunkOffset = reader.position(); ChunkHeader header = reader.readChunkHeader(marker); - IChunkMetadata chunkMetadata = chunkMetadataMap.get(chunkOffset - Byte.BYTES); + int chunkIndex = chunkCursor.find(chunkOffset - Byte.BYTES); + String metadataMeasurement = chunkIndex >= 0 ? chunkCursor.measurement(chunkIndex) : null; // Validate chunk offset matches metadata - if (chunkMetadata != null - && !chunkMetadata.getMeasurementUid().equals(header.getMeasurementID())) { + if (chunkIndex >= 0 && !metadataMeasurement.equals(header.getMeasurementID())) { errors.add( new FileError( ScanErrorType.STRUCTURE_CORRUPT, "Chunk at offset " + (chunkOffset - Byte.BYTES), "Chunk measurement ID mismatch: metadata says '" - + chunkMetadata.getMeasurementUid() + + metadataMeasurement + "' but chunk header says '" + header.getMeasurementID() + "'", @@ -302,11 +350,11 @@ private void validateDataCorrectness( long pageHeaderStartTime = isHasStatistic ? pageHeader.getStartTime() - : (chunkMetadata != null ? chunkMetadata.getStartTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.startTime(chunkIndex) : 0); long pageHeaderEndTime = isHasStatistic ? pageHeader.getEndTime() - : (chunkMetadata != null ? chunkMetadata.getEndTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.endTime(chunkIndex) : 0); validateTimeFrame( alignedTimeBatch, @@ -347,11 +395,11 @@ private void validateDataCorrectness( long pageHeaderStartTime = isHasStatistic ? pageHeader.getStartTime() - : (chunkMetadata != null ? chunkMetadata.getStartTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.startTime(chunkIndex) : 0); long pageHeaderEndTime = isHasStatistic ? pageHeader.getEndTime() - : (chunkMetadata != null ? chunkMetadata.getEndTime() : 0); + : (chunkIndex >= 0 ? chunkCursor.endTime(chunkIndex) : 0); long pageStartTime = Long.MAX_VALUE; long previousTime = Long.MIN_VALUE; @@ -532,21 +580,62 @@ private void validateTimeFrame( // --------------------------------------------------------------------------- /** - * Builds a map of chunk header offset to IChunkMetadata for cross-referencing during traversal. + * Builds the compact chunk metadata index used for cross-referencing during traversal. + * + *

      Devices are read one at a time and dropped immediately, so the transient peak stays bounded + * by a single device instead of the whole file's {@link TimeseriesMetadata} tree. The reader's + * device-metadata cache is evicted periodically; nothing in this method holds a reference into + * that cache, so eviction is safe at any point. + * + * @throws ChunkMetadataIndex.ChunkCountExceededException if the file holds more than {@code + * maxChunks} chunks */ - private Map getChunkMetadataMap(TsFileSequenceReader reader) - throws IOException { - Map offset2ChunkMetadata = new HashMap<>(); - Map> device2Metadata = - reader.getAllTimeseriesMetadata(true); - for (Map.Entry> entry : device2Metadata.entrySet()) { - for (TimeseriesMetadata timeseriesMetadata : entry.getValue()) { - for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) { - offset2ChunkMetadata.put(chunkMetadata.getOffsetOfChunkHeader(), chunkMetadata); + ChunkMetadataIndex buildChunkMetadataIndex(TsFileSequenceReader reader) throws IOException { + ChunkMetadataIndex index = new ChunkMetadataIndex(maxChunks); + int devicesSinceCacheClear = 0; + for (IDeviceID device : reader.getAllDevices()) { + for (TimeseriesMetadata timeseriesMetadata : reader.getDeviceTimeseriesMetadata(device)) { + List chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); + if (chunkMetadataList == null || chunkMetadataList.isEmpty()) { + continue; } + for (IChunkMetadata chunkMetadata : chunkMetadataList) { + index.add( + chunkMetadata.getOffsetOfChunkHeader(), + chunkMetadata.getMeasurementUid(), + chunkMetadata.getStartTime(), + chunkMetadata.getEndTime()); + } + } + if (++devicesSinceCacheClear >= DEVICE_CACHE_CLEAR_INTERVAL) { + devicesSinceCacheClear = 0; + reader.clearCachedDeviceMetadata(); + } + } + index.seal(); + return index; + } + + /** + * Returns whether the file holds at least one chunk, stopping at the first one found. + * + *

      Used by {@link #quickCheck(Path)}, which needs the "non-empty" verdict only. + */ + private boolean hasAnyChunk(TsFileSequenceReader reader) throws IOException { + int devicesSinceCacheClear = 0; + for (IDeviceID device : reader.getAllDevices()) { + for (TimeseriesMetadata timeseriesMetadata : reader.getDeviceTimeseriesMetadata(device)) { + List chunkMetadataList = timeseriesMetadata.getChunkMetadataList(); + if (chunkMetadataList != null && !chunkMetadataList.isEmpty()) { + return true; + } + } + if (++devicesSinceCacheClear >= DEVICE_CACHE_CLEAR_INTERVAL) { + devicesSinceCacheClear = 0; + reader.clearCachedDeviceMetadata(); } } - return offset2ChunkMetadata; + return false; } /** diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index 03e0860..85755fe 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -63,6 +63,9 @@ tsfile: task-timeout-minutes: 30 cache-max-size: 100 cache-ttl-minutes: 60 + # 单个文件允许建立紧凑 chunk 索引的最大 chunk 数。超过该阈值时不报错、不 OOM, + # 而是降级为只做结构与元数据检查、跳过数据区全量遍历,并在扫描结果中明确标注。 + max-chunks: 5000000 # SpringDoc OpenAPI Configuration springdoc: diff --git a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java new file mode 100644 index 0000000..070f7d6 --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerCompactIndexTest.java @@ -0,0 +1,325 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import org.apache.tsfile.file.metadata.IChunkMetadata; +import org.apache.tsfile.file.metadata.TimeseriesMetadata; +import org.apache.tsfile.read.TsFileSequenceReader; + +import org.apache.tsfile.viewer.dto.ErrorSeverity; +import org.apache.tsfile.viewer.dto.HealthStatus; +import org.apache.tsfile.viewer.dto.ScanErrorType; +import org.apache.tsfile.viewer.service.FileError; +import org.apache.tsfile.viewer.service.ScanResult; + +/** + * Tests for the compact chunk metadata index that replaced the per-chunk {@code Map} in {@link TsFileHealthChecker}. + * + *

      Covers: structure construction and lookup correctness, {@code quickCheck} early exit, the + * over-limit degradation path, and equality of the scan conclusions against the pre-change + * metadata map. + */ +class TsFileHealthCheckerCompactIndexTest { + + private static void assertEntry( + ChunkMetadataIndex index, + int position, + long offset, + String measurement, + long startTime, + long endTime) { + assertThat(index.offsetAt(position)).isEqualTo(offset); + assertThat(index.measurementAt(position)).isEqualTo(measurement); + assertThat(index.startTimeAt(position)).isEqualTo(startTime); + assertThat(index.endTimeAt(position)).isEqualTo(endTime); + } + + // ------------------------------------------------------------------------- + // Compact structure: construction and lookups + // ------------------------------------------------------------------------- + + @Nested + class CompactIndexTests { + + @Test + void seal_sortsByOffsetAndKeepsPayloadAttached() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + index.add(300L, "s3", 30L, 39L); + index.add(100L, "s1", 10L, 19L); + index.add(200L, "s2", 20L, 29L); + + index.seal(); + + assertThat(index.size()).isEqualTo(3); + assertEntry(index, 0, 100L, "s1", 10L, 19L); + assertEntry(index, 1, 200L, "s2", 20L, 29L); + assertEntry(index, 2, 300L, "s3", 30L, 39L); + } + + @Test + void cursor_forwardLookupsMatchEveryEntry() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + index.add(20L, "b", 200L, 209L); + index.add(10L, "a", 100L, 109L); + index.add(30L, "a", 300L, 309L); + index.seal(); + + ChunkMetadataIndex.Cursor cursor = index.cursor(); + + for (int position = 0; position < index.size(); position++) { + int found = cursor.find(index.offsetAt(position)); + assertThat(found).isEqualTo(position); + assertThat(cursor.measurement(found)).isEqualTo(index.measurementAt(position)); + assertThat(cursor.startTime(found)).isEqualTo(index.startTimeAt(position)); + assertThat(cursor.endTime(found)).isEqualTo(index.endTimeAt(position)); + } + } + + @Test + void cursor_returnsMinusOneForAbsentOffsetsAndKeepsAdvancing() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + index.add(100L, "a", 1L, 2L); + index.add(200L, "a", 3L, 4L); + index.add(300L, "a", 5L, 6L); + index.seal(); + + ChunkMetadataIndex.Cursor cursor = index.cursor(); + + assertThat(cursor.find(50L)).isEqualTo(-1); + assertThat(cursor.find(150L)).isEqualTo(-1); + assertThat(cursor.find(200L)).isEqualTo(1); + assertThat(cursor.find(250L)).isEqualTo(-1); + assertThat(cursor.find(350L)).isEqualTo(-1); + assertThat(cursor.find(300L)).isEqualTo(2); + } + + @Test + void cursor_backwardLookupFallsBackToBinarySearch() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + for (int i = 0; i < 40; i++) { + index.add(i * 10L, "m" + (i % 3), i, i); + } + index.seal(); + + ChunkMetadataIndex.Cursor cursor = index.cursor(); + + assertThat(cursor.find(390L)).isEqualTo(39); + // Traversal never asks for a smaller offset, but the answer must still be correct. + assertThat(cursor.find(10L)).isEqualTo(1); + assertThat(cursor.find(390L)).isEqualTo(39); + } + + @Test + void seal_sortsLargeOutOfOrderInputBeyondInitialCapacity() { + int entryCount = 5_000; + ChunkMetadataIndex index = new ChunkMetadataIndex(entryCount); + Map sourceIndexByOffset = new HashMap<>(); + // Deterministic shuffle so entries arrive far out of offset order. + for (int i = 0; i < entryCount; i++) { + int offset = (i * 7919) % entryCount; + // Payload is derived from the source index, so it must follow the entry through the sort. + index.add(offset, "s" + (i % 7), i * 1000L, i * 1000L + 999L); + sourceIndexByOffset.put(offset, i); + } + + index.seal(); + + assertThat(index.size()).isEqualTo(entryCount); + ChunkMetadataIndex.Cursor cursor = index.cursor(); + for (int offset = 0; offset < entryCount; offset++) { + int found = cursor.find(offset); + assertThat(found).as("offset %s must be indexed", offset).isGreaterThanOrEqualTo(0); + assertThat(index.offsetAt(found)).isEqualTo(offset); + + int sourceIndex = sourceIndexByOffset.get(offset); + assertThat(index.measurementAt(found)).isEqualTo("s" + (sourceIndex % 7)); + assertThat(index.startTimeAt(found)).isEqualTo(sourceIndex * 1000L); + assertThat(index.endTimeAt(found)).isEqualTo(sourceIndex * 1000L + 999L); + } + } + + @Test + void seal_toleratesDuplicateOffsets() { + ChunkMetadataIndex index = new ChunkMetadataIndex(100); + for (int i = 0; i < 40; i++) { + index.add(7L, "dup" + i, i, i); + } + + index.seal(); + + assertThat(index.size()).isEqualTo(40); + assertThat(index.cursor().find(7L)).isGreaterThanOrEqualTo(0); + } + + @Test + void add_beyondMaxChunksThrowsWithLimit() { + ChunkMetadataIndex index = new ChunkMetadataIndex(2); + index.add(1L, "a", 0L, 0L); + index.add(2L, "a", 0L, 0L); + + assertThatThrownBy(() -> index.add(3L, "a", 0L, 0L)) + .isInstanceOf(ChunkMetadataIndex.ChunkCountExceededException.class) + .hasMessageContaining("2"); + } + + @Test + void emptyIndexHasNoEntriesAndFindsNothing() { + ChunkMetadataIndex index = new ChunkMetadataIndex(10); + index.seal(); + + assertThat(index.isEmpty()).isTrue(); + assertThat(index.cursor().find(0L)).isEqualTo(-1); + } + + @Test + void add_afterSealAndCursorBeforeSealAreRejected() { + ChunkMetadataIndex index = new ChunkMetadataIndex(10); + index.add(1L, "a", 0L, 5L); + + assertThatThrownBy(index::cursor).isInstanceOf(IllegalStateException.class); + + index.seal(); + assertThatThrownBy(() -> index.add(2L, "b", 6L, 9L)) + .isInstanceOf(IllegalStateException.class); + } + } + + // ------------------------------------------------------------------------- + // Checker integration: equivalence, degradation, early exit + // ------------------------------------------------------------------------- + + @Nested + class CheckerIntegrationTests { + + private static final List DEVICE_IDS = List.of("d1", "d2", "d3"); + + private Path createSampleFile(Path tempDir) throws Exception { + return TsFileTestUtils.createTableModelMultipleDevices( + tempDir.resolve("compact-index-sample.tsfile"), "wind", DEVICE_IDS, 20) + .toPath(); + } + + /** The pre-change implementation: one map entry per chunk, built from all metadata at once. */ + private Map legacyChunkMetadataMap(Path file) throws IOException { + Map offsetToChunkMetadata = new HashMap<>(); + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.toString())) { + for (List timeseriesMetadataList : + reader.getAllTimeseriesMetadata(true).values()) { + for (TimeseriesMetadata timeseriesMetadata : timeseriesMetadataList) { + for (IChunkMetadata chunkMetadata : timeseriesMetadata.getChunkMetadataList()) { + offsetToChunkMetadata.put(chunkMetadata.getOffsetOfChunkHeader(), chunkMetadata); + } + } + } + } + return offsetToChunkMetadata; + } + + @Test + void compactIndex_matchesLegacyMetadataMap_sameFileSameConclusion(@TempDir Path tempDir) + throws Exception { + Path file = createSampleFile(tempDir); + + Map legacyMap = legacyChunkMetadataMap(file); + assertThat(legacyMap).as("the generated file must contain chunks").isNotEmpty(); + + TsFileHealthChecker checker = new TsFileHealthChecker(); + List ascendingOffsets = new ArrayList<>(legacyMap.keySet()); + ascendingOffsets.sort(null); + + try (TsFileSequenceReader reader = new TsFileSequenceReader(file.toString())) { + ChunkMetadataIndex compactIndex = checker.buildChunkMetadataIndex(reader); + + assertThat(compactIndex.isEmpty()).isEqualTo(legacyMap.isEmpty()); + assertThat(compactIndex.size()).isEqualTo(legacyMap.size()); + + ChunkMetadataIndex.Cursor cursor = compactIndex.cursor(); + for (Long offset : ascendingOffsets) { + int found = cursor.find(offset); + assertThat(found).as("offset %s must be indexed", offset).isGreaterThanOrEqualTo(0); + IChunkMetadata expected = legacyMap.get(offset); + assertThat(cursor.measurement(found)).isEqualTo(expected.getMeasurementUid()); + assertThat(cursor.startTime(found)).isEqualTo(expected.getStartTime()); + assertThat(cursor.endTime(found)).isEqualTo(expected.getEndTime()); + } + } + + // Below the cap the conclusion is unchanged: data-area traversal runs, no skip marker. + ScanResult result = checker.check(file); + assertThat(result.getHealthStatus()).isEqualTo(HealthStatus.HEALTHY); + assertThat(result.getErrors()).isEmpty(); + assertThat(result.getFilePath()).isEqualTo(file.toString()); + assertThat(result.getFileSize()).isGreaterThan(0); + } + + @Test + void check_overChunkLimit_degradesAndReportsSkip(@TempDir Path tempDir) throws Exception { + Path file = createSampleFile(tempDir); + int chunkCount = legacyChunkMetadataMap(file).size(); + assertThat(chunkCount).isGreaterThan(1); + + ScanResult result = new TsFileHealthChecker(1).check(file); + + assertThat(result.getHealthStatus()).isEqualTo(HealthStatus.WARNING); + assertThat(result.getErrors()).hasSize(1); + FileError skipError = result.getErrors().get(0); + assertThat(skipError.getErrorType()).isEqualTo(ScanErrorType.DATA_VALIDATION_SKIPPED); + assertThat(skipError.getSeverity()).isEqualTo(ErrorSeverity.WARNING); + assertThat(skipError.getDescription()).contains("max-chunks").contains("skipped"); + assertThat(result.getFilePath()).isEqualTo(file.toString()); + assertThat(result.getFileSize()).isGreaterThan(0); + assertThat(result.getScanDurationMs()).isGreaterThanOrEqualTo(0); + } + + @Test + void quickCheck_stopsAtFirstChunk(@TempDir Path tempDir) throws Exception { + Path file = createSampleFile(tempDir); + int chunkCount = legacyChunkMetadataMap(file).size(); + assertThat(chunkCount).isGreaterThan(1); + + TsFileHealthChecker oneChunkChecker = new TsFileHealthChecker(1); + ScanResult quickResult = oneChunkChecker.quickCheck(file); + assertThat(quickResult.getHealthStatus()).isEqualTo(HealthStatus.HEALTHY); + assertThat(quickResult.getErrors()).isEmpty(); + + // The same cap makes the full check degrade, which proves the file holds more than one chunk + // and that quickCheck answered without indexing all of them. + ScanResult fullResult = oneChunkChecker.check(file); + assertThat(fullResult.getErrors()) + .extracting(FileError::getErrorType) + .containsExactly(ScanErrorType.DATA_VALIDATION_SKIPPED); + } + } +} diff --git a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java index 0252d80..afd7b13 100644 --- a/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java +++ b/backend/src/test/java/org/apache/tsfile/viewer/tsfile/TsFileHealthCheckerPropertyTest.java @@ -50,6 +50,7 @@ class TsFileHealthCheckerPropertyTest { *

    • CHUNK_STATISTICS_MISMATCH → ERROR *
    • TIMESERIES_METADATA_MISMATCH → ERROR *
    • DATA_READ_ERROR → WARNING + *
    • DATA_VALIDATION_SKIPPED → WARNING *
    */ private static final Map EXPECTED_SEVERITY_MAP = @@ -58,7 +59,8 @@ class TsFileHealthCheckerPropertyTest { ScanErrorType.STRUCTURE_CORRUPT, ErrorSeverity.ERROR, ScanErrorType.CHUNK_STATISTICS_MISMATCH, ErrorSeverity.ERROR, ScanErrorType.TIMESERIES_METADATA_MISMATCH, ErrorSeverity.ERROR, - ScanErrorType.DATA_READ_ERROR, ErrorSeverity.WARNING); + ScanErrorType.DATA_READ_ERROR, ErrorSeverity.WARNING, + ScanErrorType.DATA_VALIDATION_SKIPPED, ErrorSeverity.WARNING); // --------------------------------------------------------------------------- // Property 5: ScanResult 结构完整性与严重程度映射 From 32d874308699ec6529801085ba4649900d19ca3e Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 14:38:48 +0800 Subject: [PATCH 4/9] =?UTF-8?q?fix(frontend):=20=E8=A1=A5=E6=89=AB?= =?UTF-8?q?=E6=8F=8F=E6=96=B0=E6=9E=9A=E4=B8=BE=E6=96=87=E6=A1=88=E3=80=81?= =?UTF-8?q?=E4=BF=AE=E6=B5=8B=E7=82=B9=E8=A2=AB=E6=B8=85=E7=A9=BA=E3=80=81?= =?UTF-8?q?Chunk=20=E8=A1=A8=E5=88=97=E5=8F=AF=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 补 DATA_VALIDATION_SKIPPED 的 i18n(zh/en)与 TS 联合类型,并同步 Record 分布类型;顺带补上同样缺失的 DATA_READ_ERROR 文案(后端 6 个枚举值现已与三处映射一一对应) - TreeFilterPanel 不再用 watch(selectedDevice) 清空测点:loadMetadata 会把 selectedDevice 程序化设为第一个设备,watcher 异步回调落在 chart 模式默认 测点写入之后,导致图表有数据但测点框为空。改挂 el-select-v2 的 @change, 只有用户主动切换才重置 - ChunksTable 12 列压到 11 列(编码方式+压缩方式合并为「编码/压缩」), 重排列宽使总和 1193 ≤ 容器 1196,消除 el-table-v2 的等比压缩截断; 偏移量/最小值/最大值等数值列不再出现省略号 验证:Playwright 实测线上——测点首次加载保留 8 条、手动切设备后回到占位符; Chunk 表偏移量 344172277、编码/压缩 TS_2DIFF / LZ4、最小值/最大值均完整 --- frontend/src/api/tsfile/scan-types.ts | 3 +- .../src/components/tsfile/ChunksTable.vue | 74 ++++++++++--------- .../src/components/tsfile/TreeFilterPanel.vue | 15 ++-- frontend/src/i18n/locales/en-US.json | 4 +- frontend/src/i18n/locales/zh-CN.json | 4 +- frontend/src/stores/tsfile/scan.ts | 3 + 6 files changed, 61 insertions(+), 42 deletions(-) diff --git a/frontend/src/api/tsfile/scan-types.ts b/frontend/src/api/tsfile/scan-types.ts index 166f84d..a097385 100644 --- a/frontend/src/api/tsfile/scan-types.ts +++ b/frontend/src/api/tsfile/scan-types.ts @@ -35,7 +35,8 @@ export type ScanErrorType = | "DATA_READ_ERROR" | "FORMAT_INCOMPATIBLE" | "STRUCTURE_CORRUPT" - | "TIMESERIES_METADATA_MISMATCH"; + | "TIMESERIES_METADATA_MISMATCH" + | "DATA_VALIDATION_SKIPPED"; export type ErrorSeverity = "CRITICAL" | "ERROR" | "WARNING"; diff --git a/frontend/src/components/tsfile/ChunksTable.vue b/frontend/src/components/tsfile/ChunksTable.vue index be8c1e4..0f89eeb 100644 --- a/frontend/src/components/tsfile/ChunksTable.vue +++ b/frontend/src/components/tsfile/ChunksTable.vue @@ -95,9 +95,15 @@ function formatTimeRange(startTime?: number, endTime?: number): string { } // 列宽不足时 el-table-v2 会把单元格截断成省略号,title 保证完整值仍可读 -function renderText(value: number | string | undefined): VNode { +function renderText(value: number | string | undefined, extraClass = "truncate"): VNode { const text = value === undefined || value === "" ? "-" : String(value); - return h("span", { class: "max-w-full truncate", title: text }, text); + return h("span", { class: `max-w-full ${extraClass} tnum`, title: text }, text); +} + +// 数值列统一用小一号的等宽数字:同样的像素宽度能多放 2~3 位, +// 这样 13 位的时间戳(VECTOR 的 min/max)与 9 位的偏移量不必再截断。 +function renderNumber(value: number | string | undefined): VNode { + return renderText(value, "truncate text-xs"); } // key 用已加载序列里的绝对下标:Chunk 没有天然唯一字段,不能当 row-key @@ -146,9 +152,9 @@ const columns = computed[]>(() => [ key: "measurement", dataKey: "measurement", title: t("tsfile.metadata.chunkMeasurement"), - width: 180, + width: 160, flexGrow: 1, - minWidth: 160, + minWidth: 150, cellRenderer: ({ cellData }) => h("span", { class: "block truncate", title: String(cellData ?? "") }, String(cellData || "-")), }, @@ -156,17 +162,17 @@ const columns = computed[]>(() => [ key: "device", dataKey: "device", title: t("tsfile.metadata.device"), - width: 120, + width: 99, cellRenderer: ({ cellData }) => h("span", { class: "block truncate", title: String(cellData ?? "") }, String(cellData || "-")), }, { key: "timeRange", title: t("tsfile.metadata.chunkTimeRange"), - // 12 列挤在 1194px 容器里时 el-table-v2 会按声明宽度等比压缩, - // 声明宽度给足(380)才能拿到最大份额;压缩后仍单行省略, - // 绝不换行——换行会被 32px 行高裁掉,看起来像压到下一行。 - width: 380, + // 12 列降成 11 列后,时间范围拿到的宽度仍然不足以单行放下两个完整时间戳 + // (需要 ~300px),这里保持单行省略 + title,绝不换行:换行会被 32px + // 行高裁掉,看起来像压到了下一行。 + width: 180, cellRenderer: ({ rowData }) => { const text = formatTimeRange(rowData.startTime, rowData.endTime); return h("span", { class: "max-w-full truncate font-mono text-xs", title: text }, text); @@ -175,60 +181,62 @@ const columns = computed[]>(() => [ { key: "numOfPoints", title: t("tsfile.metadata.numOfPoints"), - width: 110, - // 声明宽度给到 110,压缩后表头「数据点数」仍能排在一行 + width: 75, align: "center", cellRenderer: ({ rowData }) => - h("span", Number.isNaN(rowData.numOfPoints) ? "-" : String(rowData.numOfPoints)), + h( + "span", + { class: "text-xs tnum" }, + Number.isNaN(rowData.numOfPoints) ? "-" : String(rowData.numOfPoints), + ), }, { key: "offset", title: t("tsfile.metadata.offset"), - width: 110, - cellRenderer: ({ rowData }) => renderText(rowData.offset), + width: 89, + cellRenderer: ({ rowData }) => renderNumber(rowData.offset), }, { key: "size", title: t("tsfile.metadata.size"), - width: 80, - cellRenderer: ({ rowData }) => renderText(rowData.size), + width: 48, + cellRenderer: ({ rowData }) => renderNumber(rowData.size), }, { key: "compressionRatio", title: t("tsfile.metadata.compressionRatio"), - width: 100, + width: 79, align: "center", - cellRenderer: ({ rowData }) => h("span", formatRatio(rowData.compressionRatio)), + cellRenderer: ({ rowData }) => h("span", { class: "text-xs tnum" }, formatRatio(rowData.compressionRatio)), }, { key: "dataType", title: t("tsfile.metadata.dataType"), - width: 100, + width: 88, cellRenderer: ({ rowData }) => renderText(rowData.dataType), }, { - key: "encoding", - title: t("tsfile.metadata.encoding"), - width: 110, - cellRenderer: ({ rowData }) => renderText(rowData.encoding), - }, - { - key: "compression", - title: t("tsfile.metadata.compression"), - width: 105, - cellRenderer: ({ rowData }) => renderText(rowData.compression), + // 编码与压缩总是成对读取(同一个 Chunk 的序列化方式),合并成一列 + // 腾出的宽度正好还给被挤成「223384…」的偏移量/最小值/最大值。 + key: "encodingCompression", + title: t("tsfile.metadata.encodingCompression"), + width: 129, + cellRenderer: ({ rowData }) => { + const text = `${rowData.encoding || "-"} / ${rowData.compression || "-"}`; + return h("span", { class: "max-w-full truncate", title: text }, text); + }, }, { key: "minValue", title: t("tsfile.data.minValue"), - width: 110, - cellRenderer: ({ rowData }) => renderText(rowData.minValue), + width: 123, + cellRenderer: ({ rowData }) => renderNumber(rowData.minValue), }, { key: "maxValue", title: t("tsfile.data.maxValue"), - width: 110, - cellRenderer: ({ rowData }) => renderText(rowData.maxValue), + width: 123, + cellRenderer: ({ rowData }) => renderNumber(rowData.maxValue), }, ]); diff --git a/frontend/src/components/tsfile/TreeFilterPanel.vue b/frontend/src/components/tsfile/TreeFilterPanel.vue index 1d6dd3b..a0b332c 100644 --- a/frontend/src/components/tsfile/TreeFilterPanel.vue +++ b/frontend/src/components/tsfile/TreeFilterPanel.vue @@ -221,12 +221,14 @@ watch( }, { immediate: true }, ); -watch( - () => selectedDevice.value, - () => { - selectedMeasurements.value = []; - }, -); +// 切换设备时清空已选测点,但只在**用户主动切换**时发生。 +// 不能用 watch(selectedDevice):loadMetadata / resetFilters 都会把 selectedDevice +// 程序化地设成第一个设备,watcher 又是异步 flush,回调落在 chart 模式默认测点已经 +// 写入之后,会把它们清掉(表现为图表有数据、测点框却是空的)。 +// el-select-v2 的 change 只在用户选值/清空时发出,程序化赋值不会触发。 +function handleDeviceChange() { + selectedMeasurements.value = []; +} watch(timeRange, () => { selectedQuickRange.value = null; }); @@ -256,6 +258,7 @@ function handleAdvancedApply(conditions: AdvancedCondition[]) { :loading="loading || deviceLoading" style="width: 240px" popper-class="v2-options-compact" + @change="handleDeviceChange" /> { { name: "CHUNK_STATISTICS_MISMATCH", value: dist.CHUNK_STATISTICS_MISMATCH ?? 0 }, { name: "TIMESERIES_METADATA_MISMATCH", value: dist.TIMESERIES_METADATA_MISMATCH ?? 0 }, { name: "DATA_READ_ERROR", value: dist.DATA_READ_ERROR ?? 0 }, + { name: "DATA_VALIDATION_SKIPPED", value: dist.DATA_VALIDATION_SKIPPED ?? 0 }, ]; } const counts: Record = { @@ -126,6 +127,7 @@ export const useScanStore = defineStore("tsfile-scan", () => { CHUNK_STATISTICS_MISMATCH: 0, TIMESERIES_METADATA_MISMATCH: 0, DATA_READ_ERROR: 0, + DATA_VALIDATION_SKIPPED: 0, }; for (const result of scanResults.value) { const seen = new Set(); @@ -144,6 +146,7 @@ export const useScanStore = defineStore("tsfile-scan", () => { value: counts.TIMESERIES_METADATA_MISMATCH, }, { name: "DATA_READ_ERROR", value: counts.DATA_READ_ERROR }, + { name: "DATA_VALIDATION_SKIPPED", value: counts.DATA_VALIDATION_SKIPPED }, ]; }); From e0feb29854a9e769576138bb2f3cad07d87aad37 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 18:46:08 +0800 Subject: [PATCH 5/9] =?UTF-8?q?fix(backend):=20=E6=A0=91=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8D=E5=86=8D=E5=9B=A0=20getAllTableSche?= =?UTF-8?q?ma=20=E6=8A=9B=20NPE=20=E8=80=8C=20500?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ITsFileReader.getAllTableSchema() 在树模型文件上不解引用空的 tableSchemaMap, 直接抛 NullPointerException;而所有调用方都以"空列表"为前提来判断是否回退到 TsFileSequenceReader 的树模型读取路径,于是整条数据路径 500。 tsfile 2.3.0 与升级后的 2.5.0 均能复现,非本次依赖升级引入。 - 新增 TableSchemaSupport.readAll:把 null 与 NPE 归一化为空列表 (TsFileParser 早有同款私有实现,提取为共享工具供其余调用点复用) - TsFileDataReader 7 处、TableService 2 处裸调用改走该工具 实测:树模型文件 /api/data/preview 与 /api/data/query 由 500 变 200, 数据预览页渲染 100 行、无错误横幅。 已知未修(既有缺陷,属测点过滤语义而非崩溃):树模型文件带 devices/ measurements 过滤时仍返回空——DataService 是读后过滤,而树模型行的测点键 为全路径,与请求里的短名匹配不上。 --- .../tsfile/viewer/service/TableService.java | 5 +- .../viewer/tsfile/TableSchemaSupport.java | 63 +++++++++++++++++++ .../viewer/tsfile/TsFileDataReader.java | 14 ++--- 3 files changed, 73 insertions(+), 9 deletions(-) create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java index 71b5476..d63ac27 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/TableService.java @@ -48,6 +48,7 @@ import org.apache.tsfile.viewer.dto.TableListResponse.TableInfo; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.tsfile.TableSchemaSupport; /** * Service for table-level data operations. @@ -89,7 +90,7 @@ public TableListResponse getTableList(String fileId) List tableInfos = new ArrayList<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(filePath)).build()) { - List schemas = reader.getAllTableSchema(); + List schemas = TableSchemaSupport.readAll(reader); if (schemas == null || schemas.isEmpty()) { logger.debug("No tables found in file: {}", filePath); @@ -161,7 +162,7 @@ public DeviceListResponse getDeviceList(String fileId, String tableName) Map deviceMap = new HashMap<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(new File(filePath)).build()) { - List schemas = reader.getAllTableSchema(); + List schemas = TableSchemaSupport.readAll(reader); if (schemas == null || schemas.isEmpty()) { return DeviceListResponse.builder().devices(new ArrayList<>()).totalCount(0).build(); diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java new file mode 100644 index 0000000..8006bd9 --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TableSchemaSupport.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.tsfile; + +import java.io.IOException; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.read.v4.ITsFileReader; + +/** + * Reads the table schemas of a V4 TsFile without failing on tree model files. + * + *

    Callers use an empty result to decide that the file is a tree model file and to fall back to + * {@link org.apache.tsfile.read.TsFileSequenceReader}. {@code ITsFileReader.getAllTableSchema()} + * does not honour that contract: on a tree model file the underlying {@code + * DeviceTableModelReader} dereferences a null table schema map and throws {@link + * NullPointerException} instead of returning an empty list. + */ +public final class TableSchemaSupport { + + private static final Logger logger = LoggerFactory.getLogger(TableSchemaSupport.class); + + private TableSchemaSupport() {} + + /** + * Returns the table schemas of the file, or an empty list when the file carries no table schema. + * + * @param reader open V4 TsFile reader + * @return table schemas, never null + * @throws IOException if the metadata cannot be read + */ + public static List readAll(ITsFileReader reader) throws IOException { + try { + List schemas = reader.getAllTableSchema(); + return schemas == null ? List.of() : schemas; + } catch (NullPointerException e) { + // tsfile 2.x throws NPE here for tree model files instead of returning null or an empty list. + logger.debug("File carries no table schema (tree model): {}", e.getMessage()); + return List.of(); + } + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java index 5625cbb..8fb7c1b 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java @@ -228,7 +228,7 @@ public DataReadResult readDataByTimeRange( try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { List schemas = new ArrayList<>(); - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { schemas.add(ts); } @@ -349,7 +349,7 @@ public DataReadResult readDataByTimeRange( int skipped = 0, collected = 0; boolean hasMore = false; - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { if (collected >= limit) { hasMore = true; break; @@ -450,7 +450,7 @@ public DataReadResult readDataWithFilter(File filePath, ReadFilter filter, int l boolean hasMore = false; try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { if (collected >= limit) { hasMore = true; break; @@ -571,7 +571,7 @@ public int streamData(File filePath, ReadFilter filter, Consumer consum int processed = 0; try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { String tblName = ts.getTableName(); if (fTbls != null && !fTbls.isEmpty() && !fTbls.contains(tblName)) continue; List cols = ts.getColumnSchemas(); @@ -661,7 +661,7 @@ public DataReadResult readDataByTimestamps(File filePath, List timestamps) List colTypes = new ArrayList<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { String tblName = ts.getTableName(); List cols = ts.getColumnSchemas(); List categories = ts.getColumnTypes(); @@ -737,7 +737,7 @@ public DataReadResult readDataByTimestamps(File filePath, List timestamps) public List getTableNames(File filePath) throws IOException { List names = new ArrayList<>(); try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) names.add(ts.getTableName()); + for (TableSchema ts : TableSchemaSupport.readAll(reader)) names.add(ts.getTableName()); } return names; } @@ -766,7 +766,7 @@ public long countRows(File filePath, Long startTime, Long endTime) throws IOExce long effEnd = endTime != null ? endTime : Long.MAX_VALUE; long total = 0; try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { - for (TableSchema ts : reader.getAllTableSchema()) { + for (TableSchema ts : TableSchemaSupport.readAll(reader)) { List qCols = new ArrayList<>(); for (IMeasurementSchema c : ts.getColumnSchemas()) qCols.add(c.getMeasurementName()); if (qCols.isEmpty()) continue; From d1fef388cdb2b1ed3810df0d1b3cd4f345825a7b Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 19:00:37 +0800 Subject: [PATCH 6/9] =?UTF-8?q?fix(backend):=20=E6=A0=91=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E4=B8=8B=E6=8E=A8=E8=AE=BE=E5=A4=87/?= =?UTF-8?q?=E6=B5=8B=E7=82=B9=E8=BF=87=E6=BB=A4=EF=BC=8C=E4=BF=AE=E9=9D=9E?= =?UTF-8?q?=E9=A6=96=E4=B8=AA=E8=AE=BE=E5=A4=87=E6=9F=A5=E4=B8=8D=E5=88=B0?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 树模型读路径按存储顺序逐设备读取,用全局 limit 收集满即 break,而 deviceMeasurements 又是无序 HashMap。于是返回窗口永远被迭代顺序里的第一台 设备吃满,任何指向其它设备的过滤都匹配不到——设备筛选、测点筛选、以及依赖 它们的图表模式全部返回空(既有缺陷,2.3.0 上同样存在)。 - readDataByTimeRange 新增带 devices/measurements 的重载,由 DataService 的 数据预览与图表查询两处调用点传入 FilterConditions/请求里的条件 - readTreeModelData 用这些条件先行筛选要读的设备(按设备名精确匹配)与测点, 并改用 LinkedHashMap 让分页顺序稳定可预测 - 不带过滤时行为不变 实测(真实 25 设备 / 922 chunk 树模型文件,本地实例 + 线上各验一遍): devices=[ac_sig.a2158] 0 行 -> 3 行 measurements=[WorkState] 0 行 -> 3 行 devices+measurements 0 行 -> 3 行 图表模式:由"加载失败"/"暂无图表"变为渲染出序列 (图例 root.a.nayong.ac_sig.a2158 / WorkState,数据点 10,000) Tests: 323 run, 0 failures, 17 skipped(新增 1 个测点下推用例;现有写工具只能 产出单表带 device_name 字段的伪多设备样本,构造不出真多设备树模型文件,故设备 下推以真实样本端到端验证,测试里已注明) --- .../tsfile/viewer/service/DataService.java | 8 ++- .../viewer/tsfile/TsFileDataReader.java | 58 +++++++++++++++++-- .../integration/TreeModelIntegrationTest.java | 38 ++++++++++++ 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java index 509cfbc..2951bc2 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java @@ -131,7 +131,9 @@ public DataPreviewResponse previewData(DataPreviewRequest request) filters.getTimeRange() != null ? filters.getTimeRange().getStartTime() : null, filters.getTimeRange() != null ? filters.getTimeRange().getEndTime() : null, readLimit, - 0); + 0, + filters.getDevices(), + filters.getMeasurements()); // Check timeout checkTimeout(startTime, timeoutSeconds); @@ -206,7 +208,9 @@ public ChartDataResponse queryChartData(ChartDataRequest request) request.getStartTime(), request.getEndTime(), tsFileProperties.getQuery().getMaxResultSize(), - 0); + 0, + request.getDevices(), + request.getMeasurements()); // Check timeout checkTimeout(startTime, timeoutSeconds); diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java index 8fb7c1b..4baa939 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -217,6 +218,35 @@ public ReadFilter setMaxValue(Double maxValue) { /** Reads data from a TsFile with time range filtering and pagination. */ public DataReadResult readDataByTimeRange( File filePath, Long startTime, Long endTime, int limit, int offset) throws IOException { + return readDataByTimeRange(filePath, startTime, endTime, limit, offset, null, null); + } + + /** + * Reads data from a TsFile, optionally restricted to the given devices and measurements. + * + *

    The restriction is pushed down into the tree model read path. Without it a tree model file + * is read device by device in storage order, so the row limit is exhausted by the first device and + * a later device (or its measurements) can never be matched by the post-read filters. + * + * @param filePath file to read + * @param startTime inclusive lower time bound, or null + * @param endTime inclusive upper time bound, or null + * @param limit maximum number of rows to return + * @param offset number of rows to skip + * @param devices device names to restrict to, or null for all devices + * @param measurements measurement names to restrict to, or null for all measurements + * @return rows read, plus column metadata and warnings + * @throws IOException if the file cannot be read + */ + public DataReadResult readDataByTimeRange( + File filePath, + Long startTime, + Long endTime, + int limit, + int offset, + List devices, + List measurements) + throws IOException { long effStart = startTime != null ? startTime : Long.MIN_VALUE; long effEnd = endTime != null ? endTime : Long.MAX_VALUE; List results = new ArrayList<>(); @@ -234,7 +264,7 @@ public DataReadResult readDataByTimeRange( // If no TableSchema found, this is a tree model file - use TsFileSequenceReader if (schemas.isEmpty()) { - return readTreeModelData(filePath, effStart, effEnd, limit, offset); + return readTreeModelData(filePath, effStart, effEnd, limit, offset, devices, measurements); } for (TableSchema ts : schemas) { @@ -793,20 +823,33 @@ public long countRows(File filePath, Long startTime, Long endTime) throws IOExce * to read device/measurement structure and data. */ private DataReadResult readTreeModelData( - File filePath, long effStart, long effEnd, int limit, int offset) throws IOException { + File filePath, + long effStart, + long effEnd, + int limit, + int offset, + List selectedDevices, + List selectedMeasurements) + throws IOException { List results = new ArrayList<>(); List colNames = new ArrayList<>(); List colTypes = new ArrayList<>(); int skipped = 0, collected = 0; boolean hasMore = false; + List deviceFilter = selectedDevices != null ? selectedDevices : List.of(); + List measurementFilter = + selectedMeasurements != null ? selectedMeasurements : List.of(); try (TsFileSequenceReader seqReader = new TsFileSequenceReader(filePath.getAbsolutePath()); TsFileReader tsFileReader = new TsFileReader(seqReader)) { - // Get all devices and their measurements - Map> deviceMeasurements = new HashMap<>(); + // Get all devices and their measurements, keeping storage order so paging is stable + Map> deviceMeasurements = new LinkedHashMap<>(); for (IDeviceID device : seqReader.getAllDevices()) { + if (!deviceFilter.isEmpty() && !deviceFilter.contains(device.toString())) { + continue; + } Map> seriesMetaData = seqReader.readChunkMetadataInDevice(device); Map measurements = new HashMap<>(); @@ -820,7 +863,12 @@ private DataReadResult readTreeModelData( } } } - deviceMeasurements.put(device, measurements); + if (!measurementFilter.isEmpty()) { + measurements.keySet().retainAll(measurementFilter); + } + if (!measurements.isEmpty()) { + deviceMeasurements.put(device, measurements); + } } // Build column names from all unique measurements diff --git a/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java index d4e3e1f..0e39dbc 100644 --- a/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java +++ b/backend/src/test/java/org/apache/tsfile/viewer/integration/TreeModelIntegrationTest.java @@ -412,4 +412,42 @@ void shouldStreamFilteredTreeModelData() throws Exception { } } } + + @Nested + @DisplayName("Filter Pushdown Tests") + class FilterPushdownTests { + + private File testFile; + + @BeforeEach + void createTestFile() throws Exception { + // Note: TsFileTestUtils writes a single table carrying a device_name FIELD, so this fixture + // cannot express a genuinely multi device tree model file. The device name pushdown + // (reading only the requested devices) is therefore verified against a real multi device + // tree model file rather than here; what is covered below is that a measurement filter is + // pushed into the read path instead of being applied only after reading. + testFile = + TsFileTestUtils.createTreeModelMultipleDevices( + tempDir.resolve("multi-device-tree.tsfile"), + Arrays.asList("root.device1", "root.device2"), + 50); + } + + @Test + @DisplayName("Should return rows when filtering by measurement") + void shouldReturnRowsForMeasurementFilter() throws Exception { + DataReadResult all = dataReader.readDataByTimeRange(testFile, null, null, 1000, 0); + assertThat(all.getData()).isNotEmpty(); + String measurement = all.getData().get(0).getValues().keySet().iterator().next(); + + DataReadResult filtered = + dataReader.readDataByTimeRange( + testFile, null, null, 1000, 0, null, List.of(measurement)); + + assertThat(filtered.getData()).isNotEmpty(); + for (DataRow row : filtered.getData()) { + assertThat(row.getValues()).containsKey(measurement); + } + } + } } From 7d52e394020760939bf157c0553f8a9c87240373 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 19:07:06 +0800 Subject: [PATCH 7/9] =?UTF-8?q?feat:=20=E6=A0=91=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E6=B5=8B=E7=82=B9=E4=B8=8B=E6=8B=89=E7=BA=A7=E8=81=94=E5=88=B0?= =?UTF-8?q?=E6=89=80=E9=80=89=E8=AE=BE=E5=A4=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 树模型的层级是「设备 → 测点」,但测点下拉取的是 metadata.measurements —— 整个 文件的测点并集。以真实样本为例:文件级并集 122 个测点,而 ac_sig.a2158 只带 WorkState 一个。于是用户可以选到本设备并不存在的测点(后端过滤下推后正确返回 0 行,但 UI 无任何解释),图表模式也会把不属于该设备的测点预选进来。 - 后端新增 GET /api/meta/{fileId}/devices/{device}/measurements: 经已有的设备索引缓存解析设备名,只读该设备的元数据(readChunkMetadataInDevice), 跳过 VECTOR 时间列,按名称排序返回;设备不存在时返回空列表 - 前端测点下拉与图表默认序列改为跟随所选设备;级联请求失败时退回文件级并集, 不阻塞筛选面板 - 设备切换仍会清空已选测点(用户主动切换才触发,保持前次的修复意图) 实测(真实 25 设备树模型文件): ac_sig.a2158 -> 1 个测点(WorkState) cabinet.a2025 -> 20 个测点 不存在的设备 -> 0 个 图表模式首次加载:默认序列由 8 个(文件级)变为 1 个(该设备真实拥有) Tests: 325 run, 0 failures, 17 skipped(新增 2 个端点用例) --- .../viewer/controller/MetadataController.java | 23 ++++++++ .../viewer/dto/DeviceMeasurementsDTO.java | 57 +++++++++++++++++++ .../viewer/service/MetadataService.java | 39 +++++++++++++ .../tsfile/viewer/tsfile/TsFileParser.java | 35 ++++++++++++ .../MetadataControllerPaginationTest.java | 29 ++++++++++ frontend/src/api/tsfile/meta.ts | 19 ++++++- frontend/src/api/tsfile/types.ts | 10 ++++ .../src/components/tsfile/TreeFilterPanel.vue | 35 ++++++++++-- 8 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java diff --git a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java index 0913892..df8911f 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java @@ -20,6 +20,7 @@ package org.apache.tsfile.viewer.controller; import java.io.IOException; +import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,6 +32,8 @@ import org.springframework.web.bind.annotation.RestController; import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.DeviceMeasurementsDTO; +import org.apache.tsfile.viewer.dto.MeasurementDTO; import org.apache.tsfile.viewer.dto.PageDTO; import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.dto.TsFileMetadataDTO; @@ -82,6 +85,26 @@ public ResponseEntity getMetadata(@PathVariable String fileId return ResponseEntity.ok(metadata); } + /** + * Gets the measurements carried by one device. + * + *

    Tree model files nest measurements under devices, so a client showing a device picker needs + * the measurement choices of the selected device instead of the whole file's union. + * + * @param fileId the file identifier + * @param device device name as returned by the RowGroup paging endpoint + * @return measurements of that device, empty when the device is not part of the file + */ + @GetMapping("/{fileId}/devices/{device}/measurements") + public ResponseEntity getDeviceMeasurements( + @PathVariable String fileId, @PathVariable String device) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting measurements of device={} for fileId={}", device, fileId); + + List measurements = metadataService.getDeviceMeasurements(fileId, device); + return ResponseEntity.ok(new DeviceMeasurementsDTO(device, measurements)); + } + /** * Gets one page of RowGroups of a TsFile. * diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java new file mode 100644 index 0000000..219ac3e --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceMeasurementsDTO.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import java.util.List; + +/** + * Measurements carried by a single device. + * + *

    Tree model files nest measurements under devices, so the measurement options a client offers + * must be scoped to the selected device rather than to the whole file. + */ +public class DeviceMeasurementsDTO { + + private String device; + private List measurements; + + public DeviceMeasurementsDTO() {} + + public DeviceMeasurementsDTO(String device, List measurements) { + this.device = device; + this.measurements = measurements; + } + + public String getDevice() { + return device; + } + + public void setDevice(String device) { + this.device = device; + } + + public List getMeasurements() { + return measurements; + } + + public void setMeasurements(List measurements) { + this.measurements = measurements; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java index fec33b5..9d1fff7 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java @@ -32,9 +32,11 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import org.apache.tsfile.file.metadata.IDeviceID; import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.viewer.config.TsFileProperties; import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.MeasurementDTO; import org.apache.tsfile.viewer.dto.PageDTO; import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.dto.TimeRange; @@ -180,6 +182,43 @@ public PageDTO getRowGroupsPage(String fileId, int offset, int limi return PageDTO.of(items, total, offset, limit); } + /** + * Gets the measurements carried by one device. + * + *

    Only that device's metadata is read; the device is resolved through the cached device index, + * so this does not walk the file. + * + * @param fileId the file identifier + * @param deviceName device name as returned by the RowGroup paging endpoint + * @return measurements of that device, empty when the device is not part of the file + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public List getDeviceMeasurements(String fileId, String deviceName) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug("Getting measurements of device={} for fileId={}", deviceName, fileId); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + + IDeviceID match = null; + for (IDeviceID device : deviceIndex.devices()) { + if (device.toString().equals(deviceName)) { + match = device; + break; + } + } + if (match == null) { + logger.debug("Device {} is not part of file {}", deviceName, filePath); + return List.of(); + } + + try (TsFileSequenceReader reader = new TsFileSequenceReader(filePath)) { + return tsFileParser.readDeviceMeasurements(reader, match); + } + } + /** * Gets one page of Chunks, crossing device boundaries when a page needs to. * diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java index c6d1108..d22fa39 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileParser.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; @@ -33,6 +34,7 @@ import java.util.Set; import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -757,6 +759,39 @@ public DeviceIndex parseDeviceIndex(String filePath) throws IOException { } } + /** + * Reads the measurements carried by one device. + * + *

    Tree model files nest measurements under devices, so the measurement choices a client + * offers have to come from the selected device rather than from the whole file. Only that + * device's metadata is read. + * + * @param reader TsFile reader + * @param device device to read + * @return measurements of that device, sorted by name, excluding the VECTOR time column + * @throws IOException if reading fails + */ + public List readDeviceMeasurements( + TsFileSequenceReader reader, IDeviceID device) throws IOException { + Map> seriesMetadata = reader.readChunkMetadataInDevice(device); + List measurements = new ArrayList<>(); + for (Map.Entry> entry : seriesMetadata.entrySet()) { + List chunkMetadata = entry.getValue(); + if (chunkMetadata == null || chunkMetadata.isEmpty()) { + continue; + } + TSDataType dataType = chunkMetadata.get(0).getDataType(); + // VECTOR is the aligned time column, not a user measurement (same rule as the read path) + if (dataType == TSDataType.VECTOR) { + continue; + } + measurements.add( + MeasurementDTO.builder().name(entry.getKey()).dataType(dataType.toString()).build()); + } + measurements.sort(Comparator.comparing(MeasurementDTO::getName)); + return measurements; + } + /** * Reads one page of RowGroups. * diff --git a/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java index 0be1e36..52fd172 100644 --- a/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java +++ b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java @@ -40,6 +40,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.MeasurementDTO; import org.apache.tsfile.viewer.dto.PageDTO; import org.apache.tsfile.viewer.dto.RowGroupDTO; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; @@ -239,4 +240,32 @@ private static ChunkDTO chunk(String measurement, long offset, String device) { .maxValue("10") .build(); } + @Test + @DisplayName("device measurements are returned for a known device") + void shouldReturnDeviceMeasurements() throws Exception { + when(metadataService.getDeviceMeasurements("file-1", "root.a.b")) + .thenReturn( + List.of( + MeasurementDTO.builder().name("temperature").dataType("FLOAT").build(), + MeasurementDTO.builder().name("humidity").dataType("INT32").build())); + + mockMvc + .perform(get("/api/meta/file-1/devices/root.a.b/measurements")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.device").value("root.a.b")) + .andExpect(jsonPath("$.measurements.length()").value(2)) + .andExpect(jsonPath("$.measurements[0].name").value("temperature")); + } + + @Test + @DisplayName("device measurements are empty for an unknown device") + void shouldReturnEmptyMeasurementsForUnknownDevice() throws Exception { + when(metadataService.getDeviceMeasurements("file-1", "root.unknown")).thenReturn(List.of()); + + mockMvc + .perform(get("/api/meta/file-1/devices/root.unknown/measurements")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.device").value("root.unknown")) + .andExpect(jsonPath("$.measurements.length()").value(0)); + } } diff --git a/frontend/src/api/tsfile/meta.ts b/frontend/src/api/tsfile/meta.ts index 2a1ceac..4bc8820 100644 --- a/frontend/src/api/tsfile/meta.ts +++ b/frontend/src/api/tsfile/meta.ts @@ -17,7 +17,13 @@ * under the License. */ -import type { Chunk, PagedResponse, RowGroup, TsFileMetadata } from "./types"; +import type { + Chunk, + DeviceMeasurements, + PagedResponse, + RowGroup, + TsFileMetadata, +} from "./types"; import { apiClient } from "../request"; /** @@ -43,3 +49,14 @@ export function getChunksPage(fileId: string, offset = 0, limit = 50) { params: { offset, limit }, }); } + +/** + * 单个设备的测点清单。 + * + * 树模型的层级是「设备 → 测点」,因此测点下拉要跟随所选设备;设备不存在时返回空列表。 + */ +export function getDeviceMeasurements(fileId: string, device: string) { + return apiClient.get( + `/meta/${fileId}/devices/${encodeURIComponent(device)}/measurements`, + ); +} diff --git a/frontend/src/api/tsfile/types.ts b/frontend/src/api/tsfile/types.ts index 45ef593..73c3201 100644 --- a/frontend/src/api/tsfile/types.ts +++ b/frontend/src/api/tsfile/types.ts @@ -114,6 +114,16 @@ export interface PagedResponse { hasMore: boolean; } +/** + * 单个设备携带的测点。 + * + * 树模型文件的测点挂在设备下,测点选项应跟随所选设备,而不是取整文件并集。 + */ +export interface DeviceMeasurements { + device: string; + measurements: Measurement[]; +} + export interface TsFileMetadata { fileId: string; version: string; diff --git a/frontend/src/components/tsfile/TreeFilterPanel.vue b/frontend/src/components/tsfile/TreeFilterPanel.vue index a0b332c..a74526e 100644 --- a/frontend/src/components/tsfile/TreeFilterPanel.vue +++ b/frontend/src/components/tsfile/TreeFilterPanel.vue @@ -22,7 +22,7 @@ * TreeFilterPanel - 树模型筛选面板 * 所有下拉选择器使用 el-select-v2(虚拟化渲染),即使选项数量很大也不会卡顿。 */ -import type { AdvancedCondition, RowGroup, TsFileMetadata } from "@/api/tsfile/types"; +import type { AdvancedCondition, Measurement, RowGroup, TsFileMetadata } from "@/api/tsfile/types"; import { computed, ref, watch } from "vue"; import { useI18n } from "vue-i18n"; @@ -102,9 +102,30 @@ async function loadDeviceOptions() { } } +// 所选设备的测点;null 表示还没取到(此时退回文件级并集,保证面板仍可用) +const deviceMeasurements = ref(null); +const deviceMeasurementsLoading = ref(false); + +async function loadDeviceMeasurements(device: string) { + deviceMeasurements.value = null; + if (!device || !props.fileId) return; + deviceMeasurementsLoading.value = true; + try { + const response = await metaApi.getDeviceMeasurements(props.fileId, device); + deviceMeasurements.value = response.measurements ?? []; + } catch { + // 级联信息取不到时退回文件级列表,不阻塞测点/时间范围筛选 + deviceMeasurements.value = null; + } finally { + deviceMeasurementsLoading.value = false; + } +} + +// 树模型的层级是「设备 → 测点」:测点选项取自所选设备,拿不到时才退回文件级并集 const measurementOptions = computed(() => { - if (!metadata.value?.measurements) return []; - return metadata.value.measurements + const source = deviceMeasurements.value ?? metadata.value?.measurements; + if (!source) return []; + return source .map((m) => m.name) .filter(Boolean) .sort() as string[]; @@ -117,7 +138,8 @@ const CHART_DEFAULT_SERIES = 8; // 取前 N 个数值测点,作为图表默认序列 function defaultChartMeasurements(): string[] { - return (metadata.value?.measurements || []) + const source = deviceMeasurements.value ?? metadata.value?.measurements ?? []; + return source .filter((m) => m.name && numericTypes.has(m.dataType?.toUpperCase() || '')) .map((m) => m.name) .slice(0, CHART_DEFAULT_SERIES); @@ -158,6 +180,8 @@ async function loadMetadata() { await loadDeviceOptions(); if (deviceOptions.value.length > 0) { selectedDevice.value = deviceOptions.value[0]!.value; + // 测点选项跟随设备,默认序列也必须基于该设备的测点 + await loadDeviceMeasurements(selectedDevice.value); } // In chart mode, default to the first few numeric fields and trigger query if (props.chartMode && metadata.value?.measurements) { @@ -228,6 +252,7 @@ watch( // el-select-v2 的 change 只在用户选值/清空时发出,程序化赋值不会触发。 function handleDeviceChange() { selectedMeasurements.value = []; + void loadDeviceMeasurements(selectedDevice.value); } watch(timeRange, () => { selectedQuickRange.value = null; @@ -255,7 +280,7 @@ function handleAdvancedApply(conditions: AdvancedCondition[]) { :options="deviceOptions" filterable :placeholder="t('tsfile.metadata.searchByDevice')" - :loading="loading || deviceLoading" + :loading="loading || deviceLoading || deviceMeasurementsLoading" style="width: 240px" popper-class="v2-options-compact" @change="handleDeviceChange" From f33468eb16eb945c7abfcb6a72bed79a01bae4f3 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 19:15:50 +0800 Subject: [PATCH 8/9] =?UTF-8?q?feat:=20=E8=A1=A8=E6=A8=A1=E5=9E=8B?= =?UTF-8?q?=E8=AE=BE=E5=A4=87=E4=B8=8B=E6=8B=89=E6=8C=89=E8=A1=A8=E8=BF=87?= =?UTF-8?q?=E6=BB=A4=20+=20=E5=88=86=E9=A1=B5=E8=BF=9C=E7=A8=8B=E6=90=9C?= =?UTF-8?q?=E7=B4=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 表模型的层级是「表 → 设备(TAG 组合)→ 字段」,但设备下拉取的是全文件设备 (分页扫描 RowGroup,最多 800 条),与所选表无关;字段层则早已按表过滤。 现成的 /api/tables/{fileId}/devices?tableName= 不能直接用:它靠一次全表行查询 发现设备,实测某表 87,488 台设备时返回 16MB / 16.9s。 - 新增 GET /api/meta/{fileId}/devices?tableName=&query=&offset=&limit=: 从已缓存的设备索引推导(设备标识本身即 tableName.tag1.tag2),支持按表过滤、 大小写不敏感的子串搜索与分页;只物化请求的那一页,total 为匹配总数 - 前端设备下拉改为按所选表加载首页(50 条)并在输入时远程搜索(去抖 250ms), 切换表时清空已选设备并重新加载;已选设备的标签单独缓存,换结果集后不丢 实测:某表 87,488 台设备 —— 首页 8.4KB / 热态 13-20ms(冷态首调含设备索引构建, 与元数据摘要共用同一次构建);不存在的表名返回 0 条;搜索 "3540" 命中 464 条。 Tests: 327 run, 0 failures, 17 skipped(新增 2 个端点用例) --- .../viewer/controller/MetadataController.java | 30 +++++++ .../tsfile/viewer/dto/DeviceRefDTO.java | 67 +++++++++++++++ .../viewer/service/MetadataService.java | 76 +++++++++++++++++ .../MetadataControllerPaginationTest.java | 37 +++++++++ frontend/src/api/tsfile/meta.ts | 14 ++++ frontend/src/api/tsfile/types.ts | 7 ++ .../components/tsfile/TableFilterPanel.vue | 81 +++++++++++-------- 7 files changed, 280 insertions(+), 32 deletions(-) create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java diff --git a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java index df8911f..12a511c 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/controller/MetadataController.java @@ -33,6 +33,7 @@ import org.apache.tsfile.viewer.dto.ChunkDTO; import org.apache.tsfile.viewer.dto.DeviceMeasurementsDTO; +import org.apache.tsfile.viewer.dto.DeviceRefDTO; import org.apache.tsfile.viewer.dto.MeasurementDTO; import org.apache.tsfile.viewer.dto.PageDTO; import org.apache.tsfile.viewer.dto.RowGroupDTO; @@ -85,6 +86,35 @@ public ResponseEntity getMetadata(@PathVariable String fileId return ResponseEntity.ok(metadata); } + /** + * Gets one page of devices, optionally scoped to a table and filtered by a substring. + * + *

    Cheap enough for pickers: it is derived from the cached device index, not from a table + * query. + * + * @param fileId the file identifier + * @param tableName only devices of this table + * @param query case-insensitive substring the device name must contain + * @param offset zero-based offset, negative values are treated as 0 + * @param limit page size, defaults to 50 and is capped at 200 + * @return page of devices + */ + @GetMapping("/{fileId}/devices") + public ResponseEntity> getDevices( + @PathVariable String fileId, + @RequestParam(required = false) String tableName, + @RequestParam(required = false) String query, + @RequestParam(required = false) Integer offset, + @RequestParam(required = false) Integer limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + int effectiveOffset = PageDTO.normalizeOffset(offset); + int effectiveLimit = PageDTO.normalizeLimit(limit); + + PageDTO page = + metadataService.getDevicesPage(fileId, tableName, query, effectiveOffset, effectiveLimit); + return ResponseEntity.ok(page); + } + /** * Gets the measurements carried by one device. * diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java new file mode 100644 index 0000000..156738b --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/DeviceRefDTO.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +import java.util.List; + +/** + * A device reference for pickers: its identifier, owning table and TAG values. + * + *

    Derived from the device identifier itself, which is {@code tableName.tag1.tag2...}, so it + * needs no row scan. + */ +public class DeviceRefDTO { + + private String device; + private String tableName; + private List tagValues; + + public DeviceRefDTO() {} + + public DeviceRefDTO(String device, String tableName, List tagValues) { + this.device = device; + this.tableName = tableName; + this.tagValues = tagValues; + } + + public String getDevice() { + return device; + } + + public void setDevice(String device) { + this.device = device; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public List getTagValues() { + return tagValues; + } + + public void setTagValues(List tagValues) { + this.tagValues = tagValues; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java index 9d1fff7..2d4c61f 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/MetadataService.java @@ -23,7 +23,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.concurrent.TimeUnit; import org.slf4j.Logger; @@ -36,6 +38,7 @@ import org.apache.tsfile.read.TsFileSequenceReader; import org.apache.tsfile.viewer.config.TsFileProperties; import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.DeviceRefDTO; import org.apache.tsfile.viewer.dto.MeasurementDTO; import org.apache.tsfile.viewer.dto.PageDTO; import org.apache.tsfile.viewer.dto.RowGroupDTO; @@ -182,6 +185,79 @@ public PageDTO getRowGroupsPage(String fileId, int offset, int limi return PageDTO.of(items, total, offset, limit); } + /** + * Gets one page of devices, optionally scoped to a table and filtered by a substring. + * + *

    Derived from the cached device index rather than a row scan: a device identifier already + * carries its table name and TAG values ({@code tableName.tag1.tag2}), so listing devices of a + * table costs a pass over cached names instead of a full table query. + * + * @param fileId the file identifier + * @param tableName only devices of this table, or null for all devices + * @param query case-insensitive substring the device name must contain, or null + * @param offset zero-based offset of the first device + * @param limit maximum number of devices to return + * @return page of devices with the number of matching devices as total + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file cannot be read + */ + public PageDTO getDevicesPage( + String fileId, String tableName, String query, int offset, int limit) + throws TsFileNotFoundException, AccessDeniedException, IOException { + logger.debug( + "Getting device page for fileId={}, tableName={}, query={}, offset={}, limit={}", + fileId, + tableName, + query, + offset, + limit); + + String filePath = resolveFilePath(fileId); + DeviceIndex deviceIndex = getDeviceIndex(fileId, filePath); + + String tablePrefix = tableName == null ? "" : tableName.trim(); + String loweredQuery = query == null ? "" : query.trim().toLowerCase(Locale.ROOT); + boolean hasTable = !tablePrefix.isEmpty(); + boolean hasQuery = !loweredQuery.isEmpty(); + + List page = new ArrayList<>(); + int matched = 0; + int total = 0; + // Two passes so only the requested page is materialised + for (IDeviceID device : deviceIndex.devices()) { + String name = device.toString(); + if (hasTable && !isInTable(name, tablePrefix)) { + continue; + } + if (hasQuery && !name.toLowerCase(Locale.ROOT).contains(loweredQuery)) { + continue; + } + if (matched >= offset && page.size() < limit) { + page.add(toDeviceRef(name, hasTable ? tablePrefix : null)); + } + matched++; + total = matched; + } + + return PageDTO.of(page, total, offset, limit); + } + + private static boolean isInTable(String deviceName, String tableName) { + return deviceName.equals(tableName) || deviceName.startsWith(tableName + "."); + } + + private static DeviceRefDTO toDeviceRef(String deviceName, String tableName) { + if (tableName == null) { + return new DeviceRefDTO(deviceName, null, List.of()); + } + String remainder = deviceName.length() > tableName.length() + 1 + ? deviceName.substring(tableName.length() + 1) + : ""; + List tagValues = remainder.isEmpty() ? List.of() : List.of(remainder.split("\\.")); + return new DeviceRefDTO(deviceName, tableName, tagValues); + } + /** * Gets the measurements carried by one device. * diff --git a/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java index 52fd172..3f5fe52 100644 --- a/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java +++ b/backend/src/test/java/org/apache/tsfile/viewer/controller/MetadataControllerPaginationTest.java @@ -40,6 +40,7 @@ import org.springframework.test.web.servlet.setup.MockMvcBuilders; import org.apache.tsfile.viewer.dto.ChunkDTO; +import org.apache.tsfile.viewer.dto.DeviceRefDTO; import org.apache.tsfile.viewer.dto.MeasurementDTO; import org.apache.tsfile.viewer.dto.PageDTO; import org.apache.tsfile.viewer.dto.RowGroupDTO; @@ -268,4 +269,40 @@ void shouldReturnEmptyMeasurementsForUnknownDevice() throws Exception { .andExpect(jsonPath("$.device").value("root.unknown")) .andExpect(jsonPath("$.measurements.length()").value(0)); } + + @Test + @DisplayName("devices default to offset=0 and limit=50") + void shouldApplyDeviceDefaults() throws Exception { + when(metadataService.getDevicesPage("file-1", "t1", null, 0, 50)) + .thenReturn( + PageDTO.of( + List.of(new DeviceRefDTO("t1.d1", "t1", List.of("d1"))), 3, 0, 50)); + + mockMvc + .perform(get("/api/meta/file-1/devices").param("tableName", "t1")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(50)) + .andExpect(jsonPath("$.total").value(3)) + .andExpect(jsonPath("$.hasMore").value(true)) + .andExpect(jsonPath("$.items[0].device").value("t1.d1")) + .andExpect(jsonPath("$.items[0].tableName").value("t1")); + } + + @Test + @DisplayName("devices clamp limit to 200 and normalise a negative offset") + void shouldClampDeviceParameters() throws Exception { + when(metadataService.getDevicesPage("file-1", null, "abc", 0, 200)) + .thenReturn(PageDTO.of(List.of(), 0, 0, 200)); + + mockMvc + .perform( + get("/api/meta/file-1/devices") + .param("query", "abc") + .param("offset", "-5") + .param("limit", "500")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.offset").value(0)) + .andExpect(jsonPath("$.limit").value(200)); + } } diff --git a/frontend/src/api/tsfile/meta.ts b/frontend/src/api/tsfile/meta.ts index 4bc8820..69692f2 100644 --- a/frontend/src/api/tsfile/meta.ts +++ b/frontend/src/api/tsfile/meta.ts @@ -19,6 +19,7 @@ import type { Chunk, + DeviceRef, DeviceMeasurements, PagedResponse, RowGroup, @@ -60,3 +61,16 @@ export function getDeviceMeasurements(fileId: string, device: string) { `/meta/${fileId}/devices/${encodeURIComponent(device)}/measurements`, ); } + +/** + * 设备分页(按表过滤 + 子串搜索)。 + * + * 表模型的层级是「表 → 设备 → 字段」,设备下拉必须按所选表取。该端点由已缓存的设备索引 + * 推导(设备名本身即 tableName.tag1.tag2),因此支持分页与搜索而不做全表行扫描。 + */ +export function getDevicesPage( + fileId: string, + params: { tableName?: string; query?: string; offset?: number; limit?: number } = {}, +) { + return apiClient.get>(`/meta/${fileId}/devices`, { params }); +} diff --git a/frontend/src/api/tsfile/types.ts b/frontend/src/api/tsfile/types.ts index 73c3201..8189887 100644 --- a/frontend/src/api/tsfile/types.ts +++ b/frontend/src/api/tsfile/types.ts @@ -114,6 +114,13 @@ export interface PagedResponse { hasMore: boolean; } +/** 设备引用:设备名、所属表、TAG 值(都由设备标识本身推导,不需要行扫描) */ +export interface DeviceRef { + device: string; + tableName: string | null; + tagValues: string[]; +} + /** * 单个设备携带的测点。 * diff --git a/frontend/src/components/tsfile/TableFilterPanel.vue b/frontend/src/components/tsfile/TableFilterPanel.vue index 8b9a075..b06dcaa 100644 --- a/frontend/src/components/tsfile/TableFilterPanel.vue +++ b/frontend/src/components/tsfile/TableFilterPanel.vue @@ -24,7 +24,7 @@ * * 所有下拉选择器使用 el-select-v2(虚拟化渲染),即使选项数量很大也不会卡顿。 */ -import type { AdvancedCondition, RowGroup, Table, TsFileMetadata } from "@/api/tsfile/types"; +import type { AdvancedCondition, Table, TsFileMetadata } from "@/api/tsfile/types"; import { computed, ref, watch } from "vue"; @@ -34,7 +34,6 @@ import { Filter, RotateCcw, Search } from "lucide-vue-next"; import { ElSelectV2 } from "element-plus"; import { metaApi } from "@/api/tsfile"; -import { readPagedRange } from "@/composables/usePagedList"; import AdvancedFilterDialog from "@/components/tsfile/AdvancedFilterDialog.vue"; interface Props { @@ -85,39 +84,66 @@ const tableOptions = computed(() => { // 设备选项 // -// 元数据摘要里的 rowGroups 恒为空数组(明细改成分页接口),所以设备名只能来自 -// 分页 RowGroup 数据本身。这里不做全量拉取:最多扫 DEVICE_SCAN_MAX_PAGES 页, -// deviceScan 记录实际扫了多少行,选项不完整时由模板如实标出。 -// PAGE_SIZE 取后端 PageDTO.MAX_LIMIT(200),再大也会被服务端收窄。 -const DEVICE_SCAN_PAGE_SIZE = 200; -const DEVICE_SCAN_MAX_PAGES = 4; +// 表模型的层级是「表 → 设备(TAG 组合)→ 字段」,所以设备清单按所选表取, +// 并且只拉一页 + 输入时远程搜索:大文件的表可能有数万台设备(实测某表 87,488 台, +// 一次性返回 16MB / 17s,不可用;分页后热态约十几毫秒)。 +const DEVICE_PAGE_SIZE = 50; const deviceOptions = ref<{ label: string; value: string }[]>([]); const deviceLoading = ref(false); -const deviceScan = ref({ loaded: 0, total: 0 }); +// 已选设备的标签缓存:远程搜索换了结果集后,已选项仍要能显示名字 +const deviceLabels = new Map(); +let deviceSearchTimer: ReturnType | undefined; -async function loadDeviceOptions() { +async function loadDeviceOptions(tableName?: string, query?: string) { + if (!tableName || !props.fileId) { + deviceOptions.value = []; + return; + } deviceLoading.value = true; try { - const { items, total } = await readPagedRange( - (offset, limit) => metaApi.getRowGroupsPage(props.fileId, offset, limit), - { pageSize: DEVICE_SCAN_PAGE_SIZE, maxPages: DEVICE_SCAN_MAX_PAGES }, - ); - const devices = new Set(); - for (const rowGroup of items) { - if (rowGroup.device) devices.add(rowGroup.device); + const page = await metaApi.getDevicesPage(props.fileId, { + tableName, + query: query || undefined, + offset: 0, + limit: DEVICE_PAGE_SIZE, + }); + const options: { label: string; value: string }[] = []; + for (const item of page.items ?? []) { + if (!item.device) continue; + deviceLabels.set(item.device, item.device); + options.push({ label: item.device, value: item.device }); + } + // 保留已选项,避免远程搜索后已选标签消失 + for (const selected of selectedDevices.value) { + if (!options.some((o) => o.value === selected)) { + options.push({ label: deviceLabels.get(selected) ?? selected, value: selected }); + } } - deviceOptions.value = [...devices].map((d) => ({ label: d, value: d })); - deviceScan.value = { loaded: items.length, total }; + deviceOptions.value = options; } catch { - // 设备选项拉取失败不应拖垮面板本身:表 / 字段 / 时间范围筛选仍然可用 + // 设备清单拉取失败不应拖垮面板:表 / 字段 / 时间范围筛选仍然可用 deviceOptions.value = []; - deviceScan.value = { loaded: 0, total: 0 }; } finally { deviceLoading.value = false; } } +/** 下拉输入时远程搜索(去抖,避免每敲一个字符打一次请求) */ +function searchDevices(query: string) { + if (deviceSearchTimer) clearTimeout(deviceSearchTimer); + deviceSearchTimer = setTimeout(() => { + void loadDeviceOptions(selectedTable.value ?? undefined, query.trim()); + }, 250); +} + +// 切换表:设备属于表,旧设备选择必须失效并按新表重新取 +watch(selectedTable, (tableName) => { + selectedDevices.value = []; + deviceLabels.clear(); + void loadDeviceOptions(tableName ?? undefined); +}); + // 字段选项(根据选中的表过滤) const measurementOptions = computed(() => { if (!metadata.value?.tables || !selectedTable.value) return []; @@ -165,9 +191,6 @@ async function loadMetadata() { const response = await metaApi.getMetadata(props.fileId); metadata.value = response as TsFileMetadata; - // 设备选项来自分页 RowGroup,独立加载,不阻塞表/字段的默认选择 - void loadDeviceOptions(); - // 自动选择第一个表 if (tableOptions.value.length > 0) { const firstTable = tableOptions.value[0]; @@ -357,19 +380,13 @@ function handleAdvancedApply(conditions: AdvancedCondition[]) { :disabled="loading || !selectedTable" :loading="loading || deviceLoading" filterable + remote + :remote-method="searchDevices" collapse-tags :max-collapse-tags="1" style="min-width: 220px; max-width: 360px" popper-class="v2-options-compact" /> - - - ({{ deviceScan.loaded }} / {{ deviceScan.total }}) - From 65bfd257b050a33244c4b25717e489fccba493c5 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 19:23:37 +0800 Subject: [PATCH 9/9] =?UTF-8?q?fix(frontend):=20=E4=B8=8B=E6=8B=89?= =?UTF-8?q?=E5=8F=AA=E4=BF=9D=E7=95=99=E4=B8=80=E5=B1=82=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E6=9D=A1=EF=BC=8C=E5=B9=B6=E9=98=BB=E6=AD=A2=E6=BB=9A=E5=8A=A8?= =?UTF-8?q?=E7=A9=BF=E9=80=8F=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit el-select-v2 的下拉里有两层可滚容器:外层 .el-select-dropdown__wrap (overflow:auto + max-height:274px)与内层虚拟列表 .el-vl__window。虚拟列表 高度超过外层上限时(下拉向上弹出、可用高度被压缩时最常见)两层同时滚动, 用户会看到两条滚动条。 - .v2-options-compact(本就用在该场景的标记类,此前没有任何样式)的作用域内 把外层改为 overflow:hidden,滚动只留给内层虚拟列表 - 两层都加 overscroll-behavior:contain,滚到底不再把滚动传给页面 实测:打开「字段」下拉后 popper 内可滚容器由两层变一层,连续滚轮滚动时 window.scrollY 恒为 0(不再穿透)。 --- frontend/src/styles/index.css | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/frontend/src/styles/index.css b/frontend/src/styles/index.css index 3ab9d04..5f8dce2 100644 --- a/frontend/src/styles/index.css +++ b/frontend/src/styles/index.css @@ -404,3 +404,21 @@ html.dark { font-feature-settings: "tnum"; } } + +/* + * el-select-v2 的下拉里有两层可滚容器: + * 外层 .el-select-dropdown__wrap overflow:auto + max-height:274px + * 内层 .el-vl__window(虚拟列表) + * 当虚拟列表高度超过外层上限(下拉向上弹出、可用高度被压缩时最常见),两层会同时 + * 出现滚动条。这里固定成"只有内层滚",并阻止滚到底后把滚动穿透给页面。 + */ +.v2-options-compact .el-select-dropdown__wrap { + overflow: hidden; + overscroll-behavior: contain; +} + +.v2-options-compact .el-vl__window { + overflow-y: auto; + max-height: 274px; + overscroll-behavior: contain; +}