From 1719fe04337a7a866fec8bd4145d6ee449946b6b Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 19:46:27 +0800 Subject: [PATCH] fix: export all filtered rows instead of the current preview page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exportCSV()/exportJSON() serialized dataRows, which holds only the current preview page, so the exported subset changed with pagination — while the user guide promises everything matching the filters. Paging the preview endpoint from the browser is not a fix either: with post-read filters the backend caps a single read at query.max-result-size (10,000), so the export still truncated there (measured 10,000 rows on a 16,031-row file). - New POST /api/data/export?format=csv|json streams the whole filtered result set: the server pages through the file itself (2,000 rows per page, offset pushed down), applies exactly the filters the preview applies, and flushes each page — the file never has to fit in memory and max-result-size no longer truncates. - CSV follows RFC 4180 (commas, quotes, line breaks); columns are the file's column union in lexicographic order; timestamps are ISO-8601 UTC with milliseconds, matching what the browser used to produce. - JSON streams one row object per line, same shape as the preview rows. - Frontend requests the blob from the new endpoint and keeps the export buttons disabled while it runs. - DataPreviewRequest now accepts advancedConditions: the frontend has been sending it, but the backend silently dropped it (preview was affected too). Verified on a 16,031-row tree-model file: preview page shows 100 rows, export yields 16,031 rows for both CSV and JSON in ~100 ms; limit/offset in the request are ignored; bad format -> 400, path outside allowed dirs -> 403, missing file inside the dir -> 404. Tests: 297 run, 0 failures, 17 skipped (11 new). --- .../viewer/controller/DataController.java | 76 +++- .../tsfile/viewer/dto/DataPreviewRequest.java | 13 + .../tsfile/viewer/dto/ExportFormat.java | 77 ++++ .../tsfile/viewer/service/DataService.java | 256 ++++++++++++- .../viewer/tsfile/TsFileDataReader.java | 48 +++ .../viewer/service/DataServiceExportTest.java | 341 ++++++++++++++++++ frontend/src/api/tsfile/data.ts | 22 ++ frontend/src/components/tsfile/DataTable.vue | 7 +- .../src/views/tsfile/data-preview/index.vue | 52 +-- 9 files changed, 855 insertions(+), 37 deletions(-) create mode 100644 backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java create mode 100644 backend/src/test/java/org/apache/tsfile/viewer/service/DataServiceExportTest.java diff --git a/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java b/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java index 180d817..e3e6fe5 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java @@ -21,23 +21,28 @@ import java.io.IOException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - import org.apache.tsfile.viewer.dto.ChartDataRequest; import org.apache.tsfile.viewer.dto.ChartDataResponse; import org.apache.tsfile.viewer.dto.DataPreviewRequest; import org.apache.tsfile.viewer.dto.DataPreviewResponse; +import org.apache.tsfile.viewer.dto.ExportFormat; import org.apache.tsfile.viewer.exception.AccessDeniedException; import org.apache.tsfile.viewer.exception.QueryTimeoutException; import org.apache.tsfile.viewer.exception.TsFileNotFoundException; import org.apache.tsfile.viewer.service.DataService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.server.ResponseStatusException; +import jakarta.servlet.http.HttpServletResponse; import jakarta.validation.Valid; /** @@ -47,6 +52,7 @@ * * * @@ -80,6 +86,60 @@ public ResponseEntity previewData( return ResponseEntity.ok(response); } + /** + * Exports every row matching the given filters as a downloadable CSV or JSON file. + * + *

The request body is the preview request, so filtering behaves exactly like the preview; its + * {@code limit}/{@code offset} are ignored and the response is streamed page by page, which means + * an export is neither limited to the current preview page nor to {@code + * tsfile.query.max-result-size}. + * + * @param request the data preview request carrying the filters to apply + * @param format {@code csv} or {@code json} + * @param response the response the export is streamed to + */ + @PostMapping("/export") + public void exportData( + @RequestBody @Valid DataPreviewRequest request, + @RequestParam(name = "format", defaultValue = "csv") String format, + HttpServletResponse response) + throws TsFileNotFoundException, AccessDeniedException, IOException { + ExportFormat exportFormat = ExportFormat.from(format); + logger.debug("Exporting data for fileId={} as {}", request.getFileId(), exportFormat); + + // Resolve the file (and the CSV columns) before committing the response, so an unknown fileId + // (404) or a forbidden path (403) still yields the regular JSON error body instead of a broken + // download. + DataService.ExportPlan plan = dataService.prepareExport(request, exportFormat); + + response.setContentType(exportFormat.getContentType()); + response.setHeader(HttpHeaders.CONTENT_DISPOSITION, exportFormat.getContentDisposition()); + + long startTime = System.currentTimeMillis(); + try { + long rows = dataService.exportData(plan, response.getOutputStream()); + logger.info( + "Exported {} rows as {} for fileId={} in {} ms", + rows, + exportFormat, + request.getFileId(), + System.currentTimeMillis() - startTime); + } catch (IOException e) { + // Part of the body may already be on the wire (client gone, disk error while reading), so the + // response can only be turned into an error when nothing has been committed yet. + logger.error( + "Export failed for fileId={} as {} after {} ms", + request.getFileId(), + exportFormat, + System.currentTimeMillis() - startTime, + e); + if (!response.isCommitted()) { + throw new ResponseStatusException( + HttpStatus.INTERNAL_SERVER_ERROR, "Export failed: " + e.getMessage(), e); + } + } + } + /** * Queries chart data from a TsFile with optional aggregation. * diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java index 3a81315..8d70127 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java @@ -40,6 +40,7 @@ public class DataPreviewRequest { private List devices; private List measurements; private ValueRange valueRange; + private List advancedConditions; @Min(value = 1, message = "limit must be at least 1") @Max(value = 1000, message = "limit must not exceed 1000") @@ -99,6 +100,14 @@ public void setValueRange(ValueRange valueRange) { this.valueRange = valueRange; } + public List getAdvancedConditions() { + return advancedConditions; + } + + public void setAdvancedConditions(List advancedConditions) { + this.advancedConditions = advancedConditions; + } + public int getLimit() { return limit; } @@ -141,6 +150,10 @@ public FilterConditions toFilterConditions() { builder.valueRange(valueRange); } + if (advancedConditions != null && !advancedConditions.isEmpty()) { + builder.advancedConditions(advancedConditions); + } + return builder.build(); } } diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java new file mode 100644 index 0000000..990191e --- /dev/null +++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.dto; + +/** + * Downloadable representation of an exported data set. + * + *

Each constant carries the media type and the download file name, so the controller and the + * writer in the service layer agree on a single definition. + */ +public enum ExportFormat { + CSV("csv", "text/csv;charset=UTF-8", "data.csv"), + JSON("json", "application/json;charset=UTF-8", "data.json"); + + private final String parameterValue; + private final String contentType; + private final String fileName; + + ExportFormat(String parameterValue, String contentType, String fileName) { + this.parameterValue = parameterValue; + this.contentType = contentType; + this.fileName = fileName; + } + + /** + * Resolves the {@code format} request parameter. + * + * @param raw the raw parameter value, e.g. {@code csv}; blank means {@link #CSV} + * @return the matching format + * @throws IllegalArgumentException if the value matches no supported format (mapped to HTTP 400) + */ + public static ExportFormat from(String raw) { + if (raw == null || raw.isBlank()) { + return CSV; + } + String normalized = raw.trim(); + for (ExportFormat format : values()) { + if (format.parameterValue.equalsIgnoreCase(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported export format '" + raw + "'; expected 'csv' or 'json'"); + } + + /** Returns the {@code Content-Type} announced for the download response. */ + public String getContentType() { + return contentType; + } + + /** Returns the file name announced in the {@code Content-Disposition} response header. */ + public String getFileName() { + return fileName; + } + + /** Returns the {@code Content-Disposition} value that makes browsers save instead of render. */ + public String getContentDisposition() { + return "attachment; filename=\"" + fileName + "\""; + } +} diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java index 509cfbc..bc540b5 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java @@ -19,11 +19,25 @@ package org.apache.tsfile.viewer.service; +import java.io.BufferedWriter; +import java.io.File; import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.nio.charset.StandardCharsets; +import java.time.DateTimeException; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; +import java.util.Comparator; import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeoutException; import org.apache.tsfile.viewer.config.TsFileProperties; @@ -35,6 +49,7 @@ import org.apache.tsfile.viewer.dto.DataPreviewRequest; import org.apache.tsfile.viewer.dto.DataPreviewResponse; import org.apache.tsfile.viewer.dto.DataRow; +import org.apache.tsfile.viewer.dto.ExportFormat; import org.apache.tsfile.viewer.dto.FilterConditions; import org.apache.tsfile.viewer.dto.TimeRange; import org.apache.tsfile.viewer.dto.ValueRange; @@ -46,6 +61,8 @@ import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import com.fasterxml.jackson.databind.ObjectMapper; + /** * Service for data query operations. * @@ -73,10 +90,37 @@ public class DataService { */ private static final int MAX_CHART_SERIES = 200; + /** + * Page size used while streaming an export. The reader is asked for one page at a time, so peak + * memory depends on this constant rather than on the size of the file being exported. + */ + private static final int EXPORT_PAGE_SIZE = 2000; + + /** Buffer size of the export writer; matched to a typical socket/TCP window size. */ + private static final int EXPORT_BUFFER_SIZE = 64 * 1024; + + /** + * Timestamp layout of the CSV export: ISO-8601 in UTC with millisecond precision, i.e. what the + * browser produced with {@code new Date(ms).toISOString()}. + */ + private static final DateTimeFormatter EXPORT_TIMESTAMP_FORMAT = + DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC); + + private static final String CSV_COLUMN_SEPARATOR = ","; + private static final String CSV_LINE_SEPARATOR = "\n"; + private static final String CSV_TIMESTAMP_HEADER = "Timestamp"; + private static final String CSV_DEVICE_HEADER = "Device"; + private final TsFileProperties tsFileProperties; private final FileService fileService; private final TsFileDataReader dataReader; + /** + * ObjectMapper for the JSON export. The application context provides no ObjectMapper bean, so one + * is owned here just like {@link ScanService} does for its SSE payloads. + */ + private final ObjectMapper objectMapper = new ObjectMapper(); + public DataService( TsFileProperties tsFileProperties, FileService fileService, TsFileDataReader dataReader) { this.tsFileProperties = tsFileProperties; @@ -84,6 +128,20 @@ public DataService( this.dataReader = dataReader; } + /** + * Everything an export needs that can be resolved before a single byte is written. + * + *

Resolved up front so that an unknown file id (404) or a forbidden path (403) still produces + * a regular JSON error response instead of a download that breaks halfway. + * + * @param format the requested representation + * @param filePath the resolved absolute path of the TsFile + * @param filters the filters to apply, with {@code limit}/{@code offset} ignored + * @param columns the CSV measurement columns, in dictionary order (unused for JSON) + */ + public record ExportPlan( + ExportFormat format, String filePath, FilterConditions filters, List columns) {} + /** * Previews data from a TsFile with filtering and pagination. * @@ -179,6 +237,198 @@ public DataPreviewResponse previewData(DataPreviewRequest request) } } + /** + * Resolves everything an export needs before the first byte is written. + * + * @param request the data preview request carrying the filters; its {@code limit} and {@code + * offset} are ignored on purpose, an export always covers every matching row + * @param format the requested representation + * @return the resolved export plan + * @throws TsFileNotFoundException if the file is not found + * @throws AccessDeniedException if the path is not allowed + * @throws IOException if the file metadata cannot be read + */ + public ExportPlan prepareExport(DataPreviewRequest request, ExportFormat format) + throws TsFileNotFoundException, AccessDeniedException, IOException { + String filePath = fileService.getFilePath(request.getFileId()); + FilterConditions filters = request.toFilterConditions(); + + // CSV columns are the union of the file's measurements in dictionary order; when the user + // narrowed the selection to specific measurements only those survive, so the export does not + // grow columns that are empty for every row. + List columns = new ArrayList<>(dataReader.getAllColumnNames(new File(filePath))); + if (filters.getMeasurements() != null && !filters.getMeasurements().isEmpty()) { + Set selected = new HashSet<>(filters.getMeasurements()); + columns.removeIf(column -> !selected.contains(column)); + } + columns.sort(Comparator.naturalOrder()); + + return new ExportPlan(format, filePath, filters, columns); + } + + /** + * Streams every row matching the plan's filters as CSV or JSON. + * + *

Rows are read one page at a time with the page offset pushed down into the reader, filtered + * per page and written right away, so neither the file nor the result is ever materialized in + * memory. That is also why an export is not capped by {@code tsfile.query.max-result-size}: the + * property bounds a single read (one preview page), not the number of pages streamed here. + * + * @param plan the resolved export plan + * @param outputStream the stream to write to, flushed after every page + * @return the number of data rows written, excluding the CSV header + * @throws IOException if writing fails + */ + public long exportData(ExportPlan plan, OutputStream outputStream) throws IOException { + File file = new File(plan.filePath()); + TimeRange timeRange = plan.filters().getTimeRange(); + Long startTime = timeRange != null ? timeRange.getStartTime() : null; + Long endTime = timeRange != null ? timeRange.getEndTime() : null; + + long startedAt = System.currentTimeMillis(); + long exportedRows = 0; + int rawOffset = 0; + boolean firstJsonRow = true; + Set warnings = new LinkedHashSet<>(); + + Writer writer = + new BufferedWriter( + new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), EXPORT_BUFFER_SIZE); + + if (plan.format() == ExportFormat.CSV) { + writeCsvHeader(writer, plan.columns()); + } else { + writer.write("["); + } + + while (true) { + var page = + dataReader.readDataByTimeRange(file, startTime, endTime, EXPORT_PAGE_SIZE, rawOffset); + warnings.addAll(page.getWarnings()); + + // Exactly the filters the preview applies, so the export contains the rows it shows. + List rows = applyFilters(convertToDataRows(page.getData()), plan.filters()); + + for (DataRow row : rows) { + if (plan.format() == ExportFormat.CSV) { + writeCsvRow(writer, row, plan.columns()); + } else { + writer.write(firstJsonRow ? "\n" : ",\n"); + writer.write(objectMapper.writeValueAsString(row)); + firstJsonRow = false; + } + } + exportedRows += rows.size(); + + // Flush after every page so the browser starts receiving data immediately. + writer.flush(); + outputStream.flush(); + + rawOffset += page.getData().size(); + if (!page.hasMore() || page.getData().isEmpty()) { + break; + } + } + + if (plan.format() == ExportFormat.JSON) { + writer.write(firstJsonRow ? "]\n" : "\n]\n"); + } + writer.flush(); + + if (!warnings.isEmpty()) { + logger.warn("Export of {} finished with warnings: {}", plan.filePath(), warnings); + } + logger.info( + "Exported {} rows as {} from {} in {} ms", + exportedRows, + plan.format(), + plan.filePath(), + System.currentTimeMillis() - startedAt); + return exportedRows; + } + + /** Writes the CSV header line: {@code Timestamp,Device,}. */ + private static void writeCsvHeader(Writer writer, List columns) throws IOException { + StringBuilder line = + new StringBuilder(CSV_TIMESTAMP_HEADER) + .append(CSV_COLUMN_SEPARATOR) + .append(CSV_DEVICE_HEADER); + for (String column : columns) { + line.append(CSV_COLUMN_SEPARATOR).append(escapeCsv(column)); + } + writer.write(line.append(CSV_LINE_SEPARATOR).toString()); + } + + /** Writes one CSV data line; a value missing from a row becomes an empty field. */ + private static void writeCsvRow(Writer writer, DataRow row, List columns) + throws IOException { + Map measurements = row.getMeasurements(); + StringBuilder line = new StringBuilder(64); + line.append(formatTimestamp(row.getTimestamp())) + .append(CSV_COLUMN_SEPARATOR) + .append(escapeCsv(row.getDevice())); + for (String column : columns) { + Object value = measurements != null ? measurements.get(column) : null; + line.append(CSV_COLUMN_SEPARATOR).append(escapeCsv(csvText(value))); + } + writer.write(line.append(CSV_LINE_SEPARATOR).toString()); + } + + private static String csvText(Object value) { + return value == null ? "" : String.valueOf(value); + } + + /** + * Makes a field safe for CSV: a value containing a separator, a quote or a line break is wrapped + * in double quotes with every embedded quote doubled (RFC 4180). + */ + static String escapeCsv(String value) { + if (value == null) { + return ""; + } + boolean needsQuoting = + value.indexOf(',') >= 0 + || value.indexOf('"') >= 0 + || value.indexOf('\n') >= 0 + || value.indexOf('\r') >= 0; + return needsQuoting ? '"' + value.replace("\"", "\"\"") + '"' : value; + } + + /** + * Formats a timestamp as ISO-8601 in UTC with millisecond precision, i.e. the same text the + * export used to produce in the browser with {@code new Date(ms).toISOString()}. + */ + static String formatTimestamp(long timestamp) { + try { + return EXPORT_TIMESTAMP_FORMAT.format(Instant.ofEpochMilli(normalizeToMillis(timestamp))); + } catch (DateTimeException e) { + return Long.toString(timestamp); + } + } + + /** + * Normalizes a TsFile timestamp of unknown precision to milliseconds. + * + *

TsFile keeps whatever unit the writer chose (s/ms/us/ns) and the value's magnitude is the + * only hint available, exactly as in the frontend's {@code normalizeToMs}. + */ + static long normalizeToMillis(long timestamp) { + long magnitude = Math.abs(timestamp); + if (magnitude >= 100_000_000_000_000_000L) { + return timestamp / 1_000_000L; // nanoseconds + } + if (magnitude >= 100_000_000_000_000L) { + return timestamp / 1_000L; // microseconds + } + if (magnitude >= 100_000_000_000L) { + return timestamp; // milliseconds + } + if (magnitude >= 100_000_000L) { + return timestamp * 1_000L; // seconds + } + return timestamp; // anything else is treated as milliseconds + } + /** * Queries chart data from a TsFile with optional aggregation. * @@ -439,9 +689,9 @@ private List extractMeasurementData(List data, String measure /** * Converts a measurement value into a number that can be plotted. * - *

BOOLEAN fields are mapped to 0/1. Without this, a table whose only field is a boolean - * (e.g. a fault flag) yields no series at all, so the chart stays empty while the data preview - * shows the rows. + *

BOOLEAN fields are mapped to 0/1. Without this, a table whose only field is a boolean (e.g. + * a fault flag) yields no series at all, so the chart stays empty while the data preview shows + * the rows. * * @param value the measurement value * @return the numeric value, or null when the value cannot be plotted diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java index 5625cbb..7fb4eab 100644 --- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java +++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.function.Consumer; import org.apache.tsfile.enums.ColumnCategory; @@ -742,6 +743,53 @@ public List getTableNames(File filePath) throws IOException { return names; } + /** + * Gets the sorted union of every data column name present in the file. + * + *

Table model files contribute the union of their table schemas (TAG and FIELD columns, which + * is exactly what a read row carries); tree model files contribute the union of the measurements + * found in their device metadata. Only metadata is touched, no chunk is decoded, so this stays + * cheap even for files with millions of rows. + * + * @param filePath the TsFile to inspect + * @return the column names in ascending (dictionary) order + * @throws IOException if the file cannot be read + */ + public List getAllColumnNames(File filePath) throws IOException { + Set names = new TreeSet<>(); + + try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) { + List schemas = new ArrayList<>(); + for (TableSchema ts : reader.getAllTableSchema()) { + schemas.add(ts); + } + if (!schemas.isEmpty()) { + for (TableSchema ts : schemas) { + for (IMeasurementSchema column : ts.getColumnSchemas()) { + names.add(column.getMeasurementName()); + } + } + return new ArrayList<>(names); + } + } + + try (TsFileSequenceReader seqReader = new TsFileSequenceReader(filePath.getAbsolutePath())) { + for (IDeviceID device : seqReader.getAllDevices()) { + for (Map.Entry> entry : + seqReader.readChunkMetadataInDevice(device).entrySet()) { + if (entry.getValue().isEmpty()) { + continue; + } + // Skip VECTOR type (used for the aligned time column), it is not a data column. + if (entry.getValue().get(0).getDataType() != TSDataType.VECTOR) { + names.add(entry.getKey()); + } + } + } + } + return new ArrayList<>(names); + } + /** Gets the column schema for a specific table. */ public List> getTableColumns(File filePath, String tableName) throws IOException { diff --git a/backend/src/test/java/org/apache/tsfile/viewer/service/DataServiceExportTest.java b/backend/src/test/java/org/apache/tsfile/viewer/service/DataServiceExportTest.java new file mode 100644 index 0000000..a948a4a --- /dev/null +++ b/backend/src/test/java/org/apache/tsfile/viewer/service/DataServiceExportTest.java @@ -0,0 +1,341 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tsfile.viewer.service; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import org.apache.tsfile.enums.ColumnCategory; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.ColumnSchemaBuilder; +import org.apache.tsfile.file.metadata.TableSchema; +import org.apache.tsfile.viewer.config.TsFileProperties; +import org.apache.tsfile.viewer.dto.DataPreviewRequest; +import org.apache.tsfile.viewer.dto.ExportFormat; +import org.apache.tsfile.viewer.exception.TsFileNotFoundException; +import org.apache.tsfile.viewer.tsfile.TsFileDataReader; +import org.apache.tsfile.viewer.tsfile.TsFileTestUtils; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.v4.ITsFileWriter; +import org.apache.tsfile.write.v4.TsFileWriterBuilder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Tests for the streaming data export. + * + *

Covers the CSV layout (column order, ISO timestamps, RFC 4180 escaping), the JSON structure, + * the reuse of the preview filters, and the fact that an export is not truncated at {@code + * tsfile.query.max-result-size}. + */ +class DataServiceExportTest { + + private static final String TABLE_NAME = "fault_table"; + private static final long START_TIME_NS = 1_719_731_133_000_000_000L; + private static final long INTERVAL_NS = 1_000_000_000L; + private static final int POINT_COUNT = 5; + + /** A TAG value carrying a comma and a quote, so the CSV escaping is exercised end to end. */ + private static final String DEVICE_ID = "C919,\"A\""; + + private static final String ESCAPED_DEVICE = "\"fault_table.C919,\"\"A\"\"\""; + + /** Well above the default {@code tsfile.query.max-result-size} of 10000. */ + private static final int LARGE_ROW_COUNT = 10_500; + + /** A {@link Tablet} holds at most ~1024 rows, larger files have to be written in batches. */ + private static final int TABLET_BATCH_SIZE = 1_000; + + private Path tempDir; + private TsFileProperties properties; + private FileService fileService; + private DataService dataService; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() throws Exception { + tempDir = Files.createTempDirectory("data-service-export-test"); + + properties = new TsFileProperties(); + properties.getAllowedDirectories().add(tempDir.toString()); + properties.getQuery().setTimeoutSeconds(30); + + fileService = new FileService(properties, new PathValidationService(properties)); + objectMapper = new ObjectMapper(); + dataService = new DataService(properties, fileService, new TsFileDataReader()); + } + + @AfterEach + void tearDown() throws Exception { + if (tempDir == null || !Files.exists(tempDir)) { + return; + } + Files.walk(tempDir) + .sorted((a, b) -> -a.compareTo(b)) + .forEach( + path -> { + try { + Files.deleteIfExists(path); + } catch (Exception e) { + // Ignore cleanup errors + } + }); + } + + @Test + @DisplayName("CSV starts with Timestamp,Device followed by the sorted measurement columns") + void shouldWriteCsvHeaderWithSortedColumns() throws Exception { + String csv = exportCsv(smallFileId()); + + String[] lines = csv.split("\n"); + assertThat(lines[0]).isEqualTo("Timestamp,Device,device_id,value"); + assertThat(lines).hasSize(POINT_COUNT + 1); + } + + @Test + @DisplayName("CSV writes ISO timestamps in UTC and escapes separators and quotes") + void shouldWriteIsoTimestampsAndEscapedFields() throws Exception { + String csv = exportCsv(smallFileId()); + + String[] lines = csv.split("\n"); + assertThat(lines[1]) + .isEqualTo("2024-06-30T07:05:33.000Z," + ESCAPED_DEVICE + ",\"C919,\"\"A\"\"\",false"); + assertThat(lines[2]).startsWith("2024-06-30T07:05:34.000Z,"); + } + + @Test + @DisplayName("CSV quoting follows RFC 4180 for line breaks and quotes") + void shouldQuoteFieldsContainingLineBreaks() { + assertThat(DataService.escapeCsv("plain")).isEqualTo("plain"); + assertThat(DataService.escapeCsv(null)).isEmpty(); + assertThat(DataService.escapeCsv("a,b")).isEqualTo("\"a,b\""); + assertThat(DataService.escapeCsv("a\"b")).isEqualTo("\"a\"\"b\""); + assertThat(DataService.escapeCsv("a\nb")).isEqualTo("\"a\nb\""); + assertThat(DataService.escapeCsv("a\r\nb")).isEqualTo("\"a\r\nb\""); + } + + @Test + @DisplayName("Timestamps of every supported precision normalize to the same ISO instant") + void shouldNormalizeTimestampPrecision() { + long millis = 1_719_731_133_000L; + assertThat(DataService.formatTimestamp(millis)).isEqualTo("2024-06-30T07:05:33.000Z"); + assertThat(DataService.formatTimestamp(millis * 1_000L)).isEqualTo("2024-06-30T07:05:33.000Z"); + assertThat(DataService.formatTimestamp(millis * 1_000_000L)) + .isEqualTo("2024-06-30T07:05:33.000Z"); + assertThat(DataService.formatTimestamp(millis / 1_000L)).isEqualTo("2024-06-30T07:05:33.000Z"); + } + + @Test + @DisplayName("JSON is an array of objects shaped like DataPreviewResponse.data rows") + void shouldWriteJsonArrayOfRowObjects() throws Exception { + String json = exportToString(request(smallFileId()), ExportFormat.JSON); + + List> rows = objectMapper.readValue(json, new TypeReference<>() {}); + assertThat(rows).hasSize(POINT_COUNT); + + Map first = rows.get(0); + assertThat(first).containsOnlyKeys("timestamp", "device", "measurements"); + assertThat(first.get("timestamp")).isEqualTo(1_719_731_133_000_000_000L); + assertThat(first.get("device")).isEqualTo("fault_table." + DEVICE_ID); + + @SuppressWarnings("unchecked") + Map measurements = (Map) first.get("measurements"); + assertThat(measurements).containsOnlyKeys("device_id", "value"); + assertThat(measurements.get("device_id")).isEqualTo(DEVICE_ID); + assertThat(measurements.get("value")).isEqualTo(false); + } + + @Test + @DisplayName("An empty filter selection exports every row, not just one page") + void shouldIgnoreLimitAndOffset() throws Exception { + DataPreviewRequest request = request(smallFileId()); + request.setLimit(1); + request.setOffset(3); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + long rows = dataService.exportData(dataService.prepareExport(request, ExportFormat.CSV), out); + + assertThat(rows).isEqualTo(POINT_COUNT); + // Header + 5 data rows: the request's own limit/offset never reach the export. + assertThat(out.toString(StandardCharsets.UTF_8).split("\n")).hasSize(POINT_COUNT + 1); + } + + @Test + @DisplayName("The preview filters are reused by the export") + void shouldApplyPreviewFilters() throws Exception { + DataPreviewRequest matchingDevice = request(smallFileId()); + matchingDevice.setDevices(List.of("fault_table." + DEVICE_ID)); + assertThat(exportCsv(matchingDevice).split("\n")).hasSize(POINT_COUNT + 1); + + DataPreviewRequest unknownDevice = request(smallFileId()); + unknownDevice.setDevices(List.of("fault_table.unknown")); + assertThat(exportCsv(unknownDevice).split("\n")).hasSize(1); + } + + @Test + @DisplayName("A measurement filter keeps only the selected CSV columns") + void shouldTrimCsvColumnsToSelectedMeasurements() throws Exception { + DataPreviewRequest request = request(smallFileId()); + request.setMeasurements(List.of("value")); + + String[] lines = exportCsv(request).split("\n"); + assertThat(lines[0]).isEqualTo("Timestamp,Device,value"); + assertThat(lines).hasSize(POINT_COUNT + 1); + } + + @Test + @DisplayName("Exporting more than max-result-size rows is not truncated") + void shouldExportMoreRowsThanMaxResultSize() throws Exception { + File largeFile = createLargeTableModelFile(tempDir.resolve("large.tsfile"), LARGE_ROW_COUNT); + + assertThat(LARGE_ROW_COUNT).isGreaterThan(properties.getQuery().getMaxResultSize()); + + String csv = exportCsv(fileService.registerFile(largeFile.getAbsolutePath())); + + String[] lines = csv.split("\n"); + assertThat(lines[0]).isEqualTo("Timestamp,Device,device_id,humidity,temperature"); + assertThat(lines).hasSize(LARGE_ROW_COUNT + 1); + } + + @Test + @DisplayName("An unknown fileId keeps the preview semantics (404)") + void shouldThrowWhenFileIsUnknown() { + String missingPath = tempDir.resolve("missing.tsfile").toAbsolutePath().toString(); + String fileId = + Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(missingPath.getBytes(StandardCharsets.UTF_8)); + DataPreviewRequest request = request(fileId); + + assertThatThrownBy(() -> dataService.prepareExport(request, ExportFormat.CSV)) + .isInstanceOf(TsFileNotFoundException.class); + } + + @Test + @DisplayName("format parameter parsing accepts csv/json in any case and rejects the rest") + void shouldParseFormatParameter() { + assertThat(ExportFormat.from("csv")).isEqualTo(ExportFormat.CSV); + assertThat(ExportFormat.from("JSON")).isEqualTo(ExportFormat.JSON); + assertThat(ExportFormat.from(null)).isEqualTo(ExportFormat.CSV); + assertThatThrownBy(() -> ExportFormat.from("xlsx")) + .isInstanceOf(IllegalArgumentException.class); + } + + /** + * Creates a table model file with {@code rowCount} rows. + * + *

A single {@link Tablet} holds at most ~1024 rows, so the rows are written in batches; this + * is what makes a file larger than {@code tsfile.query.max-result-size} available to the test. + */ + private File createLargeTableModelFile(Path outputPath, int rowCount) throws Exception { + Files.deleteIfExists(outputPath); + File file = outputPath.toFile(); + + TableSchema tableSchema = + new TableSchema( + "large_table", + Arrays.asList( + new ColumnSchemaBuilder() + .name("device_id") + .dataType(TSDataType.STRING) + .category(ColumnCategory.TAG) + .build(), + new ColumnSchemaBuilder() + .name("humidity") + .dataType(TSDataType.INT32) + .category(ColumnCategory.FIELD) + .build(), + new ColumnSchemaBuilder() + .name("temperature") + .dataType(TSDataType.FLOAT) + .category(ColumnCategory.FIELD) + .build())); + + try (ITsFileWriter writer = + new TsFileWriterBuilder().file(file).tableSchema(tableSchema).build()) { + int written = 0; + while (written < rowCount) { + int batchSize = Math.min(TABLET_BATCH_SIZE, rowCount - written); + Tablet tablet = + new Tablet( + Arrays.asList("device_id", "humidity", "temperature"), + Arrays.asList(TSDataType.STRING, TSDataType.INT32, TSDataType.FLOAT)); + for (int i = 0; i < batchSize; i++) { + int row = written + i; + tablet.addTimestamp(i, row * 1_000L); + tablet.addValue(i, "device_id", "C919"); + tablet.addValue(i, "humidity", 50 + row); + tablet.addValue(i, "temperature", 20.0f + row * 0.5f); + } + writer.write(tablet); + written += batchSize; + } + } + + return file; + } + + private String smallFileId() throws Exception { + File file = + TsFileTestUtils.createTableModelBooleanField( + tempDir.resolve("small.tsfile"), + TABLE_NAME, + DEVICE_ID, + START_TIME_NS, + INTERVAL_NS, + POINT_COUNT); + return fileService.registerFile(file.getAbsolutePath()); + } + + private DataPreviewRequest request(String fileId) { + DataPreviewRequest request = new DataPreviewRequest(); + request.setFileId(fileId); + return request; + } + + private String exportCsv(String fileId) throws Exception { + return exportToString(request(fileId), ExportFormat.CSV); + } + + private String exportCsv(DataPreviewRequest request) throws Exception { + return exportToString(request, ExportFormat.CSV); + } + + private String exportToString(DataPreviewRequest request, ExportFormat format) throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + dataService.exportData(dataService.prepareExport(request, format), out); + return out.toString(StandardCharsets.UTF_8); + } +} diff --git a/frontend/src/api/tsfile/data.ts b/frontend/src/api/tsfile/data.ts index 383f319..ed82ed5 100644 --- a/frontend/src/api/tsfile/data.ts +++ b/frontend/src/api/tsfile/data.ts @@ -57,6 +57,28 @@ export function queryTableData(request: TableDataRequest) { return apiClient.post("/tables/query", request); } +/** + * 导出接口的请求体:筛选条件与预览完全一致,但不带 limit / offset——服务端始终导出全部匹配记录。 + */ +export type DataExportRequest = Omit; + +// 全量导出可能远超 axios 默认的 30s 超时,单独放宽到 10 分钟。 +const EXPORT_TIMEOUT_MS = 10 * 60_000; + +/** + * 导出当前筛选条件下的全部数据(CSV / JSON),返回文件内容。 + * + * 服务端按页流式读取并写出,因此不受预览接口 max-result-size(默认 10000)的截断影响, + * 也不需要前端翻页拼装或把整份数据读进浏览器内存。 + */ +export function exportData(request: DataExportRequest, format: "csv" | "json") { + return apiClient.post("/data/export", request, { + params: { format }, + responseType: "blob", + timeout: EXPORT_TIMEOUT_MS, + }); +} + export function queryChartData(request: ChartDataRequest) { return apiClient.post("/data/query", request); } diff --git a/frontend/src/components/tsfile/DataTable.vue b/frontend/src/components/tsfile/DataTable.vue index e9f3f82..310fb1f 100644 --- a/frontend/src/components/tsfile/DataTable.vue +++ b/frontend/src/components/tsfile/DataTable.vue @@ -45,11 +45,14 @@ interface Props { loading: boolean; error: string | null; tagColumns?: string[]; + /** 导出进行中:取全量数据期间禁用导出按钮 */ + exporting?: boolean; } const props = withDefaults(defineProps(), { limit: 100, tagColumns: () => [], + exporting: false, }); const emit = defineEmits<{ @@ -284,11 +287,11 @@ function formatValue(value: unknown): number | string {

{{ t("tsfile.data.title") }}
- + {{ t("tsfile.data.exportCsv") }} - + {{ t("tsfile.data.exportJson") }} diff --git a/frontend/src/views/tsfile/data-preview/index.vue b/frontend/src/views/tsfile/data-preview/index.vue index 90c94ed..e6b2170 100644 --- a/frontend/src/views/tsfile/data-preview/index.vue +++ b/frontend/src/views/tsfile/data-preview/index.vue @@ -19,6 +19,7 @@