diff --git a/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java b/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java
index 180d817..e3e6fe5 100644
--- a/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java
+++ b/backend/src/main/java/org/apache/tsfile/viewer/controller/DataController.java
@@ -21,23 +21,28 @@
import java.io.IOException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.http.ResponseEntity;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestBody;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
-
import org.apache.tsfile.viewer.dto.ChartDataRequest;
import org.apache.tsfile.viewer.dto.ChartDataResponse;
import org.apache.tsfile.viewer.dto.DataPreviewRequest;
import org.apache.tsfile.viewer.dto.DataPreviewResponse;
+import org.apache.tsfile.viewer.dto.ExportFormat;
import org.apache.tsfile.viewer.exception.AccessDeniedException;
import org.apache.tsfile.viewer.exception.QueryTimeoutException;
import org.apache.tsfile.viewer.exception.TsFileNotFoundException;
import org.apache.tsfile.viewer.service.DataService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.server.ResponseStatusException;
+import jakarta.servlet.http.HttpServletResponse;
import jakarta.validation.Valid;
/**
@@ -47,6 +52,7 @@
*
*
previewData(
return ResponseEntity.ok(response);
}
+ /**
+ * Exports every row matching the given filters as a downloadable CSV or JSON file.
+ *
+ * The request body is the preview request, so filtering behaves exactly like the preview; its
+ * {@code limit}/{@code offset} are ignored and the response is streamed page by page, which means
+ * an export is neither limited to the current preview page nor to {@code
+ * tsfile.query.max-result-size}.
+ *
+ * @param request the data preview request carrying the filters to apply
+ * @param format {@code csv} or {@code json}
+ * @param response the response the export is streamed to
+ */
+ @PostMapping("/export")
+ public void exportData(
+ @RequestBody @Valid DataPreviewRequest request,
+ @RequestParam(name = "format", defaultValue = "csv") String format,
+ HttpServletResponse response)
+ throws TsFileNotFoundException, AccessDeniedException, IOException {
+ ExportFormat exportFormat = ExportFormat.from(format);
+ logger.debug("Exporting data for fileId={} as {}", request.getFileId(), exportFormat);
+
+ // Resolve the file (and the CSV columns) before committing the response, so an unknown fileId
+ // (404) or a forbidden path (403) still yields the regular JSON error body instead of a broken
+ // download.
+ DataService.ExportPlan plan = dataService.prepareExport(request, exportFormat);
+
+ response.setContentType(exportFormat.getContentType());
+ response.setHeader(HttpHeaders.CONTENT_DISPOSITION, exportFormat.getContentDisposition());
+
+ long startTime = System.currentTimeMillis();
+ try {
+ long rows = dataService.exportData(plan, response.getOutputStream());
+ logger.info(
+ "Exported {} rows as {} for fileId={} in {} ms",
+ rows,
+ exportFormat,
+ request.getFileId(),
+ System.currentTimeMillis() - startTime);
+ } catch (IOException e) {
+ // Part of the body may already be on the wire (client gone, disk error while reading), so the
+ // response can only be turned into an error when nothing has been committed yet.
+ logger.error(
+ "Export failed for fileId={} as {} after {} ms",
+ request.getFileId(),
+ exportFormat,
+ System.currentTimeMillis() - startTime,
+ e);
+ if (!response.isCommitted()) {
+ throw new ResponseStatusException(
+ HttpStatus.INTERNAL_SERVER_ERROR, "Export failed: " + e.getMessage(), e);
+ }
+ }
+ }
+
/**
* Queries chart data from a TsFile with optional aggregation.
*
diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java
index 3a81315..8d70127 100644
--- a/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java
+++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/DataPreviewRequest.java
@@ -40,6 +40,7 @@ public class DataPreviewRequest {
private List devices;
private List measurements;
private ValueRange valueRange;
+ private List advancedConditions;
@Min(value = 1, message = "limit must be at least 1")
@Max(value = 1000, message = "limit must not exceed 1000")
@@ -99,6 +100,14 @@ public void setValueRange(ValueRange valueRange) {
this.valueRange = valueRange;
}
+ public List getAdvancedConditions() {
+ return advancedConditions;
+ }
+
+ public void setAdvancedConditions(List advancedConditions) {
+ this.advancedConditions = advancedConditions;
+ }
+
public int getLimit() {
return limit;
}
@@ -141,6 +150,10 @@ public FilterConditions toFilterConditions() {
builder.valueRange(valueRange);
}
+ if (advancedConditions != null && !advancedConditions.isEmpty()) {
+ builder.advancedConditions(advancedConditions);
+ }
+
return builder.build();
}
}
diff --git a/backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java b/backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java
new file mode 100644
index 0000000..990191e
--- /dev/null
+++ b/backend/src/main/java/org/apache/tsfile/viewer/dto/ExportFormat.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.tsfile.viewer.dto;
+
+/**
+ * Downloadable representation of an exported data set.
+ *
+ * Each constant carries the media type and the download file name, so the controller and the
+ * writer in the service layer agree on a single definition.
+ */
+public enum ExportFormat {
+ CSV("csv", "text/csv;charset=UTF-8", "data.csv"),
+ JSON("json", "application/json;charset=UTF-8", "data.json");
+
+ private final String parameterValue;
+ private final String contentType;
+ private final String fileName;
+
+ ExportFormat(String parameterValue, String contentType, String fileName) {
+ this.parameterValue = parameterValue;
+ this.contentType = contentType;
+ this.fileName = fileName;
+ }
+
+ /**
+ * Resolves the {@code format} request parameter.
+ *
+ * @param raw the raw parameter value, e.g. {@code csv}; blank means {@link #CSV}
+ * @return the matching format
+ * @throws IllegalArgumentException if the value matches no supported format (mapped to HTTP 400)
+ */
+ public static ExportFormat from(String raw) {
+ if (raw == null || raw.isBlank()) {
+ return CSV;
+ }
+ String normalized = raw.trim();
+ for (ExportFormat format : values()) {
+ if (format.parameterValue.equalsIgnoreCase(normalized)) {
+ return format;
+ }
+ }
+ throw new IllegalArgumentException(
+ "Unsupported export format '" + raw + "'; expected 'csv' or 'json'");
+ }
+
+ /** Returns the {@code Content-Type} announced for the download response. */
+ public String getContentType() {
+ return contentType;
+ }
+
+ /** Returns the file name announced in the {@code Content-Disposition} response header. */
+ public String getFileName() {
+ return fileName;
+ }
+
+ /** Returns the {@code Content-Disposition} value that makes browsers save instead of render. */
+ public String getContentDisposition() {
+ return "attachment; filename=\"" + fileName + "\"";
+ }
+}
diff --git a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java
index 509cfbc..bc540b5 100644
--- a/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java
+++ b/backend/src/main/java/org/apache/tsfile/viewer/service/DataService.java
@@ -19,11 +19,25 @@
package org.apache.tsfile.viewer.service;
+import java.io.BufferedWriter;
+import java.io.File;
import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.time.DateTimeException;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.TimeoutException;
import org.apache.tsfile.viewer.config.TsFileProperties;
@@ -35,6 +49,7 @@
import org.apache.tsfile.viewer.dto.DataPreviewRequest;
import org.apache.tsfile.viewer.dto.DataPreviewResponse;
import org.apache.tsfile.viewer.dto.DataRow;
+import org.apache.tsfile.viewer.dto.ExportFormat;
import org.apache.tsfile.viewer.dto.FilterConditions;
import org.apache.tsfile.viewer.dto.TimeRange;
import org.apache.tsfile.viewer.dto.ValueRange;
@@ -46,6 +61,8 @@
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
/**
* Service for data query operations.
*
@@ -73,10 +90,37 @@ public class DataService {
*/
private static final int MAX_CHART_SERIES = 200;
+ /**
+ * Page size used while streaming an export. The reader is asked for one page at a time, so peak
+ * memory depends on this constant rather than on the size of the file being exported.
+ */
+ private static final int EXPORT_PAGE_SIZE = 2000;
+
+ /** Buffer size of the export writer; matched to a typical socket/TCP window size. */
+ private static final int EXPORT_BUFFER_SIZE = 64 * 1024;
+
+ /**
+ * Timestamp layout of the CSV export: ISO-8601 in UTC with millisecond precision, i.e. what the
+ * browser produced with {@code new Date(ms).toISOString()}.
+ */
+ private static final DateTimeFormatter EXPORT_TIMESTAMP_FORMAT =
+ DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS'Z'").withZone(ZoneOffset.UTC);
+
+ private static final String CSV_COLUMN_SEPARATOR = ",";
+ private static final String CSV_LINE_SEPARATOR = "\n";
+ private static final String CSV_TIMESTAMP_HEADER = "Timestamp";
+ private static final String CSV_DEVICE_HEADER = "Device";
+
private final TsFileProperties tsFileProperties;
private final FileService fileService;
private final TsFileDataReader dataReader;
+ /**
+ * ObjectMapper for the JSON export. The application context provides no ObjectMapper bean, so one
+ * is owned here just like {@link ScanService} does for its SSE payloads.
+ */
+ private final ObjectMapper objectMapper = new ObjectMapper();
+
public DataService(
TsFileProperties tsFileProperties, FileService fileService, TsFileDataReader dataReader) {
this.tsFileProperties = tsFileProperties;
@@ -84,6 +128,20 @@ public DataService(
this.dataReader = dataReader;
}
+ /**
+ * Everything an export needs that can be resolved before a single byte is written.
+ *
+ *
Resolved up front so that an unknown file id (404) or a forbidden path (403) still produces
+ * a regular JSON error response instead of a download that breaks halfway.
+ *
+ * @param format the requested representation
+ * @param filePath the resolved absolute path of the TsFile
+ * @param filters the filters to apply, with {@code limit}/{@code offset} ignored
+ * @param columns the CSV measurement columns, in dictionary order (unused for JSON)
+ */
+ public record ExportPlan(
+ ExportFormat format, String filePath, FilterConditions filters, List columns) {}
+
/**
* Previews data from a TsFile with filtering and pagination.
*
@@ -179,6 +237,198 @@ public DataPreviewResponse previewData(DataPreviewRequest request)
}
}
+ /**
+ * Resolves everything an export needs before the first byte is written.
+ *
+ * @param request the data preview request carrying the filters; its {@code limit} and {@code
+ * offset} are ignored on purpose, an export always covers every matching row
+ * @param format the requested representation
+ * @return the resolved export plan
+ * @throws TsFileNotFoundException if the file is not found
+ * @throws AccessDeniedException if the path is not allowed
+ * @throws IOException if the file metadata cannot be read
+ */
+ public ExportPlan prepareExport(DataPreviewRequest request, ExportFormat format)
+ throws TsFileNotFoundException, AccessDeniedException, IOException {
+ String filePath = fileService.getFilePath(request.getFileId());
+ FilterConditions filters = request.toFilterConditions();
+
+ // CSV columns are the union of the file's measurements in dictionary order; when the user
+ // narrowed the selection to specific measurements only those survive, so the export does not
+ // grow columns that are empty for every row.
+ List columns = new ArrayList<>(dataReader.getAllColumnNames(new File(filePath)));
+ if (filters.getMeasurements() != null && !filters.getMeasurements().isEmpty()) {
+ Set selected = new HashSet<>(filters.getMeasurements());
+ columns.removeIf(column -> !selected.contains(column));
+ }
+ columns.sort(Comparator.naturalOrder());
+
+ return new ExportPlan(format, filePath, filters, columns);
+ }
+
+ /**
+ * Streams every row matching the plan's filters as CSV or JSON.
+ *
+ * Rows are read one page at a time with the page offset pushed down into the reader, filtered
+ * per page and written right away, so neither the file nor the result is ever materialized in
+ * memory. That is also why an export is not capped by {@code tsfile.query.max-result-size}: the
+ * property bounds a single read (one preview page), not the number of pages streamed here.
+ *
+ * @param plan the resolved export plan
+ * @param outputStream the stream to write to, flushed after every page
+ * @return the number of data rows written, excluding the CSV header
+ * @throws IOException if writing fails
+ */
+ public long exportData(ExportPlan plan, OutputStream outputStream) throws IOException {
+ File file = new File(plan.filePath());
+ TimeRange timeRange = plan.filters().getTimeRange();
+ Long startTime = timeRange != null ? timeRange.getStartTime() : null;
+ Long endTime = timeRange != null ? timeRange.getEndTime() : null;
+
+ long startedAt = System.currentTimeMillis();
+ long exportedRows = 0;
+ int rawOffset = 0;
+ boolean firstJsonRow = true;
+ Set warnings = new LinkedHashSet<>();
+
+ Writer writer =
+ new BufferedWriter(
+ new OutputStreamWriter(outputStream, StandardCharsets.UTF_8), EXPORT_BUFFER_SIZE);
+
+ if (plan.format() == ExportFormat.CSV) {
+ writeCsvHeader(writer, plan.columns());
+ } else {
+ writer.write("[");
+ }
+
+ while (true) {
+ var page =
+ dataReader.readDataByTimeRange(file, startTime, endTime, EXPORT_PAGE_SIZE, rawOffset);
+ warnings.addAll(page.getWarnings());
+
+ // Exactly the filters the preview applies, so the export contains the rows it shows.
+ List rows = applyFilters(convertToDataRows(page.getData()), plan.filters());
+
+ for (DataRow row : rows) {
+ if (plan.format() == ExportFormat.CSV) {
+ writeCsvRow(writer, row, plan.columns());
+ } else {
+ writer.write(firstJsonRow ? "\n" : ",\n");
+ writer.write(objectMapper.writeValueAsString(row));
+ firstJsonRow = false;
+ }
+ }
+ exportedRows += rows.size();
+
+ // Flush after every page so the browser starts receiving data immediately.
+ writer.flush();
+ outputStream.flush();
+
+ rawOffset += page.getData().size();
+ if (!page.hasMore() || page.getData().isEmpty()) {
+ break;
+ }
+ }
+
+ if (plan.format() == ExportFormat.JSON) {
+ writer.write(firstJsonRow ? "]\n" : "\n]\n");
+ }
+ writer.flush();
+
+ if (!warnings.isEmpty()) {
+ logger.warn("Export of {} finished with warnings: {}", plan.filePath(), warnings);
+ }
+ logger.info(
+ "Exported {} rows as {} from {} in {} ms",
+ exportedRows,
+ plan.format(),
+ plan.filePath(),
+ System.currentTimeMillis() - startedAt);
+ return exportedRows;
+ }
+
+ /** Writes the CSV header line: {@code Timestamp,Device,}. */
+ private static void writeCsvHeader(Writer writer, List columns) throws IOException {
+ StringBuilder line =
+ new StringBuilder(CSV_TIMESTAMP_HEADER)
+ .append(CSV_COLUMN_SEPARATOR)
+ .append(CSV_DEVICE_HEADER);
+ for (String column : columns) {
+ line.append(CSV_COLUMN_SEPARATOR).append(escapeCsv(column));
+ }
+ writer.write(line.append(CSV_LINE_SEPARATOR).toString());
+ }
+
+ /** Writes one CSV data line; a value missing from a row becomes an empty field. */
+ private static void writeCsvRow(Writer writer, DataRow row, List columns)
+ throws IOException {
+ Map measurements = row.getMeasurements();
+ StringBuilder line = new StringBuilder(64);
+ line.append(formatTimestamp(row.getTimestamp()))
+ .append(CSV_COLUMN_SEPARATOR)
+ .append(escapeCsv(row.getDevice()));
+ for (String column : columns) {
+ Object value = measurements != null ? measurements.get(column) : null;
+ line.append(CSV_COLUMN_SEPARATOR).append(escapeCsv(csvText(value)));
+ }
+ writer.write(line.append(CSV_LINE_SEPARATOR).toString());
+ }
+
+ private static String csvText(Object value) {
+ return value == null ? "" : String.valueOf(value);
+ }
+
+ /**
+ * Makes a field safe for CSV: a value containing a separator, a quote or a line break is wrapped
+ * in double quotes with every embedded quote doubled (RFC 4180).
+ */
+ static String escapeCsv(String value) {
+ if (value == null) {
+ return "";
+ }
+ boolean needsQuoting =
+ value.indexOf(',') >= 0
+ || value.indexOf('"') >= 0
+ || value.indexOf('\n') >= 0
+ || value.indexOf('\r') >= 0;
+ return needsQuoting ? '"' + value.replace("\"", "\"\"") + '"' : value;
+ }
+
+ /**
+ * Formats a timestamp as ISO-8601 in UTC with millisecond precision, i.e. the same text the
+ * export used to produce in the browser with {@code new Date(ms).toISOString()}.
+ */
+ static String formatTimestamp(long timestamp) {
+ try {
+ return EXPORT_TIMESTAMP_FORMAT.format(Instant.ofEpochMilli(normalizeToMillis(timestamp)));
+ } catch (DateTimeException e) {
+ return Long.toString(timestamp);
+ }
+ }
+
+ /**
+ * Normalizes a TsFile timestamp of unknown precision to milliseconds.
+ *
+ * TsFile keeps whatever unit the writer chose (s/ms/us/ns) and the value's magnitude is the
+ * only hint available, exactly as in the frontend's {@code normalizeToMs}.
+ */
+ static long normalizeToMillis(long timestamp) {
+ long magnitude = Math.abs(timestamp);
+ if (magnitude >= 100_000_000_000_000_000L) {
+ return timestamp / 1_000_000L; // nanoseconds
+ }
+ if (magnitude >= 100_000_000_000_000L) {
+ return timestamp / 1_000L; // microseconds
+ }
+ if (magnitude >= 100_000_000_000L) {
+ return timestamp; // milliseconds
+ }
+ if (magnitude >= 100_000_000L) {
+ return timestamp * 1_000L; // seconds
+ }
+ return timestamp; // anything else is treated as milliseconds
+ }
+
/**
* Queries chart data from a TsFile with optional aggregation.
*
@@ -439,9 +689,9 @@ private List extractMeasurementData(List data, String measure
/**
* Converts a measurement value into a number that can be plotted.
*
- * BOOLEAN fields are mapped to 0/1. Without this, a table whose only field is a boolean
- * (e.g. a fault flag) yields no series at all, so the chart stays empty while the data preview
- * shows the rows.
+ *
BOOLEAN fields are mapped to 0/1. Without this, a table whose only field is a boolean (e.g.
+ * a fault flag) yields no series at all, so the chart stays empty while the data preview shows
+ * the rows.
*
* @param value the measurement value
* @return the numeric value, or null when the value cannot be plotted
diff --git a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java
index 5625cbb..7fb4eab 100644
--- a/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java
+++ b/backend/src/main/java/org/apache/tsfile/viewer/tsfile/TsFileDataReader.java
@@ -27,6 +27,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.function.Consumer;
import org.apache.tsfile.enums.ColumnCategory;
@@ -742,6 +743,53 @@ public List getTableNames(File filePath) throws IOException {
return names;
}
+ /**
+ * Gets the sorted union of every data column name present in the file.
+ *
+ * Table model files contribute the union of their table schemas (TAG and FIELD columns, which
+ * is exactly what a read row carries); tree model files contribute the union of the measurements
+ * found in their device metadata. Only metadata is touched, no chunk is decoded, so this stays
+ * cheap even for files with millions of rows.
+ *
+ * @param filePath the TsFile to inspect
+ * @return the column names in ascending (dictionary) order
+ * @throws IOException if the file cannot be read
+ */
+ public List getAllColumnNames(File filePath) throws IOException {
+ Set names = new TreeSet<>();
+
+ try (ITsFileReader reader = new TsFileReaderBuilder().file(filePath).build()) {
+ List schemas = new ArrayList<>();
+ for (TableSchema ts : reader.getAllTableSchema()) {
+ schemas.add(ts);
+ }
+ if (!schemas.isEmpty()) {
+ for (TableSchema ts : schemas) {
+ for (IMeasurementSchema column : ts.getColumnSchemas()) {
+ names.add(column.getMeasurementName());
+ }
+ }
+ return new ArrayList<>(names);
+ }
+ }
+
+ try (TsFileSequenceReader seqReader = new TsFileSequenceReader(filePath.getAbsolutePath())) {
+ for (IDeviceID device : seqReader.getAllDevices()) {
+ for (Map.Entry> entry :
+ seqReader.readChunkMetadataInDevice(device).entrySet()) {
+ if (entry.getValue().isEmpty()) {
+ continue;
+ }
+ // Skip VECTOR type (used for the aligned time column), it is not a data column.
+ if (entry.getValue().get(0).getDataType() != TSDataType.VECTOR) {
+ names.add(entry.getKey());
+ }
+ }
+ }
+ }
+ return new ArrayList<>(names);
+ }
+
/** Gets the column schema for a specific table. */
public List