From 5887d5cc28c2fbcccbb8f0d3a35ae583694a019e Mon Sep 17 00:00:00 2001 From: spricoder Date: Wed, 10 Jun 2026 13:20:59 +0800 Subject: [PATCH] Fix tsfile-cli clean-build race and harden CLI validation Build: tsfile_cli_obj compiled against the staged include tree without depending on the copy_* targets that populate it, so a clean parallel build could fail with "'common/db_common.h' file not found" (deterministic via `make tsfile_cli_obj` in a fresh build dir). Add the missing add_dependencies edge. CLI hardening: - reject --offset on non-row commands and --model on write; name each read-only flag rejected by write instead of a lumped message - error when -d/-t does not match the file's data model in head/cat/sample/schema instead of silently ignoring the filter - include the storage error code in open/create failure messages - detect numeric flag overflow (ERANGE) in -n/--offset/--start/--end/--seed - cap the sample reservoir pre-allocation so a huge -n cannot abort via std::length_error; add a last-resort exception handler in main - emit null for non-finite FLOAT/DOUBLE cells in JSON output (bare nan/inf is not valid JSON) - docs: -m also applies to stats/count, count covers all tables, note that the table format buffers rows in memory --- cpp/tools/CMakeLists.txt | 6 +++ cpp/tools/README.md | 8 ++-- cpp/tools/cli/cli_args.cc | 4 +- cpp/tools/cli/run_cli.cc | 57 +++++++++++++++++++++++++-- cpp/tools/commands/cmd_write.cc | 6 ++- cpp/tools/format/output_format.cc | 19 ++++++++- cpp/tools/format/result_set_format.cc | 6 ++- cpp/tools/skills/tsfile-cli/SKILL.md | 9 +++-- cpp/tools/tools_main.cc | 10 ++++- 9 files changed, 108 insertions(+), 17 deletions(-) diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index a26e4b7129..cacc526fc8 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -30,6 +30,12 @@ target_include_directories(tsfile_cli_obj PUBLIC ${LIBRARY_INCLUDE_DIR} ${THIRD_PARTY_INCLUDE}) +# Library headers are compiled from the staged include tree +# (LIBRARY_INCLUDE_DIR), which is populated by the copy_* targets that +# tsfile depends on transitively. Without this edge a clean parallel build +# can compile these sources before the headers exist. +add_dependencies(tsfile_cli_obj tsfile) + if (ENABLE_ANTLR4) target_include_directories(tsfile_cli_obj PUBLIC ${PROJECT_SOURCE_DIR}/third_party/antlr4-cpp-runtime-4/runtime/src) diff --git a/cpp/tools/README.md b/cpp/tools/README.md index c2621fb986..bc8e5617fc 100644 --- a/cpp/tools/README.md +++ b/cpp/tools/README.md @@ -101,7 +101,7 @@ Exit codes: `0` success, `1` usage/argument error, `2` file open/corrupt, | `stats` | Per-series `count, start_time, end_time, min, max, first, last, sum` | | `count` | Per-series row counts plus a `total` row (from statistics, no page scan) | | `head` | First N rows (default 10; use `-n`) | -| `cat` | All matching rows, streamed | +| `cat` | All matching rows, streamed (`table` format buffers to align columns) | | `sample` | Reproducible reservoir sample (default 10; `-n`, `--seed`) | The metadata commands (`ls` / `schema` / `meta` / `stats` / `count`) answer most questions @@ -113,7 +113,7 @@ Shared options: |---|---| | `-f, --format csv\|tsv\|json\|table` | Output format; defaults to `table` on a TTY, `tsv` when piped | | `-d, --device ` / `-t, --table ` | Scope to one device / table (mutually exclusive) | -| `-m, --measurements a,b,c` | Column projection (`schema`, `head`, `cat`, `sample`) | +| `-m, --measurements a,b,c` | Column projection (`schema`, `stats`, `count`, `head`, `cat`, `sample`) | | `-n, --limit N` / `--offset N` | Max rows / rows to skip (`head`, `cat`; `--offset` not valid for `sample`) | | `--start ` / `--end ` | Inclusive epoch-millisecond time range (`head`, `cat`, `sample`) | | `--seed N` | Reproducible sampling seed (`sample` only) | @@ -121,7 +121,9 @@ Shared options: | `--model tree\|table` | Force the model (otherwise auto-detected) | `json` output is NDJSON (one object per line; numbers/booleans bare, other values quoted, -nulls as `null`). CSV output follows RFC 4180. Timestamps are raw epoch milliseconds. +nulls as `null`; non-finite floats — NaN/Inf — become `null`). CSV output follows RFC 4180. +Timestamps are raw epoch milliseconds. The `table` format buffers all rows in memory to +align columns, so prefer `csv`/`tsv`/`json` when dumping large files. ```bash BIN=cpp/build/Debug/bin/tsfile-cli diff --git a/cpp/tools/cli/cli_args.cc b/cpp/tools/cli/cli_args.cc index 846419f34c..c34145da95 100644 --- a/cpp/tools/cli/cli_args.cc +++ b/cpp/tools/cli/cli_args.cc @@ -19,6 +19,7 @@ #include "cli/cli_args.h" +#include #include #include @@ -42,8 +43,9 @@ bool parse_ll(const std::string& s, long long& out) { return false; } char* endp = nullptr; + errno = 0; long long v = std::strtoll(s.c_str(), &endp, 10); - if (endp == nullptr || *endp != '\0') { + if (endp == nullptr || *endp != '\0' || errno == ERANGE) { return false; } out = v; diff --git a/cpp/tools/cli/run_cli.cc b/cpp/tools/cli/run_cli.cc index d6103c82de..d7ca285a1a 100644 --- a/cpp/tools/cli/run_cli.cc +++ b/cpp/tools/cli/run_cli.cc @@ -141,9 +141,34 @@ bool validate_write_flags(const ParsedArgs& p, std::ostream& err) { err << "Error: --header-match cannot be combined with --no-header\n"; return false; } - if (!p.measurements.empty() || !p.device.empty() || p.has_start || - p.has_end || p.has_seed || p.limit != -1 || p.offset != 0) { - err << "Error: read-only flags are not valid for write\n"; + // Name the offending flag so the user does not have to guess which of + // the read-only options triggered the rejection. + if (!p.measurements.empty()) { + err << "Error: -m/--measurements is not valid for write\n"; + return false; + } + if (!p.device.empty()) { + err << "Error: -d/--device is not valid for write\n"; + return false; + } + if (p.has_start || p.has_end) { + err << "Error: --start/--end are not valid for write\n"; + return false; + } + if (p.has_seed) { + err << "Error: --seed is not valid for write\n"; + return false; + } + if (p.limit != -1) { + err << "Error: -n/--limit is not valid for write\n"; + return false; + } + if (p.offset != 0) { + err << "Error: --offset is not valid for write\n"; + return false; + } + if (!p.model.empty()) { + err << "Error: --model is not valid for write\n"; return false; } return true; @@ -178,6 +203,10 @@ bool validate_read_flag_applicability(const ParsedArgs& p, std::ostream& err) { err << "Error: -n/--limit is only valid for head/cat/sample\n"; return false; } + if (!is_row && p.offset != 0) { + err << "Error: --offset is only valid for head/cat\n"; + return false; + } if (!is_row && (p.has_start || p.has_end)) { err << "Error: --start/--end are only valid for head/cat/sample\n"; return false; @@ -253,10 +282,30 @@ int run_cli(const std::vector& args, std::ostream& out, storage::TsFileReader reader; int open_ret = reader.open(p.file); if (open_ret != 0) { - err << "Error: cannot open or corrupted file: " << p.file << "\n"; + err << "Error: cannot open " << p.file << ": " + << error_code_message(open_ret) << " (code " << open_ret << ")\n"; return kExitFile; } + // head/cat/sample/schema dispatch on the data model and would silently + // ignore the scope flag of the other model; reject that instead. + if (p.command == "head" || p.command == "cat" || p.command == "sample" || + p.command == "schema") { + const bool table_model = is_table_model(p, reader); + if (table_model && !p.device.empty()) { + err << "Error: -d/--device does not apply to the table model; " + "use -t/--table (or force --model tree)\n"; + reader.close(); + return kExitUsage; + } + if (!table_model && !p.table.empty()) { + err << "Error: -t/--table does not apply to the tree model; " + "use -d/--device (or force --model table)\n"; + reader.close(); + return kExitUsage; + } + } + bool stdout_tty = TSFILE_ISATTY(TSFILE_FILENO(stdout)) != 0; OutputFormat fmt = resolve_format(p.format, stdout_tty); diff --git a/cpp/tools/commands/cmd_write.cc b/cpp/tools/commands/cmd_write.cc index ecd703cbea..15cee62566 100644 --- a/cpp/tools/commands/cmd_write.cc +++ b/cpp/tools/commands/cmd_write.cc @@ -281,8 +281,10 @@ int cmd_write(const ParsedArgs& args, std::ostream& /*out*/, #ifdef _WIN32 flags |= O_BINARY; #endif - if (file.create(args.output, flags, 0666) != 0) { - err << "Error: cannot create output: " << args.output << "\n"; + int cret = file.create(args.output, flags, 0666); + if (cret != 0) { + err << "Error: cannot create output " << args.output << ": " + << error_code_message(cret) << " (code " << cret << ")\n"; return kExitFile; } auto* schema = new storage::TableSchema(args.table, col_schemas); diff --git a/cpp/tools/format/output_format.cc b/cpp/tools/format/output_format.cc index 67d0566bf7..e9ae8cb0bd 100644 --- a/cpp/tools/format/output_format.cc +++ b/cpp/tools/format/output_format.cc @@ -222,6 +222,19 @@ std::string json_escape(const std::string& s) { return out; } +namespace { + +// FLOAT/DOUBLE cells render non-finite values as nan/inf tokens, which have +// no JSON representation; finite numbers never contain these letters. +bool json_nonfinite(common::TSDataType t, const std::string& cell) { + if (t != common::FLOAT && t != common::DOUBLE) { + return false; + } + return cell.find_first_of("nNiI") != std::string::npos; +} + +} // namespace + RowWriter::RowWriter(std::ostream& out, OutputFormat fmt, std::vector header, std::vector types, bool no_header) @@ -287,7 +300,11 @@ void RowWriter::write(const std::vector& cells, if (i < is_null.size() && is_null[i]) { out_ << "null"; } else if (emits_json_bare(i)) { - out_ << (i < cells.size() ? cells[i] : "null"); + if (i >= cells.size() || json_nonfinite(types_[i], cells[i])) { + out_ << "null"; // NaN/Inf: match JSON serializer practice + } else { + out_ << cells[i]; + } } else { out_ << "\"" << json_escape(i < cells.size() ? cells[i] : "") << "\""; diff --git a/cpp/tools/format/result_set_format.cc b/cpp/tools/format/result_set_format.cc index 08cc3e1fc8..e73642889a 100644 --- a/cpp/tools/format/result_set_format.cc +++ b/cpp/tools/format/result_set_format.cc @@ -19,6 +19,7 @@ #include "format/result_set_format.h" +#include #include #include #include @@ -153,7 +154,10 @@ int emit_result_set_sampled(storage::ResultSet* rs, OutputFormat fmt, } std::vector reservoir; - reservoir.reserve(static_cast(limit)); + // Cap the pre-allocation: limit is user input and a huge -n would make + // reserve() throw std::length_error before any row is read. The vector + // still grows up to `limit` as rows actually arrive. + reservoir.reserve(static_cast(std::min(limit, 4096))); std::mt19937_64 rng(seed); bool has_next = false; int code = common::E_OK; diff --git a/cpp/tools/skills/tsfile-cli/SKILL.md b/cpp/tools/skills/tsfile-cli/SKILL.md index da6d8379ab..4405050cd6 100644 --- a/cpp/tools/skills/tsfile-cli/SKILL.md +++ b/cpp/tools/skills/tsfile-cli/SKILL.md @@ -47,20 +47,21 @@ Single pipe-friendly C++ binary to inspect **and** import `.tsfile` (TsFile's an | `stats` | per-series `count,start,end,min,max,first,last,sum` | no | | `count` | per-series counts + `total` row | no | | `head` | first N rows (default 10, `-n`) | yes | -| `cat` | all matching rows (streamed) | yes | +| `cat` | all matching rows (streamed; `table` format buffers) | yes | | `sample` | reservoir sample (default 10, `-n` + `--seed`) | yes | Prefer no-scan verbs (`ls/schema/meta/stats/count`) — cheap and never hit the page-decode caveat. -Table model + row verbs (`head/cat/sample/count`): without `-t`, only the **first** table is queried. Pass `-t ` to target a specific one. +Table model + row verbs (`head/cat/sample`): without `-t`, only the **first** table is queried. Pass `-t
` to target a specific one (`count` covers all tables). ``` opts: -f csv|tsv|json|table (default TTY→table, pipe→tsv) -d | -t
(mutually exclusive) -m a,b,c (projection) · -n N · --offset N · --start · --end (inclusive) --seed N · --no-header · --model tree|table (else auto) -applies: -m → schema/head/cat/sample · -d/-t → row cmds/schema/stats/count · --offset ∉ sample -json=NDJSON (num/bool bare, else quoted, null→null) · csv=RFC4180 · ts=raw epoch ms +applies: -m → schema/stats/count/head/cat/sample · -d/-t → row cmds/schema/stats/count + (-d needs tree model, -t needs table model in head/cat/sample/schema) · --offset ∉ sample +json=NDJSON (num/bool bare, else quoted, null→null, NaN/Inf→null) · csv=RFC4180 · ts=raw epoch ms exit: 0 ok · 1 usage · 2 file open/corrupt · 3 query/runtime ``` diff --git a/cpp/tools/tools_main.cc b/cpp/tools/tools_main.cc index 97c815f477..48924945d3 100644 --- a/cpp/tools/tools_main.cc +++ b/cpp/tools/tools_main.cc @@ -17,13 +17,21 @@ * under the License. */ +#include #include #include #include +#include "cli/exit_codes.h" #include "cli/run_cli.h" int main(int argc, char** argv) { std::vector args(argv + 1, argv + argc); - return tsfile_cli::run_cli(args, std::cout, std::cerr); + try { + return tsfile_cli::run_cli(args, std::cout, std::cerr); + } catch (const std::exception& e) { + // Last-resort net (e.g. std::bad_alloc): report instead of aborting. + std::cerr << "Error: " << e.what() << "\n"; + return tsfile_cli::kExitRuntime; + } }