diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9dcbd2a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Test fixtures and mpfiles depend on their exact line endings (most are CRLF, a few are deliberately LF), so never +# let Git convert them, whatever core.autocrlf is set to on the machine doing the checkout. +test/data/** -text +*.rpt -text +*.prn -text +*.fac -text diff --git a/.github/workflows/MainDistributionPipeline.yml b/.github/workflows/MainDistributionPipeline.yml index e547a57..7dad4ee 100644 --- a/.github/workflows/MainDistributionPipeline.yml +++ b/.github/workflows/MainDistributionPipeline.yml @@ -14,17 +14,26 @@ concurrency: jobs: duckdb-stable-build: name: Build extension binaries - uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v2.0-cyanoptera with: - duckdb_version: v1.5.2 - ci_tools_version: v1.5.1 + duckdb_version: v2.0.0 + ci_tools_version: v2.0-cyanoptera + extension_name: mpduck + + # The same source tree still supports DuckDB 1.5.x (see docs/UPDATING.md). + duckdb-1-5-build: + name: Build extension binaries (DuckDB 1.5.x) + uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.5.6 + with: + duckdb_version: v1.5.6 + ci_tools_version: v1.5.6 extension_name: mpduck code-quality-check: name: Code Quality Check - uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v1.5.1 + uses: duckdb/extension-ci-tools/.github/workflows/_extension_code_quality.yml@v2.0-cyanoptera with: - duckdb_version: v1.5.2 - ci_tools_version: v1.5.1 + duckdb_version: v2.0.0 + ci_tools_version: v2.0-cyanoptera extension_name: mpduck format_checks: 'format;tidy' diff --git a/.gitignore b/.gitignore index b9f264b..6de86fd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ duckdb_unittest_tempdir/ testext test/python/__pycache__/ .Rhistory +.cache/ diff --git a/CLAUDE.md b/CLAUDE.md index 0f1460f..41225f8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,13 +4,12 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -`mpduck` is a DuckDB extension for reading and writing **model point files (mpfiles)** produced by the FIS Prophet actuarial application. These files use `.rpt` or `.prn` extensions and are CSV-like, but use a first-column indicator to distinguish between header rows, data rows, and schema definition rows — which standard CSV readers cannot handle. - -The current scalar functions (`mpduck`, `mpduck_openssl_version`) are placeholders from the extension template. The real functionality will be table functions or COPY support for reading/writing `.rpt`/`.prn` files. +`mpduck` is a DuckDB extension for reading and writing **model point files (mpfiles)** produced by the FIS Prophet actuarial application. These files use `.rpt`, `.prn` or `.fac` extensions and are CSV-like, but use a first-column indicator to distinguish between header rows (`!`), data rows (`*`), comment rows (`&`) and schema rows (`VARIABLE_TYPES`) — which standard CSV readers cannot handle. ## Build commands ```bash +git submodule update --init --recursive # first time only make # Build release make test # Run SQL logic tests (release) make test_debug # Run SQL logic tests (debug) @@ -21,11 +20,6 @@ Build outputs: - `./build/release/test/unittest` — Test runner with extension linked - `./build/release/extension/mpduck/mpduck.duckdb_extension` — Loadable extension binary -To run the extension interactively: -```bash -./build/release/duckdb -``` - To run a single test: ```bash ./build/release/test/unittest "test/sql/mpduck.test" @@ -33,17 +27,18 @@ To run a single test: ## Architecture -The extension follows the standard DuckDB extension pattern: +The extension is built against DuckDB's internal C++ API (not the stable C API). One source tree builds against both DuckDB 1.5.x and 2.0: the API differences (`Identifier` vs `string` for SQL names, `ConstantExpression::FromValue`, `StarExpression::ExcludeListMutable()`, `Vector::Flatten()`) live in **`src/include/mpduck_compat.hpp`**, selected by whether `duckdb/common/enums/identifier_case_mode.hpp` exists (2.0 only; `identifier.hpp` also exists in 1.5). Route any new API-sensitive call through it rather than adding `#if`s elsewhere. -- **`src/include/mpduck_extension.hpp`** — Declares `MpduckExtension` inheriting from `Extension`, implementing `Load()`, `Name()`, and `Version()`. -- **`src/mpduck_extension.cpp`** — Registers scalar functions in `Load()` using `UnaryExecutor` and `StringVector::AddString()` for result strings. Exports the C entry point via `DUCKDB_CPP_EXTENSION_ENTRY`. +- **`src/mpduck_extension.cpp`** — `LoadInternal()` registers everything: the `MPFileSystem` sub-system, the `read_mpfile` table function (VARCHAR and LIST(VARCHAR) overloads, both via `bind_replace`), a replacement scan for `FROM 'x.rpt'`, and the COPY function. Reading works by building a parser AST that rewrites to `SELECT * EXCLUDE ("!") FROM read_csv_auto([...], dtypes=..., filename=...)`. `union_by_name` defaults to false like `read_csv`: only the first file with data is parsed for its schema and the rest are opened lazily. With `union_by_name=true` every file's schema is parsed, empty files are dropped, schemas are merged (`MergeSchemas`) and `union_by_name=true` is forwarded (`ReadMPFileMultiRef`). `ignore_errors` (default false) is forwarded to `read_csv_auto` unchanged. `skip_empty_files` (default true) drops zero-byte files from a glob or list at bind time (`IsZeroByteMPFile`, one open and size lookup per file) in both modes; it is not forwarded and does not apply to a single literal path. All options travel in `MPReadOptions`, together with the dialect (`delim`/`sep`, `quote`, `escape` defaulting to `,` `"` `"`, always forwarded so the CSV sniffer never has to search for them, which is what makes wide files and globs fast; `new_line` is forwarded only when the user sets it, because pinning it gains nothing measurable and would break LF files). All are overridable with `read_csv`'s parameter names. Only `,` is valid for `delim` because `MPFileSystem` rebuilds the `!` rows on commas. +- **`src/mpfile_filesystem.cpp` / `src/include/mpfile_filesystem.hpp`** — `MPFileSystem` claims `.rpt`/`.prn`/`.fac` paths (`IsManuallySet` gives it priority) and serves a filtered stream: a normalised in-memory header built from the `!` rows, followed by the raw `*` data block, with any footer cut off (`ScanPreDataSection`, `FindDataEnd`). A thread-local bypass flag (`ScopedMPFSBypass`) stops it from intercepting its own raw opens. Also holds `ParseMPFileSchema` (header + `VARIABLE_TYPES` → DuckDB types) and `MergeSchemas` (type widening across multiple files). +- **`src/write_mpfile.cpp`** — `COPY ... TO 'x.rpt'` / `(FORMAT mpfile)` with `KV_METADATA` and `INCLUDE_TYPES` options. Registered under the names `mpfile`, `rpt`, `prn`, `fac`. - **`extension_config.cmake`** — Registers the extension with DuckDB's build system and enables test loading. -- **`CMakeLists.txt`** — Links OpenSSL (SSL + Crypto) to both the static and loadable extension targets. OpenSSL is sourced via vcpkg (`vcpkg.json`). +- **`CMakeLists.txt`** — Builds the static and loadable extension targets. No external dependencies (`vcpkg.json` has none). ## Testing -Tests use DuckDB's SQLLogicTest format (`.test` files in `test/sql/`). Each test file uses `require mpduck` to load the extension before testing functions. +Tests use DuckDB's SQLLogicTest format (`test/sql/mpduck.test`, fixtures in `test/data/`). The test file uses `require mpduck` to load the extension. The fixtures depend on their exact line endings (most are CRLF, `lf.rpt` and `schema.fac` are LF), so `.gitattributes` marks `test/data/**` and the mpfile extensions `-text`: keep it that way so a Windows checkout with `core.autocrlf` cannot rewrite them. ## Updating DuckDB version -See `docs/UPDATING.md` — involves bumping the `duckdb` and `extension-ci-tools` submodules and updating version references in GitHub workflow files. +See `docs/UPDATING.md` — involves bumping the `duckdb` and `extension-ci-tools` submodules and updating version references in `.github/workflows/MainDistributionPipeline.yml`. diff --git a/CMakeLists.txt b/CMakeLists.txt index 18a7c62..0e3212a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,10 +9,8 @@ set(LOADABLE_EXTENSION_NAME ${TARGET_NAME}_loadable_extension) project(${TARGET_NAME}) include_directories(src/include) -set(EXTENSION_SOURCES - src/mpduck_extension.cpp - src/mpfile_filesystem.cpp - src/write_mpfile.cpp) +set(EXTENSION_SOURCES src/mpduck_extension.cpp src/mpfile_filesystem.cpp + src/write_mpfile.cpp) build_static_extension(${TARGET_NAME} ${EXTENSION_SOURCES}) build_loadable_extension(${TARGET_NAME} " " ${EXTENSION_SOURCES}) diff --git a/README.md b/README.md index 34958fb..1a12fe2 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,12 @@ INSTALL mpduck FROM community; LOAD mpduck; ``` +## Versions + +mpduck follows [semantic versioning](https://semver.org). The mpduck version is independent of the DuckDB version: +the same source builds against DuckDB 2.0 and DuckDB 1.5.x (CI builds v2.0.0 and v1.5.6), and an extension binary +has to match the DuckDB version it is loaded into. + ## Reading files Reference a `.rpt`, `.prn`, or `.fac` file directly in a `FROM` clause: @@ -31,6 +37,121 @@ SELECT * FROM read_mpfile(['mp_2024.rpt', 'mp_2025.rpt']); SELECT * FROM read_mpfile(['region_a*.rpt', 'region_b*.rpt']); ``` +### union_by_name + +> **Breaking change in mpduck 1.0.** Versions before 1.0 always combined multiple files as if +> `union_by_name = true`. From 1.0, `read_mpfile` takes a `union_by_name` parameter that **defaults to `false`**, +> matching `read_csv`, so reads that relied on the old behaviour now need `union_by_name = true`. +> See [Upgrading to mpduck 1.0](#upgrading-to-mpduck-10). + +Like `read_csv`, `read_mpfile` takes `union_by_name`, which defaults to `false`. + +- **`false` (default):** the files are expected to have the same columns. The column types come from the + `VARIABLE_TYPES` row of the first file that has data, and the remaining files are opened lazily by the scan, so + the cost of binding does not grow with the number of files. This matters most on remote storage such as S3. + Files with different columns fail with an error. Zero-byte files are skipped (see + [skip_empty_files](#skip_empty_files)), but a non-empty file with no header row is not. +- **`true`:** every file's header and `VARIABLE_TYPES` rows are read up front. Files with no data are skipped, + columns are matched by name (missing columns become `NULL`), and types that differ between files are widened + (`SMALLINT` < `INTEGER` < `DOUBLE`, anything else becomes `VARCHAR`). This costs an extra open and read per file. + +In both modes a file that cannot be opened (missing, permission denied, network or credential errors) is an +error; it is never silently skipped. + +```sql +SELECT * FROM read_mpfile('data/*.rpt', union_by_name = true); +``` + +`FROM 'data/*.rpt'` behaves like the default (`union_by_name = false`). + +#### Upgrading to mpduck 1.0 + +When moving from a version before 1.0, add `union_by_name = true` if a multi-file read (a glob, or a list of files) does any of these: + +| Your files | Before 1.0 | 1.0, by default | To keep the old behaviour | +|---|---|---|---| +| have different columns | combined by name, missing columns `NULL` | error: `COLUMN_TYPES error: Columns with names ... do not exist in the CSV File` | `union_by_name = true` | +| have the same columns with different types (e.g. `SMALLINT` in one, `INTEGER` in another) | types widened | types taken from the first file that has data (a value that does not fit is an error) | `union_by_name = true` | +| include a zero-byte file | file skipped | file skipped (`skip_empty_files`, on by default) | nothing to change | +| include a non-empty file with no header row | file skipped | error | `union_by_name = true` | +| are read with `FROM 'data/*.rpt'` | combined by name | same as `read_mpfile` with the default | use `read_mpfile('data/*.rpt', union_by_name = true)` | + +Reads of a single file, and of several files that share the same columns and types, are unaffected. + +Two further changes in 1.0 apply in both modes: + +- An unreadable file (for example a missing path, or a permission or credentials error) used to be silently dropped + from a multi-file read, returning partial results. It is now an error. +- The CSV dialect is no longer sniffed. `read_mpfile` now reads with `delim = ','`, `quote = '"'` and + `escape = '"'`, which is faster (see [delim, quote, escape and new_line](#delim-quote-escape-and-new_line)). A file + that uses a different quote or escape character, which the sniffer used to detect, needs `quote` and `escape` passed + explicitly. Line endings are still detected. + +### ignore_errors + +Like `read_csv`, `read_mpfile` takes `ignore_errors`, which defaults to `false`. When +it is `true`, rows that fail to parse (for example a non-numeric value in a `S`, `I` or `N` column) are skipped +instead of raising an error. + +```sql +SELECT * FROM read_mpfile('model_points.rpt', ignore_errors = true); +``` + +It behaves exactly as it does in `read_csv`: it skips rows, not files, so a file that cannot be opened is still an +error, and only the columns a query actually reads are converted, so a bad value in a column the query does not +use does not drop the row. + +### delim, quote, escape and new_line + +`read_mpfile` passes the CSV dialect of an mpfile to `read_csv` instead of leaving it to be sniffed. The defaults are +the standard CSV conventions (RFC 4180), which is also what `COPY ... TO` writes. Each can be overridden with the +parameter of the same name that `read_csv` takes: + +| Parameter | Default | Notes | +|---|---|---| +| `delim` (alias `sep`) | `','` | Only `','` is accepted, see below. Give `delim` or `sep`, not both. | +| `quote` | `'"'` | Quoting is optional: unquoted fields read as they are. `quote = ''` turns quote handling off. | +| `escape` | `'"'` | A quote inside a quoted field is written doubled. | +| `new_line` | not set | Detected from the file, so CRLF and LF files both read without options. | + +```sql +-- Single-quoted fields, as in 'O''Brien' +SELECT * FROM read_mpfile('model_points.rpt', quote = '''', escape = ''''); + +-- Force a line ending (the text \r\n, \n or \r, as in read_csv, not the control characters) +SELECT * FROM read_mpfile('lf_endings.rpt', new_line = '\n'); +``` + +A `new_line` that does not match the file is an error. A NULL value leaves the default in place. The settings apply to +every file in a glob or list, and `FROM 'file.rpt'` always uses the defaults, since it takes no parameters. + +`delim` and `sep` are accepted for parity with `read_csv`, but only `','` is valid: the header (`!`) rows are split and +rebuilt on commas, so any other delimiter is rejected with an error. + +Fixing the dialect makes reads faster, most of all for very wide files and for globs of many files, because the CSV +sniffer no longer has to search for it. Measurements are in [`bench/RESULTS.md`](bench/RESULTS.md). + +### skip_empty_files + +`read_mpfile` takes `skip_empty_files`, which defaults to `true`. When it is `true`, +zero-byte files in a glob or a list of files are dropped before anything is read, so one empty file does not fail +the whole read. This applies with `union_by_name` either `false` or `true`, and to `FROM 'data/*.rpt'`. + +```sql +-- Fail on empty files instead of skipping them +SELECT * FROM read_mpfile('data/*.rpt', skip_empty_files = false); +``` + +- It costs one open and size lookup per file at bind time, with no reading. On remote storage that is one extra + request per file, so set `skip_empty_files = false` if you read a very large number of files you know are not empty. +- It only covers files of exactly zero bytes. A non-empty file with no header row is still an error with the default + `union_by_name = false`, and is skipped with `union_by_name = true`. +- A single literal path, `read_mpfile('file.rpt')`, is never filtered. +- With `skip_empty_files = false` and `union_by_name = false`, an empty file fails the read with + `Columns with names ... do not exist in the CSV File`. With `union_by_name = true` it is still skipped, because + every file's header is checked. +- If every matched file is empty, the read fails with a `no files ... matched` error rather than returning no rows. + ## Writing files ```sql diff --git a/bench/RESULTS.md b/bench/RESULTS.md new file mode 100644 index 0000000..b2c4bf4 --- /dev/null +++ b/bench/RESULTS.md @@ -0,0 +1,170 @@ +# read_mpfile vs read_csv: benchmark results + +Speed and memory of `read_mpfile` against a `read_csv` that skips the metadata rows, across mpfile shapes, query +kinds and thread counts. Produced by `bench/perf_matrix.py`; this page records one run and what it showed. + +**Summary.** With the current defaults (`delim`, `quote` and `escape` pinned, `new_line` left to auto-detection), +`read_mpfile` is level with `read_csv` on ordinary files, 20-30% faster on very wide ones, and 10-20x faster on a glob +of many small files. On a glob of 100 files of about 100 MB each, `read_csv` is killed by the OOM killer on every +query while `read_mpfile` completes in about 2 GB. + +## Setup + +- **Machine:** Linux 6.18, 4 vCPU, about 16 GB RAM, virtual disk. One machine, no isolation from other load. +- **Build:** release build of DuckDB (submodule `d54cbbe3`, the 2.0 line) with the extension loaded explicitly + (`LOAD '.../mpduck.duckdb_extension'`; the shell does not preload it). Results below are commit `564a51a`. +- **Each measurement is a fresh DuckDB process**, so peak RSS belongs to one query. Wall time and CPU time are the + best of N runs, RSS is the median (`ru_maxrss` from `wait4`). N is 3, and 2 for the 10 GB sets. Differences under + about 5-10% are within run-to-run noise on this machine. +- **Queries:** `count(*)`, `agg` (count plus sums over up to 40 numeric columns), `proj1` (one column), and + `materialize` (`CREATE TEMP TABLE AS SELECT *`, skipped for the 10 GB sets because 180M rows would not fit in RAM). + All at 1 and 4 threads. +- **Baseline:** `read_csv` reading the same bytes through a symlink with a `.txt` extension (so `MPFileSystem` does + not intercept it), `header=true`, with `skip=3` (two KV metadata rows plus `VARIABLE_TYPES`) or `skip=2` when the + file has no `VARIABLE_TYPES` row. For files with types there are two baselines: `csv_auto` (types sniffed) and + `csv_typed` (explicit `dtypes` equal to what `read_mpfile` derives). The tables use `csv_typed`, or `csv_auto` for + files without types; the two were within noise of each other everywhere except where noted. +- **Every dataset exists twice**: once with a `VARIABLE_TYPES` row ("typed"), and once written with + `INCLUDE_TYPES false` ("no types", `_notypes` in the harness). Most real mpfiles carry no types. Without them + `read_mpfile` passes no `dtypes` and `read_csv_auto` sniffs the types. + +| Dataset | Shape | +|---|---| +| `narrow_100k`, `narrow_1m`, `narrow_5m` | 8 columns (2 text, SMALLINT, INTEGER, 4 DOUBLE); 5 MB, 54 MB, 270 MB | +| `narrow_footer_1m` | as `narrow_1m` plus a footer (no `read_csv` baseline: it cannot cope with the footer) | +| `wide_300c_100k` | 300 columns (100 each of DOUBLE, INTEGER, text), 100k rows, 271 MB | +| `strings_1m` | 4 columns, long text containing commas and quotes, 153 MB | +| `sparse_1m` | 8 columns, 90% empty, 18 MB | +| `files_500x2k` | glob of 500 files of 2,000 rows each, 54 MB total | +| `files_100x100mb` | glob of 100 files of 1.8M rows each, 180M rows, 9.75 GB | + +## Results: current defaults + +Wall seconds, with peak RSS in MB in brackets. `agg` query. mpfile / baseline. + +### Files with a `VARIABLE_TYPES` row (baseline `csv_typed`) + +| Dataset | 1 thread | 4 threads | +|---|---|---| +| narrow_100k | 0.10 (48) / 0.10 (48) | 0.09 (49) / 0.10 (48) | +| narrow_1m | 0.38 (97) / 0.41 (97) | 0.18 (99) / 0.17 (98) | +| narrow_5m | 1.61 (166) / 1.75 (165) | 0.54 (258) / 0.55 (285) | +| narrow_footer_1m | 0.39 (97) / n/a | 0.17 (99) / n/a | +| wide_300c_100k | **1.07 (207) / 1.39 (207)** | **0.69 (272) / 0.95 (301)** | +| strings_1m | 0.32 (165) / 0.31 (164) | 0.19 (197) / 0.19 (197) | +| sparse_1m | 0.24 (61) / 0.25 (61) | 0.14 (62) / 0.15 (62) | +| files_500x2k | **0.89 (67) / 8.92 (397)** | **0.60 (76) / 8.65 (463)** | + +### Files without types (baseline `csv_auto`) + +| Dataset | 1 thread | 4 threads | +|---|---|---| +| narrow_100k | 0.09 (49) / 0.10 (48) | 0.09 (49) / 0.10 (48) | +| narrow_1m | 0.40 (98) / 0.40 (97) | 0.18 (99) / 0.19 (98) | +| narrow_5m | 1.59 (165) / 1.57 (165) | 0.51 (286) / 0.52 (284) | +| narrow_footer_1m | 0.38 (97) / n/a | 0.17 (99) / n/a | +| wide_300c_100k | **1.12 (206) / 1.42 (209)** | **0.67 (274) / 0.88 (272)** | +| strings_1m | 0.35 (165) / 0.30 (164) | 0.18 (197) / 0.20 (196) | +| sparse_1m | 0.23 (61) / 0.24 (61) | 0.13 (62) / 0.14 (62) | +| files_500x2k | **0.76 (67) / 9.01 (399)** | **0.39 (77) / 8.93 (461)** | + +### Materialising into a table (`materialize`, typed, 1 thread) + +| Dataset | mpfile | csv_typed | +|---|---|---| +| narrow_1m | 0.49 (141) | 0.53 (139) | +| narrow_5m | 2.28 (484) | 2.57 (549) | +| wide_300c_100k | 2.20 (373) | 2.46 (374) | +| strings_1m | 0.72 (335) | 0.86 (335) | +| sparse_1m | 0.30 (111) | 0.33 (116) | +| files_500x2k | 0.96 (127) | 8.81 (466) | + +`read_csv` also materialises the `!` indicator column, which `read_mpfile` drops. + +### 100 files of about 100 MB (9.75 GB, 180M rows) + +Each set was read into the page cache before its run. `read_mpfile` wall seconds (RSS in MB). `read_csv` was +OOM-killed on **all 12 query/thread/typing combinations**, after 18-25 s at a peak RSS of about 13.6 GB. + +| Query | Typed, 1 thread | Typed, 4 threads | No types, 1 thread | No types, 4 threads | +|---|---|---|---|---| +| count | 33.6 (1588) | 9.1 (2015) | 34.4 (1738) | 9.3 (1999) | +| agg | 59.6 (1374) | 15.5 (2094) | 58.6 (1476) | 15.2 (2104) | +| proj1 | 37.1 (1601) | 10.0 (1955) | 36.6 (1600) | 9.7 (1982) | + +## Before and after pinning the dialect + +The first run used the extension as originally written, where `read_mpfile` left dialect detection to the CSV +sniffer. These are `read_mpfile` only, `count(*)`. + +| Scenario | Before | After | +|---|---|---| +| wide, typed, 1 / 4 threads | 1.94 / 1.63 s | 0.97 / 0.74 s | +| wide, no types, 1 / 4 threads | 1.81 / 1.41 s | 1.01 / 0.64 s | +| 500 small files, typed, 1 / 4 threads | 3.51 / 3.28 s | 0.77 / 0.61 s | +| 500 small files, no types, 1 / 4 threads | 0.69 / 0.42 s | 0.49 / 0.33 s | +| narrow 100k, 1 thread | 0.13 s | 0.08 s | +| 100 x 100 MB, typed, 1 / 4 threads | 66.9 / 15.0 s | 33.6 / 9.1 s | +| 100 x 100 MB, no types, 1 / 4 threads | 37.1 / 13.9 s | 34.4 / 9.3 s | + +Before the change, wide files were 30-60% slower than `read_csv` and the typed glob was 3x faster. The 10 GB +"before" runs were single runs that were not pre-read into the page cache, so part of that improvement is cache +warmth and not the change. The other rows are not affected by that. + +## What the investigation found + +**Wide-file slowness (fixed).** `callgrind` on the bind step showed 14.6 G instructions for `read_mpfile` against +7.2 G for `read_csv`; the extra 7.3 G was all in `CSVSniffer::DetectDialect -> RefineCandidates`. The sniffer +enumerates delimiter x quote x escape candidates, and when they tie it re-scans later chunks for each one, at a +cost proportional to row width. Pinning any one of the three options helps on its own (wide bind: 1.20 s unpinned, +1.00 s `delim`, 0.86 s `quote`, 0.65 s `escape`); all three together give 0.41 s. A plain file holding exactly the +stream `MPFileSystem` produces is just as slow under default `read_csv`, so the wrapper itself is not the cause. +I/O is identical (about 64 MB read at bind in both). + +**Passing `dtypes` makes globs slower (fixed by the same change).** `dtypes` replaces detected types but leaves +sniffing on. Later files in a glob normally take a cheap column-count sniff (`AdaptiveSniff`/`MinimalSniff`) and fall +back to the full sniffer if the schema does not match. On plain-CSV copies of the 500 files, adding `dtypes` took the +read from 0.71 s to 3.0 s, and pinning the dialect brought it back to 0.8 s. Why the fallback is taken when types are +set was not traced in DuckDB's source; this is inferred from the measurements. + +**`read_csv`'s `skip` is what blows up multi-file reads.** On 20 of the large files, `read_csv(skip=2, +header=true)` was OOM-killed at 13.6 GB, with or without the dialect pinned. On clean copies of the same files +(metadata rows removed) `read_csv` used 759 MB, matching `read_mpfile` at 755 MB in about the same time. So the glob +advantage of `read_mpfile` comes from `MPFileSystem` removing the metadata rows, which means `read_csv` never needs +`skip`. Anyone reading mpfiles with plain `read_csv` has to use `skip`, so the comparison reflects real use, but +the baseline is penalised by an option that `read_mpfile` avoids. The reason `skip` costs so much was not traced. + +**`new_line` is not worth pinning.** With the other three options pinned, adding `new_line='\r\n'` changed bind and +`count` times by -8% to +9%, in both directions, across six dataset/typing combinations (7 runs each). It gives no +measurable gain and would make pure-LF files fail, so it is only forwarded when the user sets it. + +**Quotes are not required.** Pinning `quote='"'` only names the character that opens a quoted field. Fully +unquoted files, files with quoted strings and unquoted numbers, and unquoted fields with a stray `"` mid-value all +read correctly with the defaults. + +## Caveats + +- One machine; noise is about 5-10%, and a handful of narrow 1M-row cells favour `read_csv` by up to 15% + (for example 0.30 s against 0.26 s for `count`, typed, 1 thread) while others favour `read_mpfile`. I treat these + as noise but cannot rule out a few percent of real cost. +- The data is machine-generated by `COPY ... (FORMAT mpfile)`. No real Prophet output was available; files that + use another quote or escape convention would need `quote`/`escape` overrides. +- CPU time exceeds wall time on globs even at `threads=1` (for example 2.2 s CPU in 0.77 s wall for the typed + 500-file glob, after the fix; before it was about 3.5x). Some work runs outside the `threads` limit. This was + not explained. +- The 10 GB results are best-of-2 and depend on the page cache: a cold read of the same glob took 444 s. +- With no types, `read_mpfile` and `read_csv` sniff types independently; the sniffed types were not compared. + +## Reproducing + +```bash +make # builds build/release/duckdb and the extension +python3 bench/perf_matrix.py --runs 3 --threads "1 4" # whole matrix; --only name,name picks datasets +python3 bench/perf_matrix.py --only wide_300c_100k,wide_300c_100k_notypes --json out.json +``` + +Datasets are generated on first use under `/tmp/mpduck-perf` (`--dir` to change): about 21 GB for the two 10 GB +sets plus about 2 GB for the rest. Each dataset has a `_notypes` twin, e.g. `--only narrow_1m_notypes`. The 10 GB +sets need `--only files_100x100mb` (and the `_notypes` twin); read them into the page cache first +(`cat dir/*.rpt > /dev/null`) for stable timings. `bench/memory_bench.sh` is an earlier, single-shape version of +the same comparison. diff --git a/bench/glob_bench.sh b/bench/glob_bench.sh new file mode 100755 index 0000000..9ea2d97 --- /dev/null +++ b/bench/glob_bench.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +# Compare peak memory of read_mpfile('dir/*.rpt') against equivalent read_csv globs as the file count grows. +# +# Two layouts: by default total data is held constant (TOTAL_ROWS) and split across N files. Set PER_FILE_ROWS +# instead to give every file the same size (total grows with N), which shows whether memory is retained per file. +# Baselines read footerless copies (.txt, so MPFileSystem does not intercept) with skip=3 to drop the +# KV_METADATA/VARIABLE_TYPES rows; the '!' row is the header. All files share one schema, so read_csv +# both with and without union_by_name is valid (read_mpfile always uses union_by_name). +# +# Usage: bench/glob_bench.sh [duckdb-binary] +# Environment: TOTAL_ROWS (default 1000000), COUNTS (default "2 5 10 20"), THREADS (default "1 2"), +# PER_FILE_ROWS (unset), RUNS (default 3), BENCH_DIR (default /tmp/mpduck-bench), +# MEM_LIMIT (default 2GB), INCLUDE_PLAIN=1 to also run read_csv without union_by_name. +# WARNING: the plain (no union_by_name) read_csv glob with skip= was observed using 4-7 GB of RAM and 13-26 s +# on just 55 MB of data, so it is excluded unless INCLUDE_PLAIN=1. Do not enable it with large files. +# Deliberately gentle: everything runs one process at a time under nice/ionice with a capped thread count +# and memory_limit, so it is safe to run on a shared or interactive machine. +set -euo pipefail + +DUCKDB="${1:-./build/release/duckdb}" +TOTAL_ROWS="${TOTAL_ROWS:-1000000}" +COUNTS="${COUNTS:-2 5 10 20}" +THREADS="${THREADS:-1 2}" +MEM_LIMIT="${MEM_LIMIT:-2GB}" +LOW="nice -n 19 ionice -c3" +RUNS="${RUNS:-3}" +BENCH_DIR="${BENCH_DIR:-/tmp/mpduck-bench}" +SKIP=3 + +DT="dtypes={'!': 'VARCHAR', POLICY: 'VARCHAR', PRODUCT: 'VARCHAR', AGE: 'SMALLINT', TERM: 'INTEGER', SA: 'DOUBLE', PREM: 'DOUBLE', RATE: 'DOUBLE', DUR: 'DOUBLE'}" +AGG="count(*) AS n, max(length(POLICY)) AS l, sum(AGE) AS a, sum(TERM) AS t, sum(SA) AS sa, sum(PREM) AS p, sum(RATE) AS r, sum(DUR) AS d" + +gen_files() { # n total_rows + local n=$1 total=$2 dir="$BENCH_DIR/glob_${2}_x$1" + if [[ -f "$dir/.done" ]]; then return; fi + rm -rf "$dir" + mkdir -p "$dir" + local per=$((total / n)) + for ((k = 0; k < n; k++)); do + local lo=$((k * per)) hi=$(((k + 1) * per)) + local f + f=$(printf '%s/part_%03d' "$dir" "$k") + $LOW "$DUCKDB" -c " +SET threads=2; SET memory_limit='$MEM_LIMIT'; +COPY ( + SELECT 'P' || lpad(i::VARCHAR, 9, '0') AS POLICY, 'PROD' || (i % 50)::VARCHAR AS PRODUCT, + (18 + i % 60)::SMALLINT AS AGE, (5 + i % 30)::INTEGER AS TERM, (i % 1000) * 1000.5 AS SA, + (i % 777) * 12.25 AS PREM, (i % 97) / 100.0 AS RATE, (i % 41) * 0.5 AS DUR + FROM range($lo, $hi) t(i) +) TO '$f.rpt' (FORMAT mpfile, KV_METADATA {run: 'bench', version: '1'});" + cp "$f.rpt" "$f.txt" # footerless CSV copy + printf 'This is a footer line\r\nMore footer, with, commas\r\n' >>"$f.rpt" # real-looking mpfile + done + touch "$dir/.done" +} + +measure() { # threads sql -> "min/med/max secs result" + local th=$1 sql=$2 rss=() best_t=999999 result="" + for _ in $(seq "$RUNS"); do + local out tm + out=$(mktemp) + tm=$(mktemp) + $LOW /usr/bin/time -f '%M %e' -o "$tm" "$DUCKDB" -csv -noheader -c "SET threads=$th; SET memory_limit='$MEM_LIMIT'; $sql;" >"$out" 2>"$out.err" || { + echo "FAILED: $sql"; cat "$out.err"; exit 1 + } + read -r kb secs <"$tm" + result=$(tail -1 "$out" | cut -c1-34) + rss+=("$((kb / 1024))") + awk -v a="$secs" -v b="$best_t" 'BEGIN{exit !(a>"$MPF" +fi +echo "data: $(du -h "$SRC" | cut -f1) head: $(head -c 200 "$SRC" | tr '\r\n' ' ' | cut -c1-120)" + +COLS="POLICY VARCHAR, PRODUCT VARCHAR, AGE SMALLINT, TERM INTEGER, SA DOUBLE, PREM DOUBLE, RATE DOUBLE, DUR DOUBLE" +AGG="count(*) AS n, max(length(POLICY)) AS l, sum(AGE) AS a, sum(TERM) AS t, sum(SA) AS sa, sum(PREM) AS p, sum(RATE) AS r, sum(DUR) AS d" + +# name | query. All aggregate so the measurement is scanner memory, not result materialisation. +declare -A Q +Q[mpfile_agg]="SELECT $AGG FROM read_mpfile('$MPF')" +Q[csv_skip_dtypes_agg]="SELECT $AGG FROM (SELECT * EXCLUDE (\"!\") FROM read_csv('$CSV', skip=$SKIP, header=true, dtypes={'!': 'VARCHAR', POLICY: 'VARCHAR', PRODUCT: 'VARCHAR', AGE: 'SMALLINT', TERM: 'INTEGER', SA: 'DOUBLE', PREM: 'DOUBLE', RATE: 'DOUBLE', DUR: 'DOUBLE'}))" +Q[csv_explicit_agg]="SELECT $AGG FROM read_csv('$CSV', skip=$SKIP, header=true, auto_detect=false, delim=',', quote='\"', escape='\"', new_line='\\r\\n', columns={'!': 'VARCHAR', $(echo "$COLS" | sed -E "s/([A-Z]+) ([A-Z]+)/'\1': '\2'/g")})" +Q[mpfile_count]="SELECT count(*) FROM read_mpfile('$MPF')" +Q[csv_skip_dtypes_count]="SELECT count(*) FROM read_csv('$CSV', skip=$SKIP, header=true, dtypes={'!': 'VARCHAR', POLICY: 'VARCHAR', PRODUCT: 'VARCHAR', AGE: 'SMALLINT', TERM: 'INTEGER', SA: 'DOUBLE', PREM: 'DOUBLE', RATE: 'DOUBLE', DUR: 'DOUBLE'})" + +ORDER=(mpfile_agg csv_skip_dtypes_agg csv_explicit_agg mpfile_count csv_skip_dtypes_count) + +# Idle baseline: the shell with the extension loaded, running a trivial query. +base_kb=$( { /usr/bin/time -f '%M' "$DUCKDB" -csv -noheader -c "SELECT 1" >/dev/null; } 2>&1 | tail -1) +echo "idle shell peak RSS: $((base_kb / 1024)) MB (subtract for scanner-only memory)" + +printf '%-24s %7s %22s %8s %s\n' query threads "rss_MB min/med/max" wall_s result +for th in $THREADS; do + for name in "${ORDER[@]}"; do + rss=() + best_t=999999 + result="" + for _ in $(seq "$RUNS"); do + out=$(mktemp) + tm=$(mktemp) + $LOW /usr/bin/time -f '%M %e' -o "$tm" "$DUCKDB" -csv -noheader -c "SET threads=$th; SET memory_limit='$MEM_LIMIT'; ${Q[$name]};" >"$out" 2>"$out.err" || { + echo "FAILED: $name"; cat "$out.err"; exit 1 + } + read -r kb secs <"$tm" + result=$(tail -1 "$out" | cut -c1-40) + rss+=("$((kb / 1024))") + awk -v a="$secs" -v b="$best_t" 'BEGIN{exit !(a (rows, select-expression list [(name, expr, duckdb_type)], files) ---- +NARROW = [("POLICY", "'P' || lpad(i::VARCHAR, 9, '0')", "VARCHAR"), + ("PRODUCT", "'PROD' || (i % 50)::VARCHAR", "VARCHAR"), + ("AGE", "(18 + i % 60)::SMALLINT", "SMALLINT"), + ("TERM", "(5 + i % 30)::INTEGER", "INTEGER"), + ("SA", "(i % 1000) * 1000.5", "DOUBLE"), + ("PREM", "(i % 777) * 12.25", "DOUBLE"), + ("RATE", "(i % 97) / 100.0", "DOUBLE"), + ("DUR", "(i % 41) * 0.5", "DOUBLE")] +WIDE = [] +for k in range(300): + if k % 3 == 0: + WIDE.append((f"D{k}", f"((i * {k + 7}) % 100003) / 7.0", "DOUBLE")) + elif k % 3 == 1: + WIDE.append((f"I{k}", f"((i * {k + 3}) % 9973)::INTEGER", "INTEGER")) + else: + WIDE.append((f"S{k}", f"'v' || ((i + {k}) % 500)::VARCHAR", "VARCHAR")) +STRINGS = [("ID", "i::BIGINT", "BIGINT"), + ("NOTE", "'note, with comma and \"quote\" ' || md5(i::VARCHAR) || md5((i+1)::VARCHAR)", "VARCHAR"), + ("DESC", "repeat('x', 20 + i % 40) || i::VARCHAR", "VARCHAR"), + ("CAT", "'C' || (i % 5)::VARCHAR", "VARCHAR")] +SPARSE = [(n, f"CASE WHEN i % 10 = 0 THEN {e} END", t) for n, e, t in NARROW] # 90% NULL (empty fields) + +# name: (rows, cols, nfiles, footer) +DATASETS = { + "narrow_100k": (100_000, NARROW, 1, False), + "narrow_1m": (1_000_000, NARROW, 1, False), + "narrow_5m": (5_000_000, NARROW, 1, False), + "narrow_footer_1m": (1_000_000, NARROW, 1, True), + "wide_300c_100k": (100_000, WIDE, 1, False), + "strings_1m": (1_000_000, STRINGS, 1, False), + "sparse_1m": (1_000_000, SPARSE, 1, False), + "files_500x2k": (2_000, NARROW, 500, False), + # ~100 MB per file (8 narrow cols, ~58 bytes/row): 100 files, ~10 GB total. Check the printed size. + "files_100x100mb": (1_800_000, NARROW, 100, False), +} +# Most real mpfiles carry no VARIABLE_TYPES row: every dataset gets a "_notypes" twin written with +# INCLUDE_TYPES false, where read_mpfile cannot pass dtypes and read_csv_auto sniffs the types. +NOTYPES = set() +for _n in list(DATASETS): + DATASETS[_n + "_notypes"] = DATASETS[_n] + NOTYPES.add(_n + "_notypes") +# Queries skipped per dataset: materialising ~180M rows would exceed RAM. +SKIP_QUERIES = {"files_100x100mb": {"materialize"}, "files_100x100mb_notypes": {"materialize"}} + + +def sh(sql, threads=None, timed=False): + pre = f"LOAD '{a.ext}'; " + (f"SET threads={threads}; " if threads else "") + cmd = [a.duckdb, "-unsigned", "-csv", "-noheader", "-c", pre + sql] + if not timed: + r = subprocess.run(cmd, capture_output=True, text=True) + if r.returncode: + sys.exit(f"duckdb failed: {r.stderr[:500]}\n{sql[:300]}") + return r.stdout + import time + with tempfile.TemporaryFile() as so, tempfile.TemporaryFile() as se: + t0 = time.time() + p = subprocess.Popen(cmd, stdout=so, stderr=se) + _, status, ru = os.wait4(p.pid, 0) + wall = time.time() - t0 + se.seek(0) + so.seek(0) + err, out = se.read().decode(errors="replace"), so.read().decode(errors="replace") + rc = os.waitstatus_to_exitcode(status) + if rc == -9: # SIGKILL: almost certainly the OOM killer; record it as a result rather than aborting the matrix + return int(ru.ru_maxrss) // 1024, wall, ru.ru_utime + ru.ru_stime, "OOM-KILLED" + if rc: + sys.exit(f"duckdb failed: {err[:500]}\n{sql[:300]}") + return int(ru.ru_maxrss) // 1024, wall, ru.ru_utime + ru.ru_stime, out.strip().splitlines()[-1][:60] + + +def gen(name): + rows, cols, nfiles, footer = DATASETS[name] + d = os.path.join(a.dir, name) + marker = os.path.join(d, ".done") + if not os.path.exists(marker): + os.makedirs(d, exist_ok=True) + sel = ", ".join(f"{e} AS {n}" for n, e, _ in cols) + for f in range(nfiles): + out = os.path.join(d, f"f{f:04d}.rpt") + sh(f"SET threads=4; COPY (SELECT {sel} FROM range({rows}) t(i)) TO '{out}' " + f"(FORMAT mpfile, KV_METADATA {{run: 'bench', version: '1'}}" + f"{', INCLUDE_TYPES false' if name in NOTYPES else ''});") + os.symlink(out, os.path.join(d, f"f{f:04d}.txt")) + if footer: + with open(out, "ab") as fh: + fh.write(b"This is a footer line\r\nMore footer, with, commas\r\n") + open(marker, "w").close() + return d + + +def queries(name): + rows, cols, nfiles, footer = DATASETS[name] + d = os.path.join(a.dir, name) + dt = ", ".join(f"'{n}': '{t}'" for n, _, t in cols) + dt = f"{{'!': 'VARCHAR', {dt}}}" + num = [n for n, _, t in cols if t != "VARCHAR"] + # footer files: csv baseline cannot cope with the footer, so it reads the symlink to the same (footer-bearing) bytes + # only when there is none; for the footer case fall back to ignore_errors-free comparison on mpfile only. + mp = f"read_mpfile('{d}/*.rpt')" if nfiles > 1 else f"read_mpfile('{d}/f0000.rpt')" + csvp = f"'{d}/*.txt'" if nfiles > 1 else f"'{d}/f0000.txt'" + skip = SKIP - 1 if name in NOTYPES else SKIP # no VARIABLE_TYPES row + sources = {"mpfile": mp} + if not footer: + sources["csv_auto"] = f"read_csv({csvp}, skip={skip}, header=true)" + if name not in NOTYPES: + sources["csv_typed"] = f"read_csv({csvp}, skip={skip}, header=true, dtypes={dt})" + first, last = cols[0][0], cols[-1][0] + agg_cols = ", ".join(f"sum(\"{n}\")" for n in num[:40]) + (f", max(length(\"{cols[0][0]}\"))" if cols[0][2] == "VARCHAR" else "") + qs = { + "count": "SELECT count(*) FROM {s}", + "agg": f"SELECT count(*), {agg_cols} FROM {{s}}", + "proj1": f"SELECT count(\"{last}\"), min(\"{last}\") FROM {{s}}", + "materialize": "CREATE TEMP TABLE t AS SELECT * FROM {s}; SELECT count(*) FROM t", + } + return sources, qs + + +results = [] +only = set(filter(None, a.only.split(","))) +for name in DATASETS: + if only and name not in only: + continue + d = gen(name) + size = sum(os.path.getsize(os.path.join(d, f)) for f in os.listdir(d) if f.endswith(".rpt")) // 2**20 + sources, qs = queries(name) + print(f"\n=== {name} ({size} MB on disk, {DATASETS[name][0] * DATASETS[name][2]} rows, {len(DATASETS[name][1])} cols) ===", flush=True) + print(f"{'query':<12}{'thr':>4} " + "".join(f"{s:>26}" for s in sources), flush=True) + print(f"{'':<12}{'':>4} " + "".join(f"{'wall_s / cpu_s / rssMB':>26}" for _ in sources), flush=True) + for q, tmpl in qs.items(): + if q in SKIP_QUERIES.get(name, ()): + continue + for th in a.threads.split(): + row = [] + for s, src in sources.items(): + runs = [sh(tmpl.format(s=src), threads=th, timed=True) for _ in range(a.runs)] + wall = min(r[1] for r in runs) + cpu = min(r[2] for r in runs) + rss = int(statistics.median(r[0] for r in runs)) + if any(r[3] == "OOM-KILLED" for r in runs): + row.append(f"OOM @ {max(r[0] for r in runs)}MB/{wall:.0f}s") + else: + row.append(f"{wall:6.2f} / {cpu:6.2f} / {rss:5d}") + results.append(dict(dataset=name, query=q, threads=int(th), source=s, wall=wall, cpu=cpu, rss_mb=rss, + result=runs[0][3], size_mb=size)) + print(f"{q:<12}{th:>4} " + "".join(f"{x:>26}" for x in row), flush=True) +if a.json: + json.dump(results, open(a.json, "w"), indent=1) diff --git a/bench/rss_profile.py b/bench/rss_profile.py new file mode 100755 index 0000000..7b3d399 --- /dev/null +++ b/bench/rss_profile.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Sample a DuckDB shell's RSS every 20 ms while it runs a query and print the memory profile. + +Usage: bench/rss_profile.py [label] +Prints peak RSS, time-to-peak, and a compact timeline (one sample per ~0.25 s). +""" +import subprocess, sys, time + +duckdb, threads, sql = sys.argv[1], sys.argv[2], sys.argv[3] +label = sys.argv[4] if len(sys.argv) > 4 else "" +p = subprocess.Popen([duckdb, "-csv", "-noheader", "-c", f"SET threads={threads}; {sql};"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +samples = [] +t0 = time.time() +while p.poll() is None: + try: + with open(f"/proc/{p.pid}/status") as f: + for line in f: + if line.startswith("VmRSS:"): + samples.append((time.time() - t0, int(line.split()[1]) // 1024)) + break + except FileNotFoundError: + break + time.sleep(0.02) +if not samples: + sys.exit("no samples") +peak_t, peak = max(((t, r) for t, r in samples), key=lambda x: x[1])[0], max(r for _, r in samples) +timeline, next_t = [], 0.0 +for t, r in samples: + if t >= next_t: + timeline.append(str(r)) + next_t += 0.25 +print(f"{label:<28} peak {peak:4d} MB @ {peak_t:5.2f}s total {samples[-1][0]:5.2f}s | " + " ".join(timeline)) diff --git a/docs/UPDATING.md b/docs/UPDATING.md index a3ac73e..ac63ec5 100644 --- a/docs/UPDATING.md +++ b/docs/UPDATING.md @@ -4,12 +4,24 @@ will inevitably come a time when a new DuckDB is released and the extension repo as follows: - Bump submodules - - `./duckdb` should be set to latest tagged release - - `./extension-ci-tools` should be set to updated branch corresponding to latest DuckDB release. So if you're building for DuckDB `v1.1.0` there will be a branch in `extension-ci-tools` named `v1.1.0` to which you should check out. -- Bump versions in `./github/workflows` - - `duckdb_version` input in `duckdb-stable-build` job in `MainDistributionPipeline.yml` should be set to latest tagged release - - `duckdb_version` input in `duckdb-stable-deploy` job in `MainDistributionPipeline.yml` should be set to latest tagged release - - the reusable workflow `duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml` for the `duckdb-stable-build` job should be set to latest tagged release + - `./duckdb` should be set to latest tagged release (currently `v2.0.0`) + - `./extension-ci-tools` tracks `main`; the workflow itself uses the `v2.0-cyanoptera` branch of extension-ci-tools +- Bump versions in `./.github/workflows/MainDistributionPipeline.yml` + - `duckdb_version` in the `duckdb-stable-build` and `code-quality-check` jobs should be set to latest tagged release + - `duckdb-1-5-build` keeps building the 1.5.x line from the same source; its `duckdb_version`, `ci_tools_version` + and `@vX.Y.Z` ref move together with the latest 1.5 patch release + - the `duckdb-stable-build` and `code-quality-check` jobs (and their `ci_tools_version`) use `v2.0-cyanoptera`, as in + duckdb/extension-template; move them to a release-named branch once one exists for 2.0.x + +# Supporting more than one DuckDB version +mpduck builds against DuckDB 1.5.x and 2.0 from the same source. Differences in DuckDB's internal C++ API are isolated +in `src/include/mpduck_compat.hpp` (see the comment at the top of that file). When a DuckDB bump breaks the build: + +- If the break is a rename or a type change, add a helper to `mpduck_compat.hpp` with one implementation per version + and call it from the sources, rather than sprinkling `#if` around. +- To try a new DuckDB, point the `duckdb` submodule at it (or use a `git worktree`, so the existing build is kept) + and build. +- Once an old version is no longer supported, delete its half of the `#if` in the compat header. # API changes DuckDB extensions built with this extension template are built against the internal C++ API of DuckDB. This API is not guaranteed to be stable. diff --git a/duckdb b/duckdb index 8a58519..d54cbbe 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 8a5851971fae891f292c2714d86046ee018e9737 +Subproject commit d54cbbe3df17e8e5ce68312f483b4155836112aa diff --git a/extension-ci-tools b/extension-ci-tools index ef15a2a..2de8029 160000 --- a/extension-ci-tools +++ b/extension-ci-tools @@ -1 +1 @@ -Subproject commit ef15a2a7453db5b4f85b7c668a545ae2f1193ff6 +Subproject commit 2de802913caf5c9bba0ef267658b20f3cc0486c3 diff --git a/src/include/mpduck_compat.hpp b/src/include/mpduck_compat.hpp new file mode 100644 index 0000000..353e414 --- /dev/null +++ b/src/include/mpduck_compat.hpp @@ -0,0 +1,82 @@ +#pragma once + +// Lets one source tree build against both DuckDB 1.5.x and DuckDB 2.0. +// +// DuckDB's C++ API is internal and not stable across releases. The differences mpduck is exposed to are kept +// here so the rest of the code reads the same for both versions: +// +// - 2.0 introduced Identifier for SQL names (column names, aliases, COPY option keys); 1.5 uses plain string. +// - 2.0 deleted ConstantExpression(const Value &) in favour of ConstantExpression::FromValue. +// - 2.0 made StarExpression::exclude_list private (ExcludeListMutable()). +// - 2.0 replaced Vector::Flatten(count) with Flatten(). +// - 2.0.0 removed TableFunction::named_parameters; named parameters are typed kwargs on the function signature. +// +// The switch is the presence of duckdb/common/enums/identifier_case_mode.hpp, which only exists from 2.0. +// (identifier.hpp is not usable for this: 1.5 already ships it, but its COPY and expression APIs still take string.) + +#include "duckdb.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/star_expression.hpp" + +#if __has_include("duckdb/common/enums/identifier_case_mode.hpp") +#define MPDUCK_DUCKDB_V2 1 +#endif + +namespace duckdb { + +#ifdef MPDUCK_DUCKDB_V2 + +//! The type DuckDB uses for SQL names: what a name in a COPY bind callback or a struct field looks like. +using MPName = Identifier; + +inline Identifier MPMakeName(const string &name) { + return Identifier(name); +} +inline const string &MPNameString(const Identifier &name) { + return name.GetIdentifierName(); +} +inline unique_ptr MPConstant(const Value &value) { + return ConstantExpression::FromValue(value); +} +inline void MPExcludeColumn(StarExpression &star, const string &column) { + star.ExcludeListMutable().insert(QualifiedColumnName(Identifier(column))); +} +inline void MPFlatten(Vector &vector, idx_t) { + vector.Flatten(); +} +inline void MPAddNamedParameters(TableFunction &function, const vector> ¶meters) { + function.GetSignature().WithTypedKwargs("options", [&](TypedKwargs &kwargs) { + for (auto ¶meter : parameters) { + kwargs.Add(Identifier(parameter.first), parameter.second); + } + }); +} + +#else + +using MPName = string; + +inline string MPMakeName(const string &name) { + return name; +} +inline const string &MPNameString(const string &name) { + return name; +} +inline unique_ptr MPConstant(const Value &value) { + return make_uniq(value); +} +inline void MPExcludeColumn(StarExpression &star, const string &column) { + star.exclude_list.insert(QualifiedColumnName(column)); +} +inline void MPFlatten(Vector &vector, idx_t count) { + vector.Flatten(count); +} +inline void MPAddNamedParameters(TableFunction &function, const vector> ¶meters) { + for (auto ¶meter : parameters) { + function.named_parameters[parameter.first] = parameter.second; + } +} + +#endif + +} // namespace duckdb diff --git a/src/include/mpfile_filesystem.hpp b/src/include/mpfile_filesystem.hpp index 005bcf0..ec3e60f 100644 --- a/src/include/mpfile_filesystem.hpp +++ b/src/include/mpfile_filesystem.hpp @@ -13,9 +13,15 @@ struct MPFileSchema { }; //! Parse the header ('!') and schema ('VARIABLE_TYPES') rows from an mpfile. -//! Reads only until the first data ('*') row, 'VARIABLE_TYPES' row, or unrecognised line. +//! Reads only until the first data ('*') row. +//! Throws if the file cannot be opened. A file that opens but has no header row yields an empty schema +//! (column_names.size() < 2), which callers treat as "no data". MPFileSchema ParseMPFileSchema(const string &path, FileSystem &fs); +//! True if the file at `path` is exactly zero bytes. Costs one open and size lookup, and does not read any content. +//! Throws if the file cannot be opened. +bool IsZeroByteMPFile(const string &path, FileSystem &fs); + //! Merge a collection of schemas into one, widening types where they differ. //! Numeric widening order: SMALLINT < INTEGER < DOUBLE. Any non-numeric mismatch //! falls back to VARCHAR. Columns present in only some files are included as-is. @@ -74,6 +80,11 @@ class MPFileSystem : public FileSystem { vector Glob(const string &path, FileOpener *opener = nullptr) override; bool FileExists(const string &filename, optional_ptr opener = nullptr) override; + + // DuckDB v2.0 probes the COPY target path with DirectoryExists and may RemoveFile it when overwriting. + // Both are claimed by MPFileSystem (it owns .rpt/.prn/.fac paths), so forward them to the real filesystem. + bool DirectoryExists(const string &directory, optional_ptr opener = nullptr) override; + void RemoveFile(const string &filename, optional_ptr opener = nullptr) override; }; } // namespace duckdb diff --git a/src/mpduck_extension.cpp b/src/mpduck_extension.cpp index 9118abf..633551c 100644 --- a/src/mpduck_extension.cpp +++ b/src/mpduck_extension.cpp @@ -1,5 +1,6 @@ #define DUCKDB_EXTENSION_MAIN +#include "mpduck_compat.hpp" #include "mpduck_extension.hpp" #include "mpfile_filesystem.hpp" #include "write_mpfile.hpp" @@ -19,36 +20,77 @@ namespace duckdb { +//! The read_mpfile named parameters that are forwarded to read_csv_auto, with read_csv's defaults. +struct MPReadOptions { + //! Bind and sniff every file up front so files may differ in columns and types (also reports a placeholder + //! cardinality to the planner). + bool union_by_name = false; + //! Skip rows that fail to parse instead of raising an error. + bool ignore_errors = false; + //! Drop zero-byte files from a glob or list before they are read, instead of letting them fail the scan. + bool skip_empty_files = true; + //! When non-null, forwarded as the filename= parameter (true, or the name of the column to add). + unique_ptr filename; + //! Dialect of the stream MPFileSystem hands to read_csv_auto. Always forwarded, so the sniffer does not have to + //! enumerate and re-scan delimiter/quote/escape candidates. + string delim = ","; + string quote = "\""; + string escape = "\""; + //! Forwarded only when set: the line ending is cheap to detect, and files may use LF or CRLF. Uses read_csv's + //! syntax: the text \r\n (backslash, r, backslash, n), not the control characters. + unique_ptr new_line; +}; + //! Build a read_csv_auto ref for the given list of paths, wrapped in SELECT * EXCLUDE ("!"). -//! When a schema is provided its column types are forwarded via the dtypes parameter. -//! When filename_opt is non-null its value is forwarded as the filename= named parameter of read_csv_auto. +//! When a schema is provided its column types are forwarded via the dtypes parameter. Each option that differs +//! from read_csv's default is forwarded as the read_csv_auto named parameter of the same name. static unique_ptr MakeReadCSVRefFromPaths(const vector &paths, const MPFileSchema &schema, - unique_ptr filename_opt = nullptr) { + MPReadOptions options) { vector> children; vector path_values; path_values.reserve(paths.size()); for (const auto &p : paths) { path_values.push_back(Value(p)); } - children.push_back(make_uniq(Value::LIST(LogicalType::VARCHAR, std::move(path_values)))); + children.push_back(MPConstant(Value::LIST(LogicalType::VARCHAR, std::move(path_values)))); + + if (options.union_by_name) { + auto union_by_name_expr = MPConstant(Value::BOOLEAN(true)); + union_by_name_expr->SetAlias("union_by_name"); + children.push_back(std::move(union_by_name_expr)); + } - auto union_by_name = make_uniq(Value::BOOLEAN(true)); - union_by_name->SetAlias("union_by_name"); - children.push_back(std::move(union_by_name)); + if (options.ignore_errors) { + auto ignore_errors_expr = MPConstant(Value::BOOLEAN(true)); + ignore_errors_expr->SetAlias("ignore_errors"); + children.push_back(std::move(ignore_errors_expr)); + } if (schema.found && !schema.column_names.empty() && !schema.column_types.empty()) { child_list_t dtypes_fields; size_t n = MinValue(schema.column_names.size(), schema.column_types.size()); for (size_t i = 0; i < n; i++) { - dtypes_fields.push_back({schema.column_names[i], Value(schema.column_types[i])}); + dtypes_fields.push_back({MPMakeName(schema.column_names[i]), Value(schema.column_types[i])}); } - auto dtypes_expr = make_uniq(Value::STRUCT(std::move(dtypes_fields))); + auto dtypes_expr = MPConstant(Value::STRUCT(std::move(dtypes_fields))); dtypes_expr->SetAlias("dtypes"); children.push_back(std::move(dtypes_expr)); } - if (filename_opt) { - auto filename_expr = make_uniq(*filename_opt); + for (const auto &dialect_option : {std::make_pair("delim", &options.delim), std::make_pair("quote", &options.quote), + std::make_pair("escape", &options.escape)}) { + auto expr = MPConstant(Value(*dialect_option.second)); + expr->SetAlias(dialect_option.first); + children.push_back(std::move(expr)); + } + if (options.new_line) { + auto new_line_expr = MPConstant(*options.new_line); + new_line_expr->SetAlias("new_line"); + children.push_back(std::move(new_line_expr)); + } + + if (options.filename) { + auto filename_expr = MPConstant(*options.filename); filename_expr->SetAlias("filename"); children.push_back(std::move(filename_expr)); } @@ -59,7 +101,7 @@ static unique_ptr MakeReadCSVRefFromPaths(const vector &paths, // Wrap in SELECT * EXCLUDE ("!") to hide the indicator column from callers. auto select_node = make_uniq(); auto star = make_uniq(); - star->exclude_list.insert(QualifiedColumnName("!")); + MPExcludeColumn(*star, "!"); select_node->select_list.push_back(std::move(star)); select_node->from_table = std::move(table_function); @@ -69,50 +111,150 @@ static unique_ptr MakeReadCSVRefFromPaths(const vector &paths, return make_uniq(std::move(select_stmt)); } -//! Build a read_csv_auto ref for a single path (wraps MakeReadCSVRefFromPaths). +//! Build a read_csv_auto ref for a single path (wraps MakeReadCSVRefFromPaths). One file is trivially +//! consistent with itself, so union_by_name is never needed. static unique_ptr MakeReadCSVRef(const string &path, const MPFileSchema &schema, - unique_ptr filename_opt = nullptr) { - return MakeReadCSVRefFromPaths({path}, schema, std::move(filename_opt)); + MPReadOptions options = MPReadOptions()) { + options.union_by_name = false; + return MakeReadCSVRefFromPaths({path}, schema, std::move(options)); } -//! Extract the filename named parameter from input, if present. -static unique_ptr ExtractFilenameOpt(TableFunctionBindInput &input) { +//! Read a BOOLEAN named parameter. Absent or NULL means `default_value`. +static bool ExtractBool(TableFunctionBindInput &input, const string &name, bool default_value = false) { + auto it = input.named_parameters.find(MPMakeName(name)); + if (it == input.named_parameters.end() || it->second.IsNull()) { + return default_value; + } + return it->second.GetValue(); +} + +//! Read a VARCHAR named parameter into target. Absent or NULL leaves the default in place; returns whether it was set. +static bool ExtractString(TableFunctionBindInput &input, const string &name, string &target) { + auto it = input.named_parameters.find(MPMakeName(name)); + if (it == input.named_parameters.end() || it->second.IsNull()) { + return false; + } + target = it->second.GetValue(); + return true; +} + +//! Collect the named parameters that are forwarded to read_csv_auto. +static MPReadOptions ExtractReadOptions(TableFunctionBindInput &input) { + MPReadOptions options; + options.union_by_name = ExtractBool(input, "union_by_name"); + options.ignore_errors = ExtractBool(input, "ignore_errors"); + options.skip_empty_files = ExtractBool(input, "skip_empty_files", true); + // delim and sep are read_csv's two spellings of one option + if (ExtractString(input, "delim", options.delim) && ExtractString(input, "sep", options.delim)) { + throw InvalidInputException("read_mpfile: specify only one of delim and sep"); + } + ExtractString(input, "sep", options.delim); + if (options.delim != ",") { + // MPFileSystem splits and rebuilds the header rows on commas, so another delimiter cannot work + throw InvalidInputException("read_mpfile: delim must be ',', mpfile header rows are always comma-separated"); + } + ExtractString(input, "quote", options.quote); + ExtractString(input, "escape", options.escape); + string new_line; + if (ExtractString(input, "new_line", new_line)) { + options.new_line = make_uniq(new_line); + } auto it = input.named_parameters.find("filename"); if (it != input.named_parameters.end()) { - return make_uniq(it->second); + options.filename = make_uniq(it->second); + } + return options; +} + +//! Concrete file paths for entries that are each either a literal path or a glob, in order. +static vector ExpandEntries(FileSystem &fs, const vector &entries) { + vector paths; + for (const auto &entry : entries) { + if (FileSystem::HasGlob(entry)) { + for (const auto &info : fs.Glob(entry)) { + paths.push_back(info.path); + } + } else { + paths.push_back(entry); + } } - return nullptr; + return paths; } -//! Redirect read_mpfile('path') to read_csv_auto('path', union_by_name=true[, dtypes=...]). +//! Shared by the glob and list forms of read_mpfile. Files are never combined by name unless asked to. +//! +//! skip_empty_files = true (default): zero-byte files among the expanded paths are dropped first, at the cost of one +//! open and size lookup per file. Files that are not empty but have no header row are not covered; see below. +//! +//! union_by_name = false (default): the entries are only expanded. The schema for the dtypes parameter comes from +//! the first matched file that has data; every other file is left for read_csv_auto to open lazily, so bind +//! cost does not grow with the number of files (which matters most on remote storage). All files must have the +//! same columns, as with read_csv. +//! +//! union_by_name = true: every file's header and VARIABLE_TYPES rows are read up front, files with no data are +//! dropped, the schemas are merged (numeric types widen, other mismatches become VARCHAR) and read_csv_auto is +//! asked to union the files by column name. This costs one extra open and read per file. +static unique_ptr ReadMPFileMultiRef(FileSystem &fs, const vector &entries, MPReadOptions options, + const string &no_match_message) { + auto paths = ExpandEntries(fs, entries); + + if (options.skip_empty_files) { + vector non_empty; + for (auto &path : paths) { + if (!IsZeroByteMPFile(path, fs)) { + non_empty.push_back(std::move(path)); + } + } + paths = std::move(non_empty); + } + + if (!options.union_by_name) { + MPFileSchema schema; + for (const auto &path : paths) { + auto s = ParseMPFileSchema(path, fs); + if (s.column_names.size() >= 2) { // "!" indicator + at least one column + schema = std::move(s); + break; + } + } + if (schema.column_names.empty()) { + throw InvalidInputException("%s", no_match_message); + } + return MakeReadCSVRefFromPaths(paths, schema, std::move(options)); + } + + vector data_paths; + vector schemas; + for (const auto &path : paths) { + auto s = ParseMPFileSchema(path, fs); + if (s.column_names.size() >= 2) { + data_paths.push_back(path); + schemas.push_back(std::move(s)); + } + } + if (data_paths.empty()) { + throw InvalidInputException("%s", no_match_message); + } + auto schema = MergeSchemas(schemas); + return MakeReadCSVRefFromPaths(data_paths, schema, std::move(options)); +} + +//! Redirect read_mpfile('path') to read_csv_auto(['path'][, dtypes=...]). A glob is handled by +//! ReadMPFileMultiRef; a literal path is a single file and needs no union. //! The registered MPFileSystem intercepts each file open and returns filtered content. static unique_ptr ReadMPFileBindReplace(ClientContext &context, TableFunctionBindInput &input) { auto path = input.inputs[0].GetValue(); auto &fs = FileSystem::GetFileSystem(context); - auto filename_opt = ExtractFilenameOpt(input); + auto options = ExtractReadOptions(input); if (FileSystem::HasGlob(path)) { - // Expand the glob, filter out empty files, parse schemas, and merge. - auto files = fs.Glob(path); - vector data_paths; - vector schemas; - for (const auto &info : files) { - auto s = ParseMPFileSchema(info.path, fs); - if (s.column_names.size() >= 2) { // "!" indicator + at least one column - data_paths.push_back(info.path); - schemas.push_back(std::move(s)); - } - } - if (data_paths.empty()) { - throw InvalidInputException("read_mpfile: no files with data matched '%s'", path); - } - auto schema = MergeSchemas(schemas); - return MakeReadCSVRefFromPaths(data_paths, schema, std::move(filename_opt)); + return ReadMPFileMultiRef(fs, {path}, std::move(options), + StringUtil::Format("read_mpfile: no files with data matched '%s'", path)); } auto schema = ParseMPFileSchema(path, fs); - auto table_function = MakeReadCSVRef(path, schema, std::move(filename_opt)); - table_function->alias = fs.ExtractBaseName(path); + auto table_function = MakeReadCSVRef(path, schema, std::move(options)); + table_function->alias = MPMakeName(fs.ExtractBaseName(path)); return std::move(table_function); } @@ -125,50 +267,40 @@ static unique_ptr ReadMPFileReplacement(ClientContext &context, Replac } auto &fs = FileSystem::GetFileSystem(context); + if (FileSystem::HasGlob(table_name)) { + // A glob pattern is not an openable path: expand it like read_mpfile does (union_by_name = false). + return ReadMPFileMultiRef(fs, {table_name}, MPReadOptions(), + StringUtil::Format("read_mpfile: no files with data matched '%s'", table_name)); + } auto schema = ParseMPFileSchema(table_name, fs); auto table_function = MakeReadCSVRef(table_name, schema); - table_function->alias = fs.ExtractBaseName(table_name); + table_function->alias = MPMakeName(fs.ExtractBaseName(table_name)); return std::move(table_function); } //! Redirect read_mpfile(['path1', 'path2', ...]) — each entry may be a literal path or glob. -//! Schemas from all matched files are merged before forwarding to read_csv_auto. +//! See ReadMPFileMultiRef for how the files are combined. No alias — list input spans multiple files. static unique_ptr ReadMPFileListBindReplace(ClientContext &context, TableFunctionBindInput &input) { auto &fs = FileSystem::GetFileSystem(context); - auto &path_entries = ListValue::GetChildren(input.inputs[0]); - auto filename_opt = ExtractFilenameOpt(input); - - // Expand globs, collect concrete paths, and filter out files with no data. - vector all_paths; - vector schemas; - for (const auto &entry : path_entries) { - auto path = entry.GetValue(); - if (FileSystem::HasGlob(path)) { - auto files = fs.Glob(path); - for (auto &info : files) { - auto s = ParseMPFileSchema(info.path, fs); - if (s.column_names.size() >= 2) { // "!" indicator + at least one column - all_paths.push_back(info.path); - schemas.push_back(std::move(s)); - } - } - } else { - auto s = ParseMPFileSchema(path, fs); - if (s.column_names.size() >= 2) { - all_paths.push_back(path); - schemas.push_back(std::move(s)); - } - } - } - - if (all_paths.empty()) { - throw InvalidInputException("read_mpfile: no files matched the provided paths"); + vector entries; + for (const auto &entry : ListValue::GetChildren(input.inputs[0])) { + entries.push_back(entry.GetValue()); } + return ReadMPFileMultiRef(fs, entries, ExtractReadOptions(input), + "read_mpfile: no files matched the provided paths"); +} - auto schema = MergeSchemas(schemas); - - // No alias — list input spans multiple files. - return MakeReadCSVRefFromPaths(all_paths, schema, std::move(filename_opt)); +//! The named parameters of both read_mpfile overloads. +static vector> MPReadParameters() { + return {{"filename", LogicalType::ANY}, + {"union_by_name", LogicalType::BOOLEAN}, + {"ignore_errors", LogicalType::BOOLEAN}, + {"delim", LogicalType::VARCHAR}, + {"sep", LogicalType::VARCHAR}, + {"quote", LogicalType::VARCHAR}, + {"escape", LogicalType::VARCHAR}, + {"new_line", LogicalType::VARCHAR}, + {"skip_empty_files", LogicalType::BOOLEAN}}; } static void LoadInternal(ExtensionLoader &loader) { @@ -180,13 +312,13 @@ static void LoadInternal(ExtensionLoader &loader) { // Explicit table function: SELECT * FROM read_mpfile('file.rpt') TableFunction read_mpfile("read_mpfile", {LogicalType::VARCHAR}, nullptr, nullptr); read_mpfile.bind_replace = ReadMPFileBindReplace; - read_mpfile.named_parameters["filename"] = LogicalType::ANY; + MPAddNamedParameters(read_mpfile, MPReadParameters()); loader.RegisterFunction(read_mpfile); // List overload: SELECT * FROM read_mpfile(['file1.rpt', 'glob_*.rpt', ...]) TableFunction read_mpfile_list("read_mpfile", {LogicalType::LIST(LogicalType::VARCHAR)}, nullptr, nullptr); read_mpfile_list.bind_replace = ReadMPFileListBindReplace; - read_mpfile_list.named_parameters["filename"] = LogicalType::ANY; + MPAddNamedParameters(read_mpfile_list, MPReadParameters()); loader.RegisterFunction(read_mpfile_list); // Implicit scan: SELECT * FROM 'file.rpt' diff --git a/src/mpfile_filesystem.cpp b/src/mpfile_filesystem.cpp index 80a9846..745278d 100644 --- a/src/mpfile_filesystem.cpp +++ b/src/mpfile_filesystem.cpp @@ -133,15 +133,23 @@ static string MapMPType(const string &mp_type, size_t col_idx) { throw IOException("Unknown mpfile type specifier '%s' at column %llu", mp_type, (unsigned long long)col_idx); } +bool IsZeroByteMPFile(const string &path, FileSystem &fs) { + // The bypass makes this the raw file size rather than the size of the filtered stream MPFileSystem would serve. + // As in ParseMPFileSchema, open errors propagate so an unreadable file is never mistaken for an empty one. + ScopedMPFSBypass bypass; + auto handle = fs.OpenFile(path, FileOpenFlags::FILE_FLAGS_READ); + return fs.GetFileSize(*handle) == 0; +} + MPFileSchema ParseMPFileSchema(const string &path, FileSystem &fs) { MPFileSchema schema; + // Errors opening the file (missing, permission denied, network, credentials) propagate: swallowing them + // would make callers treat an unreadable file as one with no data and silently drop it from a glob. unique_ptr handle; - try { + { ScopedMPFSBypass bypass; handle = fs.OpenFile(path, FileOpenFlags::FILE_FLAGS_READ); - } catch (...) { - return schema; } ForEachLine(*handle, [&](const string &line) -> bool { @@ -618,4 +626,25 @@ bool MPFileSystem::FileExists(const string &filename, optional_ptr o return local_fs->FileExists(filename, opener); } +bool MPFileSystem::DirectoryExists(const string &directory, optional_ptr opener) { + auto db = FileOpener::TryGetDatabase(opener); + if (db) { + ScopedMPFSBypass bypass; + return DBConfig::GetConfig(*db).file_system->DirectoryExists(directory, opener); + } + auto local_fs = FileSystem::CreateLocal(); + return local_fs->DirectoryExists(directory, opener); +} + +void MPFileSystem::RemoveFile(const string &filename, optional_ptr opener) { + auto db = FileOpener::TryGetDatabase(opener); + if (db) { + ScopedMPFSBypass bypass; + DBConfig::GetConfig(*db).file_system->RemoveFile(filename, opener); + return; + } + auto local_fs = FileSystem::CreateLocal(); + local_fs->RemoveFile(filename, opener); +} + } // namespace duckdb diff --git a/src/write_mpfile.cpp b/src/write_mpfile.cpp index 81ab5da..fcdb9ef 100644 --- a/src/write_mpfile.cpp +++ b/src/write_mpfile.cpp @@ -1,3 +1,4 @@ +#include "mpduck_compat.hpp" #include "write_mpfile.hpp" #include "mpfile_filesystem.hpp" @@ -18,13 +19,14 @@ namespace duckdb { //! Map a DuckDB column type to the corresponding mpfile VARIABLE_TYPES code. //! T = text (VARCHAR and anything else), S = smallint, I = integer, N = numeric. +//! BIGINT maps to N (read back as DOUBLE, exact to 2^53) because I is a 32-bit INTEGER on read. static string DuckTypeToMPCode(const LogicalType &type) { switch (type.id()) { case LogicalTypeId::SMALLINT: return "S"; case LogicalTypeId::INTEGER: - case LogicalTypeId::BIGINT: return "I"; + case LogicalTypeId::BIGINT: case LogicalTypeId::FLOAT: case LogicalTypeId::DOUBLE: case LogicalTypeId::DECIMAL: @@ -69,7 +71,7 @@ struct WriteMPFileData : FunctionData { } else if (code == "I") { estimate += 12; // max int64: sign + 19 digits (typical much shorter) } else { - estimate += 15; // N: std::to_string double gives ~12 chars + headroom + estimate += 15; // N: shortest round-trip double is up to ~24 chars; typical values are much shorter } } return estimate; @@ -106,17 +108,32 @@ struct WriteMPFileLocalState : LocalFunctionData { }; static unique_ptr WriteMPFileBind(ClientContext &context, CopyFunctionBindInput &input, - const vector &names, const vector &sql_types) { + const vector &identifiers, + const vector &sql_types) { auto data = make_uniq(); + vector names; + names.reserve(identifiers.size()); + for (const auto &identifier : identifiers) { + names.push_back(MPNameString(identifier)); + } data->column_names = names; data->column_types = sql_types; data->mp_type_codes.reserve(sql_types.size()); for (const auto &type : sql_types) { data->mp_type_codes.push_back(DuckTypeToMPCode(type)); } + // The header row is written unquoted and split on commas by the reader, so a name + // containing a delimiter, quote or line break would corrupt the file. + for (const auto &name : names) { + if (name.find_first_of(",\"\r\n") != string::npos) { + throw BinderException("Column name '%s' cannot be written to an mpfile: names must not contain commas, " + "double quotes or line breaks", + name); + } + } for (auto &option : input.info.options) { - auto loption = StringUtil::Lower(option.first); + auto loption = StringUtil::Lower(MPNameString(option.first)); if (loption == "kv_metadata") { auto &kv_struct = option.second[0]; if (kv_struct.type().id() != LogicalTypeId::STRUCT) { @@ -124,7 +141,7 @@ static unique_ptr WriteMPFileBind(ClientContext &context, CopyFunc } auto &values = StructValue::GetChildren(kv_struct); for (idx_t i = 0; i < values.size(); i++) { - string key = StructType::GetChildName(kv_struct.type(), i); + string key = MPNameString(StructType::GetChildName(kv_struct.type(), i)); data->metadata_kv.emplace_back(key, values[i].ToString()); } } else if (loption == "include_types") { @@ -134,7 +151,7 @@ static unique_ptr WriteMPFileBind(ClientContext &context, CopyFunc } data->include_types = BooleanValue::Get(val); } else { - throw NotImplementedException("Unrecognised mpfile COPY option: %s", option.first); + throw NotImplementedException("Unrecognised mpfile COPY option: %s", MPNameString(option.first)); } } @@ -196,7 +213,7 @@ static void WriteMPFileSink(ExecutionContext &context, FunctionData &bind_data, // Flatten all vectors once up front so FlatVector accessors are valid. for (idx_t col = 0; col < ncols; col++) { - input.data[col].Flatten(nrows); + MPFlatten(input.data[col], nrows); } auto &output = lstate.buffer; //! accumulates formatted rows; flushed when threshold is reached @@ -258,11 +275,13 @@ static void WriteMPFileSink(ExecutionContext &context, FunctionData &bind_data, output.append(start, UnsafeNumericCast(end - start)); break; } + // Value::ToString gives the shortest round-trip representation; + // std::to_string uses %f (6 decimal places) and silently loses small values. case LogicalTypeId::FLOAT: - output += std::to_string(FlatVector::GetData(vec)[row]); + output += Value::FLOAT(FlatVector::GetData(vec)[row]).ToString(); break; case LogicalTypeId::DOUBLE: - output += std::to_string(FlatVector::GetData(vec)[row]); + output += Value::DOUBLE(FlatVector::GetData(vec)[row]).ToString(); break; case LogicalTypeId::DECIMAL: { auto &col_type = data.column_types[col]; @@ -297,11 +316,14 @@ static void WriteMPFileSink(ExecutionContext &context, FunctionData &bind_data, } break; } - default: - // Fallback for any remaining types — correct but not maximally fast. - output += input.GetValue(col, row).ToString(); + default: { + // Fallback for any remaining types (written as text, so quote them: + // lists, structs and timestamps may contain commas or quotes). + auto str = input.GetValue(col, row).ToString(); + AppendQuotedMPString(str.data(), str.size(), output); break; } + } } output += "\r\n"; } @@ -339,8 +361,13 @@ static void WriteMPFileCombine(ExecutionContext & /*context*/, FunctionData & /* } static void WriteMPFileFinalize(ClientContext & /*context*/, FunctionData & /*bind_data*/, - GlobalFunctionData & /*gstate*/) { - // FileHandle destructor closes the file. + GlobalFunctionData &gstate_ref) { + // Close explicitly rather than relying on the destructor so write errors surface here. + auto &state = gstate_ref.Cast(); + if (state.handle) { + state.handle->Close(); + state.handle.reset(); + } } static void WriteMPFileCopyOptions(ClientContext & /*context*/, CopyOptionsInput &input) { diff --git a/test/data/bad_rows.rpt b/test/data/bad_rows.rpt new file mode 100644 index 0000000..b2c7e6f --- /dev/null +++ b/test/data/bad_rows.rpt @@ -0,0 +1,6 @@ +VARIABLE_TYPES,T1,T20,S,N +!,NAME,AGE,SCORE +*,Alice,30,98.5 +*,Bob,notanumber,72.0 +*,Carol,41,abc +*,Dave,25,60.5 diff --git a/test/data/quote_single.rpt b/test/data/quote_single.rpt new file mode 100644 index 0000000..deb416b --- /dev/null +++ b/test/data/quote_single.rpt @@ -0,0 +1,4 @@ +VARIABLE_TYPES,T1,T20,I +!,NAME,N +*,'Smith, J',1 +*,'O''Brien',2 diff --git a/test/data/skip_empty/a_empty.rpt b/test/data/skip_empty/a_empty.rpt new file mode 100644 index 0000000..e69de29 diff --git a/test/data/skip_empty/b_data.rpt b/test/data/skip_empty/b_data.rpt new file mode 100644 index 0000000..6db8943 --- /dev/null +++ b/test/data/skip_empty/b_data.rpt @@ -0,0 +1,3 @@ +VARIABLE_TYPES,T1,T20,S,N,T5 +!,NAME,AGE,SCORE,CODE +*,Alice,30,98.5,ABC diff --git a/test/data/skip_empty/c_empty.rpt b/test/data/skip_empty/c_empty.rpt new file mode 100644 index 0000000..e69de29 diff --git a/test/data/skip_empty/d_data.rpt b/test/data/skip_empty/d_data.rpt new file mode 100644 index 0000000..cb90f97 --- /dev/null +++ b/test/data/skip_empty/d_data.rpt @@ -0,0 +1,3 @@ +VARIABLE_TYPES,T1,T20,S,N,T5 +!,NAME,AGE,SCORE,CODE +*,Dave,41,60.5,DDD diff --git a/test/sql/mpduck.test b/test/sql/mpduck.test index a0e20c9..c75f166 100644 --- a/test/sql/mpduck.test +++ b/test/sql/mpduck.test @@ -31,7 +31,7 @@ SCORE DOUBLE # SALARY=DOUBLE, DEPT=VARCHAR, RATING=DOUBLE, GRADE=VARCHAR query II SELECT column_name, column_type -FROM (DESCRIBE SELECT * FROM read_mpfile('test/data/glob_*.rpt')) +FROM (DESCRIBE SELECT * FROM read_mpfile('test/data/glob_*.rpt', union_by_name=true)) ORDER BY column_name; ---- AGE INTEGER @@ -76,7 +76,7 @@ Velociraptor 1.8 1924 # Confirm data round-trips correctly: glob_*.rpt now covers all four glob files; # NAME is the column present in every file. query T rowsort -SELECT NAME FROM read_mpfile('test/data/glob_*.rpt'); +SELECT NAME FROM read_mpfile('test/data/glob_*.rpt', union_by_name=true); ---- Alice Bob @@ -85,14 +85,14 @@ Diana # List overload: explicit list of files — schemas merged, rows combined query IIIR rowsort -SELECT NAME, AGE, CODE, SCORE FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt']); +SELECT NAME, AGE, CODE, SCORE FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt'], union_by_name=true); ---- Alice 30 ABC 98.5 Bob 25 XYZ 72.0 # List overload: single glob entry expands to all matching files; NAME is present in all four query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_*.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_*.rpt'], union_by_name=true); ---- Alice Bob @@ -102,7 +102,7 @@ Diana # List overload: schema merging across list entries (AGE widened SMALLINT→INTEGER, SCORE widened →DOUBLE) query II SELECT column_name, column_type -FROM (DESCRIBE SELECT * FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt'])) +FROM (DESCRIBE SELECT * FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt'], union_by_name=true)) ORDER BY column_name; ---- AGE INTEGER @@ -116,7 +116,7 @@ SCORE DOUBLE # NAME is the only column present in all four files; selecting it gives a clean equality check. # Results must equal the equivalent explicit file list. query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_a*.rpt', 'test/data/glob_b*.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_a*.rpt', 'test/data/glob_b*.rpt'], union_by_name=true); ---- Alice Bob @@ -124,7 +124,7 @@ Charlie Diana query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt', 'test/data/glob_b.rpt', 'test/data/glob_b2.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt', 'test/data/glob_b.rpt', 'test/data/glob_b2.rpt'], union_by_name=true); ---- Alice Bob @@ -135,7 +135,7 @@ Diana # footer.rpt contributes Alice and Bob; all six rows are union'd via union_by_name. # Results must equal the equivalent explicit file list. query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_a*.rpt', 'test/data/glob_b*.rpt', 'test/data/footer.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_a*.rpt', 'test/data/glob_b*.rpt', 'test/data/footer.rpt'], union_by_name=true); ---- Alice Alice @@ -145,7 +145,7 @@ Charlie Diana query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt', 'test/data/glob_b.rpt', 'test/data/glob_b2.rpt', 'test/data/footer.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt', 'test/data/glob_b.rpt', 'test/data/glob_b2.rpt', 'test/data/footer.rpt'], union_by_name=true); ---- Alice Alice @@ -176,7 +176,7 @@ Bob 25 72.3 false 1995-06-20 # Mixed indicator variants across files: '!' and '!1' both normalise to the same indicator # column '!', so schema merging produces exactly one '!' column and the EXCLUDE step works. query TIT rowsort -SELECT NAME, AGE, CITY FROM read_mpfile(['test/data/mixed_indicator_a.rpt', 'test/data/mixed_indicator_b.rpt']); +SELECT NAME, AGE, CITY FROM read_mpfile(['test/data/mixed_indicator_a.rpt', 'test/data/mixed_indicator_b.rpt'], union_by_name=true); ---- Alice 30 London Bob 25 Paris @@ -184,7 +184,7 @@ Bob 25 Paris # The merged schema must contain no '!' column — only the data columns. query II SELECT column_name, column_type -FROM (DESCRIBE SELECT * FROM read_mpfile(['test/data/mixed_indicator_a.rpt', 'test/data/mixed_indicator_b.rpt'])) +FROM (DESCRIBE SELECT * FROM read_mpfile(['test/data/mixed_indicator_a.rpt', 'test/data/mixed_indicator_b.rpt'], union_by_name=true)) ORDER BY column_name; ---- AGE SMALLINT @@ -419,7 +419,7 @@ SCORE DOUBLE # only the two files with data contribute columns. query II SELECT column_name, column_type -FROM (DESCRIBE SELECT * FROM read_mpfile('test/data/glob_a*.rpt')) +FROM (DESCRIBE SELECT * FROM read_mpfile('test/data/glob_a*.rpt', union_by_name=true)) ORDER BY column_name; ---- AGE SMALLINT @@ -431,19 +431,19 @@ SCORE DOUBLE # Empty-file filtering: file with no '!' column header is silently ignored via list input. query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/no_columns.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/no_columns.rpt'], union_by_name=true); ---- Alice # Empty-file filtering: zero-byte file is silently ignored via list input. query T rowsort -SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/empty.rpt']); +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/empty.rpt'], union_by_name=true); ---- Alice # Empty-file filtering: list containing only empty/no-column files throws an error. statement error -SELECT * FROM read_mpfile(['test/data/empty.rpt', 'test/data/no_columns.rpt']); +SELECT * FROM read_mpfile(['test/data/empty.rpt', 'test/data/no_columns.rpt'], union_by_name=true); ---- # filename=true adds a filename column containing the file path for each row @@ -454,14 +454,14 @@ Alice 30 glob_a.rpt # filename=true tracks per-file paths when reading a glob query TT rowsort -SELECT NAME, parse_filename(filename) FROM read_mpfile('test/data/glob_a*.rpt', filename=true); +SELECT NAME, parse_filename(filename) FROM read_mpfile('test/data/glob_a*.rpt', filename=true, union_by_name=true); ---- Alice glob_a.rpt Charlie glob_a2.rpt # filename=true works with the list overload query TT rowsort -SELECT NAME, parse_filename(filename) FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt'], filename=true); +SELECT NAME, parse_filename(filename) FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt'], filename=true, union_by_name=true); ---- Alice glob_a.rpt Bob glob_b.rpt @@ -471,3 +471,437 @@ query TT SELECT NAME, parse_filename(source_file) FROM read_mpfile('test/data/glob_a.rpt', filename='source_file'); ---- Alice glob_a.rpt + +# write_mpfile: small doubles keep full precision (std::to_string would write 0.000000) +statement ok +COPY ( + SELECT 1e-9::DOUBLE AS TINY, 123456789.123456789::DOUBLE AS BIG +) TO '__TEST_DIR__/write_double_precision.rpt' (FORMAT mpfile); + +query II +SELECT TINY = 1e-9, BIG = 123456789.123456789 FROM read_mpfile('__TEST_DIR__/write_double_precision.rpt'); +---- +true true + +# write_mpfile: BIGINT values above the INTEGER range survive a round-trip (written as N, read as DOUBLE) +statement ok +COPY ( + SELECT 5000000000::BIGINT AS BIG +) TO '__TEST_DIR__/write_bigint.rpt' (FORMAT mpfile); + +query R +SELECT BIG FROM read_mpfile('__TEST_DIR__/write_bigint.rpt'); +---- +5000000000.0 + +# write_mpfile: types without a native mpfile code are written as quoted text +statement ok +COPY ( + SELECT DATE '2024-01-31' AS D, [1, 2, 3] AS L +) TO '__TEST_DIR__/write_fallback_types.rpt' (FORMAT mpfile); + +query TT +SELECT D, L FROM read_mpfile('__TEST_DIR__/write_fallback_types.rpt'); +---- +2024-01-31 [1, 2, 3] + +# write_mpfile: NULLs round-trip as NULL in numeric and text columns +statement ok +COPY ( + SELECT * FROM (VALUES ('a', 1.5), (NULL, NULL)) t(S, N) +) TO '__TEST_DIR__/write_nulls.rpt' (FORMAT mpfile); + +query TR rowsort +SELECT S, N FROM read_mpfile('__TEST_DIR__/write_nulls.rpt'); +---- +NULL NULL +a 1.5 + +# write_mpfile: column names that cannot be represented in the header row are rejected +statement error +COPY (SELECT 1 AS "a,b") TO '__TEST_DIR__/write_bad_name.rpt' (FORMAT mpfile); +---- +cannot be written to an mpfile + +# write_mpfile: unknown COPY options are rejected +statement error +COPY (SELECT 1 AS A) TO '__TEST_DIR__/write_bad_option.rpt' (FORMAT mpfile, NOT_AN_OPTION true); +---- + +# read_mpfile: a glob of files with identical VARIABLE_TYPES schemas is scanned without union_by_name +statement ok +COPY (SELECT 'a' AS N, CAST(1 AS SMALLINT) AS V) TO '__TEST_DIR__/same_1.rpt'; + +statement ok +COPY (SELECT 'b' AS N, CAST(2 AS SMALLINT) AS V) TO '__TEST_DIR__/same_2.rpt'; + +query TI rowsort +SELECT N, V FROM read_mpfile('__TEST_DIR__/same_*.rpt'); +---- +a 1 +b 2 + +query II +SELECT column_name, column_type FROM (DESCRIBE SELECT * FROM read_mpfile('__TEST_DIR__/same_*.rpt')) ORDER BY column_name; +---- +N VARCHAR +V SMALLINT + +# read_mpfile: list form over identical-schema files also works +query TI rowsort +SELECT N, V FROM read_mpfile(['__TEST_DIR__/same_1.rpt', '__TEST_DIR__/same_2.rpt']); +---- +a 1 +b 2 + +# read_mpfile: same columns in a different order still need union_by_name and must match by name +statement ok +COPY (SELECT CAST(3 AS SMALLINT) AS V, 'c' AS N) TO '__TEST_DIR__/order_1.rpt'; + +statement ok +COPY (SELECT 'd' AS N, CAST(4 AS SMALLINT) AS V) TO '__TEST_DIR__/order_2.rpt'; + +query TI rowsort +SELECT N, V FROM read_mpfile('__TEST_DIR__/order_*.rpt'); +---- +c 3 +d 4 + +# read_mpfile: the filename column still works on the identical-schema (no union_by_name) path +query TT rowsort +SELECT N, parse_filename(filename) FROM read_mpfile('__TEST_DIR__/same_*.rpt', filename=true); +---- +a same_1.rpt +b same_2.rpt + +# read_mpfile: union_by_name defaults to false, like read_csv. Files with different columns are an error... +statement error +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt']); +---- +do not exist in the CSV File + +statement error +SELECT NAME FROM read_mpfile('test/data/glob_a*.rpt'); +---- +do not exist in the CSV File + +# ...and union_by_name := false is the same as omitting it. +statement error +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt'], union_by_name=false); +---- +do not exist in the CSV File + +# ...while union_by_name := true combines them by column name. +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_a2.rpt'], union_by_name=true); +---- +Alice +Charlie + +# Without union_by_name the schema (and so the column types) comes from the first file only. +# glob_a has AGE SMALLINT / SCORE DOUBLE; glob_b has AGE INTEGER / SCORE INTEGER. +query II +SELECT column_name, column_type +FROM (DESCRIBE SELECT * FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt'])) +ORDER BY column_name; +---- +AGE SMALLINT +CODE VARCHAR +NAME VARCHAR +SCORE DOUBLE + +query TI rowsort +SELECT NAME, AGE FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/glob_b.rpt']); +---- +Alice 30 +Bob 25 + +# skip_empty_files (default true) drops zero-byte files from a list or glob before reading. It is exercised with +# the empty file first and not first, with union_by_name both false and true. +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/empty.rpt']); +---- +Alice + +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/empty.rpt', 'test/data/glob_a.rpt']); +---- +Alice + +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/empty.rpt'], union_by_name=true); +---- +Alice + +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/empty.rpt', 'test/data/glob_a.rpt'], union_by_name=true); +---- +Alice + +# Empty files between and around files with data. +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/skip_empty/b_data.rpt', 'test/data/empty.rpt', 'test/data/skip_empty/d_data.rpt']); +---- +Alice +Dave + +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/empty.rpt', 'test/data/skip_empty/b_data.rpt', 'test/data/skip_empty/c_empty.rpt', 'test/data/skip_empty/d_data.rpt'], union_by_name=true); +---- +Alice +Dave + +# Globs: a_empty.rpt and c_empty.rpt are empty, b_data.rpt and d_data.rpt have data. +query T rowsort +SELECT NAME FROM read_mpfile('test/data/skip_empty/*.rpt'); +---- +Alice +Dave + +query T rowsort +SELECT NAME FROM read_mpfile('test/data/skip_empty/*.rpt', union_by_name=true); +---- +Alice +Dave + +query T rowsort +SELECT NAME FROM 'test/data/skip_empty/*.rpt'; +---- +Alice +Dave + +# skip_empty_files=false restores the failure without union_by_name, whether the empty file is first or not: +# the first file with data supplies the schema and the empty file then fails when it is read. +statement error +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/empty.rpt'], skip_empty_files=false); +---- +do not exist in the CSV File + +statement error +SELECT NAME FROM read_mpfile(['test/data/empty.rpt', 'test/data/glob_a.rpt'], skip_empty_files=false); +---- +do not exist in the CSV File + +# With union_by_name=true the header check drops empty files whatever skip_empty_files says. +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/empty.rpt'], union_by_name=true, skip_empty_files=false); +---- +Alice + +query T rowsort +SELECT NAME FROM read_mpfile(['test/data/empty.rpt', 'test/data/glob_a.rpt'], union_by_name=true, skip_empty_files=false); +---- +Alice + +# A non-empty file with no header row is not covered by skip_empty_files. +statement error +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/no_columns.rpt']); +---- +do not exist in the CSV File + +# skip_empty_files must be a boolean. +statement error +SELECT * FROM read_mpfile('test/data/glob_a.rpt', skip_empty_files='maybe'); +---- + +# A missing file is an error rather than being silently dropped. +statement error +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/does_not_exist.rpt']); +---- +does_not_exist.rpt + +# Input with no file that has any data is still an error with a clear message. +statement error +SELECT * FROM read_mpfile(['test/data/empty.rpt', 'test/data/no_columns.rpt']); +---- +no files matched + +# union_by_name must be a boolean. +statement error +SELECT * FROM read_mpfile('test/data/glob_a.rpt', union_by_name='maybe'); +---- + +# An unreadable file is an error even with union_by_name=true; it used to be silently dropped from the result. +statement error +SELECT NAME FROM read_mpfile(['test/data/glob_a.rpt', 'test/data/does_not_exist.rpt'], union_by_name=true); +---- +does_not_exist.rpt + +statement error +SELECT NAME FROM read_mpfile('test/data/does_not_exist.rpt'); +---- +does_not_exist.rpt + +statement error +SELECT * FROM 'test/data/does_not_exist.rpt'; +---- +does_not_exist.rpt + +# The implicit FROM 'glob' scan expands the glob and takes column types from VARIABLE_TYPES. +query II +SELECT column_name, column_type FROM (DESCRIBE SELECT * FROM '__TEST_DIR__/same_*.rpt') ORDER BY column_name; +---- +N VARCHAR +V SMALLINT + +query TI rowsort +SELECT N, V FROM '__TEST_DIR__/same_*.rpt'; +---- +a 1 +b 2 + +statement error +SELECT * FROM 'test/data/nomatch_*.rpt'; +---- +no files with data matched + +# read_mpfile: ignore_errors defaults to false, like read_csv. A row that fails to convert is an error... +# bad_rows.rpt: Bob's AGE (SMALLINT) and Carol's SCORE (DOUBLE) do not parse; Alice and Dave are fine. +statement error +SELECT NAME, AGE, SCORE FROM read_mpfile('test/data/bad_rows.rpt'); +---- +Could not convert string "notanumber" + +statement error +SELECT NAME, AGE, SCORE FROM read_mpfile('test/data/bad_rows.rpt', ignore_errors=false); +---- +Could not convert string "notanumber" + +# ...and with ignore_errors=true the rows that fail to convert are skipped. +query TIR rowsort +SELECT NAME, AGE, SCORE FROM read_mpfile('test/data/bad_rows.rpt', ignore_errors=true); +---- +Alice 30 98.5 +Dave 25 60.5 + +# It behaves the same as read_csv's ignore_errors. +query TIR rowsort +SELECT NAME, AGE, SCORE +FROM read_csv('test/data/bad_rows.rpt', header=true, + dtypes={'!': 'VARCHAR', NAME: 'VARCHAR', AGE: 'SMALLINT', SCORE: 'DOUBLE'}, ignore_errors=true); +---- +Alice 30 98.5 +Dave 25 60.5 + +# As in read_csv, only the columns a query reads are converted: Carol's bad SCORE is never looked at here, +# so her row is kept, while Bob (bad AGE) is still skipped. +query TI rowsort +SELECT NAME, AGE FROM read_mpfile('test/data/bad_rows.rpt', ignore_errors=true); +---- +Alice 30 +Carol 41 +Dave 25 + +# ignore_errors works with the glob and list forms, with union_by_name, and with filename. + +query TIR rowsort +SELECT NAME, AGE, SCORE FROM read_mpfile(['test/data/bad_rows.rpt', 'test/data/bad_rows.rpt'], ignore_errors=true); +---- +Alice 30 98.5 +Alice 30 98.5 +Dave 25 60.5 +Dave 25 60.5 + +query TIR rowsort +SELECT NAME, AGE, SCORE FROM read_mpfile('test/data/bad_row*.rpt', union_by_name=true, ignore_errors=true); +---- +Alice 30 98.5 +Dave 25 60.5 + +query TRT rowsort +SELECT NAME, SCORE, parse_filename(filename) FROM read_mpfile('test/data/bad_rows.rpt', ignore_errors=true, filename=true); +---- +Alice 98.5 bad_rows.rpt +Bob 72.0 bad_rows.rpt +Dave 60.5 bad_rows.rpt + +# ignore_errors skips rows, not files: an unreadable file is still an error, as in read_csv. +statement error +SELECT NAME FROM read_mpfile(['test/data/bad_rows.rpt', 'test/data/does_not_exist.rpt'], ignore_errors=true); +---- +does_not_exist.rpt + +# ignore_errors must be a boolean. +statement error +SELECT * FROM read_mpfile('test/data/bad_rows.rpt', ignore_errors='maybe'); +---- + +# Dialect parameters: delim, quote, escape and new_line are forwarded to read_csv. The defaults are +# delim=',', quote='"', escape='"' and new_line='\r\n', so a CRLF file with double-quoted strings needs no options. +query TI rowsort +SELECT NAME, AGE FROM read_mpfile('test/data/crlf.rpt', delim=',', quote='"', escape='"', new_line='\r\n'); +---- +Alice 30 +Bob 25 + +# The defaults apply to the replacement scan too. +query TI rowsort +SELECT NAME, AGE FROM 'test/data/crlf.rpt'; +---- +Alice 30 +Bob 25 + +# Defaults do not understand single-quoted fields: the comma inside 'Smith, J' splits the row. +statement error +SELECT * FROM read_mpfile('test/data/quote_single.rpt'); +---- + +# quote and escape override the defaults. +query TI rowsort +SELECT NAME, N FROM read_mpfile('test/data/quote_single.rpt', quote='''', escape=''''); +---- +O'Brien 2 +Smith, J 1 + +# ... and work for the list form and for several files at once. +query TI rowsort +SELECT NAME, N FROM read_mpfile(['test/data/quote_single.rpt', 'test/data/quote_single.rpt'], quote='''', escape=''''); +---- +O'Brien 2 +O'Brien 2 +Smith, J 1 +Smith, J 1 + +# sep is read_csv's alias for delim. +query TI rowsort +SELECT NAME, AGE FROM read_mpfile('test/data/crlf.rpt', sep=','); +---- +Alice 30 +Bob 25 + +# delim and sep together are ambiguous. +statement error +SELECT * FROM read_mpfile('test/data/crlf.rpt', delim=',', sep=','); +---- +specify only one of delim and sep + +# Only the comma delimiter works: MPFileSystem rebuilds the header rows on commas. +statement error +SELECT * FROM read_mpfile('test/data/crlf.rpt', delim=';'); +---- +delim must be ',' + +# new_line is not pinned by default (LF and CRLF files both read); it can be given explicitly. +query TIR rowsort +SELECT NAME, AGE, SCORE FROM read_mpfile('test/data/lf.rpt', new_line='\n'); +---- +Alice 30 98.5 +Bob 25 72.0 + +# A new_line that does not match the file is an error. +statement error +SELECT * FROM read_mpfile('test/data/lf.rpt', new_line='\r\n'); +---- + +# new_line is validated by read_csv. +statement error +SELECT * FROM read_mpfile('test/data/crlf.rpt', new_line='x'); +---- +This is not accepted as a newline + +# NULL leaves the default in place. +query TI rowsort +SELECT NAME, AGE FROM read_mpfile('test/data/crlf.rpt', quote=NULL, new_line=NULL); +---- +Alice 30 +Bob 25