diff --git a/.claude/skills/krishiv-golden-category/SKILL.md b/.claude/skills/krishiv-golden-category/SKILL.md index 14893e37e..840b4d5a6 100644 --- a/.claude/skills/krishiv-golden-category/SKILL.md +++ b/.claude/skills/krishiv-golden-category/SKILL.md @@ -39,8 +39,11 @@ or by hand (ADR 0008). Three kinds exist; pick the one that fits. `tests/fixtures/analysis/type-coercion/` (one file per coercion input file) and `registry-matrix/` (one file per function) list the cases that fail today, each with a reason; `crates/krishiv-testkit/tests/{type_coercion,registry_matrix}.rs` require the list to -match the actual failures exactly, so a fix shrinks it and a regression fails. `tests/spark-golden/known-failures.json` -holds the categories and rules while `tests/spark-golden/known-failures-cases/` holds one file per golden input file. +match the actual failures exactly, so a fix shrinks it and a regression fails. The golden baseline is split the same +way: `tests/spark-golden/known-failures-categories/` holds one file per category with the rules that assign it (each +rule has an `order`; rules are tried in ascending order across all files, so leave gaps) and +`tests/spark-golden/known-failures-cases/` one file per golden input file; `known-failures.json` itself holds only the +format version, Spark tag and freeze date. - Regenerate with `KRISHIV_UPDATE_MANIFEST=1 cargo test -p krishiv-testkit --test ` (or `cargo xtask golden --engine session --ratchet update` for the golden baseline), then read the diff: entries removed are progress, entries added are regressions to explain or fix. Only the files whose content changed diff --git a/crates/krishiv-io/CLAUDE.md b/crates/krishiv-io/CLAUDE.md index c9dd77e85..551b66773 100644 --- a/crates/krishiv-io/CLAUDE.md +++ b/crates/krishiv-io/CLAUDE.md @@ -76,4 +76,4 @@ Paths are relative to this crate. ## Tests `cargo nextest run -p krishiv-io --no-tests=pass` · `cargo clippy -p krishiv-io --all-targets -- -D warnings` · -conformance: `cargo nextest run -p krishiv-testkit --test file_format_mock` · bench: `cargo bench -p krishiv-io --bench io` +conformance: `cargo nextest run -p krishiv-testkit --test file_format_mock` · bench: `cargo bench -p krishiv-io` (targets `io`, `coalesce` and `runtime_filter`) diff --git a/crates/krishiv-io/Cargo.toml b/crates/krishiv-io/Cargo.toml index b3c65c263..b70aa6e5a 100644 --- a/crates/krishiv-io/Cargo.toml +++ b/crates/krishiv-io/Cargo.toml @@ -60,6 +60,10 @@ tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread", "sync [target.'cfg(loom)'.dev-dependencies] loom.workspace = true +[[bench]] +name = "coalesce" +harness = false + [[bench]] name = "io" harness = false diff --git a/crates/krishiv-io/benches/coalesce.rs b/crates/krishiv-io/benches/coalesce.rs new file mode 100644 index 000000000..2539d38ef --- /dev/null +++ b/crates/krishiv-io/benches/coalesce.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Coalescing 10,000 column-chunk ranges of one file into requests, run +//! under CodSpeed in CI in its own binary (single-threaded, no I/O) so +//! allocator state from the Parquet encode/scan benchmarks cannot swing it. + +#![expect( + missing_docs, + reason = "criterion_group! generates an undocumented function" +)] + +use std::hint::black_box; +use std::ops::Range; + +use criterion::{Criterion, criterion_group, criterion_main}; +use krishiv_io::io::coalesce; + +/// Column chunks of 500 row groups × 20 columns, a few KiB to a few MiB +/// apart, listed in column-major order (as a projection asks for them). +fn ranges() -> Vec> { + let mut out = Vec::with_capacity(10_000); + for col in 0..20_u64 { + for rg in 0..500_u64 { + let start = rg * (8 << 20) + col * (300 << 10) + (rg * col) % 4096; + out.push(start..start + (200 << 10) + (col * 977) % 65_536); + } + } + out +} + +fn bench(c: &mut Criterion) { + let ranges = ranges(); + let mut group = c.benchmark_group("io"); + group.bench_function("coalesce_10k_ranges", |b| { + b.iter(|| coalesce(black_box(&ranges), 1 << 20, 16 << 20)); + }); + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/crates/krishiv-io/benches/io.rs b/crates/krishiv-io/benches/io.rs index 6a112b5a3..27de9894a 100644 --- a/crates/krishiv-io/benches/io.rs +++ b/crates/krishiv-io/benches/io.rs @@ -3,7 +3,6 @@ //! (single-threaded, no I/O): //! - Spark split planning of 10,000 listed files (sizes up to 300 MiB, a //! third not splittable); -//! - coalescing 10,000 column-chunk ranges of one file into requests; //! - encoding 16,384 rows (an integer and a string column) as one Parquet //! file with the default (snappy) settings, footer and page indexes //! included: arrow-rs's encoder plus the writer's own per-batch work; @@ -22,7 +21,6 @@ )] use std::hint::black_box; -use std::ops::Range; use std::sync::Arc; @@ -30,7 +28,6 @@ use arrow_array::{Int64Array, RecordBatch, StringViewArray}; use arrow_schema::{DataType, Field, Schema}; use criterion::{Criterion, criterion_group, criterion_main}; use krishiv_common::config::Config; -use krishiv_io::io::coalesce; use krishiv_io::parquet::ParquetFormat; use krishiv_io::split::{SplitConfig, SplitFile, plan}; use krishiv_io::{ @@ -53,19 +50,6 @@ fn files() -> Vec { .collect() } -/// Column chunks of 500 row groups × 20 columns, a few KiB to a few MiB -/// apart, listed in column-major order (as a projection asks for them). -fn ranges() -> Vec> { - let mut out = Vec::with_capacity(10_000); - for col in 0..20_u64 { - for rg in 0..500_u64 { - let start = rg * (8 << 20) + col * (300 << 10) + (rg * col) % 4096; - out.push(start..start + (200 << 10) + (col * 977) % 65_536); - } - } - out -} - /// 16,384 rows in batches of 2,048: a key and a short, repetitive string. fn rows() -> Vec { let schema = Arc::new(Schema::new(vec![ @@ -181,14 +165,10 @@ fn scan(rt: &tokio::runtime::Runtime, io: &IoContext, file: &FileRef, req: &Scan fn bench(c: &mut Criterion) { let files = files(); let cfg = SplitConfig::from_config(&Config::new(), 16); - let ranges = ranges(); let mut group = c.benchmark_group("io"); group.bench_function("split_plan_10k_files", |b| { b.iter(|| plan(black_box(&cfg), black_box(&files))); }); - group.bench_function("coalesce_10k_ranges", |b| { - b.iter(|| coalesce(black_box(&ranges), 1 << 20, 16 << 20)); - }); let batches = rows(); let o = FormatOptions::defaults(ParquetFormat.options()); group.bench_function("parquet_encode_16k_rows", |b| { diff --git a/crates/krishiv-testkit/CLAUDE.md b/crates/krishiv-testkit/CLAUDE.md index 9cad18284..7d003a79f 100644 --- a/crates/krishiv-testkit/CLAUDE.md +++ b/crates/krishiv-testkit/CLAUDE.md @@ -9,10 +9,12 @@ the `SQLQueryTestSuite` golden runner over `tests/spark-golden//` (`.sql.ou three-number report, exclusion manifest `tests/spark-golden/exclusions.json`; `cargo xtask golden` writes `target/golden-report.json`; `--engine sql-parse` measures the `krishiv-sql` parser on the DQL statements, `golden::sql_parse`; `--engine session` runs every statement through `krishiv-session`, `golden::session_engine`), -the frozen statement-level baseline `tests/spark-golden/known-failures.json` (categories and classification rules) with -one file per golden input file under `tests/spark-golden/known-failures-cases/` (`golden::known_failures`: every failing -statement with a category — unsupported feature with its plan task, divergence with ADR 0023, or bug with its owner; -classification rules in the same file, `golden::classify`) and its CI ratchet (test `golden_ratchet`: a newly failing +the frozen statement-level baseline, one file per key (`golden::known_failures`): `tests/spark-golden/known-failures.json` +holds only the format version, Spark tag and freeze date; `known-failures-categories/.json` one category +each (unsupported feature with its plan task, divergence with ADR 0023, or bug with its owner) with the classification +rules that assign it (`golden::classify`; each rule has an `order`, rules are tried in ascending order across all +files, so leave gaps); `known-failures-cases/.json` one golden input file's failing statements by category. +Its CI ratchet (test `golden_ratchet`: a newly failing statement fails, a listed statement that passes must be removed; refresh with `cargo run -p krishiv-testkit --bin golden-report -- --engine session --ratchet update`), the batch == bounded-stream invariant harness, a sqllogictest runner over `tests/sqllogictest/`, and the Spark function matrix (P2-T106a): diff --git a/crates/krishiv-testkit/src/bin/golden-report.rs b/crates/krishiv-testkit/src/bin/golden-report.rs index c385d4a4b..4637ad91e 100644 --- a/crates/krishiv-testkit/src/bin/golden-report.rs +++ b/crates/krishiv-testkit/src/bin/golden-report.rs @@ -12,7 +12,8 @@ //! `krishiv-session` ([`krishiv_testkit::golden::session_engine`]). //! * `--cases FILE` writes every statement that did not pass, with why. //! * `--ratchet check` compares the run with the frozen baseline -//! (`tests/spark-golden/known-failures.json`) and fails on any difference; +//! (`tests/spark-golden/known-failures.json` with its per-category and per-file directories) +//! and fails on any difference; //! `--ratchet update` rewrites the baseline: passing statements leave it, new failures are //! classified by its rules (the command fails, writing nothing, if a failure matches no rule). #![forbid(unsafe_code)] diff --git a/crates/krishiv-testkit/src/golden/classify.rs b/crates/krishiv-testkit/src/golden/classify.rs index 5af51a8e8..31f47608b 100644 --- a/crates/krishiv-testkit/src/golden/classify.rs +++ b/crates/krishiv-testkit/src/golden/classify.rs @@ -32,6 +32,9 @@ use super::parse::Statement; pub struct Rule { /// Category key the rule assigns. pub category: String, + /// Position among all rules: lower matches first. Not part of the rule's + /// identity ([`PartialEq`] compares the rule as written). + pub order: u32, command: Option, depends: Option, file: Option, @@ -108,6 +111,7 @@ impl Rule { detail: re("detail")?, family, source: v.clone(), + order: 0, category, }; if rule.source.as_object().is_some_and(|o| o.len() == 1) { diff --git a/crates/krishiv-testkit/src/golden/known_failures.rs b/crates/krishiv-testkit/src/golden/known_failures.rs index 7d9aa8a91..401fa7bd5 100644 --- a/crates/krishiv-testkit/src/golden/known_failures.rs +++ b/crates/krishiv-testkit/src/golden/known_failures.rs @@ -1,21 +1,34 @@ // SPDX-License-Identifier: Apache-2.0 -//! The frozen statement-level baseline of the session engine's golden run: -//! `tests/spark-golden/known-failures.json` holds the categories and the -//! classification rules, and `tests/spark-golden/known-failures-cases/` -//! holds one file per golden input file with that file's failing statement -//! indices by category. Two PRs touching different golden files never touch -//! the same file; totals are computed from the run, never stored. +//! The frozen statement-level baseline of the session engine's golden run. +//! It is split into one file per key so that two PRs never touch the same +//! file unless they touch the same key: +//! +//! - `tests/spark-golden/known-failures.json` holds only the format version, +//! the Spark tag and the freeze date; +//! - `tests/spark-golden/known-failures-categories/` holds one file per +//! category, with the classification rules that assign it; +//! - `tests/spark-golden/known-failures-cases/` holds one file per golden +//! input file with that file's failing statement indices by category. +//! +//! Totals are computed from the run, never stored. //! //! ```json -//! {"format_version": 1, "spark_tag": "v4.2.0", "frozen_at": "2026-10-22", -//! "categories": { -//! "ddl": {"kind": "unsupported", "task": "P2-T075a", "summary": "DDL statements"}, -//! "float-text": {"kind": "divergence", "adr": "0023", "summary": "..."}, -//! "wrong-sum": {"kind": "bug", "owner": "U-agg", "summary": "..."}}} +//! {"format_version": 1, "spark_tag": "v4.2.0", "frozen_at": "2026-10-22"} +//! ``` +//! ```json +//! {"format_version": 1, "category": "ddl", "kind": "unsupported", "task": "P2-T075a", +//! "summary": "DDL statements", "rules": [{"order": 10, "command": true}]} //! ``` //! ```json //! {"format_version": 1, "file": "group-by", "cases": {"ddl": [0, 1], "wrong-sum": [7]}} //! ``` +//! A category's kind is `unsupported` (with `task`), `divergence` (with `adr`) or +//! `bug` (with `owner`). +//! +//! Rules are tried in ascending `order` across all category files, and the first that +//! matches a failure assigns its category; rules with equal `order` are tried by +//! category key, then by position in the file. Leave gaps between orders so that a +//! new rule can go anywhere without renumbering the others. //! //! A category is one of three kinds, each with a mandatory reference: an unsupported feature //! names the plan task that adds it (`P2-T075a`), a known divergence names the ADR that records @@ -24,10 +37,10 @@ //! ([`KnownFailures::check`]) fails on a statement that fails but is not listed (a regression) //! and on a listed statement that passes (the list must shrink with it). //! -//! [`KnownFailures::parse`] reads the legacy single-file form (header and -//! cases in one document, as the unit tests use); [`KnownFailures::load`] -//! reads the split on-disk form, where the cases directory next to the -//! header file supplies the per-file cases. +//! [`KnownFailures::parse`] reads the legacy single-file form (header, +//! categories, rules and cases in one document, as the unit tests use); +//! [`KnownFailures::load`] reads the split on-disk form, where the +//! categories and cases directories next to the header file supply them. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -45,6 +58,9 @@ pub const KNOWN_FAILURES_FORMAT_VERSION: u64 = 1; /// Name of the per-file cases directory next to the header file. pub const CASES_DIR_NAME: &str = "known-failures-cases"; +/// Name of the per-category directory next to the header file. +pub const CATEGORIES_DIR_NAME: &str = "known-failures-categories"; + /// The ADR a known divergence must cite. pub const DIVERGENCE_ADR: &str = "0023"; @@ -229,6 +245,154 @@ fn is_task_id(s: &str) -> bool { && task.len() - digits.len() <= 1 } +fn invalid(message: String) -> KnownFailuresError { + KnownFailuresError::Invalid(message) +} + +fn text_field(v: &Value, k: &str) -> Option { + v.get(k).and_then(Value::as_str).map(str::to_owned) +} + +fn check_format_version(v: &Value) -> bool { + v.get("format_version").and_then(Value::as_u64) == Some(KNOWN_FAILURES_FORMAT_VERSION) +} + +/// The Spark tag and freeze date of a header document. +fn parse_meta(root: &Value) -> Result<(String, String), KnownFailuresError> { + if !check_format_version(root) { + return Err(invalid("unsupported or missing `format_version`".into())); + } + let spark_tag = + text_field(root, "spark_tag").ok_or_else(|| invalid("missing `spark_tag`".into()))?; + let frozen_at = + text_field(root, "frozen_at").ok_or_else(|| invalid("missing `frozen_at`".into()))?; + Ok((spark_tag, frozen_at)) +} + +/// One category from its JSON object (`kind`, the reference it needs, `summary`). +fn parse_category(key: &str, c: &Value) -> Result { + let summary = text_field(c, "summary") + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| invalid(format!("category `{key}` without a `summary`")))?; + let reference = |k: &str| { + text_field(c, k) + .filter(|s| !s.trim().is_empty()) + .ok_or_else(|| invalid(format!("category `{key}` without `{k}`"))) + }; + let kind = match c.get("kind").and_then(Value::as_str) { + Some("unsupported") => { + let task = reference("task")?; + if !is_task_id(&task) { + return Err(invalid(format!( + "category `{key}`: `{task}` is not a task ID" + ))); + } + CategoryKind::Unsupported { task } + } + Some("divergence") => { + let adr = reference("adr")?; + if adr != DIVERGENCE_ADR { + return Err(invalid(format!( + "category `{key}`: a divergence cites ADR {DIVERGENCE_ADR}, not `{adr}`" + ))); + } + CategoryKind::Divergence { adr } + } + Some("bug") => CategoryKind::Bug { + owner: reference("owner")?, + }, + other => { + return Err(invalid(format!("category `{key}`: unknown kind {other:?}"))); + } + }; + Ok(Category { kind, summary }) +} + +/// A category as the JSON object that holds it (`kind`, its reference, `summary`). +fn category_json(c: &Category) -> Map { + let value = match &c.kind { + CategoryKind::Unsupported { task } => { + json!({"kind": "unsupported", "task": task, "summary": c.summary}) + } + CategoryKind::Divergence { adr } => { + json!({"kind": "divergence", "adr": adr, "summary": c.summary}) + } + CategoryKind::Bug { owner } => { + json!({"kind": "bug", "owner": owner, "summary": c.summary}) + } + }; + match value { + Value::Object(o) => o, + _ => Map::new(), + } +} + +/// The key of a per-key file: its path relative to the directory without `.json`. +fn file_key(rel: &Path) -> Result { + rel.to_str() + .and_then(|s| s.strip_suffix(".json")) + .map(|s| s.replace('\\', "/")) + .ok_or_else(|| invalid(format!("{}: not a JSON file name", rel.display()))) +} + +/// Every category file of `dir` as the categories and the rules of all of them in +/// matching order (see the module docs). +fn load_categories( + dir: &Path, +) -> Result<(BTreeMap, Vec), KnownFailuresError> { + let files = + crate::manifest_dir::read_json_dir(dir).map_err(|source| KnownFailuresError::Io { + path: dir.to_path_buf(), + source, + })?; + let mut categories = BTreeMap::new(); + let mut ordered: Vec<(u32, String, usize, Rule)> = Vec::new(); + for (rel, text) in &files { + let name = rel.display(); + let stem = file_key(rel)?; + let value: Value = + serde_json::from_str(text).map_err(|e| invalid(format!("{name}: {e}")))?; + if !check_format_version(&value) { + return Err(invalid(format!( + "{name}: unsupported or missing `format_version`" + ))); + } + let key = text_field(&value, "category") + .ok_or_else(|| invalid(format!("{name}: missing `category`")))?; + if key != stem { + return Err(invalid(format!( + "{name}: names category `{key}`, not its file" + ))); + } + let category = parse_category(&key, &value)?; + let rules = match value.get("rules") { + None => &[][..], + Some(v) => v + .as_array() + .ok_or_else(|| invalid(format!("{name}: `rules` is not an array")))?, + }; + for (i, rule) in rules.iter().enumerate() { + let mut object = rule + .as_object() + .cloned() + .ok_or_else(|| invalid(format!("{name}: rule {i} is not an object")))?; + let order = object + .remove("order") + .and_then(|o| o.as_u64()) + .and_then(|o| u32::try_from(o).ok()) + .ok_or_else(|| invalid(format!("{name}: rule {i} without a numeric `order`")))?; + object.insert("category".to_owned(), Value::String(key.clone())); + let mut rule = Rule::parse(&Value::Object(object)) + .map_err(|e| invalid(format!("{name}: rule {i}: {e}")))?; + rule.order = order; + ordered.push((order, key.clone(), i, rule)); + } + categories.insert(key, category); + } + ordered.sort_by(|a, b| (a.0, &a.1, a.2).cmp(&(b.0, &b.1, b.2))); + Ok((categories, ordered.into_iter().map(|o| o.3).collect())) +} + impl KnownFailures { /// Default path of the header file in the repo. #[must_use] @@ -244,17 +408,44 @@ impl KnownFailures { crate::tests_dir().join("spark-golden").join(CASES_DIR_NAME) } - /// Load the split baseline: the header file at `path` plus the - /// per-file cases in the [`CASES_DIR_NAME`] directory next to it. When - /// that directory exists it supplies every file's cases (the header's - /// own `cases`, if any, are ignored); otherwise the header's embedded - /// `cases` apply (the legacy single-file form the unit tests use). + /// Default path of the per-category directory in the repo. + #[must_use] + pub fn default_categories_dir() -> PathBuf { + crate::tests_dir() + .join("spark-golden") + .join(CATEGORIES_DIR_NAME) + } + + /// Load the split baseline: the header file at `path`, the categories + /// and rules in the [`CATEGORIES_DIR_NAME`] directory next to it, and the + /// per-file cases in the [`CASES_DIR_NAME`] directory next to it. Each + /// directory, when it exists, supplies everything of its kind (the + /// header's own `categories`, `rules` and `cases`, if any, are ignored); + /// otherwise the header's embedded ones apply (the legacy single-file + /// form the unit tests use). pub fn load(path: &Path) -> Result { let text = std::fs::read_to_string(path).map_err(|source| KnownFailuresError::Io { path: path.to_path_buf(), source, })?; - let mut known = Self::parse(&text)?; + let categories_dir = path + .parent() + .map(|p| p.join(CATEGORIES_DIR_NAME)) + .filter(|d| d.is_dir()); + let mut known = if let Some(dir) = &categories_dir { + let root: Value = serde_json::from_str(&text).map_err(|e| invalid(e.to_string()))?; + let (spark_tag, frozen_at) = parse_meta(&root)?; + let (categories, rules) = load_categories(dir)?; + Self { + spark_tag, + frozen_at, + categories, + rules, + cases: BTreeMap::new(), + } + } else { + Self::parse(&text)? + }; let Some(dir) = path.parent().map(|p| p.join(CASES_DIR_NAME)) else { return Ok(known); }; @@ -313,73 +504,38 @@ impl KnownFailures { Ok(known) } - /// Parse JSON text, validating every category and case. A document - /// without `cases` parses to an empty baseline (the split header form). + /// Parse JSON text (the legacy single-file form), validating every + /// category and case. A document without `cases` parses to a baseline + /// without cases. pub fn parse(text: &str) -> Result { - let bad = |m: String| KnownFailuresError::Invalid(m); + let bad = invalid; let root: Value = serde_json::from_str(text).map_err(|e| bad(e.to_string()))?; - if root.get("format_version").and_then(Value::as_u64) != Some(KNOWN_FAILURES_FORMAT_VERSION) - { - return Err(bad("unsupported or missing `format_version`".into())); - } - let text_field = |v: &Value, k: &str| v.get(k).and_then(Value::as_str).map(str::to_owned); - let spark_tag = - text_field(&root, "spark_tag").ok_or_else(|| bad("missing `spark_tag`".into()))?; - let frozen_at = - text_field(&root, "frozen_at").ok_or_else(|| bad("missing `frozen_at`".into()))?; + let (spark_tag, frozen_at) = parse_meta(&root)?; let mut categories = BTreeMap::new(); for (key, c) in root .get("categories") .and_then(Value::as_object) .ok_or_else(|| bad("missing `categories` object".into()))? { - let summary = text_field(c, "summary") - .filter(|s| !s.trim().is_empty()) - .ok_or_else(|| bad(format!("category `{key}` without a `summary`")))?; - let reference = |k: &str| { - text_field(c, k) - .filter(|s| !s.trim().is_empty()) - .ok_or_else(|| bad(format!("category `{key}` without `{k}`"))) - }; - let kind = match c.get("kind").and_then(Value::as_str) { - Some("unsupported") => { - let task = reference("task")?; - if !is_task_id(&task) { - return Err(bad(format!("category `{key}`: `{task}` is not a task ID"))); - } - CategoryKind::Unsupported { task } - } - Some("divergence") => { - let adr = reference("adr")?; - if adr != DIVERGENCE_ADR { - return Err(bad(format!( - "category `{key}`: a divergence cites ADR {DIVERGENCE_ADR}, not `{adr}`" - ))); - } - CategoryKind::Divergence { adr } - } - Some("bug") => CategoryKind::Bug { - owner: reference("owner")?, - }, - other => { - return Err(bad(format!("category `{key}`: unknown kind {other:?}"))); - } - }; - categories.insert(key.clone(), Category { kind, summary }); + categories.insert(key.clone(), parse_category(key, c)?); } let mut rules = Vec::new(); - for r in root + for (i, r) in root .get("rules") .and_then(Value::as_array) .ok_or_else(|| bad("missing `rules` array".into()))? + .iter() + .enumerate() { - let rule = Rule::parse(r).map_err(bad)?; + let mut rule = Rule::parse(r).map_err(bad)?; if !categories.contains_key(&rule.category) { return Err(bad(format!( "a rule names the unknown category `{}`", rule.category ))); } + // The document order is the match order; gaps leave room to insert. + rule.order = u32::try_from((i + 1) * 10).map_err(|e| bad(e.to_string()))?; rules.push(rule); } let mut cases = BTreeMap::new(); @@ -425,27 +581,14 @@ impl KnownFailures { header } - /// The header JSON: everything but the per-file cases (the on-disk - /// `known-failures.json`). + /// The legacy single-file header JSON: everything but the per-file + /// cases, in one document. #[must_use] pub fn header_json(&self) -> Value { let categories: Map = self .categories .iter() - .map(|(k, c)| { - let v = match &c.kind { - CategoryKind::Unsupported { task } => { - json!({"kind": "unsupported", "task": task, "summary": c.summary}) - } - CategoryKind::Divergence { adr } => { - json!({"kind": "divergence", "adr": adr, "summary": c.summary}) - } - CategoryKind::Bug { owner } => { - json!({"kind": "bug", "owner": owner, "summary": c.summary}) - } - }; - (k.clone(), v) - }) + .map(|(k, c)| (k.clone(), Value::Object(category_json(c)))) .collect(); json!({ "format_version": KNOWN_FAILURES_FORMAT_VERSION, @@ -456,10 +599,54 @@ impl KnownFailures { }) } - /// The header file text: pretty JSON with a trailing newline. + /// The on-disk header file text (`known-failures.json`): the format + /// version, the Spark tag and the freeze date; pretty JSON with a + /// trailing newline. + #[must_use] + pub fn meta_text(&self) -> String { + let doc = json!({ + "format_version": KNOWN_FAILURES_FORMAT_VERSION, + "spark_tag": self.spark_tag, + "frozen_at": self.frozen_at, + }); + serde_json::to_string_pretty(&doc).unwrap_or_default() + "\n" + } + + /// One category's file (`known-failures-categories/.json`): the + /// category and the rules that assign it, each with its `order`; `None` + /// for a key that is not a category. #[must_use] - pub fn header_text(&self) -> String { - serde_json::to_string_pretty(&self.header_json()).unwrap_or_default() + "\n" + pub fn category_file_json(&self, key: &str) -> Option { + let category = self.categories.get(key)?; + let rules: Vec = self + .rules + .iter() + .filter(|r| r.category == key) + .map(|r| { + let mut rule = r.to_json().clone(); + if let Some(o) = rule.as_object_mut() { + o.remove("category"); + o.insert("order".to_owned(), json!(r.order)); + } + rule + }) + .collect(); + let mut doc = category_json(category); + doc.insert( + "format_version".to_owned(), + json!(KNOWN_FAILURES_FORMAT_VERSION), + ); + doc.insert("category".to_owned(), json!(key)); + doc.insert("rules".to_owned(), Value::Array(rules)); + Some(Value::Object(doc)) + } + + /// The category file text: pretty JSON with a trailing newline, or + /// `None` for a key that is not a category. + #[must_use] + pub fn category_file_text(&self, key: &str) -> Option { + self.category_file_json(key) + .map(|doc| serde_json::to_string_pretty(&doc).unwrap_or_default() + "\n") } /// One golden file's indices as `category -> sorted indices`, the value @@ -561,29 +748,39 @@ impl KnownFailures { } /// [`Self::update_with_set`] on the file at `path`, rewriting the header - /// and syncing the per-file cases directory next to it (only changed - /// files are rewritten; case files with no failures left are deleted); - /// `Err` (and nothing written) when a failure matches no rule. + /// and syncing the per-category and per-file cases directories next to + /// it (only changed files are rewritten; category and case files with + /// nothing left are deleted); `Err` (and nothing written) when a failure + /// matches no rule. pub fn update_file(path: &Path, run: &RunResult, set: &GoldenSet) -> Result { let mut known = Self::load(path).map_err(|e| e.to_string())?; let up = known.update_with_set(run, set)?; if !up.unclassified.is_empty() { let mut msg = format!( - "{} failure(s) match no rule; add a rule or category to {} (nothing written):\n", + "{} failure(s) match no rule; add a rule or category under {} (nothing written):\n", up.unclassified.len(), - path.display() + path.with_file_name(CATEGORIES_DIR_NAME).display() ); for d in &up.unclassified { msg.push_str(&format!(" {}#{}: {}\n", d.file, d.index, d.detail)); } return Err(msg); } - crate::manifest_dir::write_if_changed(path, &known.header_text()) + crate::manifest_dir::write_if_changed(path, &known.meta_text()) .map_err(|e| format!("{}: {e}", path.display()))?; - let dir = path + let parent = path .parent() - .map(|p| p.join(CASES_DIR_NAME)) .ok_or_else(|| format!("{}: no parent directory", path.display()))?; + let mut categories = BTreeMap::new(); + for key in known.categories.keys() { + let text = known + .category_file_text(key) + .ok_or_else(|| format!("category `{key}` vanished while writing"))?; + categories.insert(PathBuf::from(format!("{key}.json")), text); + } + crate::manifest_dir::sync_json_dir(&parent.join(CATEGORIES_DIR_NAME), &categories) + .map_err(|e| e.to_string())?; + let dir = parent.join(CASES_DIR_NAME); let mut wanted = BTreeMap::new(); for file in known.cases.keys() { let text = known @@ -1116,25 +1313,46 @@ mod tests { Ok(()) } - #[test] - fn header_and_case_files_round_trip_through_the_split_form() -> R { - let k = KnownFailures::parse(DOC)?; - // The header alone parses, with the same categories and rules but no cases. - let header = KnownFailures::parse(&k.header_text())?; - assert!(header.cases.is_empty()); - assert_eq!(header.categories, k.categories); - assert_eq!(header.rules, k.rules); - assert!(!k.header_text().contains("cases"), "{}", k.header_text()); - // Header plus one case file per listed file loads back to the same baseline. - let dir = tempfile::tempdir()?; - let path = dir.path().join("known.json"); - std::fs::write(&path, k.header_text())?; - let cases = dir.path().join(super::CASES_DIR_NAME); + /// Writes the split on-disk form of `k` into `dir`: `known.json`, one file per category and + /// one per golden file. + fn write_split(dir: &Path, k: &KnownFailures) -> R { + std::fs::write(dir.join("known.json"), k.meta_text())?; + let categories = dir.join(super::CATEGORIES_DIR_NAME); + std::fs::create_dir_all(&categories)?; + for key in k.categories.keys() { + let text = k.category_file_text(key).ok_or("missing category file")?; + std::fs::write(categories.join(format!("{key}.json")), text)?; + } + let cases = dir.join(super::CASES_DIR_NAME); for file in k.cases.keys() { let text = k.case_file_text(file).ok_or("missing case file")?; std::fs::create_dir_all(cases.join(file).parent().ok_or("no parent")?)?; std::fs::write(cases.join(format!("{file}.json")), text)?; } + Ok(()) + } + + #[test] + fn header_category_and_case_files_round_trip_through_the_split_form() -> R { + let k = KnownFailures::parse(DOC)?; + // The header holds the version, tag and date only. + let meta = k.meta_text(); + assert!( + ["categories", "rules", "cases"] + .iter() + .all(|w| !meta.contains(w)), + "{meta}" + ); + assert!( + KnownFailures::parse(&meta).is_err(), + "the header alone is not a legacy document" + ); + // Header plus one category file per category and one case file per listed file loads + // back to the same baseline. + let dir = tempfile::tempdir()?; + let path = dir.path().join("known.json"); + write_split(dir.path(), &k)?; + let cases = dir.path().join(super::CASES_DIR_NAME); // A nested golden name lands in a subdirectory, like the real baseline. std::fs::create_dir_all(cases.join("nonansi"))?; std::fs::write( @@ -1155,7 +1373,7 @@ mod tests { let k = KnownFailures::parse(DOC)?; let dir = tempfile::tempdir()?; let path = dir.path().join("known.json"); - std::fs::write(&path, k.header_text())?; + write_split(dir.path(), &k)?; let cases = dir.path().join(super::CASES_DIR_NAME); std::fs::create_dir_all(&cases)?; let write_case = |name: &str, text: &str| -> R { @@ -1187,6 +1405,121 @@ mod tests { Ok(()) } + fn category_dir( + rules: &[(&str, &str)], + ) -> Result> { + let dir = tempfile::tempdir()?; + std::fs::write( + dir.path().join("known.json"), + r#"{"format_version":1,"spark_tag":"v4.2.0","frozen_at":"2026-10-22"}"#, + )?; + let categories = dir.path().join(super::CATEGORIES_DIR_NAME); + std::fs::create_dir_all(&categories)?; + for (name, text) in rules { + std::fs::write(categories.join(name), text)?; + } + Ok(dir) + } + + #[test] + fn rules_match_in_global_order_across_category_files() -> R { + let file = |key: &str, orders: &[u32]| { + let rules: Vec = orders + .iter() + .map(|o| format!(r#"{{"order":{o},"detail":"x{o}"}}"#)) + .collect(); + ( + format!("{key}.json"), + format!( + r#"{{"format_version":1,"category":"{key}","kind":"bug","owner":"o", + "summary":"s","rules":[{}]}}"#, + rules.join(",") + ), + ) + }; + let (a, b) = (file("a", &[30, 5]), file("b", &[10, 30])); + let dir = category_dir(&[(&a.0, &a.1), (&b.0, &b.1)])?; + let k = KnownFailures::load(&dir.path().join("known.json"))?; + // Ascending `order`; equal orders go by category key, then position. + let got: Vec<(&str, u32)> = k + .rules + .iter() + .map(|r| (r.category.as_str(), r.order)) + .collect(); + assert_eq!(got, [("a", 5), ("b", 10), ("a", 30), ("b", 30)]); + assert!(k.cases.is_empty()); + Ok(()) + } + + #[test] + fn category_files_must_name_their_category_and_number_their_rules() -> R { + let ok = r#"{"format_version":1,"category":"a","kind":"bug","owner":"o","summary":"s", + "rules":[{"order":1,"command":true}]}"#; + let cases = [ + ( + ok.replace(r#""category":"a""#, r#""category":"b""#), + "not its file", + ), + ( + ok.replace(r#""format_version":1"#, r#""format_version":2"#), + "format_version", + ), + (ok.replace(r#""order":1,"#, ""), "without a numeric `order`"), + ( + ok.replace(r#""order":1"#, r#""order":-1"#), + "without a numeric `order`", + ), + ( + ok.replace(r#"{"order":1,"command":true}"#, r#"{"order":1}"#), + "no condition", + ), + ( + ok.replace(r#""kind":"bug""#, r#""kind":"later""#), + "unknown kind", + ), + (ok.replace(r#""owner":"o","#, ""), "without `owner`"), + ( + ok.replace(r#""rules":[{"order":1,"command":true}]"#, r#""rules":{}"#), + "not an array", + ), + ]; + for (text, msg) in cases { + let dir = category_dir(&[("a.json", &text)])?; + let err = KnownFailures::load(&dir.path().join("known.json")).unwrap_err(); + assert!(err.to_string().contains(msg), "{msg}: {err}"); + } + let dir = category_dir(&[("a.json", ok)])?; + assert_eq!( + KnownFailures::load(&dir.path().join("known.json"))? + .rules + .len(), + 1 + ); + Ok(()) + } + + /// The checked-in baseline is exactly what update mode writes, so a hand-edited or stale + /// per-key file fails here without running the whole golden suite. + #[test] + fn the_checked_in_baseline_is_in_canonical_form() -> R { + let path = KnownFailures::default_path(); + let known = KnownFailures::load(&path)?; + assert_eq!(std::fs::read_to_string(&path)?, known.meta_text()); + let on_disk = + |dir: PathBuf, key: &str| std::fs::read_to_string(dir.join(format!("{key}.json"))); + for key in known.categories.keys() { + let want = known.category_file_text(key).ok_or("missing category")?; + let got = on_disk(KnownFailures::default_categories_dir(), key)?; + assert_eq!(got, want, "category `{key}` is not in canonical form"); + } + for file in known.cases.keys() { + let want = known.case_file_text(file).ok_or("missing case file")?; + let got = on_disk(KnownFailures::default_cases_dir(), file)?; + assert_eq!(got, want, "case file `{file}` is not in canonical form"); + } + Ok(()) + } + #[test] fn update_file_writes_the_split_and_rewrites_nothing_twice() -> R { let dir = tempfile::tempdir()?; @@ -1201,19 +1534,29 @@ mod tests { ..RunResult::default() }; KnownFailures::update_file(&path, &fewer, &set)?; - // The header holds no cases; each listed file has its own case file. + // The header holds no cases, categories or rules; each category and each listed file + // has its own file. let header: Value = serde_json::from_str(&std::fs::read_to_string(&path)?).map_err(|e| e.to_string())?; - assert!(header.get("cases").is_none(), "{header}"); + for key in ["cases", "categories", "rules"] { + assert!(header.get(key).is_none(), "{header}"); + } let cases = dir.path().join(super::CASES_DIR_NAME); + let categories = dir.path().join(super::CATEGORIES_DIR_NAME); assert!(cases.join("a.json").is_file()); assert!(cases.join("b.json").is_file()); - assert_eq!(KnownFailures::load(&path)?.listed(), 3); + for key in ["ddl", "sum", "text"] { + assert!(categories.join(format!("{key}.json")).is_file(), "{key}"); + } + let loaded = KnownFailures::load(&path)?; + assert_eq!(loaded.listed(), 3); + assert_eq!(loaded.rules.len(), 2); // A second update changes no bytes and touches no file. let before: BTreeMap, std::time::SystemTime)> = ["known.json"] .into_iter() .map(|n| dir.path().join(n)) .chain([cases.join("a.json"), cases.join("b.json")]) + .chain(["ddl", "sum", "text"].map(|k| categories.join(format!("{k}.json")))) .map(|p| { Ok(( p.to_string_lossy().into_owned(), @@ -1231,6 +1574,19 @@ mod tests { "{name} was touched" ); } + // A category with no statements left loses its file, and only its file. + let without_b = RunResult { + files: vec![file( + "a", + vec![case(0, false), case(1, true), case(3, false)], + )], + ..RunResult::default() + }; + KnownFailures::update_file(&path, &without_b, &set)?; + assert!(!categories.join("text.json").exists()); + assert!(categories.join("ddl.json").is_file()); + assert!(!cases.join("b.json").exists()); + assert_eq!(KnownFailures::load(&path)?.categories.len(), 2); Ok(()) } } diff --git a/crates/krishiv-testkit/src/manifest_dir.rs b/crates/krishiv-testkit/src/manifest_dir.rs index 4b8142bd3..09a679bab 100644 --- a/crates/krishiv-testkit/src/manifest_dir.rs +++ b/crates/krishiv-testkit/src/manifest_dir.rs @@ -2,12 +2,13 @@ //! Split test manifests: one small JSON file per key. //! //! The analysis manifests (`tests/fixtures/analysis/`) and the golden -//! baseline (`tests/spark-golden/known-failures-cases/`) each used to be a -//! single JSON file, so any two PRs touching different functions, coercion -//! rules or golden files conflicted on the same lines. They are now +//! baseline (`tests/spark-golden/known-failures-cases/` and +//! `known-failures-categories/`) each used to be a single JSON file, so any +//! two PRs touching different functions, coercion rules, golden files or +//! failure categories conflicted on the same lines. They are now //! directories with one file per key (function name, coercion input file, -//! golden input file); two PRs touching different keys never touch the same -//! file. The tests read the whole directory (sorted, so deterministic); +//! golden input file, category); two PRs touching different keys never touch +//! the same file. The tests read the whole directory (sorted, so deterministic); //! totals and counts are computed from what they read, never stored. //! //! [`sync_json_dir`] is the update mode (`KRISHIV_UPDATE_MANIFEST=1`, the diff --git a/crates/krishiv-testkit/tests/golden_ratchet.rs b/crates/krishiv-testkit/tests/golden_ratchet.rs index 63093892b..a257081af 100644 --- a/crates/krishiv-testkit/tests/golden_ratchet.rs +++ b/crates/krishiv-testkit/tests/golden_ratchet.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 //! The Spark golden-file ratchet: the whole vendored Spark 4.2 `SQLQueryTestSuite` runs through //! `krishiv-session`, and the statements that do not pass must be exactly those of the frozen -//! baseline `tests/spark-golden/known-failures.json`. A statement that newly fails is a +//! baseline `tests/spark-golden/known-failures.json` (with its per-category and per-file +//! directories). A statement that newly fails is a //! regression; a listed statement that passes must leave the list (it only shrinks). Refresh //! the list with `cargo run -p krishiv-testkit --bin golden-report -- --engine session //! --ratchet update`. diff --git a/tests/spark-golden/known-failures-categories/cast-matrix.json b/tests/spark-golden/known-failures-categories/cast-matrix.json new file mode 100644 index 000000000..4acb574b2 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/cast-matrix.json @@ -0,0 +1,19 @@ +{ + "category": "cast-matrix", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature (functions of timestamps in a session time zone|casts that read a session time zone|CAST from )", + "order": 280 + }, + { + "family": [ + "conversion" + ], + "order": 410 + } + ], + "summary": "the full cast matrix: casts not in P2a, cast-named functions, region time zones", + "task": "P2-T021b" +} diff --git a/tests/spark-golden/known-failures-categories/coercion.json b/tests/spark-golden/known-failures-categories/coercion.json new file mode 100644 index 000000000..45bd28f5f --- /dev/null +++ b/tests/spark-golden/known-failures-categories/coercion.json @@ -0,0 +1,13 @@ +{ + "category": "coercion", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature implicit string casts", + "order": 290 + } + ], + "summary": "full Spark type coercion (implicit string casts with ANSI off)", + "task": "P2-T071d" +} diff --git a/tests/spark-golden/known-failures-categories/datetime.json b/tests/spark-golden/known-failures-categories/datetime.json new file mode 100644 index 000000000..778918eba --- /dev/null +++ b/tests/spark-golden/known-failures-categories/datetime.json @@ -0,0 +1,28 @@ +{ + "category": "datetime", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "WRONG_NUM_ARGS.*`(date_add|dateadd|datediff|date_diff)`|Cannot resolve routine `(timestampadd|timestampdiff|timediff)`", + "order": 120 + }, + { + "detail": "INVALID_TYPED_LITERAL", + "file": "^postgreSQL/date$", + "order": 130 + }, + { + "detail": "The feature (TIMESTAMP|TIMESTAMP_NTZ|TIMESTAMP_LTZ) literals|The feature date and timestamp arithmetic", + "order": 270 + }, + { + "family": [ + "datetime" + ], + "order": 400 + } + ], + "summary": "datetime functions and literals beyond P2a (TIMESTAMP literals, date/timestamp arithmetic, patterns, unit-argument forms)", + "task": "P2-T025" +} diff --git a/tests/spark-golden/known-failures-categories/depends-on-command.json b/tests/spark-golden/known-failures-categories/depends-on-command.json new file mode 100644 index 000000000..4e9f992b0 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/depends-on-command.json @@ -0,0 +1,17 @@ +{ + "category": "depends-on-command", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "depends": true, + "order": 50 + }, + { + "detail": "TABLE_OR_VIEW_NOT_FOUND.*`(tab_renamed)`", + "order": 90 + } + ], + "summary": "queries that read a table, view, function or variable an earlier command of the file creates (the command failed, so the query fails or reads the shared fixture of the same name)", + "task": "P2-T179" +} diff --git a/tests/spark-golden/known-failures-categories/files.json b/tests/spark-golden/known-failures-categories/files.json new file mode 100644 index 000000000..00b30a9b6 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/files.json @@ -0,0 +1,13 @@ +{ + "category": "files", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature (glob patterns|reading the subdirectory)", + "order": 330 + } + ], + "summary": "partitioned and glob reads of directories", + "task": "P2-T082b" +} diff --git a/tests/spark-golden/known-failures-categories/fn-aggregate.json b/tests/spark-golden/known-failures-categories/fn-aggregate.json new file mode 100644 index 000000000..00fb9696f --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-aggregate.json @@ -0,0 +1,20 @@ +{ + "category": "fn-aggregate", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature WITHIN GROUP", + "order": 370 + }, + { + "family": [ + "agg", + "sketch" + ], + "order": 420 + } + ], + "summary": "aggregate and sketch functions beyond P2a, WITHIN GROUP", + "task": "P2-T032" +} diff --git a/tests/spark-golden/known-failures-categories/fn-array.json b/tests/spark-golden/known-failures-categories/fn-array.json new file mode 100644 index 000000000..45d51aaae --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-array.json @@ -0,0 +1,16 @@ +{ + "category": "fn-array", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "family": [ + "array", + "collection" + ], + "order": 430 + } + ], + "summary": "array and collection functions", + "task": "P2-T028" +} diff --git a/tests/spark-golden/known-failures-categories/fn-json.json b/tests/spark-golden/known-failures-categories/fn-json.json new file mode 100644 index 000000000..50e11c804 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-json.json @@ -0,0 +1,15 @@ +{ + "category": "fn-json", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "family": [ + "json" + ], + "order": 440 + } + ], + "summary": "JSON functions", + "task": "P2-T031" +} diff --git a/tests/spark-golden/known-failures-categories/fn-misc.json b/tests/spark-golden/known-failures-categories/fn-misc.json new file mode 100644 index 000000000..03b0bfd74 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-misc.json @@ -0,0 +1,18 @@ +{ + "category": "fn-misc", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "family": [ + "misc", + "vector", + "hash", + "conditional" + ], + "order": 460 + } + ], + "summary": "misc functions (typeof, uuid, rand, raise_error, vector, current_*)", + "task": "P2-T035" +} diff --git a/tests/spark-golden/known-failures-categories/fn-regex.json b/tests/spark-golden/known-failures-categories/fn-regex.json new file mode 100644 index 000000000..8a6722fce --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-regex.json @@ -0,0 +1,17 @@ +{ + "category": "fn-regex", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature the operator rlike", + "order": 190 + }, + { + "detail": "(The feature function |Cannot resolve routine `)(?i)(regexp\\w*|rlike|split)\\b", + "order": 380 + } + ], + "summary": "regular-expression functions and operators", + "task": "P2-T024" +} diff --git a/tests/spark-golden/known-failures-categories/fn-string.json b/tests/spark-golden/known-failures-categories/fn-string.json new file mode 100644 index 000000000..096b0c915 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-string.json @@ -0,0 +1,20 @@ +{ + "category": "fn-string", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature the operator ilike", + "order": 180 + }, + { + "family": [ + "string", + "math" + ], + "order": 390 + } + ], + "summary": "string and math-tail functions not in P2a (hex, conv, bin, width_bucket, ...)", + "task": "P2-T023" +} diff --git a/tests/spark-golden/known-failures-categories/fn-url-xml-csv.json b/tests/spark-golden/known-failures-categories/fn-url-xml-csv.json new file mode 100644 index 000000000..cd03d3494 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/fn-url-xml-csv.json @@ -0,0 +1,18 @@ +{ + "category": "fn-url-xml-csv", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "family": [ + "url", + "xml", + "csv", + "protobuf" + ], + "order": 480 + } + ], + "summary": "URL, XML, CSV and protobuf functions", + "task": "P2-T038" +} diff --git a/tests/spark-golden/known-failures-categories/generators.json b/tests/spark-golden/known-failures-categories/generators.json new file mode 100644 index 000000000..894810be1 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/generators.json @@ -0,0 +1,20 @@ +{ + "category": "generators", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature (multi-name aliases|LATERAL VIEW)", + "order": 350 + }, + { + "family": [ + "generator", + "table" + ], + "order": 450 + } + ], + "summary": "generators: explode, LATERAL VIEW, multi-name aliases, stack, inline", + "task": "P2-T034" +} diff --git a/tests/spark-golden/known-failures-categories/grouping-sets.json b/tests/spark-golden/known-failures-categories/grouping-sets.json new file mode 100644 index 000000000..06e804dac --- /dev/null +++ b/tests/spark-golden/known-failures-categories/grouping-sets.json @@ -0,0 +1,13 @@ +{ + "category": "grouping-sets", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature ROLLUP", + "order": 310 + } + ], + "summary": "ROLLUP, CUBE, GROUPING SETS, grouping()/grouping_id()", + "task": "P2-T053b" +} diff --git a/tests/spark-golden/known-failures-categories/hof.json b/tests/spark-golden/known-failures-categories/hof.json new file mode 100644 index 000000000..91e65a8fd --- /dev/null +++ b/tests/spark-golden/known-failures-categories/hof.json @@ -0,0 +1,19 @@ +{ + "category": "hof", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature lambda functions", + "order": 360 + }, + { + "family": [ + "lambda" + ], + "order": 470 + } + ], + "summary": "lambda functions and higher-order functions", + "task": "P2-T036" +} diff --git a/tests/spark-golden/known-failures-categories/legacy-configs.json b/tests/spark-golden/known-failures-categories/legacy-configs.json new file mode 100644 index 000000000..645074f66 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/legacy-configs.json @@ -0,0 +1,24 @@ +{ + "category": "legacy-configs", + "format_version": 1, + "kind": "bug", + "owner": "config map follow-up of P2-T004", + "rules": [ + { + "file": "^cte-(legacy|nonlegacy)$", + "order": 20 + }, + { + "detail": "^rows differ", + "file": "literals$", + "order": 30, + "query": "\\\"S\\\"\\\"par|'S''par" + }, + { + "detail": "^no error|WRONG_NUM_ARGS", + "file": "^count$", + "order": 40 + } + ], + "summary": "legacy behaviour flags the session ignores instead of rejecting (ctePrecedencePolicy, consecutiveStringLiterals, allowStarWithSingleTableIdentifierInCount, allowParameterlessCount)" +} diff --git a/tests/spark-golden/known-failures-categories/nested-interval-types.json b/tests/spark-golden/known-failures-categories/nested-interval-types.json new file mode 100644 index 000000000..401801bf9 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/nested-interval-types.json @@ -0,0 +1,21 @@ +{ + "category": "nested-interval-types", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature (INTERVAL literals|interval data types|nested data types|row and struct constructors|subscripts|field access|char/varchar)", + "order": 220 + }, + { + "detail": "INVALID_EXTRACT_BASE_FIELD_TYPE|interval literals in physical plans", + "order": 230 + }, + { + "detail": "TABLE_OR_VIEW_NOT_FOUND.*`(arraydata|mapdata)`", + "order": 240 + } + ], + "summary": "nested types, intervals, CHAR/VARCHAR, struct constructors, subscripts and field access", + "task": "P2-T001b" +} diff --git a/tests/spark-golden/known-failures-categories/object-stores.json b/tests/spark-golden/known-failures-categories/object-stores.json new file mode 100644 index 000000000..ebc4180a8 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/object-stores.json @@ -0,0 +1,13 @@ +{ + "category": "object-stores", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature paths with the scheme", + "order": 340 + } + ], + "summary": "remote paths (https and object stores)", + "task": "P2-T084" +} diff --git a/tests/spark-golden/known-failures-categories/output-names.json b/tests/spark-golden/known-failures-categories/output-names.json new file mode 100644 index 000000000..8b92cb231 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/output-names.json @@ -0,0 +1,13 @@ +{ + "category": "output-names", + "format_version": 1, + "kind": "bug", + "owner": "analyzer output-name follow-up of P2-T071a", + "rules": [ + { + "detail": "^schema ", + "order": 500 + } + ], + "summary": "auto-generated column names differ from Spark's toPrettySQL (Java float/decimal text, default arguments, TRIM/CEIL/LOG/mod spellings, +year dates)" +} diff --git a/tests/spark-golden/known-failures-categories/parser-pipe-collate.json b/tests/spark-golden/known-failures-categories/parser-pipe-collate.json new file mode 100644 index 000000000..190286710 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/parser-pipe-collate.json @@ -0,0 +1,17 @@ +{ + "category": "parser-pipe-collate", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "order": 150, + "query": "(?i)\\|>|\\bcollat(e|ion)\\b|\\|\\s*(select|where|aggregate|extend|set|drop|as|join|union|order|limit|pivot)\\b" + }, + { + "file": "^(pipe-operators|collations)", + "order": 160 + } + ], + "summary": "pipe syntax |> and COLLATE / collations", + "task": "P2-T075b" +} diff --git a/tests/spark-golden/known-failures-categories/parser-tail.json b/tests/spark-golden/known-failures-categories/parser-tail.json new file mode 100644 index 000000000..66547c389 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/parser-tail.json @@ -0,0 +1,18 @@ +{ + "category": "parser-tail", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "file": "^transform$", + "order": 110 + }, + { + "detail": "^parse failed", + "order": 170, + "query": "(?i)\\b(transform|identifier|tablesample|nearest)\\b|\\btable\\s*\\(|:\\w" + } + ], + "summary": "TRANSFORM, IDENTIFIER(), TABLE(...) arguments, TABLESAMPLE, NEAREST BY, variant paths", + "task": "P2-T075c" +} diff --git a/tests/spark-golden/known-failures-categories/query-features.json b/tests/spark-golden/known-failures-categories/query-features.json new file mode 100644 index 000000000..a1c3bc95a --- /dev/null +++ b/tests/spark-golden/known-failures-categories/query-features.json @@ -0,0 +1,17 @@ +{ + "category": "query-features", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature (VALUES|table-valued functions|NATURAL JOIN|recursive CTEs|LATERAL subqueries|LATERAL joins|PIVOT and UNPIVOT|OFFSET|SELECT \\* EXCEPT|qualified function names|named function arguments|LIKE ANY|FROM with several SELECT|QUALIFY|aggregate FILTER|DISTINCT aggregates over different|JOIN \\.\\.\\. USING)", + "order": 200 + }, + { + "detail": "UNRESOLVED_COLUMN.*`ALL`", + "order": 210 + } + ], + "summary": "query features the analyzer or planner lacks: inline VALUES, table-valued functions, NATURAL and USING joins, recursive CTEs, LATERAL, PIVOT/UNPIVOT, OFFSET, GROUP BY ALL, SELECT * EXCEPT, QUALIFY, named and qualified function names, LIKE ANY/ALL, aggregate FILTER, multi-SELECT FROM", + "task": "P2-T180" +} diff --git a/tests/spark-golden/known-failures-categories/sql-commands.json b/tests/spark-golden/known-failures-categories/sql-commands.json new file mode 100644 index 000000000..20efeea66 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/sql-commands.json @@ -0,0 +1,17 @@ +{ + "category": "sql-commands", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "command": true, + "order": 10 + }, + { + "order": 80, + "query": "(?i)^\\s*explain\\s+(extended\\s+|formatted\\s+)?(desc|describe|show|create|insert|set)\\b" + } + ], + "summary": "SQL commands (DDL, DML, SET/RESET, DESCRIBE/SHOW, session variables, EXECUTE IMMEDIATE, SQL functions): not parsed or executed by the session yet", + "task": "P2-T179" +} diff --git a/tests/spark-golden/known-failures-categories/subqueries.json b/tests/spark-golden/known-failures-categories/subqueries.json new file mode 100644 index 000000000..b5f397226 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/subqueries.json @@ -0,0 +1,17 @@ +{ + "category": "subqueries", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "list-query|UNRESOLVED_COLUMN.*`a`\\.`four`", + "order": 140 + }, + { + "detail": "The feature correlated scalar subqueries", + "order": 320 + } + ], + "summary": "general subquery unnesting (correlated shapes beyond TPC-H)", + "task": "P2-T115" +} diff --git a/tests/spark-golden/known-failures-categories/test-udfs.json b/tests/spark-golden/known-failures-categories/test-udfs.json new file mode 100644 index 000000000..0ed074547 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/test-udfs.json @@ -0,0 +1,17 @@ +{ + "category": "test-udfs", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "file": "^(udf|udaf|udtf)/", + "order": 60 + }, + { + "detail": "Cannot resolve routine `(udf|udaf|vol|booleq|boolne)`", + "order": 70 + } + ], + "summary": "the suite's Python/Scala test UDFs (udf/, udaf/, udtf/ folders)", + "task": "P2-T059" +} diff --git a/tests/spark-golden/known-failures-categories/time-geo-types.json b/tests/spark-golden/known-failures-categories/time-geo-types.json new file mode 100644 index 000000000..63e3f8192 --- /dev/null +++ b/tests/spark-golden/known-failures-categories/time-geo-types.json @@ -0,0 +1,23 @@ +{ + "category": "time-geo-types", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "file": "^st-functions$", + "order": 100 + }, + { + "detail": "The feature (TIME literals|the time data type)", + "order": 260 + }, + { + "family": [ + "st" + ], + "order": 490 + } + ], + "summary": "Spark 4.x TIME type and literals; GEOMETRY/GEOGRAPHY types and ST functions", + "task": "P2-T181" +} diff --git a/tests/spark-golden/known-failures-categories/variant-type.json b/tests/spark-golden/known-failures-categories/variant-type.json new file mode 100644 index 000000000..c697b7d4e --- /dev/null +++ b/tests/spark-golden/known-failures-categories/variant-type.json @@ -0,0 +1,13 @@ +{ + "category": "variant-type", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature the variant data type", + "order": 250 + } + ], + "summary": "the VARIANT type", + "task": "P2-T001c" +} diff --git a/tests/spark-golden/known-failures-categories/window.json b/tests/spark-golden/known-failures-categories/window.json new file mode 100644 index 000000000..0aa4eeefe --- /dev/null +++ b/tests/spark-golden/known-failures-categories/window.json @@ -0,0 +1,13 @@ +{ + "category": "window", + "format_version": 1, + "kind": "unsupported", + "rules": [ + { + "detail": "The feature (window functions|WINDOW clauses)", + "order": 300 + } + ], + "summary": "window functions and WINDOW clauses", + "task": "P2-T055" +} diff --git a/tests/spark-golden/known-failures.json b/tests/spark-golden/known-failures.json index 28877c582..cee16562e 100644 --- a/tests/spark-golden/known-failures.json +++ b/tests/spark-golden/known-failures.json @@ -1,386 +1,5 @@ { - "categories": { - "cast-matrix": { - "kind": "unsupported", - "summary": "the full cast matrix: casts not in P2a, cast-named functions, region time zones", - "task": "P2-T021b" - }, - "coercion": { - "kind": "unsupported", - "summary": "full Spark type coercion (implicit string casts with ANSI off)", - "task": "P2-T071d" - }, - "datetime": { - "kind": "unsupported", - "summary": "datetime functions and literals beyond P2a (TIMESTAMP literals, date/timestamp arithmetic, patterns, unit-argument forms)", - "task": "P2-T025" - }, - "depends-on-command": { - "kind": "unsupported", - "summary": "queries that read a table, view, function or variable an earlier command of the file creates (the command failed, so the query fails or reads the shared fixture of the same name)", - "task": "P2-T179" - }, - "files": { - "kind": "unsupported", - "summary": "partitioned and glob reads of directories", - "task": "P2-T082b" - }, - "fn-aggregate": { - "kind": "unsupported", - "summary": "aggregate and sketch functions beyond P2a, WITHIN GROUP", - "task": "P2-T032" - }, - "fn-array": { - "kind": "unsupported", - "summary": "array and collection functions", - "task": "P2-T028" - }, - "fn-json": { - "kind": "unsupported", - "summary": "JSON functions", - "task": "P2-T031" - }, - "fn-misc": { - "kind": "unsupported", - "summary": "misc functions (typeof, uuid, rand, raise_error, vector, current_*)", - "task": "P2-T035" - }, - "fn-regex": { - "kind": "unsupported", - "summary": "regular-expression functions and operators", - "task": "P2-T024" - }, - "fn-string": { - "kind": "unsupported", - "summary": "string and math-tail functions not in P2a (hex, conv, bin, width_bucket, ...)", - "task": "P2-T023" - }, - "fn-url-xml-csv": { - "kind": "unsupported", - "summary": "URL, XML, CSV and protobuf functions", - "task": "P2-T038" - }, - "generators": { - "kind": "unsupported", - "summary": "generators: explode, LATERAL VIEW, multi-name aliases, stack, inline", - "task": "P2-T034" - }, - "grouping-sets": { - "kind": "unsupported", - "summary": "ROLLUP, CUBE, GROUPING SETS, grouping()/grouping_id()", - "task": "P2-T053b" - }, - "hof": { - "kind": "unsupported", - "summary": "lambda functions and higher-order functions", - "task": "P2-T036" - }, - "legacy-configs": { - "kind": "bug", - "owner": "config map follow-up of P2-T004", - "summary": "legacy behaviour flags the session ignores instead of rejecting (ctePrecedencePolicy, consecutiveStringLiterals, allowStarWithSingleTableIdentifierInCount, allowParameterlessCount)" - }, - "nested-interval-types": { - "kind": "unsupported", - "summary": "nested types, intervals, CHAR/VARCHAR, struct constructors, subscripts and field access", - "task": "P2-T001b" - }, - "object-stores": { - "kind": "unsupported", - "summary": "remote paths (https and object stores)", - "task": "P2-T084" - }, - "output-names": { - "kind": "bug", - "owner": "analyzer output-name follow-up of P2-T071a", - "summary": "auto-generated column names differ from Spark's toPrettySQL (Java float/decimal text, default arguments, TRIM/CEIL/LOG/mod spellings, +year dates)" - }, - "parser-pipe-collate": { - "kind": "unsupported", - "summary": "pipe syntax |> and COLLATE / collations", - "task": "P2-T075b" - }, - "parser-tail": { - "kind": "unsupported", - "summary": "TRANSFORM, IDENTIFIER(), TABLE(...) arguments, TABLESAMPLE, NEAREST BY, variant paths", - "task": "P2-T075c" - }, - "query-features": { - "kind": "unsupported", - "summary": "query features the analyzer or planner lacks: inline VALUES, table-valued functions, NATURAL and USING joins, recursive CTEs, LATERAL, PIVOT/UNPIVOT, OFFSET, GROUP BY ALL, SELECT * EXCEPT, QUALIFY, named and qualified function names, LIKE ANY/ALL, aggregate FILTER, multi-SELECT FROM", - "task": "P2-T180" - }, - "sql-commands": { - "kind": "unsupported", - "summary": "SQL commands (DDL, DML, SET/RESET, DESCRIBE/SHOW, session variables, EXECUTE IMMEDIATE, SQL functions): not parsed or executed by the session yet", - "task": "P2-T179" - }, - "subqueries": { - "kind": "unsupported", - "summary": "general subquery unnesting (correlated shapes beyond TPC-H)", - "task": "P2-T115" - }, - "test-udfs": { - "kind": "unsupported", - "summary": "the suite's Python/Scala test UDFs (udf/, udaf/, udtf/ folders)", - "task": "P2-T059" - }, - "time-geo-types": { - "kind": "unsupported", - "summary": "Spark 4.x TIME type and literals; GEOMETRY/GEOGRAPHY types and ST functions", - "task": "P2-T181" - }, - "variant-type": { - "kind": "unsupported", - "summary": "the VARIANT type", - "task": "P2-T001c" - }, - "window": { - "kind": "unsupported", - "summary": "window functions and WINDOW clauses", - "task": "P2-T055" - } - }, "format_version": 1, "frozen_at": "2026-10-22", - "rules": [ - { - "category": "sql-commands", - "command": true - }, - { - "category": "legacy-configs", - "file": "^cte-(legacy|nonlegacy)$" - }, - { - "category": "legacy-configs", - "detail": "^rows differ", - "file": "literals$", - "query": "\\\"S\\\"\\\"par|'S''par" - }, - { - "category": "legacy-configs", - "detail": "^no error|WRONG_NUM_ARGS", - "file": "^count$" - }, - { - "category": "depends-on-command", - "depends": true - }, - { - "category": "test-udfs", - "file": "^(udf|udaf|udtf)/" - }, - { - "category": "test-udfs", - "detail": "Cannot resolve routine `(udf|udaf|vol|booleq|boolne)`" - }, - { - "category": "sql-commands", - "query": "(?i)^\\s*explain\\s+(extended\\s+|formatted\\s+)?(desc|describe|show|create|insert|set)\\b" - }, - { - "category": "depends-on-command", - "detail": "TABLE_OR_VIEW_NOT_FOUND.*`(tab_renamed)`" - }, - { - "category": "time-geo-types", - "file": "^st-functions$" - }, - { - "category": "parser-tail", - "file": "^transform$" - }, - { - "category": "datetime", - "detail": "WRONG_NUM_ARGS.*`(date_add|dateadd|datediff|date_diff)`|Cannot resolve routine `(timestampadd|timestampdiff|timediff)`" - }, - { - "category": "datetime", - "detail": "INVALID_TYPED_LITERAL", - "file": "^postgreSQL/date$" - }, - { - "category": "subqueries", - "detail": "list-query|UNRESOLVED_COLUMN.*`a`\\.`four`" - }, - { - "category": "parser-pipe-collate", - "query": "(?i)\\|>|\\bcollat(e|ion)\\b|\\|\\s*(select|where|aggregate|extend|set|drop|as|join|union|order|limit|pivot)\\b" - }, - { - "category": "parser-pipe-collate", - "file": "^(pipe-operators|collations)" - }, - { - "category": "parser-tail", - "detail": "^parse failed", - "query": "(?i)\\b(transform|identifier|tablesample|nearest)\\b|\\btable\\s*\\(|:\\w" - }, - { - "category": "fn-string", - "detail": "The feature the operator ilike" - }, - { - "category": "fn-regex", - "detail": "The feature the operator rlike" - }, - { - "category": "query-features", - "detail": "The feature (VALUES|table-valued functions|NATURAL JOIN|recursive CTEs|LATERAL subqueries|LATERAL joins|PIVOT and UNPIVOT|OFFSET|SELECT \\* EXCEPT|qualified function names|named function arguments|LIKE ANY|FROM with several SELECT|QUALIFY|aggregate FILTER|DISTINCT aggregates over different|JOIN \\.\\.\\. USING)" - }, - { - "category": "query-features", - "detail": "UNRESOLVED_COLUMN.*`ALL`" - }, - { - "category": "nested-interval-types", - "detail": "The feature (INTERVAL literals|interval data types|nested data types|row and struct constructors|subscripts|field access|char/varchar)" - }, - { - "category": "nested-interval-types", - "detail": "INVALID_EXTRACT_BASE_FIELD_TYPE|interval literals in physical plans" - }, - { - "category": "nested-interval-types", - "detail": "TABLE_OR_VIEW_NOT_FOUND.*`(arraydata|mapdata)`" - }, - { - "category": "variant-type", - "detail": "The feature the variant data type" - }, - { - "category": "time-geo-types", - "detail": "The feature (TIME literals|the time data type)" - }, - { - "category": "datetime", - "detail": "The feature (TIMESTAMP|TIMESTAMP_NTZ|TIMESTAMP_LTZ) literals|The feature date and timestamp arithmetic" - }, - { - "category": "cast-matrix", - "detail": "The feature (functions of timestamps in a session time zone|casts that read a session time zone|CAST from )" - }, - { - "category": "coercion", - "detail": "The feature implicit string casts" - }, - { - "category": "window", - "detail": "The feature (window functions|WINDOW clauses)" - }, - { - "category": "grouping-sets", - "detail": "The feature ROLLUP" - }, - { - "category": "subqueries", - "detail": "The feature correlated scalar subqueries" - }, - { - "category": "files", - "detail": "The feature (glob patterns|reading the subdirectory)" - }, - { - "category": "object-stores", - "detail": "The feature paths with the scheme" - }, - { - "category": "generators", - "detail": "The feature (multi-name aliases|LATERAL VIEW)" - }, - { - "category": "hof", - "detail": "The feature lambda functions" - }, - { - "category": "fn-aggregate", - "detail": "The feature WITHIN GROUP" - }, - { - "category": "fn-regex", - "detail": "(The feature function |Cannot resolve routine `)(?i)(regexp\\w*|rlike|split)\\b" - }, - { - "category": "fn-string", - "family": [ - "string", - "math" - ] - }, - { - "category": "datetime", - "family": [ - "datetime" - ] - }, - { - "category": "cast-matrix", - "family": [ - "conversion" - ] - }, - { - "category": "fn-aggregate", - "family": [ - "agg", - "sketch" - ] - }, - { - "category": "fn-array", - "family": [ - "array", - "collection" - ] - }, - { - "category": "fn-json", - "family": [ - "json" - ] - }, - { - "category": "generators", - "family": [ - "generator", - "table" - ] - }, - { - "category": "fn-misc", - "family": [ - "misc", - "vector", - "hash", - "conditional" - ] - }, - { - "category": "hof", - "family": [ - "lambda" - ] - }, - { - "category": "fn-url-xml-csv", - "family": [ - "url", - "xml", - "csv", - "protobuf" - ] - }, - { - "category": "time-geo-types", - "family": [ - "st" - ] - }, - { - "category": "output-names", - "detail": "^schema " - } - ], "spark_tag": "v4.2.0" }