diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..4870508 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,14 @@ +--- +name: Bug report +about: Report a sqlmesh-spec-kit issue +--- + +## Version + +## Command + +## Expected Behavior + +## Actual Behavior + +## Minimal Reproduction diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..e9d902c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,10 @@ +--- +name: Feature request +about: Suggest a sqlmesh-spec-kit improvement +--- + +## Use Case + +## Proposed Behavior + +## SQLMesh Context diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..199c4a1 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ +## Summary + +## SQLMesh Evidence + +- `sqlmesh plan `: +- tests/audits: +- apply/promote evidence: + +## Spec Boundary + +- spec: +- plan: +- tasks: +- files outside approved plan: no diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5adf3da --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,38 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: uv run --extra dev ruff check src tests + - run: uv run --extra dev mypy src tests + - run: uv run --extra dev pytest + - run: uv run --extra dev python -m build --sdist --wheel + - run: uv run --extra dev python -m twine check dist/* + - name: Check sdist exclusions + run: | + python - <<'PY' + import tarfile + from pathlib import Path + sdist = next(Path("dist").glob("*.tar.gz")) + with tarfile.open(sdist) as tar: + names = tar.getnames() + forbidden = [ + name + for name in names + if "/.git/" in name + or name.endswith(("/PLAN.md", "/FOLLOWUP.md", "/QUESTIONS.md", "/BUILD-SUMMARY.md")) + ] + if forbidden: + raise SystemExit(f"forbidden paths in sdist: {forbidden}") + PY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..59f5596 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,28 @@ +name: Release + +on: + release: + types: [published] + +permissions: + contents: read + id-token: write + +jobs: + publish: + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/sqlmesh-spec-kit + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: uv run --extra dev ruff check src tests + - run: uv run --extra dev mypy src tests + - run: uv run --extra dev pytest + - run: uv run --extra dev python -m build --sdist --wheel + - run: uv run --extra dev python -m twine check dist/* + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1d049ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,58 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtualenvs +.venv/ +venv/ +env/ +ENV/ + +# Testing / coverage +.pytest_cache/ +.coverage +.coverage.* +htmlcov/ +coverage.xml +.tox/ +.nox/ + +# Type checking +.mypy_cache/ +.dmypy.json +dmypy.json +.ruff_cache/ + +# IDE +.idea/ +.vscode/ + +# OS +.DS_Store +Thumbs.db + +# Local-only build artifacts +WAITING.md +uv.lock + +# sqlmesh-spec-kit test outputs inside fixtures +tests/fixtures/**/.sqlmesh-specify/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e4282aa --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## 0.1.0 + +- Initial SQLMesh-native bootstrap. +- Added installable `sqlmesh-specify` CLI. +- Added lifecycle, spec, SQLMesh project, doctor, report, Jira, and Confluence support. +- Added templates, commands, agents, skills, engine presets, docs, tutorials, examples, CI, and release workflow. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..cbdec8b --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,13 @@ +# Contributing + +Use Python 3.11 or newer. + +```bash +uv run --extra dev ruff check src tests +uv run --extra dev mypy src tests +uv run --extra dev pytest +uv run --extra dev python -m build --sdist --wheel +uv run --extra dev python -m twine check dist/* +``` + +Engine presets live in `presets//` and may add constitution and plan sections. They must not weaken the base lifecycle gates. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6880e8c --- /dev/null +++ b/README.md @@ -0,0 +1,79 @@ +# sqlmesh-spec-kit + +AI SDLC and durable memory for SQLMesh teams: specs are contracts, SQLMesh plans are deployment evidence, and agents implement only inside approved task boundaries. + +## Install + +After the first PyPI release: + +```bash +uvx --from sqlmesh-spec-kit sqlmesh-specify --help +``` + +Before PyPI release, install directly from this repository: + +```bash +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify --help +``` + +## Initialize + +Run inside an existing SQLMesh project with `config.yaml`, `config.yml`, or `config.py` and a `models/` directory. + +```bash +sqlmesh-specify init analytics --engine duckdb --target . +sqlmesh-specify doctor --target . +sqlmesh-specify validate project --target . +sqlmesh-specify validate sqlmesh --target . +sqlmesh-specify report --target . --format markdown +``` + +Supported engine presets: `duckdb`, `snowflake`, `databricks`, `bigquery`, `trino`, `redshift`, `postgres`, `mysql`, `mssql`, `athena`, `spark`, and `clickhouse`. + +## Verified SQLMesh Example + +The full SDLC flow has been tested against the official SQLMesh examples repository: + +```bash +git clone https://github.com/TobikoData/sqlmesh-examples.git +cd sqlmesh-examples/001_sushi/2_moderate +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify init sushi-moderate --engine duckdb --target . +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify ci --target . +``` + +For the complete spec, plan, tasks, implementation, SQLMesh test, and `sqlmesh plan dev --auto-apply --no-prompts` flow, follow [Tutorial 02](docs/tutorials/02-ship-a-sqlmesh-change.md). + +## Workflow + +1. Draft `spec.md` with EARS acceptance criteria. +2. Human approves the spec by setting status exactly to `approved`. +3. Create `plan.md` with SQLMesh environment, changed models, audits/tests, backfill scope, forward-only/restatement decision, and plan/apply evidence expectations. +4. Human approves the plan by setting status exactly to `approved`. +5. Create `tasks.md` and implement only approved files. +6. Review final diff against `spec.md`, `plan.md`, `tasks.md`, and SQLMesh plan/audit/test evidence. + +Validators do not execute SQLMesh in v0.1. Run `sqlmesh plan `, tests, and audits in your project workflow and attach the evidence to the spec directory or PR. + +## CLI + +```bash +sqlmesh-specify init --engine --target . [--force] +sqlmesh-specify doctor --target . +sqlmesh-specify validate +sqlmesh-specify validate project --target . +sqlmesh-specify validate sqlmesh --target . +sqlmesh-specify report --target . --format markdown +sqlmesh-specify ci --target . +sqlmesh-specify jira pull|attach|create-tasks|sync +sqlmesh-specify confluence pull-page|publish|sync +``` + +## Docs + +- [Getting started](docs/getting-started.md) +- [Methodology](docs/methodology.md) +- [Enterprise CI](docs/enterprise-ci.md) +- [SQLMesh AI SDLC walkthrough](docs/sqlmesh-ai-sdlc-walkthrough.md) +- [Memory and repo hygiene](docs/sqlmesh-memory-and-repo-hygiene.md) +- [Tutorials](docs/tutorials/README.md) +- [Roadmap](ROADMAP.md) diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..1a95745 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,28 @@ +# Roadmap + +## Phase 1: Bootstrap and Parity + +- SQLMesh-native CLI, init, lifecycle validation, doctor, report, CI, Jira, Confluence, docs, templates, commands, agents, skills, examples, and release workflow. +- Validators check project shape without executing SQLMesh. +- Humans provide `sqlmesh plan`, audit, test, apply, and promote evidence. + +## Phase 2: Policy Configuration + +- Add `.sqlmesh-specify/config.yml`. +- Support severity overrides and project-specific policy toggles. + +## Phase 3: Plan/Diff Enforcement + +- Validate changed files stay inside approved `plan.md` file lists. +- Add PR evidence for changed files, AC coverage, SQLMesh plan output, tests, and downstream impact. +- Candidate command: `sqlmesh-specify validate diff --target . --base-ref main`. + +## Phase 4: SQLMesh Artifact Intelligence + +- Compare before/after SQLMesh manifests or state artifacts. +- Detect changed models, downstream children, removed/renamed models, grain changes, missing tests, and semantic-layer impact. + +## Phase 5: Enterprise Integration Hardening + +- Improve Jira/Confluence idempotency, dry-run detail, version tracking, and failure-mode tests. +- Add stronger redaction guidance before external publishing. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b8ec52c --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security + +Do not commit secrets, credentials, customer data, or raw production extracts to spec artifacts. + +Before publishing to Jira or Confluence, redact sensitive SQL, PII samples, environment credentials, and private endpoint names that are not required for review. + +Report security issues privately through the repository security advisory flow. diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..154addd --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,9 @@ +# Support + +Open an issue with: + +- sqlmesh-spec-kit version +- SQLMesh version +- engine preset +- command and output +- minimal repo shape or sanitized spec directory diff --git a/agents/engine-optimizer.md b/agents/engine-optimizer.md new file mode 100644 index 0000000..95d2539 --- /dev/null +++ b/agents/engine-optimizer.md @@ -0,0 +1,10 @@ +# Engine Optimizer + +Reviews engine-specific SQLMesh tradeoffs. + +Responsibilities: + +- check materialization, partitioning, clustering, catalog, and cost decisions for the selected engine +- verify engine preset additions are addressed in `plan.md` +- flag query patterns that are unsafe for the target engine +- produce review notes without editing implementation files diff --git a/agents/governance-reviewer.md b/agents/governance-reviewer.md new file mode 100644 index 0000000..1d31e7e --- /dev/null +++ b/agents/governance-reviewer.md @@ -0,0 +1,10 @@ +# Governance Reviewer + +Reviews privacy, access, and publishing risk. + +Responsibilities: + +- identify PII, regulated fields, and access policy changes +- require masking, redaction, and downstream owner approval when needed +- check Jira and Confluence output for sensitive content before publishing +- block promotion when governance evidence is absent diff --git a/agents/implementation-agent.md b/agents/implementation-agent.md new file mode 100644 index 0000000..8a83ec5 --- /dev/null +++ b/agents/implementation-agent.md @@ -0,0 +1,10 @@ +# Implementation Agent + +Implements approved tasks only. + +Responsibilities: + +- read `spec.md`, `plan.md`, and `tasks.md` +- edit only files listed in the approved plan +- stop on unexpected SQLMesh plan output, failed audits/tests, or unclear AC traceability +- update tasks only after evidence exists diff --git a/agents/review-agent.md b/agents/review-agent.md new file mode 100644 index 0000000..58077cf --- /dev/null +++ b/agents/review-agent.md @@ -0,0 +1,10 @@ +# Review Agent + +Performs final change review. + +Responsibilities: + +- compare final diff to `spec.md`, `plan.md`, `tasks.md`, and SQLMesh evidence +- verify every AC has implementation and evidence +- block files outside the approved plan +- summarize residual risk and missing evidence diff --git a/agents/spec-steward.md b/agents/spec-steward.md new file mode 100644 index 0000000..f84707d --- /dev/null +++ b/agents/spec-steward.md @@ -0,0 +1,10 @@ +# Spec Steward + +Owns spec clarity before implementation. + +Responsibilities: + +- enforce exact lifecycle statuses +- keep acceptance criteria EARS-formatted +- ensure SQLMesh models, environments, audits/tests, and downstream owners are named +- block plans from draft, invalid, or missing-status specs diff --git a/agents/sqlmesh-architect.md b/agents/sqlmesh-architect.md new file mode 100644 index 0000000..3351ca6 --- /dev/null +++ b/agents/sqlmesh-architect.md @@ -0,0 +1,10 @@ +# SQLMesh Architect + +Reviews SQLMesh design before tasks are written. + +Responsibilities: + +- validate model boundaries, snapshot impact, backfill windows, restatements, and forward-only decisions +- ensure `plan.md` lists every file to add, modify, or delete +- require `sqlmesh plan ` evidence for review +- block work when plan output is unclear or risky diff --git a/commands/sqlmesh.analyze.md b/commands/sqlmesh.analyze.md new file mode 100644 index 0000000..77252cd --- /dev/null +++ b/commands/sqlmesh.analyze.md @@ -0,0 +1,19 @@ +# /sqlmesh.analyze + +Analyze a SQLMesh repo or proposed change without editing files. + +Report: + +- SQLMesh config and model inventory +- missing audits/tests +- downstream model and semantic impact +- likely backfill/restatement risk +- gaps in spec, plan, tasks, or SQLMesh evidence + +Useful commands: + +```bash +sqlmesh-specify doctor --target . +sqlmesh-specify validate project --target . +sqlmesh-specify validate sqlmesh --target . +``` diff --git a/commands/sqlmesh.implement-all.md b/commands/sqlmesh.implement-all.md new file mode 100644 index 0000000..f1e26b6 --- /dev/null +++ b/commands/sqlmesh.implement-all.md @@ -0,0 +1,11 @@ +# /sqlmesh.implement-all + +Implement all tasks in `tasks.md` after `plan.md` is exactly `approved`. + +Required loop: + +1. Implement only approved file changes. +2. Run or document the project SQLMesh parse/test/audit commands. +3. Run `sqlmesh plan ` and inspect changed snapshots, backfill scope, restatement scope, and forward-only behavior. +4. Stop on failed tests/audits, unexpected plan output, unclear evidence, or files outside the approved plan. +5. Update task checkboxes only after evidence exists. diff --git a/commands/sqlmesh.implement.md b/commands/sqlmesh.implement.md new file mode 100644 index 0000000..d2419ac --- /dev/null +++ b/commands/sqlmesh.implement.md @@ -0,0 +1,11 @@ +# /sqlmesh.implement + +Implement one approved task. + +Before editing: + +1. Read `spec.md`, `plan.md`, `tasks.md`, and `.sqlmesh-specify/constitution.md`. +2. Confirm `plan.md` status is exactly `approved`. +3. Confirm the target file is listed in the approved plan. + +Stop if SQLMesh plan output, backfill scope, restatement scope, or required tests/audits are unclear. diff --git a/commands/sqlmesh.plan.md b/commands/sqlmesh.plan.md new file mode 100644 index 0000000..c3965c8 --- /dev/null +++ b/commands/sqlmesh.plan.md @@ -0,0 +1,10 @@ +# /sqlmesh.plan + +Create `plan.md` only when `spec.md` status is exactly `approved` or `shipped`. + +1. Read `spec.md`, `.sqlmesh-specify/constitution.md`, and engine additions. +2. Identify target SQLMesh environment, changed models, snapshots, audits, tests, backfill window, and downstream impact. +3. List every file to add, modify, or delete. +4. Map every planned change and test to AC ids. +5. Include the `sqlmesh plan ` command that must be run and the evidence reviewers need. +6. Set status to `proposed` until a human approves it. diff --git a/commands/sqlmesh.review.md b/commands/sqlmesh.review.md new file mode 100644 index 0000000..28fee09 --- /dev/null +++ b/commands/sqlmesh.review.md @@ -0,0 +1,19 @@ +# /sqlmesh.review + +Review a completed SQLMesh change. + +Compare: + +1. final diff +2. `spec.md` +3. `plan.md` +4. `tasks.md` +5. SQLMesh plan/audit/test evidence + +Block when: + +- final files are outside the approved plan +- an AC lacks implementation or evidence +- `sqlmesh plan ` shows unexpected backfill, restatement, or forward-only behavior +- audits/tests failed or are missing +- downstream impact is undocumented diff --git a/commands/sqlmesh.specify.md b/commands/sqlmesh.specify.md new file mode 100644 index 0000000..748e572 --- /dev/null +++ b/commands/sqlmesh.specify.md @@ -0,0 +1,9 @@ +# /sqlmesh.specify + +Create or revise `specs/-/spec.md`. + +1. Read `.sqlmesh-specify/constitution.md` and relevant project conventions. +2. Translate the request into business intent, affected SQLMesh concepts, and downstream impact. +3. Write EARS acceptance criteria only in the Acceptance Criteria section. +4. Set status to `draft` until a human explicitly approves it. +5. Do not write a plan or tasks from an unapproved spec. diff --git a/commands/sqlmesh.tasks.md b/commands/sqlmesh.tasks.md new file mode 100644 index 0000000..a270762 --- /dev/null +++ b/commands/sqlmesh.tasks.md @@ -0,0 +1,9 @@ +# /sqlmesh.tasks + +Create `tasks.md` only when `plan.md` status is exactly `approved`. + +1. Convert the approved plan into bounded implementation tasks. +2. Include tasks for SQLMesh models, audits/tests, `sqlmesh plan dev`, and review evidence. +3. Add stop conditions for unclear plan output, unexpected backfill/restatement, failed audits/tests, or files outside plan. +4. Map every task to AC ids. +5. Set status to `in progress` until all tasks and evidence are complete. diff --git a/docs/brownfield-onboarding.md b/docs/brownfield-onboarding.md new file mode 100644 index 0000000..c14e27d --- /dev/null +++ b/docs/brownfield-onboarding.md @@ -0,0 +1,11 @@ +# Brownfield Onboarding + +1. Run `sqlmesh-specify doctor --target .`. +2. Initialize with the closest engine preset: + +```bash +sqlmesh-specify init analytics --engine snowflake --target . +``` + +3. Capture existing conventions in `.sqlmesh-specify/constitution.md` or skills. +4. Start new work under `specs/-/`; do not retroactively force every old model into a spec. diff --git a/docs/ears-cheatsheet.md b/docs/ears-cheatsheet.md new file mode 100644 index 0000000..7e21e29 --- /dev/null +++ b/docs/ears-cheatsheet.md @@ -0,0 +1,69 @@ +# EARS cheatsheet — testable acceptance criteria for SQLMesh specs + +EARS = **Easy Approach to Requirements Syntax**. Five patterns. Every AC in every spec uses one. + +## The five patterns + +### 1. Ubiquitous — always true + +> The system shall . + +Use for invariants. The thing is always required, no triggering condition. + +**SQLMesh examples:** +- The system shall produce a model named `dim_customers`. +- The system shall enforce `unique` and `not_null` on `customer_sk`. + +### 2. Event-driven — triggered by an event + +> When , the system shall . + +Use for "when X happens, do Y." + +**SQLMesh examples:** +- When `SQLMesh run` completes, the system shall write a `query_tag` to the audit table. +- When a source row arrives more than 7 days late, the system shall reject it. + +### 3. State-driven — true while a state holds + +> While , the system shall . + +Use for ongoing conditions. + +**SQLMesh examples:** +- While the model is materialized as an incremental, the system shall use a `unique_key` of ``. +- While the engine is paused, the system shall not trigger refreshes. + +### 4. Unwanted condition — error handling + +> If , then the system shall . + +Use for failure modes. + +**SQLMesh examples:** +- If a source row has `is_test = true`, then the system shall exclude it from the staging output. +- If `SQLMesh parse` fails, then the system shall not deploy. + +### 5. Optional — feature-flagged + +> Where , the system shall . + +Use for behavior that's conditional on configuration. + +**SQLMesh examples:** +- Where Snowflake is the engine, the system shall apply the `mask_email` policy to PII columns. +- Where the semantic layer is enabled, the system shall preserve `` with stable name and type. + +## Anti-patterns + +- "It should probably work" — not EARS. Not testable. +- "We want a fast model" — not EARS. Define "fast" as a measurable AC. +- "The user can do X" — not EARS. Reword as "When the user does Y, the system shall X." + +## Validating + +```bash +sqlmesh-specify validate specs/-/spec.md +``` + +Exits 0 if all AC lines match one of the five patterns. Exits 1 with a list of non-conformant lines otherwise. diff --git a/docs/engine-guides/athena.md b/docs/engine-guides/athena.md new file mode 100644 index 0000000..b5b3e76 --- /dev/null +++ b/docs/engine-guides/athena.md @@ -0,0 +1,26 @@ +# Athena guide + +Use this preset when SQLMesh targets Amazon Athena and S3-backed tables. + +## What the preset adds + +- S3 table and file-layout planning +- partitioning and partition-projection decisions +- Glue catalog and Lake Formation/IAM governance review +- workgroup, output location, and scan-cost guardrails +- Iceberg versus Hive-style table format decisions + +## Good fit + +- lakehouse tables queried through Athena +- Iceberg or external Hive-style datasets +- teams that need scan-cost and S3 layout evidence in PRs + +## Use + +```bash +sqlmesh-specify init analytics --engine athena +``` + +sqlmesh-spec-kit does not query Athena. SQLMesh, the adapter, AWS credentials, and workgroup configuration +remain outside this toolkit. diff --git a/docs/engine-guides/bigquery.md b/docs/engine-guides/bigquery.md new file mode 100644 index 0000000..bca2beb --- /dev/null +++ b/docs/engine-guides/bigquery.md @@ -0,0 +1,39 @@ +# BigQuery guide + +The BigQuery preset adds planning requirements for partitioning, clustering, materialization, cost, +policy tags, authorized views, and BI Engine expectations. + +Use it with: + +```bash +sqlmesh-specify init analytics --engine bigquery +``` + +## Partitioning + +Large fact-like tables should partition by the date or timestamp column that bounds routine +production scans. If the model is not partitioned, document the size estimate and why partitioning is +not useful. + +```sql +{{ config( + materialized='incremental', + partition_by={"field": "event_date", "data_type": "date"}, + cluster_by=["customer_id", "region"] +) }} +``` + +## Cost guardrails + +Plans for large tables should name the normal scan boundary. Incremental models should document the +incremental predicate, unique key, and late-arriving data behavior. + +## Governance + +Restricted columns should use BigQuery policy tags or be exposed through authorized views. The plan +should list the protected column or view and how access behavior is tested. + +## Semantic consumers + +If a model supports dashboards or semantic-layer metrics, call out whether BI Engine acceleration is +expected, unnecessary, or out of scope. diff --git a/docs/engine-guides/clickhouse.md b/docs/engine-guides/clickhouse.md new file mode 100644 index 0000000..15dc08d --- /dev/null +++ b/docs/engine-guides/clickhouse.md @@ -0,0 +1,9 @@ +# clickhouse engine guide + +Use this guide when SQLMesh runs against `clickhouse`. + +```bash +sqlmesh-specify init analytics --engine clickhouse +``` + +Document target environment, gateway, change category, audits, tests, and backfill/restatement scope in `plan.md`. diff --git a/docs/engine-guides/databricks.md b/docs/engine-guides/databricks.md new file mode 100644 index 0000000..7874beb --- /dev/null +++ b/docs/engine-guides/databricks.md @@ -0,0 +1,83 @@ +# Databricks guide for sqlmesh-spec-kit + +## What `--engine databricks` adds + +`sqlmesh-specify init --engine databricks` appends Databricks-specific articles to the constitution and Databricks-specific tables to the plan template. The wedge is the same as for Snowflake: engine-specific patterns SQLMesh Labs intentionally doesn't ship. + +## What's covered + +| Topic | Where | +|---|---| +| Liquid Clustering | Constitution Article D1 + plan additions + clustering-decisions skill | +| Photon | Constitution Article D2 + plan additions | +| Unity Catalog | Constitution Article D3 + plan additions | +| Materialized Views | Constitution Article D4 | +| Query tags via system tables | Constitution Article D5 | +| AI/ML governance (`ai_query`, Mosaic AI) | Constitution Article D6 | + +## Liquid Clustering — the modern default + +Liquid Clustering replaces rigid partitioning. Use the `databricks-liquid-clustering-decisions` skill for the decision rules. Short version: + +- Use Liquid Clustering for tables >1 GB with selective query predicates +- One column is usually enough; up to 4 supported +- Compose with Predictive Optimization at the schema level + +```sql +ALTER SCHEMA . ENABLE PREDICTIVE OPTIMIZATION; +``` + +## Unity Catalog placement + +All models use three-part names: + +```sql +{{ config( + materialized='table', + catalog='analytics', + schema='marts' +) }} +``` + +Grants live in `models/_governance/grants.sql` and apply as a post-hook: + +```yaml +on-run-end: + - "{{ apply_grants() }}" +``` + +## Materialized Views over hand-rolled incrementals + +Where the engine supports it, prefer: + +```python +{{ config(materialized='materialized_view') }} +``` + +Over: + +```python +{{ config( + materialized='incremental', + unique_key='order_id', + on_schema_change='sync_all_columns' +) }} +``` + +The plan must justify any choice of `incremental` over `materialized_view`. + +## Query tags via system tables + +Databricks doesn't have inline `query_tag` like Snowflake. Use SQL engine tags + `system.access.audit`: + +```sql +SELECT + request_params:statement AS sql, + request_params:engine_id AS engine, + request_params:statement_id AS statement_id +FROM system.access.audit +WHERE service_name = 'databrickssql' + AND date_trunc('day', event_time) = current_date(); +``` + +Pair with workspace-level tags applied to the engine for full attribution. diff --git a/docs/engine-guides/duckdb.md b/docs/engine-guides/duckdb.md new file mode 100644 index 0000000..1f66d34 --- /dev/null +++ b/docs/engine-guides/duckdb.md @@ -0,0 +1,26 @@ +# DuckDB guide + +Use this preset when SQLMesh targets DuckDB for local analytics, CI fixtures, or file-based workflows. + +## What the preset adds + +- local file and object-store path review +- memory and single-process limits +- DuckDB extension planning +- Parquet staging and export expectations +- governance handling for local files and CI artifacts + +## Good fit + +- local development examples +- lightweight analytics projects +- CI smoke tests with deterministic data files + +## Use + +```bash +sqlmesh-specify init analytics --engine duckdb +``` + +This preset is useful for AI-agent demos because it makes local file dependencies and reproducibility +explicit. diff --git a/docs/engine-guides/mssql.md b/docs/engine-guides/mssql.md new file mode 100644 index 0000000..4ae6f99 --- /dev/null +++ b/docs/engine-guides/mssql.md @@ -0,0 +1,9 @@ +# mssql engine guide + +Use this guide when SQLMesh runs against `mssql`. + +```bash +sqlmesh-specify init analytics --engine mssql +``` + +Document target environment, gateway, change category, audits, tests, and backfill/restatement scope in `plan.md`. diff --git a/docs/engine-guides/mysql.md b/docs/engine-guides/mysql.md new file mode 100644 index 0000000..1d5b33c --- /dev/null +++ b/docs/engine-guides/mysql.md @@ -0,0 +1,25 @@ +# MySQL guide + +Use this preset when SQLMesh reads from or writes to MySQL and operational safety matters. + +## What the preset adds + +- primary versus replica versus analytics database review +- index and `EXPLAIN` expectations for heavy joins +- incremental watermark, batch size, and lock planning +- engine, charset, collation, and timezone assumptions +- grants and PII access review + +## Good fit + +- operational-source staging projects +- MySQL replicas used for analytics +- teams that need to prevent AI agents from overloading OLTP databases + +## Use + +```bash +sqlmesh-specify init analytics --engine mysql +``` + +sqlmesh-spec-kit does not run MySQL queries. SQLMesh and the selected adapter own execution. diff --git a/docs/engine-guides/postgres.md b/docs/engine-guides/postgres.md new file mode 100644 index 0000000..4dab40a --- /dev/null +++ b/docs/engine-guides/postgres.md @@ -0,0 +1,25 @@ +# Postgres guide + +Use this preset when SQLMesh targets Postgres or uses Postgres as an analytics-safe serving database. + +## What the preset adds + +- OLTP safety checks for primary versus replica versus analytics database +- index and materialization planning +- lock and transaction impact review +- schema/grant/access planning +- extension and non-portable SQL callouts + +## Good fit + +- smaller analytics projects on Postgres +- product analytics replicas +- local or self-managed SQLMesh deployments where lock safety matters + +## Use + +```bash +sqlmesh-specify init analytics --engine postgres +``` + +The preset does not connect to Postgres. SQLMesh and its adapter handle database execution. diff --git a/docs/engine-guides/redshift.md b/docs/engine-guides/redshift.md new file mode 100644 index 0000000..33ca3f6 --- /dev/null +++ b/docs/engine-guides/redshift.md @@ -0,0 +1,26 @@ +# Redshift guide + +Use this preset when SQLMesh runs on Amazon Redshift or Redshift is the analytics serving database. + +## What the preset adds + +- distribution and sort key planning +- vacuum and analyze expectations +- Spectrum/external table scan boundaries +- workload queue and full-refresh cost guardrails +- schema, grants, late binding view, and restricted-data review + +## Good fit + +- enterprise marts on Redshift +- Redshift Spectrum plus internal table joins +- teams that need PR evidence for dist/sort decisions + +## Use + +```bash +sqlmesh-specify init analytics --engine redshift +``` + +SQLMesh still owns execution through the Redshift adapter and the user's normal profile. This preset only +changes planning guidance and generated agent context. diff --git a/docs/engine-guides/snowflake.md b/docs/engine-guides/snowflake.md new file mode 100644 index 0000000..51478c8 --- /dev/null +++ b/docs/engine-guides/snowflake.md @@ -0,0 +1,67 @@ +# Snowflake guide for sqlmesh-spec-kit + +## What `--engine snowflake` adds + +`sqlmesh-specify init --engine snowflake` appends Snowflake-specific articles to the constitution and Snowflake-specific tables to the plan template. These cover the concerns sqlmesh-spec-kit's wedge is built around: engine-specific patterns SQLMesh Labs intentionally doesn't ship. + +## What's covered + +| Topic | Where | +|---|---| +| Clustering keys | Constitution Article S1 + plan additions + clustering-decisions skill | +| Engine sizing | Constitution Article S2 + plan additions | +| Query tags | Constitution Article S3 + plan additions | +| Masking & row-access policies | Constitution Article S4 + plan additions | +| Dynamic tables | Constitution Article S5 | +| Cortex / LLM-powered transformations | Constitution Article S6 | + +## Clustering — the most common question + +Use the `snowflake-clustering-decisions` skill (installed at `.sqlmesh-specify/skills/snowflake-clustering-decisions/SKILL.md`). The short version: + +- Tables under 1 GB → no clustering +- Tables over 1 GB → cluster on the column most often in WHERE / JOIN +- Verify via `SYSTEM$CLUSTERING_INFORMATION` after first build + +## Query tags — getting cost attribution right + +Add to `config.yaml`: + +```yaml +on-run-start: + - "{{ set_query_tag() }}" +``` + +Where `set_query_tag` is a macro that emits: + +```jinja +{% macro set_query_tag() %} + {% set tag %} + project={{ project_name }}, model={{ this.name }}, env={{ target.name }}, run_id={{ invocation_id }} + {% endset %} + {% do run_query("ALTER SESSION SET QUERY_TAG = '" ~ tag ~ "'") %} +{% endmacro %} +``` + +Then attribute cost via: + +```sql +SELECT query_tag, SUM(credits_used) +FROM snowflake.account_usage.query_history +GROUP BY 1; +``` + +## Masking — when staging is the right boundary + +The constitution requires masking at the staging layer, not the mart. Reason: marts are often built from intermediate models that have already lost track of which columns are PII. Masking at staging keeps the boundary clean. + +```yaml +# models/staging//___models.yml +- name: stg___customers + columns: + - name: email + meta: + masking_policy: mask_email +``` + +See `models/_governance/masking_policies.sql` for the policy definitions and `tests/unit/test_masking_policies.sql` for the deterministic-fixture tests. diff --git a/docs/engine-guides/spark.md b/docs/engine-guides/spark.md new file mode 100644 index 0000000..62279bb --- /dev/null +++ b/docs/engine-guides/spark.md @@ -0,0 +1,9 @@ +# spark engine guide + +Use this guide when SQLMesh runs against `spark`. + +```bash +sqlmesh-specify init analytics --engine spark +``` + +Document target environment, gateway, change category, audits, tests, and backfill/restatement scope in `plan.md`. diff --git a/docs/engine-guides/trino.md b/docs/engine-guides/trino.md new file mode 100644 index 0000000..f6cd429 --- /dev/null +++ b/docs/engine-guides/trino.md @@ -0,0 +1,115 @@ +# Trino guide for sqlmesh-spec-kit + +## What `--engine trino` adds + +`sqlmesh-specify init --engine trino` appends Trino-specific articles to the constitution and Trino-specific tables to the plan template. + +**Trino is not a engine.** It's a federated query engine. The constitution and plan additions reflect that — concerns center on catalogs, connectors, federation, pushdown, and session properties, not on engines or clustering. + +## What's covered + +| Topic | Where | +|---|---| +| Three-part naming (`catalog.schema.table`) | Constitution Article T2 + plan additions | +| Federation and cross-catalog joins | Constitution Article T3 + plan additions + federated-query-patterns skill | +| Connector pushdown | Constitution Article T4 + plan additions + skill | +| Session properties | Constitution Article T5 + plan additions | +| `on_table_exists: rename / drop` | Constitution Article T6 + plan additions | +| View security (`definer` / `invoker`) | Constitution Article T7 + plan additions | +| Iceberg as preferred destination | Constitution Article T8 | +| Adapter cadence (Starburst-maintained) | Constitution Article T9 | +| Cost attribution | Constitution Article T10 + plan additions | + +## Three-part naming — the single most common Trino-on-SQLMesh mistake + +Always reference data with three parts: + +```sql +SELECT * FROM iceberg.analytics.fct_orders; -- right +SELECT * FROM postgresql.public.customers; -- right +SELECT * FROM fct_orders; -- wrong: silent routing risk +``` + +The constitution enforces this. The plan template's catalog table makes every catalog touched visible. + +## Federation — three good questions to ask before any cross-catalog join + +1. **Should this join exist in Trino at all, or should one side be landed in Iceberg first?** Materializing the smaller side into Iceberg is often the right answer. +2. **What's the pushdown story?** Run `EXPLAIN (TYPE DISTRIBUTED)` and confirm the predicate appears in a `ScanFilterProject` node, not after the scan. +3. **What's the network cost?** Cross-catalog joins move data over the network. Estimate the rows pulled into Trino. + +See `.sqlmesh-specify/skills/trino-federated-query-patterns/SKILL.md` for the full decision rules. + +## Session properties — replacing "engine sizing" + +Trino doesn't have Snowflake's named engines. Adjust execution behavior via session properties: + +```yaml +# profiles.yml +my_SQLMesh_trino_project: + target: prod + outputs: + prod: + type: trino + host: trino.example.com + catalog: iceberg + schema: analytics + session_properties: + query_max_run_time: '30m' + query_max_memory: '4GB' + task_concurrency: 8 +``` + +Per-model overrides via `pre_hook`: + +```python +{{ config( + materialized='table', + pre_hook=["set session query_max_run_time='10m'"] +) }} +``` + +## `on_table_exists` — choose deliberately, especially on Glue + +```python +{{ config( + materialized='table', + on_table_exists='rename' -- default; safer +) }} +``` + +But if your underlying metastore is AWS Glue and the target catalog is Hive-style: + +```python +{{ config( + materialized='table', + on_table_exists='drop' -- required; Glue cannot rename +) }} +``` + +Constitution Article T6 makes this an explicit choice in the plan, not a default-and-hope. + +## View security + +```python +{{ config( + materialized='view', + view_security='definer' -- default; view runs as creator +) }} +``` + +Use `'invoker'` only when downstream users must be enforced against their own access controls. The plan declares the choice. + +## Cost attribution (best-effort) + +Trino has no native cost-tag mechanism. The viable patterns: + +1. Set `application_name` or a custom session property at run time so queries are identifiable in coordinator logs. +2. Aggregate from coordinator query logs into an Iceberg table for analysis. +3. Attribute storage cost separately via the underlying connector's billing (Iceberg/S3 lifecycle policies). + +The plan's "Cost attribution" table makes the chosen approach explicit. + +## Composition with the lakehouse pattern + +The most common Trino-on-SQLMesh deployment is the **lakehouse pattern**: Iceberg tables in S3 (or equivalent object store), Trino as the query engine, SQLMesh as the transformation layer. The constitution and plan additions assume this pattern as default but do not require it — non-Iceberg catalogs are supported with explicit annotation. diff --git a/docs/enterprise-ci.md b/docs/enterprise-ci.md new file mode 100644 index 0000000..b790879 --- /dev/null +++ b/docs/enterprise-ci.md @@ -0,0 +1,21 @@ +# Enterprise CI + +Recommended checks: + +```bash +ruff check src tests +mypy src tests +pytest +python -m build --sdist --wheel +python -m twine check dist/* +sqlmesh-specify ci --target . +``` + +PR evidence should include: + +- spec, plan, and tasks lifecycle status +- `sqlmesh plan ` output +- audit and test results +- unexpected backfill/restatement review +- downstream owner approval when needed +- apply/promote evidence before marking a spec `shipped` diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..37ec9ab --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,50 @@ +# Getting Started + +## Install + +After the first PyPI release: + +```bash +uvx --from sqlmesh-spec-kit sqlmesh-specify --help +``` + +Before PyPI release, install directly from GitHub: + +```bash +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify --help +``` + +## Initialize a SQLMesh repo + +Run from a SQLMesh project root with `config.yaml`, `config.yml`, or `config.py` and `models/`. + +```bash +sqlmesh-specify init analytics --engine duckdb --target . +sqlmesh-specify doctor --target . +``` + +`init` writes `.sqlmesh-specify/`, `CLAUDE.md` or `CLAUDE.md.sqlmesh-specify-suggested`, and `specs/.gitkeep`. + +## Validate + +```bash +sqlmesh-specify validate path/to/spec.md +sqlmesh-specify validate project --target . +sqlmesh-specify validate sqlmesh --target . +sqlmesh-specify report --target . --format markdown +``` + +Status values are exact, not free-form prose. Use only the legal values shown in the templates. + +## Try the Verified Example + +Use SQLMesh's official sushi example as the quickest end-to-end test target: + +```bash +git clone https://github.com/TobikoData/sqlmesh-examples.git +cd sqlmesh-examples/001_sushi/2_moderate +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify init sushi-moderate --engine duckdb --target . +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify ci --target . +``` + +The full implementation flow is documented in [Tutorial 02: Ship a SQLMesh Change](tutorials/02-ship-a-sqlmesh-change.md). diff --git a/docs/integrations/confluence.md b/docs/integrations/confluence.md new file mode 100644 index 0000000..5e3dda9 --- /dev/null +++ b/docs/integrations/confluence.md @@ -0,0 +1,11 @@ +# Confluence Integration + +Set `CONFLUENCE_BASE_URL`, `CONFLUENCE_EMAIL`, and `CONFLUENCE_API_TOKEN`. + +```bash +sqlmesh-specify confluence pull-page 12345 --to specs/001-change/context/source.md +sqlmesh-specify confluence publish --spec-dir specs/001-change --space-key DATA --dry-run +sqlmesh-specify confluence sync --spec-dir specs/001-change --dry-run +``` + +Published pages are summaries. The local spec directory remains the source of truth. diff --git a/docs/integrations/jira.md b/docs/integrations/jira.md new file mode 100644 index 0000000..c36ba11 --- /dev/null +++ b/docs/integrations/jira.md @@ -0,0 +1,12 @@ +# Jira Integration + +Set `JIRA_BASE_URL`, `JIRA_EMAIL`, and `JIRA_API_TOKEN`. + +```bash +sqlmesh-specify jira pull DATA-123 --target . +sqlmesh-specify jira attach DATA-123 --spec specs/001-change/spec.md +sqlmesh-specify jira create-tasks DATA-123 --from specs/001-change/tasks.md +sqlmesh-specify jira sync DATA-123 --spec-dir specs/001-change --dry-run +``` + +Generated specs mention SQLMesh models, audits, environments, and plan evidence. Redact sensitive context before syncing externally. diff --git a/docs/methodology.md b/docs/methodology.md new file mode 100644 index 0000000..af2878c --- /dev/null +++ b/docs/methodology.md @@ -0,0 +1,19 @@ +# Methodology + +sqlmesh-spec-kit treats AI SDLC artifacts as durable repo memory. + +## Spec + +`spec.md` is the contract. It describes the problem, users, SQLMesh-facing constraints, and EARS acceptance criteria. A plan may exist only when the spec status is exactly `approved` or `shipped`. + +## Plan + +`plan.md` is the implementation boundary. It names target environment, changed models, snapshots, audits/tests, backfill window, forward-only/restatement decision, downstream impact, and required SQLMesh evidence. Tasks may exist only when the plan status is exactly `approved`. + +## Tasks + +`tasks.md` is the bounded implementation checklist. Agents implement only approved files and stop when SQLMesh plan output, audit/test results, or AC traceability is unclear. + +## Evidence + +The v0.1 validators do not execute SQLMesh. Teams run `sqlmesh plan `, SQLMesh tests, audits, apply, and promote commands in their normal workflow and attach evidence to the PR or spec directory. diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 0000000..723d0ae --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,15 @@ +# Releasing + +The package is PyPI-ready through Hatchling and GitHub Trusted Publishing. + +Local verification: + +```bash +uv run --extra dev ruff check src tests +uv run --extra dev mypy src tests +uv run --extra dev pytest +uv run --extra dev python -m build --sdist --wheel +uv run --extra dev python -m twine check dist/* +``` + +Create a GitHub release tag such as `v0.1.0` to publish. diff --git a/docs/skills-and-sub-agents.md b/docs/skills-and-sub-agents.md new file mode 100644 index 0000000..cf92437 --- /dev/null +++ b/docs/skills-and-sub-agents.md @@ -0,0 +1,21 @@ +# Skills and Sub-Agents + +Bundled agents: + +- `spec-steward` +- `sqlmesh-architect` +- `engine-optimizer` +- `implementation-agent` +- `governance-reviewer` +- `review-agent` + +Bundled skills: + +- `writing-sqlmesh-model-specs` +- `writing-incremental-model-specs` +- `reviewing-audits-and-tests` +- `reviewing-plan-apply-evidence` +- `reviewing-pii-access-governance` +- `capturing-project-conventions` + +Use them as focused context for AI agents. They do not replace lifecycle gates. diff --git a/docs/sqlmesh-ai-sdlc-walkthrough.md b/docs/sqlmesh-ai-sdlc-walkthrough.md new file mode 100644 index 0000000..1feb5b4 --- /dev/null +++ b/docs/sqlmesh-ai-sdlc-walkthrough.md @@ -0,0 +1,20 @@ +# SQLMesh AI SDLC Walkthrough + +Scenario: add a customer order segmentation model to a small sushi-style SQLMesh project. + +1. Run `sqlmesh-specify init sushi --engine duckdb --target .`. +2. Draft `specs/001-customer-segmentation/spec.md`. +3. Approve the spec by setting `**Status:** approved`. +4. Write `plan.md` with changed models, audits/tests, backfill window, and SQLMesh plan evidence requirements. +5. Approve the plan by setting `**Status:** approved`. +6. Write `tasks.md`, implement only the approved files, and run: + +```bash +sqlmesh plan dev +sqlmesh test +sqlmesh-specify report --target . --format markdown +``` + +7. Attach plan/test/audit evidence and mark the spec `shipped` after promotion. + +See `examples/sushi-customer-segmentation/` for complete artifacts. diff --git a/docs/sqlmesh-memory-and-repo-hygiene.md b/docs/sqlmesh-memory-and-repo-hygiene.md new file mode 100644 index 0000000..1bb71a7 --- /dev/null +++ b/docs/sqlmesh-memory-and-repo-hygiene.md @@ -0,0 +1,17 @@ +# SQLMesh Memory and Repo Hygiene + +Keep: + +- `.sqlmesh-specify/constitution.md` +- approved `spec.md`, `plan.md`, `tasks.md`, review notes, and reports +- concise project convention skills +- sanitized Jira and Confluence context needed for implementation + +Do not keep: + +- raw agent scratch work +- secrets or credentials +- production data samples +- stale plan output superseded by final review evidence + +Specs and plans are memory because they explain why a model exists, what boundaries were approved, and what SQLMesh evidence made the change safe to promote. diff --git a/docs/tutorials/01-initialize-a-sqlmesh-repo.md b/docs/tutorials/01-initialize-a-sqlmesh-repo.md new file mode 100644 index 0000000..c25f9f1 --- /dev/null +++ b/docs/tutorials/01-initialize-a-sqlmesh-repo.md @@ -0,0 +1,14 @@ +# 01: Initialize a SQLMesh Repo + +```bash +sqlmesh-specify init analytics --engine duckdb --target . +sqlmesh-specify doctor --target . +sqlmesh-specify validate sqlmesh --target . +``` + +Record: + +- the SQLMesh config file used +- engine preset +- first gaps from doctor +- where specs will live diff --git a/docs/tutorials/02-ship-a-sqlmesh-change.md b/docs/tutorials/02-ship-a-sqlmesh-change.md new file mode 100644 index 0000000..a46aad7 --- /dev/null +++ b/docs/tutorials/02-ship-a-sqlmesh-change.md @@ -0,0 +1,146 @@ +# 02: Ship a SQLMesh Change + +This tutorial uses the official `TobikoData/sqlmesh-examples` repository and the `001_sushi/2_moderate` project. + +## 1. Clone the example repo + +```bash +git clone https://github.com/TobikoData/sqlmesh-examples.git +cd sqlmesh-examples/001_sushi/2_moderate +``` + +## 2. Initialize sqlmesh-spec-kit + +After PyPI release: + +```bash +sqlmesh-specify init sushi-moderate --engine duckdb --target . +sqlmesh-specify doctor --target . +``` + +Before PyPI release, run the same commands directly from GitHub: + +```bash +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify init sushi-moderate --engine duckdb --target . +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify doctor --target . +``` + +## 3. Write and approve the spec + +Create `specs/001-customer-segments/spec.md`. + +The spec should use exact status `approved` only after human approval: + +```markdown +**Status:** approved + +## Acceptance Criteria + +- AC1: When `sushimoderate.customer_segments` is queried, the system shall return one row per customer from `sushimoderate.customers`. +- AC2: Where a customer's latest lifetime revenue is at least 100, the system shall classify the customer as `vip`. +- AC3: Where a customer's latest lifetime revenue is at least 25 and less than 100, the system shall classify the customer as `repeat`. +- AC4: Where a customer has no lifetime revenue below 25, the system shall classify the customer as `new`. +- AC5: When SQLMesh validation runs, the system shall include unit-test evidence and plan/apply evidence for the new model. +``` + +Validate it: + +```bash +sqlmesh-specify validate specs/001-customer-segments/spec.md +``` + +## 4. Write and approve the plan + +Create `specs/001-customer-segments/plan.md` only after the spec is approved. + +The plan should list exactly the implementation files: + +```text +models/customer_segments.sql +tests/test_customer_segments.yaml +``` + +It should include: + +- target environment: `dev` +- changed model: `sushimoderate.customer_segments` +- change category: non-breaking +- restatement required: no +- forward-only decision: no +- SQLMesh evidence: `sqlmesh test` and `sqlmesh plan dev --auto-apply --no-prompts` + +Set status to `approved` only after human approval. + +## 5. Write tasks + +Create `specs/001-customer-segments/tasks.md` only after the plan is approved. + +Every task should map to AC ids and stop if implementation requires files outside the approved plan. + +## 6. Implement the model + +Create `models/customer_segments.sql`: + +```sql +MODEL ( + name sushimoderate.customer_segments, + kind FULL, + cron '@daily', + grain customer_id, + audits ( + not_null(columns=[customer_id, segment]), + unique_values(columns=[customer_id]) + ), +); + +WITH latest_revenue AS ( + SELECT + customer_id, + revenue, + ROW_NUMBER() OVER (PARTITION BY customer_id ORDER BY ds DESC) AS revenue_rank + FROM sushimoderate.customer_revenue_lifetime +) + +SELECT + c.customer_id::INT AS customer_id, + CASE + WHEN COALESCE(lr.revenue, 0) >= 100 THEN 'vip' + WHEN COALESCE(lr.revenue, 0) >= 25 THEN 'repeat' + ELSE 'new' + END::TEXT AS segment, + COALESCE(lr.revenue, 0)::DOUBLE AS lifetime_revenue, + c.status::TEXT AS status, + c.zip::TEXT AS zip +FROM sushimoderate.customers AS c +LEFT JOIN latest_revenue AS lr + ON c.customer_id = lr.customer_id + AND lr.revenue_rank = 1; +``` + +Create `tests/test_customer_segments.yaml` with rows that verify `vip`, `repeat`, and `new`. + +## 7. Validate and execute + +```bash +sqlmesh-specify ci --target . +sqlmesh test +sqlmesh plan dev --auto-apply --no-prompts +``` + +Expected evidence: + +- `sqlmesh-specify ci`: 0 errors +- `sqlmesh test`: customer-segments unit test passes +- `sqlmesh plan dev`: `sushimoderate__dev.customer_segments` is added, full-refreshed, and audits pass + +## 8. Review and close + +Write `specs/001-customer-segments/review.md` with: + +- final diff +- AC-to-evidence table +- SQLMesh test output +- SQLMesh plan/apply output +- notes on backfill/restatement behavior + +Then mark `tasks.md` status `done`. Mark `spec.md` status `shipped` only after the team accepts the result. diff --git a/docs/tutorials/03-brownfield-enterprise-adoption.md b/docs/tutorials/03-brownfield-enterprise-adoption.md new file mode 100644 index 0000000..681965b --- /dev/null +++ b/docs/tutorials/03-brownfield-enterprise-adoption.md @@ -0,0 +1,10 @@ +# 03: Brownfield Enterprise Adoption + +Start with diagnostics, not a rewrite. + +```bash +sqlmesh-specify doctor --target . +sqlmesh-specify init analytics --engine snowflake --target . +``` + +Capture only conventions that affect future AI-assisted changes: environment strategy, model ownership, audit/test expectations, and promotion evidence. diff --git a/docs/tutorials/04-skills-and-agent-handoffs.md b/docs/tutorials/04-skills-and-agent-handoffs.md new file mode 100644 index 0000000..d5d566c --- /dev/null +++ b/docs/tutorials/04-skills-and-agent-handoffs.md @@ -0,0 +1,9 @@ +# 04: Skills and Agent Handoffs + +Use specialized agents for bounded review: + +- spec steward for EARS and status gates +- SQLMesh architect for snapshots, backfills, and restatements +- engine optimizer for platform-specific choices +- governance reviewer for PII and publishing risk +- review agent for final AC/evidence coverage diff --git a/docs/tutorials/05-jira-to-spec-workflow.md b/docs/tutorials/05-jira-to-spec-workflow.md new file mode 100644 index 0000000..708d839 --- /dev/null +++ b/docs/tutorials/05-jira-to-spec-workflow.md @@ -0,0 +1,9 @@ +# 05: Jira to Spec Workflow + +```bash +sqlmesh-specify jira pull DATA-123 --target . +sqlmesh-specify jira attach DATA-123 --spec specs/001-change/spec.md --plan specs/001-change/plan.md +sqlmesh-specify jira create-tasks DATA-123 --from specs/001-change/tasks.md --dry-run +``` + +Review and redact generated specs before attaching them back to Jira. diff --git a/docs/tutorials/06-confluence-context-workflow.md b/docs/tutorials/06-confluence-context-workflow.md new file mode 100644 index 0000000..98e39b4 --- /dev/null +++ b/docs/tutorials/06-confluence-context-workflow.md @@ -0,0 +1,8 @@ +# 06: Confluence Context Workflow + +```bash +sqlmesh-specify confluence pull-page 12345 --to specs/001-change/context/source.md +sqlmesh-specify confluence publish --spec-dir specs/001-change --space-key DATA --dry-run +``` + +Use Confluence as source context and summary publishing. Keep the approved local spec directory as the implementation source of truth. diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md new file mode 100644 index 0000000..acd8692 --- /dev/null +++ b/docs/tutorials/README.md @@ -0,0 +1,20 @@ +# Tutorial Series + +Install from PyPI after release: + +```bash +uvx --from sqlmesh-spec-kit sqlmesh-specify --help +``` + +Until the first release is published, run the tutorials directly from GitHub: + +```bash +uvx --from git+https://github.com/duckcode-ai/sqlmesh-spec-kit.git sqlmesh-specify --help +``` + +1. [Initialize a SQLMesh repo](01-initialize-a-sqlmesh-repo.md) +2. [Ship a SQLMesh change](02-ship-a-sqlmesh-change.md) +3. [Brownfield enterprise adoption](03-brownfield-enterprise-adoption.md) +4. [Skills and agent handoffs](04-skills-and-agent-handoffs.md) +5. [Jira to spec workflow](05-jira-to-spec-workflow.md) +6. [Confluence context workflow](06-confluence-context-workflow.md) diff --git a/examples/sushi-customer-segmentation/plan.md b/examples/sushi-customer-segmentation/plan.md new file mode 100644 index 0000000..c34ea92 --- /dev/null +++ b/examples/sushi-customer-segmentation/plan.md @@ -0,0 +1,57 @@ +# Plan: Customer Segmentation + +**Author:** Example +**Date:** 2026-05-26 +**Status:** approved +**Spec:** spec.md + +## Architecture + +Add `models/customer_segments.sql` in the `dev` environment with one row per customer. + +## SQLMesh Plan + +| Field | Decision | +|---|---| +| Target environment | dev | +| Change category | non-breaking | +| Forward-only decision | no | +| Restatement required | no | +| Backfill window | none | +| Snapshots affected | `sushi.customer_segments` | +| Apply/promote path | review `sqlmesh plan dev`, then apply after approval | + +## Files to Add + +| Path | Purpose | ACs | +|---|---|---| +| `models/customer_segments.sql` | customer segmentation model | AC1, AC2 | +| `audits/customer_segments.sql` | dormant/active classification audit | AC2 | + +## Files to Modify + +| Path | Purpose | ACs | +|---|---|---| +| None | | | + +## Files to Delete + +| Path | Reason | ACs | +|---|---|---| +| None | | | + +## Audits and Tests + +| Evidence | Command or artifact | ACs | +|---|---|---| +| SQLMesh plan | `sqlmesh plan dev` | AC3 | +| SQLMesh tests | `sqlmesh test` | AC1, AC2 | +| Audits | customer segment audit output | AC2 | + +## Downstream Impact + +Growth analytics dashboards can use the new model after promotion. + +## SQLMesh Plan/Apply Evidence + +Attach `sqlmesh plan dev`, audit/test output, and apply evidence before shipping. diff --git a/examples/sushi-customer-segmentation/review.md b/examples/sushi-customer-segmentation/review.md new file mode 100644 index 0000000..a682921 --- /dev/null +++ b/examples/sushi-customer-segmentation/review.md @@ -0,0 +1,3 @@ +# Review + +Final diff matches the approved plan. SQLMesh plan output showed only `sushi.customer_segments`; tests and audits passed. diff --git a/examples/sushi-customer-segmentation/spec.md b/examples/sushi-customer-segmentation/spec.md new file mode 100644 index 0000000..c14bb65 --- /dev/null +++ b/examples/sushi-customer-segmentation/spec.md @@ -0,0 +1,25 @@ +# Customer Segmentation + +**Author:** Example +**Date:** 2026-05-26 +**Status:** approved + +## Problem + +The sushi team needs a customer segmentation model for order-frequency analysis. + +## Users + +| User | Job to be done | +|---|---| +| Growth analyst | Identify active, repeat, and dormant customers. | + +## Acceptance Criteria + +- AC1: When the customer segmentation model is queried, the system shall return one row per customer. +- AC2: Where a customer has no orders in the lookback window, the system shall classify the customer as dormant. +- AC3: When SQLMesh plan evidence is reviewed, the system shall show only the approved segmentation model and audit changes. + +## Out of Scope + +- New marketing activation exports. diff --git a/examples/sushi-customer-segmentation/sqlmesh-specify-report.md b/examples/sushi-customer-segmentation/sqlmesh-specify-report.md new file mode 100644 index 0000000..0f07a8c --- /dev/null +++ b/examples/sushi-customer-segmentation/sqlmesh-specify-report.md @@ -0,0 +1,3 @@ +# sqlmesh-specify CI report + +No blocking lifecycle findings in the example artifacts. diff --git a/examples/sushi-customer-segmentation/tasks.md b/examples/sushi-customer-segmentation/tasks.md new file mode 100644 index 0000000..e80ab36 --- /dev/null +++ b/examples/sushi-customer-segmentation/tasks.md @@ -0,0 +1,15 @@ +# Tasks: Customer Segmentation + +**Author:** Example +**Date:** 2026-05-26 +**Status:** done +**Plan:** plan.md + +## Task list + +- [x] T-01 - Add `models/customer_segments.sql`. + - **Validates:** AC1, AC2 +- [x] T-02 - Add SQLMesh audit coverage. + - **Validates:** AC2 +- [x] T-03 - Capture `sqlmesh plan dev` evidence. + - **Validates:** AC3 diff --git a/memory/constitution.md b/memory/constitution.md new file mode 100644 index 0000000..a5ee42e --- /dev/null +++ b/memory/constitution.md @@ -0,0 +1,23 @@ +# sqlmesh-spec-kit Constitution + +This file is durable project memory for AI-assisted SQLMesh development. + +## Lifecycle + +- `spec.md` status must be exactly `draft`, `approved`, `shipped`, or `superseded`. +- `plan.md` status must be exactly `proposed`, `approved`, or `superseded`. +- `tasks.md` status must be exactly `in progress` or `done`. +- `plan.md` may exist only after an approved or shipped spec. +- `tasks.md` may exist only after an approved plan. + +## SQLMesh Evidence + +- Approved plans identify target environment, changed models, snapshots, audits/tests, backfill scope, forward-only decision, restatement decision, and downstream impact. +- Implementations attach `sqlmesh plan ` evidence before review. +- Unexpected backfills, restatements, or forward-only changes require human review before continuing. + +## Boundaries + +- Agents implement only files listed in the approved plan. +- Every acceptance criterion must be traceable through plan, tasks, implementation, and evidence. +- Jira and Confluence publishing must redact secrets, credentials, customer data, and unnecessary PII. diff --git a/presets/athena/constitution-additions.md b/presets/athena/constitution-additions.md new file mode 100644 index 0000000..cadd4ac --- /dev/null +++ b/presets/athena/constitution-additions.md @@ -0,0 +1,24 @@ +## Article A1 — S3 file layout is part of model design + +Athena models document table format, file format, target file size, and small-file risk. Plans for +large tables explain how files are compacted or kept query-efficient. + +## Article A2 — Partitioning and projection are explicit + +Large fact-like tables document partition columns, partition projection usage, and expected partition +pruning. Unpartitioned large scans require an approved justification. + +## Article A3 — Glue and Lake Formation governance is reviewed + +Plans identify Glue catalog/database/table ownership and Lake Formation or IAM access boundaries for +governed outputs. + +## Article A4 — Workgroups and query cost guardrails are required + +Plans name the Athena workgroup, expected scanned data, output location, and cost guardrails for +large models. + +## Article A5 — Iceberg versus Hive table format is deliberate + +Plans choose Iceberg, Hive-style external tables, or another format deliberately and document +incremental, compaction, and schema evolution behavior. diff --git a/presets/athena/plan-additions.md b/presets/athena/plan-additions.md new file mode 100644 index 0000000..9c93d91 --- /dev/null +++ b/presets/athena/plan-additions.md @@ -0,0 +1,25 @@ +## Athena-specific concerns + +### Table and file layout + +| Model | Table format | File format/size | Small-file risk | +|---|---|---|---| +| | Iceberg / Hive external | Parquet/ORC/other, | low/medium/high | + +### Partitioning and pruning + +| Model | Partition columns | Projection used? | Expected scan boundary | +|---|---|---|---| +| | | yes/no | | + +### Governance + +| Output | Glue database/table | Lake Formation/IAM boundary | Sensitive? | +|---|---|---|---| +| | | | yes/no | + +### Workgroup and cost + +| Job | Workgroup | Output location | Scan guardrail | +|---|---|---|---| +| | | s3://... | | diff --git a/presets/bigquery/constitution-additions.md b/presets/bigquery/constitution-additions.md new file mode 100644 index 0000000..4fb7a8a --- /dev/null +++ b/presets/bigquery/constitution-additions.md @@ -0,0 +1,31 @@ +## Article B1 — Partitioning decisions are explicit + +Large fact-like tables declare a partitioning strategy in the model config. Date or timestamp +partitioning is preferred when the model is queried by time range. Tables that are not partitioned +must justify that choice in the plan. + +## Article B2 — Clustering supports the dominant access path + +Cluster columns are chosen from high-cardinality filter or join columns. Clustering is documented in +the plan with the expected query pattern it supports. + +## Article B3 — Cost guardrails are mandatory + +Plans for large models document the expected scan boundary and require partition filters where +applicable. Queries must not rely on unbounded full-table scans for routine transformations. + +## Article B4 — Materialized views and incremental models are deliberate choices + +If the model can be represented as a BigQuery materialized view, the plan explains why that is or is +not the right choice. Hand-rolled incremental models document the unique key, partition filter, and +late-arriving data behavior. + +## Article B5 — Governance lives in policy tags and authorized views + +Columns carrying PII or restricted data use BigQuery policy tags or authorized views. The plan lists +the tag or view boundary and the test evidence for masked or restricted access. + +## Article B6 — BI Engine and semantic consumers are called out + +Models serving dashboards or semantic-layer metrics document whether BI Engine acceleration is +expected, unnecessary, or explicitly out of scope. diff --git a/presets/bigquery/plan-additions.md b/presets/bigquery/plan-additions.md new file mode 100644 index 0000000..565780c --- /dev/null +++ b/presets/bigquery/plan-additions.md @@ -0,0 +1,31 @@ +## BigQuery-specific concerns + +### Partitioning and clustering decisions + +| Model | Size estimate | Partitioning | Clustering | Justification | +|---|---|---|---|---| +| | | `` or none | `[col1, col2]` or none | | + +### Cost guardrails + +| Model/job | Scan boundary | Guardrail | Tested? | +|---|---|---|---| +| | | require partition filter / incremental predicate | yes/no | + +### Materialization strategy + +| Model | Materialization | Alternative considered | Why this choice | +|---|---|---|---| +| | table / incremental / materialized_view / view | | | + +### Policy tags and authorized views + +| Column/view | Source | Policy tag or authorized view | Tested? | +|---|---|---|---| +| | | | yes/no | + +### BI Engine and semantic consumers + +| Consumer | BI Engine expected? | Impact | +|---|---|---| +| | yes/no | | diff --git a/presets/clickhouse/constitution-additions.md b/presets/clickhouse/constitution-additions.md new file mode 100644 index 0000000..771fc7c --- /dev/null +++ b/presets/clickhouse/constitution-additions.md @@ -0,0 +1,3 @@ +## clickhouse SQLMesh engine guardrails + +- Plans must document engine-specific cost, catalog, and promotion risks. diff --git a/presets/clickhouse/plan-additions.md b/presets/clickhouse/plan-additions.md new file mode 100644 index 0000000..55d9b12 --- /dev/null +++ b/presets/clickhouse/plan-additions.md @@ -0,0 +1,6 @@ +## clickhouse-specific SQLMesh concerns + +| Decision | Choice | Rationale | +|---|---|---| +| Gateway / connection | | | +| Backfill cost guardrail | | | diff --git a/presets/databricks/constitution-additions.md b/presets/databricks/constitution-additions.md new file mode 100644 index 0000000..c251703 --- /dev/null +++ b/presets/databricks/constitution-additions.md @@ -0,0 +1,23 @@ +## Article D1 — Liquid Clustering, not partitioning + +New tables use Liquid Clustering (`CLUSTER BY (col)`), not rigid partition columns. The clustering column is declared in `config(liquid_clustered_by=[...])`. Tables migrated from partitioning to Liquid Clustering call out the migration in the plan. + +## Article D2 — Photon is the default execution engine + +Production engines use Photon. The spec or plan calls out any model that *must* run on classic compute (e.g., uses a UDF Photon doesn't support). + +## Article D3 — Unity Catalog is the governance layer + +All models are addressed via three-part names: `..`. Grants are declared in `_governance/grants.sql` and applied as post-hooks. Hive metastore is not used for new work. + +## Article D4 — Materialized Views for incremental work + +For incremental processing, prefer Materialized Views (`materialized='materialized_view'`) over hand-rolled incremental SQLMesh models when the engine supports them. Specs that propose `materialized='incremental'` justify why an MV won't work. + +## Article D5 — Query tags via system tables + +Cost attribution uses Databricks query tags written to `system.access.audit`. The tag schema mirrors Snowflake's: `project`, `model`, `env`, `run_id`. + +## Article D6 — AI/ML governance + +LLM calls via `ai_query()` or Mosaic AI endpoints are: (a) called out in the spec, (b) tested with deterministic fixtures, (c) governed by per-endpoint cost limits. diff --git a/presets/databricks/plan-additions.md b/presets/databricks/plan-additions.md new file mode 100644 index 0000000..682b744 --- /dev/null +++ b/presets/databricks/plan-additions.md @@ -0,0 +1,32 @@ +## Databricks-specific concerns + +### Liquid Clustering decisions + +| Model | Size estimate | Clustering key | Justification | +|---|---|---|---| +| | | `` or none | | + +### Materialization choice + +| Model | Materialization | Justification | +|---|---|---| +| | view / table / materialized_view / streaming_table | | + +### Unity Catalog placement + +| Model | Catalog | Schema | Grants | +|---|---|---|---| +| | | | `_governance/grants.sql` entry: | + +### Photon compatibility + +| Model | Photon-compatible? | If no, why | +|---|---|---| +| | yes / no | | + +### Cost guardrails + +| Risk | Mitigation | +|---|---| +| Predictive Optimization not enabled on changed tables | Verify via `DESCRIBE TABLE EXTENDED` after first build | +| Streaming Table without watermark | Plan declares watermark column explicitly | diff --git a/presets/duckdb/constitution-additions.md b/presets/duckdb/constitution-additions.md new file mode 100644 index 0000000..398aee8 --- /dev/null +++ b/presets/duckdb/constitution-additions.md @@ -0,0 +1,24 @@ +## Article D1 — Local file boundaries are explicit + +Plans name every local file, object-store path, or external source read by DuckDB models. Paths must +be reproducible for CI or clearly marked local-only. + +## Article D2 — Memory and single-process limits are respected + +Large transformations document expected input size, memory risk, and whether data should be staged as +Parquet before downstream models consume it. + +## Article D3 — Extensions are deliberate dependencies + +Use of DuckDB extensions such as httpfs, spatial, iceberg, or json is named in the plan with install +and CI expectations. + +## Article D4 — Outputs are portable or intentionally local + +Plans say whether outputs are local development artifacts, CI artifacts, Parquet exports, or inputs +to another engine. Local-only marts are not presented as enterprise serving tables. + +## Article D5 — Governance still applies to files + +PII in local files, Parquet exports, or CI artifacts requires masking, exclusion, or an explicit +handling decision. diff --git a/presets/duckdb/plan-additions.md b/presets/duckdb/plan-additions.md new file mode 100644 index 0000000..66ad2eb --- /dev/null +++ b/presets/duckdb/plan-additions.md @@ -0,0 +1,25 @@ +## DuckDB-specific concerns + +### File and source boundaries + +| Input/output | Location | Reproducible in CI? | Notes | +|---|---|---|---| +| | local / object storage / generated | yes/no | | + +### Memory and data size + +| Model | Input size estimate | Memory risk | Mitigation | +|---|---|---|---| +| | | low/medium/high | | + +### Extensions + +| Extension | Why needed | CI install covered? | +|---|---|---| +| | | yes/no | + +### Governance for file outputs + +| File/model | Sensitive data? | Handling decision | +|---|---|---| +| | yes/no | mask / exclude / approved | diff --git a/presets/mssql/constitution-additions.md b/presets/mssql/constitution-additions.md new file mode 100644 index 0000000..85c819f --- /dev/null +++ b/presets/mssql/constitution-additions.md @@ -0,0 +1,3 @@ +## mssql SQLMesh engine guardrails + +- Plans must document engine-specific cost, catalog, and promotion risks. diff --git a/presets/mssql/plan-additions.md b/presets/mssql/plan-additions.md new file mode 100644 index 0000000..84767b3 --- /dev/null +++ b/presets/mssql/plan-additions.md @@ -0,0 +1,6 @@ +## mssql-specific SQLMesh concerns + +| Decision | Choice | Rationale | +|---|---|---| +| Gateway / connection | | | +| Backfill cost guardrail | | | diff --git a/presets/mysql/constitution-additions.md b/presets/mysql/constitution-additions.md new file mode 100644 index 0000000..cf3bac1 --- /dev/null +++ b/presets/mysql/constitution-additions.md @@ -0,0 +1,24 @@ +## Article M1 — Operational source safety comes first + +Plans identify whether MySQL is a production OLTP source, replica, or analytics database. Heavy SQLMesh +work must not run against an unprotected production primary. + +## Article M2 — Indexes and explain plans support every heavy model + +Large joins and filters document required indexes and `EXPLAIN` evidence or expected plan behavior. +Views over heavy joins require a justification. + +## Article M3 — Incremental loads avoid large locks + +Incremental models document watermark, primary/unique key, batch size, and lock impact. Full refreshes +of large tables require an approved window. + +## Article M4 — Engine, charset, collation, and timezone assumptions are visible + +Plans call out InnoDB assumptions, collation-sensitive comparisons, and timezone/date handling when +they affect business logic. + +## Article M5 — Grants and PII access are reviewed + +Governed marts document schemas/databases, grants, serving views, and sensitive columns before +implementation. diff --git a/presets/mysql/plan-additions.md b/presets/mysql/plan-additions.md new file mode 100644 index 0000000..7283406 --- /dev/null +++ b/presets/mysql/plan-additions.md @@ -0,0 +1,25 @@ +## MySQL-specific concerns + +### Operational safety + +| Source/model | Database role | Lock/load risk | Mitigation | +|---|---|---|---| +| | primary / replica / analytics | low/medium/high | | + +### Index and query plan + +| Model/query | Required indexes | EXPLAIN expected? | Notes | +|---|---|---|---| +| | | yes/no | | + +### Incremental strategy + +| Model | Watermark/key | Batch size | Full-refresh window | +|---|---|---|---| +| | | | | + +### Compatibility assumptions + +| Area | Assumption | Risk | +|---|---|---| +| engine/charset/collation/timezone | | | diff --git a/presets/postgres/constitution-additions.md b/presets/postgres/constitution-additions.md new file mode 100644 index 0000000..f3eee31 --- /dev/null +++ b/presets/postgres/constitution-additions.md @@ -0,0 +1,25 @@ +## Article P1 — Production analytics do not harm OLTP systems + +Plans identify whether Postgres is an analytics engine, replica, or operational source. Heavy SQLMesh +models must run on an analytics-safe database or replica, not an unconstrained production OLTP +primary. + +## Article P2 — Index and materialization choices are explicit + +Tables, incremental models, and materialized views document indexes needed for joins, filters, and +refresh behavior. Views over expensive joins require a justification. + +## Article P3 — Locking and transaction impact are reviewed + +Plans for large table rebuilds, materialized view refreshes, or schema changes document lock risk and +deployment timing. Blocking production workloads is not acceptable without an approved window. + +## Article P4 — Grants and schemas are part of the plan + +Marts document schema ownership, grants, and whether access is through direct tables, views, or a +serving schema. Sensitive columns require an access decision. + +## Article P5 — Extensions and non-portable SQL are called out + +Use of Postgres-specific extensions, custom functions, JSON operators, or advanced indexes is named +in the plan with compatibility and maintenance expectations. diff --git a/presets/postgres/plan-additions.md b/presets/postgres/plan-additions.md new file mode 100644 index 0000000..9ba5330 --- /dev/null +++ b/presets/postgres/plan-additions.md @@ -0,0 +1,25 @@ +## Postgres-specific concerns + +### OLTP safety + +| Model/source | Database role | Workload risk | Mitigation | +|---|---|---|---| +| | analytics / replica / primary | low/medium/high | | + +### Index and materialization plan + +| Model | Materialization | Indexes | Refresh/load behavior | +|---|---|---|---| +| | view / table / incremental / materialized view | `` or none | | + +### Locking and deployment + +| Change | Lock risk | Deployment window needed? | Mitigation | +|---|---|---|---| +| | low/medium/high | yes/no | | + +### Grants and schemas + +| Output | Schema | Grant/access path | Sensitive columns | +|---|---|---|---| +| | | table / view / role | | diff --git a/presets/redshift/constitution-additions.md b/presets/redshift/constitution-additions.md new file mode 100644 index 0000000..37505d3 --- /dev/null +++ b/presets/redshift/constitution-additions.md @@ -0,0 +1,25 @@ +## Article R1 — Distribution and sort keys are explicit decisions + +Large fact and mart tables document distribution style, distribution key, sort key, and why the +choice matches the dominant joins and filters. `AUTO` is acceptable only when the plan names the +expected table size and review evidence. + +## Article R2 — Vacuum and analyze expectations are planned + +Incremental and append-heavy models document whether `VACUUM` and `ANALYZE` are expected after +production builds. Plans call out the owner of table maintenance when SQLMesh is not responsible. + +## Article R3 — Spectrum and external data are bounded + +Models reading external tables document partition pruning, file layout, and expected scan size. +External joins to large internal tables require an explicit materialization or staging decision. + +## Article R4 — Workload and cost impact are visible + +Plans identify the production workload queue, concurrency scaling expectations, and any large +full-refresh risk. Production plans do not rely on unbounded full scans as routine behavior. + +## Article R5 — Grants, schemas, and late binding views are deliberate + +Governed marts document schema ownership, grants, and whether late binding views are used for +consumer stability. Restricted data requires an access decision before implementation. diff --git a/presets/redshift/plan-additions.md b/presets/redshift/plan-additions.md new file mode 100644 index 0000000..bbf3fe6 --- /dev/null +++ b/presets/redshift/plan-additions.md @@ -0,0 +1,25 @@ +## Redshift-specific concerns + +### Distribution and sort decisions + +| Model | Size estimate | Dist style/key | Sort key | Justification | +|---|---|---|---|---| +| | | AUTO / KEY(``) / EVEN / ALL | `` or none | | + +### Maintenance expectations + +| Model | Load pattern | Vacuum needed? | Analyze needed? | Owner | +|---|---|---|---|---| +| | full / incremental / append | yes/no | yes/no | SQLMesh / platform | + +### External and Spectrum access + +| Source | Partition boundary | File layout risk | Mitigation | +|---|---|---|---| +| | | small files / unbounded scan | | + +### Workload and cost guardrails + +| Job | Queue/workload | Full-refresh risk | Guardrail | +|---|---|---|---| +| | | low/medium/high | | diff --git a/presets/snowflake/constitution-additions.md b/presets/snowflake/constitution-additions.md new file mode 100644 index 0000000..6c5f4e2 --- /dev/null +++ b/presets/snowflake/constitution-additions.md @@ -0,0 +1,23 @@ +## Article S1 — Clustering keys are explicit decisions + +Tables larger than 1 GB declare a clustering key in `config(cluster_by=[...])`. The choice of clustering key is justified in the plan. Tables smaller than 1 GB do not need clustering. + +## Article S2 — Engine sizing is documented + +Production builds use a named engine. The spec or plan declares which engine and at what size (X-Small, Small, Medium, ...). Auto-suspend is set to ≤ 5 minutes. + +## Article S3 — Query tags are required + +All production runs set `query_tag` via a `config.yaml` pre-hook so spend can be attributed by team and project. The query tag includes at minimum: project name, model name, environment (dev|prod), and invocation id. + +## Article S4 — Masking and row access policies + +Any column carrying PII or restricted data is masked via a Snowflake masking policy attached at the table level. Masking policies live in `models/_governance/masking_policies.sql` and are tested for correctness via SQLMesh unit tests. + +## Article S5 — Dynamic tables, when used, are first-class + +If a model is materialized as a Snowflake dynamic table, the spec calls out: target lag, refresh mode (incremental vs. full), and downstream dependencies. Dynamic tables are NOT the default — explicit choice required. + +## Article S6 — Cortex usage is governed + +LLM-powered transformations (Cortex `COMPLETE`, `SUMMARIZE`, etc.) are: (a) called out in the spec, (b) tested with deterministic fixtures, (c) cost-capped via engine sizing. No Cortex calls outside `models/_ai/` without an explicit waiver in the plan. diff --git a/presets/snowflake/plan-additions.md b/presets/snowflake/plan-additions.md new file mode 100644 index 0000000..06b8997 --- /dev/null +++ b/presets/snowflake/plan-additions.md @@ -0,0 +1,30 @@ +## Snowflake-specific concerns + +### Clustering decisions + +| Model | Size estimate | Clustering key | Justification | +|---|---|---|---| +| | | `[col1, col2]` or none | | + +### Engine sizing + +| Job | Engine | Size | Auto-suspend (s) | Justification | +|---|---|---|---|---| +| | | | | | + +### Query tag plan + +The `query_tag` for this work will include: `project=, model=, env=, run_id=`. + +### Masking & governance + +| Column | Source | Policy | Tested? | +|---|---|---|---| +| | | | yes/no | + +### Cost guardrails + +| Risk | Mitigation | +|---|---| +| Unbounded scan on large source | Add date predicate via incremental config | +| Unintended engine upsize | Use named engine, not USE WAREHOUSE inline | diff --git a/presets/spark/constitution-additions.md b/presets/spark/constitution-additions.md new file mode 100644 index 0000000..18d0055 --- /dev/null +++ b/presets/spark/constitution-additions.md @@ -0,0 +1,3 @@ +## spark SQLMesh engine guardrails + +- Plans must document engine-specific cost, catalog, and promotion risks. diff --git a/presets/spark/plan-additions.md b/presets/spark/plan-additions.md new file mode 100644 index 0000000..781fdb2 --- /dev/null +++ b/presets/spark/plan-additions.md @@ -0,0 +1,6 @@ +## spark-specific SQLMesh concerns + +| Decision | Choice | Rationale | +|---|---|---| +| Gateway / connection | | | +| Backfill cost guardrail | | | diff --git a/presets/trino/constitution-additions.md b/presets/trino/constitution-additions.md new file mode 100644 index 0000000..f82e9d7 --- /dev/null +++ b/presets/trino/constitution-additions.md @@ -0,0 +1,39 @@ +## Article T1 — Trino is a query engine, not a engine + +Models built on Trino are not "stored in Trino." Storage lives in the underlying connector (Iceberg, Delta, Hive, Postgres, Kafka, etc.). Spec and plan documents declare both the **catalog** (= connector + storage) and any storage-format concerns for that catalog. Storage decisions (partitioning, file size, table format options) belong in plan-additions sections aligned to the catalog used. + +## Article T2 — Three-part naming is mandatory + +All references use three-part names: `..`. Two-part references are forbidden, even when the default catalog is set. Reason: silent routing to the wrong catalog is one of the most common Trino-on-SQLMesh failure modes, and three-part names make every cross-catalog join visible at code-review time. + +## Article T3 — Cross-catalog joins are deliberate, never accidental + +Any model that joins across two or more catalogs is called out in the spec's "Constraints" section AND in the plan's "Federation impact" table. Cross-catalog joins move data over the network and the cost model is opaque. Reviewers reject plans that introduce cross-catalog joins without explicit justification. + +## Article T4 — Connector pushdown is a planning concern, not a runtime hope + +For every model that filters or aggregates on a non-Iceberg-native catalog (Postgres, MySQL, Hive with Glue, Kafka), the plan declares **which predicates are expected to push down** and how this was verified (via `EXPLAIN`). Models that scan an entire underlying table because pushdown silently failed are a Constitution §11 violation (silent breaking change to performance contract). + +## Article T5 — Session properties replace engine sizing + +Trino has no equivalent of Snowflake's named engines or Databricks's compute clusters at the per-query level. Session properties (`query_max_run_time`, `query_max_memory`, `task_concurrency`, etc.) are set via `session_properties` in `profiles.yml` or `pre_hook` for one-off overrides. The plan declares non-default session properties. + +## Article T6 — `on_table_exists` is an explicit choice + +SQLMesh-trino's table materialization supports two modes: `rename` (default) and `drop`. Plans for table-materialized models declare which mode applies, especially where the AWS Glue Metastore is the backend (Glue cannot rename — `drop` is required there). + +## Article T7 — View security is declared + +View materializations declare `view_security` as either `definer` (default — view runs as its creator) or `invoker` (runs as the caller). The default is fine for most cases; specs that require `invoker` semantics call this out explicitly because the downstream access pattern changes. + +## Article T8 — Iceberg is the preferred destination format + +When the destination catalog is configurable, models materialize into an Iceberg-backed catalog rather than Hive or external relational connectors. Reasons: Iceberg supports time travel, hidden partitioning, schema evolution, and works cleanly with SQLMesh's incremental materializations. Plans that materialize into Hive or relational backends justify the choice (often: legacy constraint). + +## Article T9 — Adapter is community-maintained + +`SQLMesh-trino` is maintained by Starburst, not by SQLMesh Labs. Adapter-specific concerns (new features, breaking changes, version pinning) follow the SQLMesh-trino release cadence, not SQLMesh-core's. The `dependencies.yml` of any project using this preset pins `SQLMesh-trino` to a known-good minor version range. + +## Article T10 — Cost attribution is best-effort + +Trino has no native query-tag-and-bill mechanism. Cost attribution depends on (a) session properties that include identifying tags, (b) log aggregation from coordinator-side query logs, and (c) connector-side billing for the underlying storage. Specs and plans note when cost attribution matters and how it will be achieved for this specific work. diff --git a/presets/trino/plan-additions.md b/presets/trino/plan-additions.md new file mode 100644 index 0000000..c84b8c1 --- /dev/null +++ b/presets/trino/plan-additions.md @@ -0,0 +1,52 @@ +## Trino-specific concerns + +### Catalogs and federation + +| Catalog | Connector | Role in this work | Read / write | +|---|---|---|---| +| `iceberg` | Iceberg + S3 | destination for materialized models | read + write | +| `` | | source data | read-only | + +### Cross-catalog joins (Article T3) + +| Model | Catalogs joined | Justification | Estimated rows moved over network | +|---|---|---|---| +| | `` × `` | | | + +If this table is empty, mark "none" — do not omit the table. + +### Pushdown plan (Article T4) + +| Model | Non-Iceberg sources | Predicates expected to push down | Verified via `EXPLAIN`? | +|---|---|---|---| +| | `` | `, ` | yes / no | + +### Storage-format concerns + +For each model materialized into an Iceberg-backed catalog: + +| Model | Partitioning | File format | Compaction concerns | +|---|---|---|---| +| | `` (e.g., `day(event_ts)`) | parquet / orc | | + +For non-Iceberg destinations, document the equivalent concerns for that table format. + +### Session properties + +| Property | Value | Justification | +|---|---|---| +| `query_max_run_time` | `` | | +| `task_concurrency` | `` | | + +### Materialization choices + +| Model | Materialization | `on_table_exists` | View security | Justification | +|---|---|---|---|---| +| | table / view / incremental / materialized_view | rename / drop / n/a | definer / invoker / n/a | | + +### Cost attribution + +| Risk | Mitigation | +|---|---| +| Cost of large cross-catalog scan invisible at review time | Add `EXPLAIN` output to the plan for any model with cross-catalog joins | +| Underlying connector (e.g., Hive on EMR) billed separately | Tag query via session property `application_name`; coordinator logs aggregated weekly | diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..82f6165 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,104 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "sqlmesh-spec-kit" +version = "0.1.0" +description = "Enterprise AI SDLC toolkit for SQLMesh projects, with spec-driven workflows, CI validation, and engine-specific presets." +readme = "README.md" +license = { file = "LICENSE" } +requires-python = ">=3.11" +authors = [ + { name = "duckcode-ai", email = "open-source@duckcode.ai" } +] +keywords = [ + "sqlmesh", + "spec-driven-development", + "ai-agents", + "analytics-engineering", + "data-engineering", + "sdlc", +] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: Software Development :: Libraries :: Python Modules", +] +dependencies = [ + "click>=8.1", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = [ + "build>=1.2", + "pytest>=7.0", + "pytest-cov>=4.0", + "ruff>=0.4", + "mypy>=1.10", + "twine>=5.0", + "types-PyYAML", +] + +[project.scripts] +sqlmesh-specify = "sqlmesh_specify.cli:main" + +[project.urls] +Homepage = "https://github.com/duckcode-ai/sqlmesh-spec-kit" +Repository = "https://github.com/duckcode-ai/sqlmesh-spec-kit" +Issues = "https://github.com/duckcode-ai/sqlmesh-spec-kit/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/sqlmesh_specify"] + +[tool.hatch.build.targets.wheel.force-include] +"memory" = "sqlmesh_specify/_assets/memory" +"templates" = "sqlmesh_specify/_assets/templates" +"presets" = "sqlmesh_specify/_assets/presets" +"skills" = "sqlmesh_specify/_assets/skills" +"commands" = "sqlmesh_specify/_assets/commands" +"agents" = "sqlmesh_specify/_assets/agents" + +[tool.hatch.build.targets.sdist] +exclude = [ + "/.git", + "/.github", + "/.mypy_cache", + "/.pytest_cache", + "/.ruff_cache", + "/.venv", + "/build", + "/dist", + "/htmlcov", + "/.coverage", + "/.coverage.*", + "/BUILD-SUMMARY.md", + "/FOLLOWUP.md", + "/PLAN.md", + "/QUESTIONS.md", + "/uv.lock", +] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "W", "UP", "B", "A", "C4", "RET", "SIM"] +ignore = [] + +[tool.mypy] +python_version = "3.11" +strict = true +warn_return_any = true +warn_unused_configs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = "test_*.py" +addopts = "-v --cov=sqlmesh_specify --cov-report=term-missing" diff --git a/skills/capturing-project-conventions/SKILL.md b/skills/capturing-project-conventions/SKILL.md new file mode 100644 index 0000000..2724fe2 --- /dev/null +++ b/skills/capturing-project-conventions/SKILL.md @@ -0,0 +1,10 @@ +# Capturing Project Conventions + +Use this when turning repo-specific SQLMesh practices into durable memory. + +Checklist: + +- Capture naming, model layout, environment strategy, audit/test expectations, and engine rules. +- Keep project memory concise and actionable. +- Add conventions to `.sqlmesh-specify/constitution.md` or a focused skill. +- Do not store secrets, raw customer data, or private credentials. diff --git a/skills/reviewing-audits-and-tests/SKILL.md b/skills/reviewing-audits-and-tests/SKILL.md new file mode 100644 index 0000000..b46b1a2 --- /dev/null +++ b/skills/reviewing-audits-and-tests/SKILL.md @@ -0,0 +1,10 @@ +# Reviewing Audits and Tests + +Use this when checking SQLMesh validation coverage. + +Checklist: + +- Map every acceptance criterion to at least one test, audit, review check, or plan artifact. +- Prefer SQLMesh audits for data quality invariants that must run with the project. +- Check tests cover changed model grain, joins, edge cases, and downstream contracts. +- Block when evidence is missing or not tied to AC ids. diff --git a/skills/reviewing-pii-access-governance/SKILL.md b/skills/reviewing-pii-access-governance/SKILL.md new file mode 100644 index 0000000..c591717 --- /dev/null +++ b/skills/reviewing-pii-access-governance/SKILL.md @@ -0,0 +1,10 @@ +# Reviewing PII Access Governance + +Use this when models touch PII, regulated data, or access policy. + +Checklist: + +- Identify sensitive columns and required masking, hashing, tokenization, or access controls. +- Confirm downstream consumers and environment promotion are approved for the data class. +- Require audits/tests that prevent accidental exposure or grain expansion. +- Add redaction notes before publishing Jira or Confluence artifacts. diff --git a/skills/reviewing-plan-apply-evidence/SKILL.md b/skills/reviewing-plan-apply-evidence/SKILL.md new file mode 100644 index 0000000..ee3b177 --- /dev/null +++ b/skills/reviewing-plan-apply-evidence/SKILL.md @@ -0,0 +1,10 @@ +# Reviewing Plan and Apply Evidence + +Use this when reviewing `sqlmesh plan`, apply, or promotion output. + +Checklist: + +- Confirm the target environment matches `plan.md`. +- Compare changed snapshots to the approved file list. +- Flag unexpected backfill, restatement, forward-only, or destructive changes. +- Verify apply/promote evidence is attached before a spec is marked shipped. diff --git a/skills/writing-incremental-model-specs/SKILL.md b/skills/writing-incremental-model-specs/SKILL.md new file mode 100644 index 0000000..a794c00 --- /dev/null +++ b/skills/writing-incremental-model-specs/SKILL.md @@ -0,0 +1,10 @@ +# Writing Incremental Model Specs + +Use this when a SQLMesh change may affect incremental behavior. + +Checklist: + +- Define the incremental key, time column, late-arriving data policy, and backfill window. +- Call out forward-only changes and restatement risk. +- Require SQLMesh plan evidence that shows expected snapshot and backfill behavior. +- Add audits/tests for uniqueness, freshness, null handling, and boundary dates. diff --git a/skills/writing-sqlmesh-model-specs/SKILL.md b/skills/writing-sqlmesh-model-specs/SKILL.md new file mode 100644 index 0000000..5206542 --- /dev/null +++ b/skills/writing-sqlmesh-model-specs/SKILL.md @@ -0,0 +1,10 @@ +# Writing SQLMesh Model Specs + +Use this when drafting specs for SQLMesh model changes. + +Checklist: + +- Name affected models, upstream dependencies, downstream consumers, and one-row grain. +- Keep acceptance criteria in EARS format. +- State required audits, tests, and `sqlmesh plan ` evidence. +- Do not prescribe implementation until the spec is approved. diff --git a/src/sqlmesh_specify/__init__.py b/src/sqlmesh_specify/__init__.py new file mode 100644 index 0000000..c9ce4ca --- /dev/null +++ b/src/sqlmesh_specify/__init__.py @@ -0,0 +1,6 @@ +"""sqlmesh-spec-kit - spec-driven development for SQLMesh projects.""" +from __future__ import annotations + +from sqlmesh_specify._version import __version__ + +__all__ = ["__version__"] diff --git a/src/sqlmesh_specify/__main__.py b/src/sqlmesh_specify/__main__.py new file mode 100644 index 0000000..771c17f --- /dev/null +++ b/src/sqlmesh_specify/__main__.py @@ -0,0 +1,7 @@ +"""Entry point for `python -m sqlmesh_specify`.""" +from __future__ import annotations + +from sqlmesh_specify.cli import main + +if __name__ == "__main__": + main() diff --git a/src/sqlmesh_specify/_version.py b/src/sqlmesh_specify/_version.py new file mode 100644 index 0000000..47014dd --- /dev/null +++ b/src/sqlmesh_specify/_version.py @@ -0,0 +1,4 @@ +"""Single source of truth for the package version.""" +from __future__ import annotations + +__version__ = "0.1.0" diff --git a/src/sqlmesh_specify/cli.py b/src/sqlmesh_specify/cli.py new file mode 100644 index 0000000..84135cd --- /dev/null +++ b/src/sqlmesh_specify/cli.py @@ -0,0 +1,430 @@ +"""Click command group for the sqlmesh-specify CLI.""" +from __future__ import annotations + +from pathlib import Path + +import click + +from sqlmesh_specify._version import __version__ +from sqlmesh_specify.confluence import ( + ConfluenceError, + make_confluence_client, + pull_page_to_context, +) +from sqlmesh_specify.confluence import publish_spec_dir as publish_confluence_spec_dir +from sqlmesh_specify.confluence import sync_spec_dir as sync_confluence_spec_dir +from sqlmesh_specify.doctor import doctor_project +from sqlmesh_specify.init import SUPPORTED_ENGINES, init_project +from sqlmesh_specify.jira import ( + JiraError, + attach_artifacts, + create_subtasks_from_tasks, + make_jira_client, + pull_issue_to_spec, + sync_spec_dir, +) +from sqlmesh_specify.lifecycle import validate_lifecycle +from sqlmesh_specify.reporting import ValidationReport, combine_reports +from sqlmesh_specify.sqlmesh_artifacts import validate_sqlmesh_project +from sqlmesh_specify.validate import validate_spec + + +@click.group() +@click.version_option(__version__, prog_name="sqlmesh-specify") +def main() -> None: + """AI SDLC and durable memory for SQLMesh projects.""" + + +@main.command() +@click.argument("project_name") +@click.option( + "--engine", + type=click.Choice(SUPPORTED_ENGINES, case_sensitive=False), + required=True, + help="SQLMesh engine preset to install alongside the base constitution.", +) +@click.option( + "--target", + "target_dir", + type=click.Path(file_okay=False, path_type=Path), + default=Path("."), + help="Target SQLMesh project directory.", +) +@click.option("--force", is_flag=True, help="Overwrite an existing .sqlmesh-specify/ directory.") +def init(project_name: str, engine: str, target_dir: Path, force: bool) -> None: + """Initialize sqlmesh-specify in an existing SQLMesh project.""" + init_project(project_name, engine.lower(), target_dir.resolve(), force) + + +@main.command(context_settings={"ignore_unknown_options": True, "allow_extra_args": True}) +@click.argument("args", nargs=-1, type=click.UNPROCESSED) +def validate(args: tuple[str, ...]) -> None: + """Validate a spec file, lifecycle artifacts, or SQLMesh project structure.""" + if not args: + click.echo( + "Usage:\n" + " sqlmesh-specify validate path/to/spec.md\n" + " sqlmesh-specify validate project --target .\n" + " sqlmesh-specify validate sqlmesh --target ." + ) + raise SystemExit(2) + + subcommand = args[0].lower() + if subcommand == "project": + target_dir = _path_option(args[1:], "--target", Path(".")).resolve() + report = validate_lifecycle(target_dir) + _echo_report(report, "markdown") + raise SystemExit(report.exit_code) + if subcommand == "sqlmesh": + target_dir = _path_option(args[1:], "--target", Path(".")).resolve() + report = validate_sqlmesh_project(target_dir) + _echo_report(report, "markdown") + raise SystemExit(report.exit_code) + if len(args) != 1: + click.echo( + "error: expected `validate `, `validate project`, or `validate sqlmesh`", + err=True, + ) + raise SystemExit(2) + + spec_path = Path(args[0]) + if not spec_path.exists() or not spec_path.is_file(): + click.echo(f"error: spec file not found: {spec_path}", err=True) + raise SystemExit(1) + raise SystemExit(validate_spec(spec_path.resolve())) + + +@main.command() +@click.option( + "--target", + "target_dir", + type=click.Path(file_okay=False, path_type=Path), + default=Path("."), + help="Target SQLMesh project directory.", +) +def doctor(target_dir: Path) -> None: + """Inspect a brownfield SQLMesh repo for adoption gaps.""" + report = doctor_project(target_dir.resolve()) + _echo_report(report, "markdown") + raise SystemExit(report.exit_code) + + +@main.command(name="report") +@click.option( + "--target", + "target_dir", + type=click.Path(file_okay=False, path_type=Path), + default=Path("."), + help="Target SQLMesh project directory.", +) +@click.option( + "--format", + "output_format", + type=click.Choice(["markdown", "json"], case_sensitive=False), + default="markdown", + help="Output format.", +) +def report(target_dir: Path, output_format: str) -> None: + """Emit CI-ready validation evidence.""" + combined = _combined_validation_report(target_dir.resolve()) + _echo_report(combined, output_format.lower()) + raise SystemExit(combined.exit_code) + + +@main.command() +@click.option( + "--target", + "target_dir", + type=click.Path(file_okay=False, path_type=Path), + default=Path("."), + help="Target SQLMesh project directory.", +) +def ci(target_dir: Path) -> None: + """Run release-blocking sqlmesh-specify checks for CI.""" + combined = _combined_validation_report(target_dir.resolve()) + _echo_report(combined, "markdown") + raise SystemExit(combined.exit_code) + + +@main.group() +def jira() -> None: + """Read from and publish sqlmesh-specify artifacts to Jira Cloud.""" + + +@jira.command("pull") +@click.argument("issue_key") +@click.option( + "--target", + "target_dir", + type=click.Path(file_okay=False, path_type=Path), + default=Path("."), + help="Target SQLMesh project directory.", +) +@click.option("--slug", default=None, help="Optional slug for specs/-/.") +@click.option("--force", is_flag=True, help="Overwrite existing generated files when present.") +def jira_pull(issue_key: str, target_dir: Path, slug: str | None, force: bool) -> None: + """Create a local spec draft from a Jira issue.""" + try: + spec_dir = pull_issue_to_spec( + client=make_jira_client(), + issue_key=issue_key, + target_dir=target_dir.resolve(), + slug=slug, + force=force, + ) + except JiraError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + click.echo(f"created {spec_dir / 'spec.md'}") + click.echo(f"created {spec_dir / 'jira.yml'}") + click.echo("next: review the draft spec, then run /sqlmesh.plan after approval") + + +@jira.command("attach") +@click.argument("issue_key") +@click.option("--spec", "spec_path", type=click.Path(dir_okay=False, path_type=Path)) +@click.option("--plan", "plan_path", type=click.Path(dir_okay=False, path_type=Path)) +@click.option("--tasks", "tasks_path", type=click.Path(dir_okay=False, path_type=Path)) +@click.option( + "--file", + "extra_files", + multiple=True, + type=click.Path(dir_okay=False, path_type=Path), +) +@click.option("--no-comment", is_flag=True, help="Do not add a Jira comment after upload.") +def jira_attach( + issue_key: str, + spec_path: Path | None, + plan_path: Path | None, + tasks_path: Path | None, + extra_files: tuple[Path, ...], + no_comment: bool, +) -> None: + """Attach approved local artifacts to a Jira issue.""" + files = _artifact_paths(spec_path, plan_path, tasks_path, extra_files) + try: + uploaded = attach_artifacts( + client=make_jira_client(), issue_key=issue_key, files=files, comment=not no_comment + ) + except JiraError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + for filename in uploaded: + click.echo(f"attached {filename} to {issue_key}") + + +@jira.command("create-tasks") +@click.argument("issue_key") +@click.option( + "--from", + "tasks_path", + required=True, + type=click.Path(dir_okay=False, path_type=Path), +) +@click.option("--issue-type", "issue_type_name", default="Sub-task", show_default=True) +@click.option("--include-done", is_flag=True, help="Also create subtasks for checked tasks.") +@click.option("--dry-run", is_flag=True, help="Print subtasks that would be created.") +def jira_create_tasks( + issue_key: str, + tasks_path: Path, + issue_type_name: str, + include_done: bool, + dry_run: bool, +) -> None: + """Create Jira subtasks from a tasks.md file.""" + try: + results = create_subtasks_from_tasks( + client=make_jira_client(), + issue_key=issue_key, + tasks_path=tasks_path.resolve(), + issue_type_name=issue_type_name, + include_done=include_done, + dry_run=dry_run, + ) + except JiraError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + if not results: + click.echo("no pending tasks found") + return + for result in results: + if result.created: + click.echo(f"created {result.key}: {result.summary}") + elif result.key: + click.echo(f"skipped existing {result.key}: {result.summary}") + else: + click.echo(f"would create: {result.summary}") + + +@jira.command("sync") +@click.argument("issue_key") +@click.option("--spec-dir", required=True, type=click.Path(file_okay=False, path_type=Path)) +@click.option("--issue-type", "issue_type_name", default="Sub-task", show_default=True) +@click.option("--dry-run", is_flag=True, help="Print sync actions without writing to Jira.") +def jira_sync(issue_key: str, spec_dir: Path, issue_type_name: str, dry_run: bool) -> None: + """Attach spec/plan and create Jira subtasks for a spec directory.""" + try: + uploaded, subtasks = sync_spec_dir( + client=make_jira_client(), + issue_key=issue_key, + spec_dir=spec_dir.resolve(), + issue_type_name=issue_type_name, + dry_run=dry_run, + ) + except JiraError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + for filename in uploaded: + click.echo(f"attached {filename} to {issue_key}") + for result in subtasks: + if result.created: + click.echo(f"created {result.key}: {result.summary}") + elif result.key: + click.echo(f"skipped existing {result.key}: {result.summary}") + else: + click.echo(f"would create: {result.summary}") + if not uploaded and not subtasks: + click.echo("nothing to sync") + + +@main.group() +def confluence() -> None: + """Read from and publish sqlmesh-specify context to Confluence Cloud.""" + + +@confluence.command("pull-page") +@click.argument("page_id") +@click.option("--to", "output_path", required=True, type=click.Path(dir_okay=False, path_type=Path)) +@click.option("--spec-dir", type=click.Path(file_okay=False, path_type=Path), default=None) +def confluence_pull_page(page_id: str, output_path: Path, spec_dir: Path | None) -> None: + """Pull a Confluence page into local markdown context.""" + try: + page = pull_page_to_context( + client=make_confluence_client(), + page_id=page_id, + output_path=output_path.resolve(), + spec_dir=spec_dir.resolve() if spec_dir else None, + ) + except ConfluenceError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + click.echo(f"pulled {page.title} ({page.page_id})") + click.echo(f"wrote {output_path}") + + +@confluence.command("publish") +@click.option("--spec-dir", required=True, type=click.Path(file_okay=False, path_type=Path)) +@click.option("--space-key", default=None) +@click.option("--space-id", default=None) +@click.option("--parent-id", default=None) +@click.option("--page-id", default=None) +@click.option("--title", default=None) +@click.option("--dry-run", is_flag=True) +def confluence_publish( + spec_dir: Path, + space_key: str | None, + space_id: str | None, + parent_id: str | None, + page_id: str | None, + title: str | None, + dry_run: bool, +) -> None: + """Create or update a Confluence summary page for a spec directory.""" + try: + page = publish_confluence_spec_dir( + client=None if dry_run else make_confluence_client(), + spec_dir=spec_dir.resolve(), + space_key=space_key, + space_id=space_id, + parent_id=parent_id, + page_id=page_id, + title=title, + dry_run=dry_run, + ) + except ConfluenceError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + if dry_run: + action = "would create" if page.created else "would update" + else: + action = "created" if page.created else "updated" + click.echo(f"{action} Confluence page {page.page_id}: {page.title}") + if page.page_url != "": + click.echo(page.page_url) + + +@confluence.command("sync") +@click.option("--spec-dir", required=True, type=click.Path(file_okay=False, path_type=Path)) +@click.option("--dry-run", is_flag=True) +def confluence_sync(spec_dir: Path, dry_run: bool) -> None: + """Update the Confluence page recorded in specs//confluence.yml.""" + try: + page = sync_confluence_spec_dir( + client=None if dry_run else make_confluence_client(), + spec_dir=spec_dir.resolve(), + dry_run=dry_run, + ) + except ConfluenceError as error: + click.echo(f"error: {error}", err=True) + raise SystemExit(1) from error + if dry_run: + action = "would create" if page.created else "would update" + else: + action = "created" if page.created else "updated" + click.echo(f"{action} Confluence page {page.page_id}: {page.title}") + if page.page_url != "": + click.echo(page.page_url) + + +@main.command() +def version() -> None: + """Print the installed version.""" + click.echo(__version__) + + +def _combined_validation_report(target_dir: Path) -> ValidationReport: + return combine_reports( + "sqlmesh-specify CI report", + [validate_lifecycle(target_dir), validate_sqlmesh_project(target_dir)], + ) + + +def _echo_report(report: ValidationReport, output_format: str) -> None: + if output_format == "json": + click.echo(report.to_json()) + return + click.echo(report.to_markdown()) + + +def _artifact_paths( + spec_path: Path | None, + plan_path: Path | None, + tasks_path: Path | None, + extra_files: tuple[Path, ...], +) -> list[Path]: + files = [path for path in (spec_path, plan_path, tasks_path) if path is not None] + files.extend(extra_files) + return [path.resolve() for path in files] + + +def _path_option(args: tuple[str, ...], flag: str, default: Path) -> Path: + value = _optional_path_option(args, flag) + return value or default + + +def _optional_path_option(args: tuple[str, ...], flag: str) -> Path | None: + for index, arg in enumerate(args): + if arg == flag: + try: + return Path(args[index + 1]) + except IndexError: + click.echo(f"error: {flag} requires a path", err=True) + raise SystemExit(2) from None + prefix = f"{flag}=" + if arg.startswith(prefix): + return Path(arg.removeprefix(prefix)) + return None + + +if __name__ == "__main__": + main() diff --git a/src/sqlmesh_specify/confluence.py b/src/sqlmesh_specify/confluence.py new file mode 100644 index 0000000..6ac0045 --- /dev/null +++ b/src/sqlmesh_specify/confluence.py @@ -0,0 +1,593 @@ +"""Confluence Cloud bridge for sqlmesh-specify knowledge artifacts.""" +from __future__ import annotations + +import base64 +import html +import json +import os +from dataclasses import dataclass +from datetime import UTC, datetime +from html.parser import HTMLParser +from pathlib import Path +from typing import Any, Protocol +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + +import yaml + + +class ConfluenceError(RuntimeError): + """Raised when Confluence integration cannot complete safely.""" + + +@dataclass(frozen=True) +class ConfluenceConfig: + """Confluence Cloud connection settings.""" + + base_url: str + email: str + api_token: str + + +@dataclass(frozen=True) +class ConfluencePage: + """Normalized Confluence page fields used by sqlmesh-specify.""" + + page_id: str + title: str + space_id: str | None + space_key: str | None + version: int + page_url: str + storage: str + + +@dataclass(frozen=True) +class PublishedPage: + """Result of creating or updating a Confluence page.""" + + page_id: str + title: str + page_url: str + created: bool + + +class ConfluenceApi(Protocol): + """Methods required by spec publishing helpers.""" + + def get_page(self, page_id: str) -> ConfluencePage: + """Fetch a Confluence page with storage-format body.""" + + def resolve_space_id(self, space_key: str) -> str: + """Resolve a Confluence space key to a v2 API space id.""" + + def create_page( + self, + *, + space_id: str, + title: str, + storage: str, + parent_id: str | None = None, + ) -> PublishedPage: + """Create a Confluence page with storage-format body.""" + + def update_page(self, *, page_id: str, title: str, storage: str) -> PublishedPage: + """Update an existing Confluence page with storage-format body.""" + + +class ConfluenceClient: + """Small Confluence Cloud REST client backed by the Python standard library.""" + + def __init__(self, config: ConfluenceConfig) -> None: + self._config = config + + def get_page(self, page_id: str) -> ConfluencePage: + """Fetch a Confluence page with storage-format body.""" + raw = self._request_json( + "GET", + f"/wiki/api/v2/pages/{quote(page_id)}?body-format=storage", + ) + return _parse_page(raw, self._config.base_url) + + def resolve_space_id(self, space_key: str) -> str: + """Resolve a Confluence space key to a v2 API space id.""" + raw = self._request_json("GET", f"/wiki/api/v2/spaces?{urlencode({'keys': space_key})}") + if not isinstance(raw, dict): + raise ConfluenceError("Confluence spaces response was not an object.") + results = raw.get("results") + if not isinstance(results, list) or not results: + raise ConfluenceError(f"Confluence space not found for key: {space_key}") + first = results[0] + if not isinstance(first, dict): + raise ConfluenceError(f"Confluence space response missing id for key: {space_key}") + space_id = first.get("id") + if not isinstance(space_id, str): + raise ConfluenceError(f"Confluence space response missing id for key: {space_key}") + return space_id + + def create_page( + self, + *, + space_id: str, + title: str, + storage: str, + parent_id: str | None = None, + ) -> PublishedPage: + """Create a Confluence page with storage-format body.""" + payload: dict[str, object] = { + "spaceId": space_id, + "status": "current", + "title": title, + "body": {"representation": "storage", "value": storage}, + } + if parent_id: + payload["parentId"] = parent_id + + raw = self._request_json("POST", "/wiki/api/v2/pages", json_body=payload) + page = _parse_page(raw, self._config.base_url) + return PublishedPage( + page_id=page.page_id, + title=page.title, + page_url=page.page_url, + created=True, + ) + + def update_page(self, *, page_id: str, title: str, storage: str) -> PublishedPage: + """Update an existing Confluence page with storage-format body.""" + current = self.get_page(page_id) + raw = self._request_json( + "PUT", + f"/wiki/api/v2/pages/{quote(page_id)}", + json_body={ + "id": page_id, + "status": "current", + "title": title, + "body": {"representation": "storage", "value": storage}, + "version": { + "number": current.version + 1, + "message": "Updated by sqlmesh-specify", + }, + }, + ) + page = _parse_page(raw, self._config.base_url) + return PublishedPage( + page_id=page.page_id, + title=page.title, + page_url=page.page_url, + created=False, + ) + + def _request_json( + self, + method: str, + path: str, + *, + json_body: dict[str, object] | None = None, + ) -> dict[str, Any] | list[Any]: + data = None + headers = { + "Authorization": _basic_auth(self._config.email, self._config.api_token), + "Accept": "application/json", + } + if json_body is not None: + data = json.dumps(json_body).encode() + headers["Content-Type"] = "application/json" + + request = Request( + f"{self._config.base_url.rstrip('/')}{path}", + data=data, + headers=headers, + method=method, + ) + try: + with urlopen(request, timeout=30) as response: + response_body = response.read() + except HTTPError as error: + detail = error.read().decode(errors="replace") + raise ConfluenceError( + f"Confluence API request failed with HTTP {error.code}: {detail}" + ) from error + except URLError as error: + raise ConfluenceError(f"Confluence API request failed: {error.reason}") from error + + if not response_body: + return {} + parsed = json.loads(response_body.decode()) + if not isinstance(parsed, (dict, list)): + raise ConfluenceError("Confluence API returned an unexpected JSON response.") + return parsed + + +def config_from_env() -> ConfluenceConfig: + """Load Confluence Cloud credentials from environment variables.""" + base_url = os.environ.get("CONFLUENCE_BASE_URL", "").strip() + email = os.environ.get("CONFLUENCE_EMAIL", "").strip() + api_token = os.environ.get("CONFLUENCE_API_TOKEN", "").strip() + missing = [ + name + for name, value in ( + ("CONFLUENCE_BASE_URL", base_url), + ("CONFLUENCE_EMAIL", email), + ("CONFLUENCE_API_TOKEN", api_token), + ) + if not value + ] + if missing: + joined = ", ".join(missing) + raise ConfluenceError(f"Missing Confluence environment variable(s): {joined}") + return ConfluenceConfig( + base_url=base_url.rstrip("/"), + email=email, + api_token=api_token, + ) + + +def make_confluence_client() -> ConfluenceClient: + """Create a Confluence client from environment variables.""" + return ConfluenceClient(config_from_env()) + + +def pull_page_to_context( + *, + client: ConfluenceApi, + page_id: str, + output_path: Path, + spec_dir: Path | None = None, +) -> ConfluencePage: + """Pull a Confluence page into a local markdown context file.""" + page = client.get_page(page_id) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(_context_markdown(page)) + + resolved_spec_dir = spec_dir or infer_spec_dir(output_path) + if resolved_spec_dir is not None: + _record_source_page(resolved_spec_dir, page, output_path) + + return page + + +def publish_spec_dir( + *, + client: ConfluenceApi | None, + spec_dir: Path, + space_key: str | None = None, + space_id: str | None = None, + parent_id: str | None = None, + page_id: str | None = None, + title: str | None = None, + dry_run: bool = False, +) -> PublishedPage: + """Create or update a Confluence summary page for a spec directory.""" + if not spec_dir.exists() or not spec_dir.is_dir(): + raise ConfluenceError(f"spec directory not found: {spec_dir}") + + manifest = read_confluence_manifest(spec_dir) + resolved_page_id = page_id or _optional_str(manifest.get("page_id")) + resolved_space_id = space_id or _optional_str(manifest.get("space_id")) + resolved_space_key = space_key or _optional_str(manifest.get("space_key")) + resolved_parent_id = parent_id or _optional_str(manifest.get("parent_id")) + resolved_title = title or _optional_str(manifest.get("title")) or _default_title(spec_dir) + storage = spec_dir_to_storage(spec_dir, resolved_title) + + if dry_run: + return PublishedPage( + page_id=resolved_page_id or "", + title=resolved_title, + page_url="", + created=resolved_page_id is None, + ) + + if client is None: + raise ConfluenceError("Confluence client is required unless --dry-run is used.") + + if resolved_page_id is not None: + published = client.update_page( + page_id=resolved_page_id, + title=resolved_title, + storage=storage, + ) + else: + if resolved_space_id is None: + if resolved_space_key is None: + raise ConfluenceError("Provide --space-id or --space-key when creating a page.") + resolved_space_id = client.resolve_space_id(resolved_space_key) + published = client.create_page( + space_id=resolved_space_id, + title=resolved_title, + storage=storage, + parent_id=resolved_parent_id, + ) + + write_confluence_manifest( + spec_dir, + { + **manifest, + "space_id": resolved_space_id, + "space_key": resolved_space_key, + "parent_id": resolved_parent_id, + "page_id": published.page_id, + "page_url": published.page_url, + "title": published.title, + "spec_dir": _display_path(spec_dir), + "last_synced_at": _now_iso(), + }, + ) + return published + + +def sync_spec_dir( + *, + client: ConfluenceApi | None, + spec_dir: Path, + dry_run: bool = False, +) -> PublishedPage: + """Update a previously published Confluence page for a spec directory.""" + manifest = read_confluence_manifest(spec_dir) + page_id = _optional_str(manifest.get("page_id")) + if page_id is None: + raise ConfluenceError("confluence.yml is missing page_id. Run confluence publish first.") + return publish_spec_dir( + client=client, + spec_dir=spec_dir, + page_id=page_id, + dry_run=dry_run, + ) + + +def read_confluence_manifest(spec_dir: Path) -> dict[str, object]: + """Read specs//confluence.yml if it exists.""" + path = spec_dir / "confluence.yml" + if not path.exists(): + return {} + loaded = yaml.safe_load(path.read_text()) + return loaded if isinstance(loaded, dict) else {} + + +def write_confluence_manifest(spec_dir: Path, manifest: dict[str, object]) -> None: + """Write specs//confluence.yml.""" + spec_dir.mkdir(parents=True, exist_ok=True) + cleaned = {key: value for key, value in manifest.items() if value is not None} + (spec_dir / "confluence.yml").write_text(yaml.safe_dump(cleaned, sort_keys=False)) + + +def spec_dir_to_storage(spec_dir: Path, title: str) -> str: + """Render local spec artifacts into safe Confluence storage XHTML.""" + sections = [ + f"

{html.escape(title)}

", + "

Generated by: sqlmesh-spec-kit

", + f"

Spec directory: {html.escape(_display_path(spec_dir))}

", + ( + "

This page is a Confluence knowledge summary. The approved local " + "spec.md, plan.md, and tasks.md files remain " + "the source of truth for implementation.

" + ), + ] + for filename in ( + "spec.md", + "plan.md", + "tasks.md", + "review.md", + "governance-review.md", + "sqlmesh-specify-report.md", + ): + path = spec_dir / filename + if path.exists() and path.is_file(): + sections.append(f"

{html.escape(filename)}

") + sections.append(f"
{html.escape(path.read_text())}
") + + context_dir = spec_dir / "context" + if context_dir.is_dir(): + context_files = sorted(path for path in context_dir.glob("*.md") if path.is_file()) + if context_files: + sections.append("

Confluence context pulled into this spec

") + for path in context_files: + sections.append(f"

{html.escape(path.name)}

") + sections.append(f"
{html.escape(path.read_text())}
") + + return "\n".join(sections) + + +def storage_to_markdown(storage: str) -> str: + """Convert Confluence storage XHTML into lightweight markdown context.""" + parser = _StorageToMarkdownParser() + parser.feed(storage) + parser.close() + return parser.markdown() + + +def infer_spec_dir(path: Path) -> Path | None: + """Infer specs/-/ from a file path when possible.""" + parts = path.resolve().parts + if "specs" not in parts: + return None + index = parts.index("specs") + if index + 1 >= len(parts): + return None + return Path(*parts[: index + 2]) + + +def _record_source_page(spec_dir: Path, page: ConfluencePage, output_path: Path) -> None: + manifest = read_confluence_manifest(spec_dir) + raw_sources = manifest.get("source_pages") + sources = raw_sources if isinstance(raw_sources, list) else [] + next_source = { + "page_id": page.page_id, + "title": page.title, + "page_url": page.page_url, + "context_file": _display_path(output_path), + "last_synced_at": _now_iso(), + } + updated_sources = [ + source + for source in sources + if not (isinstance(source, dict) and source.get("page_id") == page.page_id) + ] + updated_sources.append(next_source) + write_confluence_manifest( + spec_dir, + { + **manifest, + "spec_dir": _display_path(spec_dir), + "source_pages": updated_sources, + "last_synced_at": _now_iso(), + }, + ) + + +def _context_markdown(page: ConfluencePage) -> str: + body = storage_to_markdown(page.storage).strip() + return f"""# {page.title} + +**Confluence page:** {page.page_url} +**Page ID:** {page.page_id} +**Synced at:** {_now_iso()} + +## Context + +{body} +""" + + +def _parse_page(raw: dict[str, Any] | list[Any], base_url: str) -> ConfluencePage: + if not isinstance(raw, dict): + raise ConfluenceError("Confluence page response was not an object.") + page_id = _string(raw.get("id"), "page id") + body = _mapping(raw.get("body")) + storage = _mapping(body.get("storage")) + version = _mapping(raw.get("version")) + links = _mapping(raw.get("_links")) + webui = links.get("webui") + page_url = f"{base_url.rstrip('/')}{webui}" if isinstance(webui, str) else "" + if not page_url: + page_url = f"{base_url.rstrip('/')}/wiki/spaces/~pages/{page_id}" + + version_number = version.get("number") + if not isinstance(version_number, int): + version_number = 1 + + return ConfluencePage( + page_id=page_id, + title=_string(raw.get("title"), "title"), + space_id=_optional_str(raw.get("spaceId")), + space_key=_optional_str(raw.get("spaceKey")), + version=version_number, + page_url=page_url, + storage=_optional_str(storage.get("value")) or "", + ) + + +class _StorageToMarkdownParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self._parts: list[str] = [] + self._href: str | None = None + self._in_pre = False + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + attr_map = dict(attrs) + if tag in {"h1", "h2", "h3"}: + self._break() + self._parts.append("#" * int(tag[1])) + self._parts.append(" ") + elif tag == "p": + self._break() + elif tag == "br": + self._parts.append("\n") + elif tag == "li": + self._break() + self._parts.append("- ") + elif tag == "a": + self._href = attr_map.get("href") + elif tag == "pre": + self._in_pre = True + self._break() + self._parts.append("```text\n") + elif tag == "strong": + self._parts.append("**") + elif tag == "em": + self._parts.append("_") + elif tag == "code" and not self._in_pre: + self._parts.append("`") + + def handle_endtag(self, tag: str) -> None: + if tag in {"h1", "h2", "h3", "p", "li", "ul", "ol"}: + self._break() + elif tag == "a": + if self._href: + self._parts.append(f" ({self._href})") + self._href = None + elif tag == "pre": + self._parts.append("\n```") + self._in_pre = False + self._break() + elif tag == "strong": + self._trim_trailing_space() + self._parts.append("** ") + elif tag == "em": + self._trim_trailing_space() + self._parts.append("_ ") + elif tag == "code" and not self._in_pre: + self._trim_trailing_space() + self._parts.append("` ") + + def handle_data(self, data: str) -> None: + if self._in_pre: + self._parts.append(data) + return + collapsed = " ".join(data.split()) + if collapsed: + self._parts.append(collapsed) + self._parts.append(" ") + + def markdown(self) -> str: + text = "".join(self._parts) + lines = [line.rstrip() for line in text.splitlines()] + output: list[str] = [] + previous_blank = False + for line in lines: + blank = not line.strip() + if blank and previous_blank: + continue + output.append(line) + previous_blank = blank + return "\n".join(output).strip() + "\n" + + def _break(self) -> None: + if self._parts and not self._parts[-1].endswith("\n"): + self._parts.append("\n\n") + + def _trim_trailing_space(self) -> None: + if self._parts: + self._parts[-1] = self._parts[-1].rstrip(" ") + + +def _default_title(spec_dir: Path) -> str: + return f"sqlmesh-spec-kit: {spec_dir.name}" + + +def _display_path(path: Path) -> str: + return str(path) + + +def _now_iso() -> str: + return datetime.now(UTC).replace(microsecond=0).isoformat() + + +def _basic_auth(email: str, api_token: str) -> str: + token = base64.b64encode(f"{email}:{api_token}".encode()).decode() + return f"Basic {token}" + + +def _string(value: object, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise ConfluenceError(f"Confluence response is missing {field_name}.") + return value + + +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _optional_str(value: object) -> str | None: + return value if isinstance(value, str) and value else None diff --git a/src/sqlmesh_specify/doctor.py b/src/sqlmesh_specify/doctor.py new file mode 100644 index 0000000..a2f74ba --- /dev/null +++ b/src/sqlmesh_specify/doctor.py @@ -0,0 +1,73 @@ +"""Brownfield onboarding diagnostics for SQLMesh projects.""" +from __future__ import annotations + +from pathlib import Path + +from sqlmesh_specify.lifecycle import relpath +from sqlmesh_specify.reporting import Finding, ValidationReport +from sqlmesh_specify.sqlmesh_artifacts import CONFIG_FILENAMES + + +def doctor_project(target_dir: Path) -> ValidationReport: + """Inspect a SQLMesh repo and report adoption gaps without changing files.""" + findings: list[Finding] = [] + if not any((target_dir / name).exists() for name in CONFIG_FILENAMES): + findings.append( + Finding( + "error", + "SQLMESH_CONFIG_MISSING", + "No config.yaml, config.yml, or config.py found. Run doctor at a SQLMesh root.", + relpath(target_dir, target_dir), + ) + ) + return ValidationReport("sqlmesh-specify doctor", tuple(findings)) + + checks = [ + ( + target_dir / ".sqlmesh-specify" / "constitution.md", + "SQLMESH_SPECIFY_MISSING", + "No .sqlmesh-specify/constitution.md found. Run `sqlmesh-specify init`.", + ), + (target_dir / "CLAUDE.md", "AGENT_CONTEXT_MISSING", "No CLAUDE.md found."), + (target_dir / "specs", "SPECS_DIR_MISSING", "No specs/ directory found."), + ( + target_dir / ".sqlmesh-specify" / "agents", + "AGENTS_DIR_MISSING", + "No .sqlmesh-specify/agents/ directory found.", + ), + (target_dir / "audits", "AUDITS_DIR_MISSING", "No audits/ directory found."), + (target_dir / "tests", "TESTS_DIR_MISSING", "No tests/ directory found."), + ] + for path, code, message in checks: + if not path.exists(): + findings.append(Finding("warning", code, message, relpath(path, target_dir))) + + models_dir = target_dir / "models" + if not models_dir.exists(): + findings.append( + Finding("error", "MODELS_DIR_MISSING", "No models/ directory found.", "models") + ) + else: + sql_count = len(list(models_dir.rglob("*.sql"))) + py_count = len(list(models_dir.rglob("*.py"))) + findings.append( + Finding( + "info", + "MODEL_INVENTORY", + f"Found {sql_count} SQL model file(s) and {py_count} Python model file(s).", + relpath(models_dir, target_dir), + ) + ) + if sql_count + py_count == 0: + findings.append( + Finding( + "error", + "NO_MODEL_FILES", + "models/ has no .sql or .py model files.", + "models", + ) + ) + + if not findings: + findings.append(Finding("info", "DOCTOR_OK", "No adoption gaps found.")) + return ValidationReport("sqlmesh-specify doctor", tuple(findings)) diff --git a/src/sqlmesh_specify/ears.py b/src/sqlmesh_specify/ears.py new file mode 100644 index 0000000..4467d96 --- /dev/null +++ b/src/sqlmesh_specify/ears.py @@ -0,0 +1,72 @@ +"""EARS pattern classification. + +EARS = Easy Approach to Requirements Syntax. Five patterns: + + 1. Ubiquitous: "The system shall ." + 2. Event-driven: "When , the system shall ." + 3. State-driven: "While , the system shall ." + 4. Unwanted: "If , then the system shall ." + 5. Optional: "Where , the system shall ." + +We use loose matching — the requirement phrasing must contain the right +trigger keyword *and* the "shall" verb. Punctuation and capitalization +are tolerant. +""" +from __future__ import annotations + +import re + +# All patterns require " shall " or " must " somewhere after the trigger. +_RESPONSE_VERB = r"(shall|must)" + + +# Order matters: more specific (Event/State/Unwanted/Optional) before Ubiquitous. +_PATTERNS = [ + ( + "event_driven", + re.compile( + rf"^\s*when\b.*?,?\s*the\s+system\s+{_RESPONSE_VERB}\b", + re.IGNORECASE | re.DOTALL, + ), + ), + ( + "state_driven", + re.compile( + rf"^\s*while\b.*?,?\s*the\s+system\s+{_RESPONSE_VERB}\b", + re.IGNORECASE | re.DOTALL, + ), + ), + ( + "unwanted", + re.compile( + rf"^\s*if\b.*?,?\s*then\s+the\s+system\s+{_RESPONSE_VERB}\b", + re.IGNORECASE | re.DOTALL, + ), + ), + ( + "optional", + re.compile( + rf"^\s*where\b.*?,?\s*the\s+system\s+{_RESPONSE_VERB}\b", + re.IGNORECASE | re.DOTALL, + ), + ), + ( + "ubiquitous", + re.compile( + rf"^\s*the\s+system\s+{_RESPONSE_VERB}\b", + re.IGNORECASE | re.DOTALL, + ), + ), +] + + +def classify_ears(line: str) -> str | None: + """Return the EARS pattern name for a given line, or None if not EARS. + + Recognised pattern names: "ubiquitous", "event_driven", "state_driven", + "unwanted", "optional". + """ + for name, pattern in _PATTERNS: + if pattern.search(line): + return name + return None diff --git a/src/sqlmesh_specify/init.py b/src/sqlmesh_specify/init.py new file mode 100644 index 0000000..6f103ad --- /dev/null +++ b/src/sqlmesh_specify/init.py @@ -0,0 +1,103 @@ +"""Implementation of `sqlmesh-specify init`.""" +from __future__ import annotations + +import shutil +from pathlib import Path + +import click + +from sqlmesh_specify.sqlmesh_artifacts import CONFIG_FILENAMES +from sqlmesh_specify.templates_loader import asset_dir + +_IGNORE_JUNK = shutil.ignore_patterns(".DS_Store", "__pycache__", "*.pyc", ".gitkeep") +SUPPORTED_ENGINES = ( + "duckdb", + "snowflake", + "databricks", + "bigquery", + "trino", + "redshift", + "postgres", + "mysql", + "mssql", + "athena", + "spark", + "clickhouse", +) + + +def init_project(project_name: str, engine: str, target_dir: Path, force: bool) -> None: + """Initialize SQLMesh spec assets in an existing SQLMesh project.""" + if engine not in SUPPORTED_ENGINES: + click.echo(f"error: unknown engine '{engine}'", err=True) + raise SystemExit(2) + if not any((target_dir / name).is_file() for name in CONFIG_FILENAMES): + click.echo( + "error: no SQLMesh config.yaml, config.yml, or config.py found at " + f"{target_dir}.\n" + " sqlmesh-specify init must be run inside an existing SQLMesh project.", + err=True, + ) + raise SystemExit(1) + models_dir = target_dir / "models" + if not models_dir.is_dir(): + click.echo("error: no models/ directory found in the SQLMesh project.", err=True) + raise SystemExit(1) + + specify_dir = target_dir / ".sqlmesh-specify" + if specify_dir.exists(): + if not force: + click.echo(f"error: {specify_dir} already exists. Re-run with --force.", err=True) + raise SystemExit(1) + shutil.rmtree(specify_dir) + specify_dir.mkdir(parents=True, exist_ok=False) + + _copy_file(asset_dir("memory") / "constitution.md", specify_dir / "constitution.md") + additions = (asset_dir("presets") / engine / "constitution-additions.md").read_text() + constitution_path = specify_dir / "constitution.md" + constitution_path.write_text( + constitution_path.read_text() + + f"\n\n\n\n" + + additions + + f"\n\n\n" + ) + + templates_dst = specify_dir / "templates" + shutil.copytree(asset_dir("templates"), templates_dst, ignore=_IGNORE_JUNK) + plan_additions = (asset_dir("presets") / engine / "plan-additions.md").read_text() + plan_path = templates_dst / "plan-template.md" + plan_path.write_text( + plan_path.read_text() + + f"\n\n\n\n" + + plan_additions + + f"\n\n\n" + ) + + shutil.copytree(asset_dir("skills"), specify_dir / "skills", ignore=_IGNORE_JUNK) + shutil.copytree(asset_dir("commands"), specify_dir / "commands", ignore=_IGNORE_JUNK) + shutil.copytree(asset_dir("agents"), specify_dir / "agents", ignore=_IGNORE_JUNK) + + claude_template = ( + (asset_dir("templates") / "CLAUDE.md.template") + .read_text() + .replace("{{ project_name }}", project_name) + .replace("{{ engine }}", engine) + ) + claude_target = target_dir / "CLAUDE.md" + if claude_target.exists(): + suggested = target_dir / "CLAUDE.md.sqlmesh-specify-suggested" + suggested.write_text(claude_template) + click.echo(f"note: {claude_target.name} already exists; wrote {suggested.name}") + else: + claude_target.write_text(claude_template) + click.echo("wrote CLAUDE.md") + + (target_dir / "specs").mkdir(exist_ok=True) + (target_dir / "specs" / ".gitkeep").touch() + click.echo(f"\nsqlmesh-specify initialized in {target_dir}") + click.echo(f" engine preset: {engine}") + + +def _copy_file(src: Path, dst: Path) -> None: + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst) diff --git a/src/sqlmesh_specify/jira.py b/src/sqlmesh_specify/jira.py new file mode 100644 index 0000000..bde8ca9 --- /dev/null +++ b/src/sqlmesh_specify/jira.py @@ -0,0 +1,648 @@ +"""Jira Cloud bridge for sqlmesh-specify artifacts.""" +from __future__ import annotations + +import base64 +import json +import mimetypes +import os +import re +import uuid +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +import yaml + + +class JiraError(RuntimeError): + """Raised when Jira integration cannot complete safely.""" + + +@dataclass(frozen=True) +class JiraConfig: + """Jira Cloud connection settings.""" + + base_url: str + email: str + api_token: str + + +@dataclass(frozen=True) +class JiraSubtask: + """Existing Jira subtask summary.""" + + key: str + summary: str + + +@dataclass(frozen=True) +class JiraIssue: + """Normalized Jira issue fields used by sqlmesh-specify.""" + + key: str + summary: str + description: str + project_key: str + issue_url: str + labels: tuple[str, ...] + components: tuple[str, ...] + priority: str | None + issue_type: str | None + status: str | None + subtasks: tuple[JiraSubtask, ...] + + +@dataclass(frozen=True) +class TaskItem: + """Task parsed from tasks.md.""" + + task_id: str + description: str + checked: bool + validates: str | None + + +@dataclass(frozen=True) +class CreatedSubtask: + """Result for a task pushed to Jira.""" + + task_id: str + summary: str + key: str | None + created: bool + + +class JiraClient: + """Small Jira Cloud REST client backed by the Python standard library.""" + + def __init__(self, config: JiraConfig) -> None: + self._config = config + + def get_issue(self, issue_key: str) -> JiraIssue: + """Fetch an issue and normalize fields needed by the bridge.""" + fields = ",".join( + [ + "summary", + "description", + "project", + "labels", + "components", + "priority", + "issuetype", + "status", + "subtasks", + ] + ) + raw = self._request_json( + "GET", + f"/rest/api/3/issue/{quote(issue_key)}?fields={quote(fields)}", + ) + return _parse_issue(raw, self._config.base_url) + + def attach_file(self, issue_key: str, path: Path) -> str: + """Attach a file to a Jira issue and return the uploaded filename.""" + if not path.exists() or not path.is_file(): + raise JiraError(f"attachment file not found: {path}") + + boundary = f"sqlmesh-specify-{uuid.uuid4().hex}" + filename = path.name + content_type = mimetypes.guess_type(filename)[0] or "application/octet-stream" + file_bytes = path.read_bytes() + body = b"".join( + [ + f"--{boundary}\r\n".encode(), + ( + f'Content-Disposition: form-data; name="file"; filename="{filename}"\r\n' + ).encode(), + f"Content-Type: {content_type}\r\n\r\n".encode(), + file_bytes, + b"\r\n", + f"--{boundary}--\r\n".encode(), + ] + ) + self._request_json( + "POST", + f"/rest/api/3/issue/{quote(issue_key)}/attachments", + body=body, + headers={ + "Accept": "application/json", + "Content-Type": f"multipart/form-data; boundary={boundary}", + "X-Atlassian-Token": "no-check", + }, + ) + return filename + + def add_comment(self, issue_key: str, text: str) -> None: + """Add a plain-text comment to a Jira issue.""" + self._request_json( + "POST", + f"/rest/api/3/issue/{quote(issue_key)}/comment", + json_body={"body": text_to_adf(text)}, + ) + + def create_subtask( + self, + *, + parent_issue_key: str, + project_key: str, + issue_type_name: str, + summary: str, + description: str, + ) -> str: + """Create a Jira subtask and return its issue key.""" + response = self._request_json( + "POST", + "/rest/api/3/issue", + json_body={ + "fields": { + "project": {"key": project_key}, + "parent": {"key": parent_issue_key}, + "issuetype": {"name": issue_type_name}, + "summary": summary, + "description": text_to_adf(description), + "labels": ["sqlmesh-specify"], + } + }, + ) + if not isinstance(response, dict): + raise JiraError("Jira did not return an object for the created subtask.") + key = response.get("key") + if not isinstance(key, str): + raise JiraError("Jira did not return a key for the created subtask.") + return key + + def _request_json( + self, + method: str, + path: str, + *, + json_body: dict[str, object] | None = None, + body: bytes | None = None, + headers: dict[str, str] | None = None, + ) -> dict[str, Any] | list[Any]: + data = body + request_headers = { + "Authorization": _basic_auth(self._config.email, self._config.api_token), + "Accept": "application/json", + } + if json_body is not None: + data = json.dumps(json_body).encode() + request_headers["Content-Type"] = "application/json" + if headers: + request_headers.update(headers) + + request = Request( + f"{self._config.base_url.rstrip('/')}{path}", + data=data, + headers=request_headers, + method=method, + ) + try: + with urlopen(request, timeout=30) as response: + response_body = response.read() + except HTTPError as error: + detail = error.read().decode(errors="replace") + raise JiraError(f"Jira API request failed with HTTP {error.code}: {detail}") from error + except URLError as error: + raise JiraError(f"Jira API request failed: {error.reason}") from error + + if not response_body: + return {} + parsed = json.loads(response_body.decode()) + if not isinstance(parsed, (dict, list)): + raise JiraError("Jira API returned an unexpected JSON response.") + return parsed + + +def config_from_env() -> JiraConfig: + """Load Jira Cloud credentials from environment variables.""" + base_url = os.environ.get("JIRA_BASE_URL", "").strip() + email = os.environ.get("JIRA_EMAIL", "").strip() + api_token = os.environ.get("JIRA_API_TOKEN", "").strip() + missing = [ + name + for name, value in ( + ("JIRA_BASE_URL", base_url), + ("JIRA_EMAIL", email), + ("JIRA_API_TOKEN", api_token), + ) + if not value + ] + if missing: + joined = ", ".join(missing) + raise JiraError(f"Missing Jira environment variable(s): {joined}") + return JiraConfig(base_url=base_url.rstrip("/"), email=email, api_token=api_token) + + +def make_jira_client() -> JiraClient: + """Create a Jira client from environment variables.""" + return JiraClient(config_from_env()) + + +def pull_issue_to_spec( + *, + client: JiraClient, + issue_key: str, + target_dir: Path, + slug: str | None = None, + force: bool = False, +) -> Path: + """Create a local spec directory from a Jira issue.""" + issue = client.get_issue(issue_key) + specs_dir = target_dir / "specs" + specs_dir.mkdir(exist_ok=True) + spec_dir = specs_dir / f"{_next_spec_number(specs_dir)}-{_slug_for_issue(issue, slug)}" + if spec_dir.exists() and not force: + raise JiraError(f"spec directory already exists: {spec_dir}") + spec_dir.mkdir(parents=True, exist_ok=True) + + (spec_dir / "jira.yml").write_text(_jira_yml(issue, spec_dir, target_dir)) + spec_path = spec_dir / "spec.md" + if spec_path.exists() and not force: + raise JiraError(f"spec.md already exists: {spec_path}") + spec_path.write_text(_spec_from_issue(issue)) + return spec_dir + + +def attach_artifacts( + *, + client: JiraClient, + issue_key: str, + files: list[Path], + comment: bool = True, +) -> list[str]: + """Attach local artifacts to a Jira issue.""" + if not files: + raise JiraError("No files were provided to attach.") + + uploaded: list[str] = [] + for path in files: + uploaded.append(client.attach_file(issue_key, path)) + + if comment: + client.add_comment( + issue_key, + "sqlmesh-specify attached approved artifact(s):\n" + + "\n".join(f"- {filename}" for filename in uploaded), + ) + return uploaded + + +def create_subtasks_from_tasks( + *, + client: JiraClient, + issue_key: str, + tasks_path: Path, + issue_type_name: str = "Sub-task", + include_done: bool = False, + dry_run: bool = False, +) -> list[CreatedSubtask]: + """Create Jira subtasks from a sqlmesh-specify tasks.md file.""" + if not tasks_path.exists() or not tasks_path.is_file(): + raise JiraError(f"tasks file not found: {tasks_path}") + + issue = client.get_issue(issue_key) + tasks = parse_tasks(tasks_path.read_text()) + existing = { + _extract_task_id_from_summary(subtask.summary): subtask + for subtask in issue.subtasks + if _extract_task_id_from_summary(subtask.summary) is not None + } + + results: list[CreatedSubtask] = [] + for task in tasks: + if task.checked and not include_done: + continue + + summary = _subtask_summary(task) + existing_subtask = existing.get(task.task_id) + if existing_subtask is not None: + results.append( + CreatedSubtask( + task_id=task.task_id, + summary=summary, + key=existing_subtask.key, + created=False, + ) + ) + continue + + if dry_run: + results.append( + CreatedSubtask(task_id=task.task_id, summary=summary, key=None, created=False) + ) + continue + + key = client.create_subtask( + parent_issue_key=issue.key, + project_key=issue.project_key, + issue_type_name=issue_type_name, + summary=summary, + description=_subtask_description(task, tasks_path), + ) + results.append(CreatedSubtask(task_id=task.task_id, summary=summary, key=key, created=True)) + + return results + + +def sync_spec_dir( + *, + client: JiraClient, + issue_key: str, + spec_dir: Path, + issue_type_name: str = "Sub-task", + dry_run: bool = False, +) -> tuple[list[str], list[CreatedSubtask]]: + """Attach approved artifacts and create Jira subtasks for a spec directory.""" + files = [path for path in (spec_dir / "spec.md", spec_dir / "plan.md") if path.exists()] + uploaded: list[str] = [] + if files and not dry_run: + uploaded = attach_artifacts(client=client, issue_key=issue_key, files=files) + + subtasks: list[CreatedSubtask] = [] + tasks_path = spec_dir / "tasks.md" + if tasks_path.exists(): + subtasks = create_subtasks_from_tasks( + client=client, + issue_key=issue_key, + tasks_path=tasks_path, + issue_type_name=issue_type_name, + dry_run=dry_run, + ) + + return uploaded, subtasks + + +_TASK_RE = re.compile( + r"^\s*-\s+\[(?P[ xX])\]\s+(?:\*\*)?(?PT[-_]?\d+)(?:\*\*)?\s+[—-]\s+" + r"(?P.+?)\s*$" +) +_VALIDATES_RE = re.compile(r"^\s*-\s+\*\*Validates:\*\*\s+(?P.+?)\s*$") +_SUBTASK_ID_RE = re.compile(r"^\[sqlmesh-specify (?PT-\d+)\]") + + +def parse_tasks(text: str) -> list[TaskItem]: + """Parse sqlmesh-specify task list entries from tasks.md.""" + tasks: list[TaskItem] = [] + current_index: int | None = None + + for line in text.splitlines(): + task_match = _TASK_RE.match(line) + if task_match is not None: + task = TaskItem( + task_id=_normalize_task_id(task_match.group("id")), + description=task_match.group("description").strip(), + checked=task_match.group("checked").lower() == "x", + validates=None, + ) + tasks.append(task) + current_index = len(tasks) - 1 + continue + + validates_match = _VALIDATES_RE.match(line) + if validates_match is not None and current_index is not None: + previous = tasks[current_index] + tasks[current_index] = TaskItem( + task_id=previous.task_id, + description=previous.description, + checked=previous.checked, + validates=validates_match.group("validates").strip(), + ) + + return tasks + + +def text_to_adf(text: str) -> dict[str, object]: + """Convert plain text to a minimal Atlassian Document Format document.""" + content: list[dict[str, object]] = [] + for raw_line in text.splitlines(): + line = raw_line.strip() + if not line: + continue + content.append( + { + "type": "paragraph", + "content": [{"type": "text", "text": line}], + } + ) + if not content: + content.append({"type": "paragraph", "content": [{"type": "text", "text": " "}]}) + return {"type": "doc", "version": 1, "content": content} + + +def _parse_issue(raw: dict[str, Any] | list[Any], base_url: str) -> JiraIssue: + if not isinstance(raw, dict): + raise JiraError("Jira issue response was not an object.") + fields = raw.get("fields") + if not isinstance(fields, dict): + raise JiraError("Jira issue response did not include fields.") + + key = _string(raw.get("key"), "issue key") + project = _mapping(fields.get("project")) + priority = _optional_name(fields.get("priority")) + issue_type = _optional_name(fields.get("issuetype")) + status = _optional_name(fields.get("status")) + components = tuple( + name + for item in _list(fields.get("components")) + if (name := _optional_name(item)) is not None + ) + labels = tuple(value for value in _list(fields.get("labels")) if isinstance(value, str)) + subtasks = tuple(_parse_subtask(item) for item in _list(fields.get("subtasks"))) + + return JiraIssue( + key=key, + summary=_string(fields.get("summary"), "summary"), + description=adf_to_text(fields.get("description")), + project_key=_string(project.get("key"), "project key"), + issue_url=f"{base_url.rstrip('/')}/browse/{key}", + labels=labels, + components=components, + priority=priority, + issue_type=issue_type, + status=status, + subtasks=subtasks, + ) + + +def _parse_subtask(value: object) -> JiraSubtask: + mapping = _mapping(value) + fields = _mapping(mapping.get("fields")) + return JiraSubtask( + key=_string(mapping.get("key"), "subtask key"), + summary=_string(fields.get("summary"), "subtask summary"), + ) + + +def adf_to_text(value: object) -> str: + """Convert a Jira ADF document, string, or null value to readable text.""" + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, dict): + node_type = value.get("type") + if node_type == "text": + text = value.get("text") + return text if isinstance(text, str) else "" + parts = [adf_to_text(child) for child in _list(value.get("content"))] + joined = " ".join(part for part in parts if part).strip() + if node_type in {"paragraph", "heading", "blockquote"}: + return joined + if node_type == "listItem": + return f"- {joined}" if joined else "" + if node_type in {"bulletList", "orderedList", "doc"}: + return "\n".join(part for part in parts if part) + return joined + if isinstance(value, list): + return "\n".join(adf_to_text(item) for item in value) + return "" + + +def _jira_yml(issue: JiraIssue, spec_dir: Path, target_dir: Path) -> str: + payload = { + "issue_key": issue.key, + "issue_url": issue.issue_url, + "summary": issue.summary, + "project_key": issue.project_key, + "spec_dir": str(spec_dir.relative_to(target_dir)), + "last_synced_at": datetime.now(UTC).replace(microsecond=0).isoformat(), + } + return yaml.safe_dump(payload, sort_keys=False) + + +def _spec_from_issue(issue: JiraIssue) -> str: + today = datetime.now(UTC).date().isoformat() + labels = ", ".join(issue.labels) if issue.labels else "none" + components = ", ".join(issue.components) if issue.components else "none" + description = issue.description.strip() or "No Jira description was provided." + return f"""# {issue.summary} + +**Ticket:** {issue.issue_url} +**Author:** Jira +**Date:** {today} +**Status:** draft + +## Problem + +Jira issue `{issue.key}` requests: {issue.summary} + +Source Jira description: + +{description} + +## Users + +| User | Job to be done | +|---|---| +| | | + +## What this is + + + +## Acceptance criteria + +- AC1: The system shall implement `{issue.key}` without unrelated SQLMesh changes. +- AC2: When affected SQLMesh models are parsed, the system shall preserve downstream references. +- AC3: Where grain, metrics, or semantic behavior changes, the system shall document approval. + +## Out of scope + +- Raw agent scratch notes are not part of the approved Jira record. +- Unrelated model cleanup is not included unless added to the approved plan. + +## Constraints + +- Warehouse: +- Materialization: +- Grain: +- Refresh cadence: +- Downstream consumers: +- Jira priority: {issue.priority or "not set"} +- Jira issue type: {issue.issue_type or "not set"} +- Jira status: {issue.status or "not set"} +- Jira labels: {labels} +- Jira components: {components} + +## Open questions + +- [ ] Confirm the business owner who can approve this spec. +- [ ] Confirm affected SQLMesh models, metrics, and downstream consumers. +- [ ] Confirm the validation evidence that must be attached back to Jira. +""" + + +def _next_spec_number(specs_dir: Path) -> str: + highest = 0 + for path in specs_dir.iterdir(): + if not path.is_dir(): + continue + prefix = path.name.split("-", 1)[0] + if prefix.isdigit(): + highest = max(highest, int(prefix)) + return f"{highest + 1:03d}" + + +def _slug_for_issue(issue: JiraIssue, requested_slug: str | None) -> str: + base = requested_slug or f"{issue.key}-{issue.summary}" + slug = re.sub(r"[^a-z0-9]+", "-", base.lower()).strip("-") + return slug[:80] or issue.key.lower() + + +def _subtask_summary(task: TaskItem) -> str: + return f"[sqlmesh-specify {task.task_id}] {task.description}"[:255] + + +def _subtask_description(task: TaskItem, tasks_path: Path) -> str: + lines = [ + f"Created from {tasks_path}", + "", + f"Task: {task.task_id}", + f"Description: {task.description}", + ] + if task.validates: + lines.append(f"Validates: {task.validates}") + return "\n".join(lines) + + +def _extract_task_id_from_summary(summary: str) -> str | None: + match = _SUBTASK_ID_RE.match(summary) + if match is None: + return None + return match.group("id") + + +def _normalize_task_id(task_id: str) -> str: + digits = re.sub(r"\D", "", task_id) + return f"T-{int(digits):02d}" + + +def _basic_auth(email: str, api_token: str) -> str: + token = base64.b64encode(f"{email}:{api_token}".encode()).decode() + return f"Basic {token}" + + +def _string(value: object, field_name: str) -> str: + if not isinstance(value, str) or not value: + raise JiraError(f"Jira issue response is missing {field_name}.") + return value + + +def _mapping(value: object) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _list(value: object) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _optional_name(value: object) -> str | None: + mapping = _mapping(value) + name = mapping.get("name") + return name if isinstance(name, str) else None diff --git a/src/sqlmesh_specify/lifecycle.py b/src/sqlmesh_specify/lifecycle.py new file mode 100644 index 0000000..8ec71a3 --- /dev/null +++ b/src/sqlmesh_specify/lifecycle.py @@ -0,0 +1,311 @@ +"""Validate sqlmesh-spec-kit lifecycle artifacts.""" +from __future__ import annotations + +import re +from pathlib import Path + +from sqlmesh_specify.ears import classify_ears +from sqlmesh_specify.reporting import Finding, ValidationReport +from sqlmesh_specify.validate import _extract_ac_lines + +_STATUS_RE = re.compile(r"^\*\*Status:\*\*\s*(?P.+?)\s*$", re.IGNORECASE | re.MULTILINE) +_AC_ID_RE = re.compile(r"\bAC(?P\d+[A-Za-z]?)\b") +_REQUIRED_PLAN_SECTIONS = ( + ("## Architecture",), + ("## Files to add",), + ("## Files to modify",), + ("## Files to delete",), + ("## Audits and Tests", "## Tests"), + ("## Downstream impact",), +) +_SPEC_STATUSES = frozenset({"draft", "approved", "shipped", "superseded"}) +_PLAN_STATUSES = frozenset({"proposed", "approved", "superseded"}) +_TASKS_STATUSES = frozenset({"in progress", "done"}) +_SPEC_STATUSES_ALLOWING_PLAN = frozenset({"approved", "shipped"}) +_PLAN_STATUSES_ALLOWING_TASKS = frozenset({"approved"}) + + +def validate_lifecycle(target_dir: Path) -> ValidationReport: + """Validate spec -> plan -> tasks traceability for a SQLMesh project.""" + findings: list[Finding] = [] + specs_dir = target_dir / "specs" + if not specs_dir.exists(): + return ValidationReport( + "sqlmesh-specify project validation", + ( + Finding( + "warning", + "PROJECT_NO_SPECS_DIR", + "No specs/ directory found. Run `sqlmesh-specify init` before using the " + "enterprise workflow.", + relpath(specs_dir, target_dir), + ), + ), + ) + + spec_dirs = sorted(path for path in specs_dir.iterdir() if path.is_dir()) + if not spec_dirs: + findings.append( + Finding( + "warning", + "PROJECT_NO_SPECS", + "specs/ exists but contains no feature specs yet.", + relpath(specs_dir, target_dir), + ) + ) + + for spec_dir in spec_dirs: + findings.extend(_validate_spec_dir(spec_dir, target_dir)) + + return ValidationReport("sqlmesh-specify project validation", tuple(findings)) + + +def _validate_spec_dir(spec_dir: Path, target_dir: Path) -> list[Finding]: + findings: list[Finding] = [] + spec_path = spec_dir / "spec.md" + plan_path = spec_dir / "plan.md" + tasks_path = spec_dir / "tasks.md" + + if not spec_path.exists(): + return [ + Finding( + "error", + "SPEC_MISSING", + "Spec directory is missing spec.md.", + relpath(spec_path, target_dir), + ) + ] + + spec_text = spec_path.read_text() + ac_lines = _extract_ac_lines(spec_text) + ac_ids = _extract_referenced_ac_ids(spec_text) + spec_status = _read_status(spec_text) + findings.extend( + _validate_status( + status=spec_status, + allowed_statuses=_SPEC_STATUSES, + missing_code="SPEC_STATUS_MISSING", + invalid_code="SPEC_STATUS_INVALID", + artifact_name="spec.md", + path=spec_path, + target_dir=target_dir, + ) + ) + + if not ac_lines: + findings.append( + Finding( + "error", + "SPEC_NO_ACCEPTANCE_CRITERIA", + "spec.md has no Acceptance Criteria section.", + relpath(spec_path, target_dir), + ) + ) + for line_no, line in ac_lines: + if classify_ears(line) is None: + findings.append( + Finding( + "error", + "SPEC_NON_EARS_AC", + f"Acceptance criterion on line {line_no} is not EARS-formatted.", + relpath(spec_path, target_dir), + ) + ) + if ac_lines and not ac_ids: + findings.append( + Finding( + "warning", + "SPEC_UNLABELED_ACS", + "Acceptance criteria should use stable AC ids such as AC1, AC2, AC3.", + relpath(spec_path, target_dir), + ) + ) + + if plan_path.exists(): + findings.extend(_validate_plan(plan_path, spec_status, ac_ids, target_dir)) + if tasks_path.exists(): + findings.extend(_validate_tasks(tasks_path, plan_path, ac_ids, target_dir)) + + return findings + + +def _validate_plan( + plan_path: Path, + spec_status: str | None, + ac_ids: set[str], + target_dir: Path, +) -> list[Finding]: + findings: list[Finding] = [] + plan_text = plan_path.read_text() + plan_status = _read_status(plan_text) + + if not _status_allows(spec_status, _SPEC_STATUSES_ALLOWING_PLAN): + findings.append( + Finding( + "error", + "PLAN_BEFORE_SPEC_APPROVAL", + "plan.md exists before spec.md is marked exactly `approved` or `shipped`.", + relpath(plan_path, target_dir), + ) + ) + findings.extend( + _validate_status( + status=plan_status, + allowed_statuses=_PLAN_STATUSES, + missing_code="PLAN_STATUS_MISSING", + invalid_code="PLAN_STATUS_INVALID", + artifact_name="plan.md", + path=plan_path, + target_dir=target_dir, + ) + ) + for section_options in _REQUIRED_PLAN_SECTIONS: + if not any(section.lower() in plan_text.lower() for section in section_options): + section = section_options[0] + findings.append( + Finding( + "error", + "PLAN_SECTION_MISSING", + f"plan.md is missing required section `{section}`.", + relpath(plan_path, target_dir), + ) + ) + + findings.extend(_validate_ac_traceability(ac_ids, plan_text, plan_path, target_dir, "plan.md")) + return findings + + +def _validate_tasks( + tasks_path: Path, + plan_path: Path, + ac_ids: set[str], + target_dir: Path, +) -> list[Finding]: + findings: list[Finding] = [] + tasks_text = tasks_path.read_text() + plan_text = plan_path.read_text() if plan_path.exists() else "" + plan_status = _read_status(plan_text) + tasks_status = _read_status(tasks_text) + + if not plan_path.exists(): + findings.append( + Finding( + "error", + "TASKS_WITHOUT_PLAN", + "tasks.md exists before plan.md.", + relpath(tasks_path, target_dir), + ) + ) + elif not _status_allows(plan_status, _PLAN_STATUSES_ALLOWING_TASKS): + findings.append( + Finding( + "error", + "TASKS_BEFORE_PLAN_APPROVAL", + "tasks.md exists before plan.md is marked exactly `approved`.", + relpath(tasks_path, target_dir), + ) + ) + findings.extend( + _validate_status( + status=tasks_status, + allowed_statuses=_TASKS_STATUSES, + missing_code="TASKS_STATUS_MISSING", + invalid_code="TASKS_STATUS_INVALID", + artifact_name="tasks.md", + path=tasks_path, + target_dir=target_dir, + ) + ) + + if "## Task list" not in tasks_text: + findings.append( + Finding( + "error", + "TASKS_SECTION_MISSING", + "tasks.md must include a `## Task list` section.", + relpath(tasks_path, target_dir), + ) + ) + findings.extend( + _validate_ac_traceability(ac_ids, tasks_text, tasks_path, target_dir, "tasks.md") + ) + return findings + + +def _validate_ac_traceability( + ac_ids: set[str], + text: str, + path: Path, + target_dir: Path, + artifact_name: str, +) -> list[Finding]: + if not ac_ids: + return [] + referenced = _extract_referenced_ac_ids(text) + findings: list[Finding] = [] + for ac_id in sorted(ac_ids): + if ac_id not in referenced: + findings.append( + Finding( + "error", + "AC_NOT_TRACED", + f"{artifact_name} does not reference {ac_id}.", + relpath(path, target_dir), + ) + ) + return findings + + +def _validate_status( + *, + status: str | None, + allowed_statuses: frozenset[str], + missing_code: str, + invalid_code: str, + artifact_name: str, + path: Path, + target_dir: Path, +) -> list[Finding]: + if status is None: + return [ + Finding( + "error", + missing_code, + f"{artifact_name} must include a `Status:` line.", + relpath(path, target_dir), + ) + ] + if status not in allowed_statuses: + allowed = ", ".join(f"`{value}`" for value in sorted(allowed_statuses)) + return [ + Finding( + "error", + invalid_code, + f"{artifact_name} has invalid status `{status}`. Allowed statuses: {allowed}.", + relpath(path, target_dir), + ) + ] + return [] + + +def _read_status(text: str) -> str | None: + match = _STATUS_RE.search(text) + if match is None: + return None + return " ".join(match.group("status").strip().lower().split()) + + +def _status_allows(status: str | None, allowed_statuses: frozenset[str]) -> bool: + return status in allowed_statuses + + +def _extract_referenced_ac_ids(text: str) -> set[str]: + return {f"AC{match.group('num')}" for match in _AC_ID_RE.finditer(text)} + + +def relpath(path: Path, base: Path) -> str: + """Return a stable display path relative to the target project when possible.""" + try: + return str(path.relative_to(base)) + except ValueError: + return str(path) diff --git a/src/sqlmesh_specify/reporting.py b/src/sqlmesh_specify/reporting.py new file mode 100644 index 0000000..498eca1 --- /dev/null +++ b/src/sqlmesh_specify/reporting.py @@ -0,0 +1,111 @@ +"""Shared validation report primitives.""" +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Literal + +Severity = Literal["error", "warning", "info"] + + +@dataclass(frozen=True) +class Finding: + """A single validation or diagnostic finding.""" + + severity: Severity + code: str + message: str + path: str | None = None + + def to_dict(self) -> dict[str, str]: + """Return a JSON-serializable representation.""" + data = { + "severity": self.severity, + "code": self.code, + "message": self.message, + } + if self.path is not None: + data["path"] = self.path + return data + + +@dataclass(frozen=True) +class ValidationReport: + """A collection of findings for one CLI command.""" + + title: str + findings: tuple[Finding, ...] = () + + @property + def error_count(self) -> int: + """Number of error findings.""" + return sum(1 for finding in self.findings if finding.severity == "error") + + @property + def warning_count(self) -> int: + """Number of warning findings.""" + return sum(1 for finding in self.findings if finding.severity == "warning") + + @property + def info_count(self) -> int: + """Number of info findings.""" + return sum(1 for finding in self.findings if finding.severity == "info") + + @property + def exit_code(self) -> int: + """Return 1 when the report has release-blocking errors.""" + return 1 if self.error_count else 0 + + def with_findings(self, findings: list[Finding]) -> ValidationReport: + """Return a copy with more findings appended.""" + return ValidationReport(self.title, self.findings + tuple(findings)) + + def to_markdown(self) -> str: + """Render the report as GitHub-flavored Markdown.""" + lines = [ + f"# {self.title}", + "", + "## Summary", + "", + f"- Errors: {self.error_count}", + f"- Warnings: {self.warning_count}", + f"- Info: {self.info_count}", + "", + ] + if not self.findings: + lines.extend(["No findings.", ""]) + return "\n".join(lines) + + lines.extend(["## Findings", ""]) + for finding in self.findings: + location = f" — `{finding.path}`" if finding.path else "" + lines.append( + f"- **{finding.severity.upper()} [{finding.code}]**{location}: " + f"{finding.message}" + ) + lines.append("") + return "\n".join(lines) + + def to_json(self) -> str: + """Render the report as stable JSON.""" + return json.dumps( + { + "title": self.title, + "summary": { + "errors": self.error_count, + "warnings": self.warning_count, + "info": self.info_count, + }, + "findings": [finding.to_dict() for finding in self.findings], + }, + indent=2, + sort_keys=True, + ) + + +def combine_reports(title: str, reports: list[ValidationReport]) -> ValidationReport: + """Combine multiple validation reports into one.""" + findings: list[Finding] = [] + for report in reports: + findings.extend(report.findings) + return ValidationReport(title, tuple(findings)) diff --git a/src/sqlmesh_specify/sqlmesh_artifacts.py b/src/sqlmesh_specify/sqlmesh_artifacts.py new file mode 100644 index 0000000..06d20aa --- /dev/null +++ b/src/sqlmesh_specify/sqlmesh_artifacts.py @@ -0,0 +1,97 @@ +"""Validate SQLMesh project structure for sqlmesh-spec-kit.""" +from __future__ import annotations + +from pathlib import Path + +from sqlmesh_specify.lifecycle import relpath +from sqlmesh_specify.reporting import Finding, ValidationReport + +CONFIG_FILENAMES = ("config.yaml", "config.yml", "config.py") +MODEL_SUFFIXES = (".sql", ".py") +ENGINE_DIRS = ("seeds", "macros", "external_models") + + +def validate_sqlmesh_project(target_dir: Path) -> ValidationReport: + """Validate SQLMesh project structure without executing SQLMesh.""" + findings: list[Finding] = [] + config_paths = [target_dir / name for name in CONFIG_FILENAMES] + if not any(path.exists() and path.is_file() for path in config_paths): + findings.append( + Finding( + "error", + "SQLMESH_CONFIG_MISSING", + "No config.yaml, config.yml, or config.py found. " + "Run this at a SQLMesh project root.", + relpath(target_dir, target_dir), + ) + ) + + models_dir = target_dir / "models" + if not models_dir.exists() or not models_dir.is_dir(): + findings.append( + Finding( + "error", + "MODELS_DIR_MISSING", + "No models/ directory found for SQLMesh models.", + relpath(models_dir, target_dir), + ) + ) + else: + model_files = _model_files(models_dir) + if not model_files: + findings.append( + Finding( + "error", + "NO_MODEL_FILES", + "models/ contains no .sql or .py SQLMesh model files.", + relpath(models_dir, target_dir), + ) + ) + elif not _has_inline_audit_reference(model_files) and not (target_dir / "audits").is_dir(): + findings.append( + Finding( + "warning", + "AUDITS_MISSING", + "No audits/ directory or inline audit references found.", + relpath(target_dir / "audits", target_dir), + ) + ) + + tests_dir = target_dir / "tests" + if not tests_dir.exists() or not tests_dir.is_dir(): + findings.append( + Finding( + "warning", + "TESTS_DIR_MISSING", + "No tests/ directory found for SQLMesh tests.", + relpath(tests_dir, target_dir), + ) + ) + + for dirname in ENGINE_DIRS: + path = target_dir / dirname + if not path.exists(): + findings.append( + Finding( + "info", + f"{dirname.upper()}_DIR_MISSING", + f"No {dirname}/ directory found. Add one when the project needs it.", + relpath(path, target_dir), + ) + ) + + return ValidationReport("SQLMesh project validation", tuple(findings)) + + +def _model_files(models_dir: Path) -> list[Path]: + return sorted(path for path in models_dir.rglob("*") if path.suffix in MODEL_SUFFIXES) + + +def _has_inline_audit_reference(model_files: list[Path]) -> bool: + for path in model_files: + if path.suffix != ".sql": + continue + text = path.read_text(errors="ignore").lower() + if "audits" in text or "audit(" in text or "audit (" in text: + return True + return False diff --git a/src/sqlmesh_specify/templates_loader.py b/src/sqlmesh_specify/templates_loader.py new file mode 100644 index 0000000..652cdba --- /dev/null +++ b/src/sqlmesh_specify/templates_loader.py @@ -0,0 +1,68 @@ +"""Load packaged template assets from the installed sqlmesh-specify package.""" +from __future__ import annotations + +from functools import cache +from importlib import resources +from pathlib import Path + +_KNOWN_ASSETS = {"memory", "templates", "presets", "skills", "commands", "agents"} + + +@cache +def asset_dir(kind: str) -> Path: + """Return the on-disk path to a packaged asset directory. + + Resolves in two phases so both wheel installs and editable installs work: + 1. Look inside the installed package at `sqlmesh_specify/_assets//`. + This is where hatch's force-include lands assets in the wheel. + 2. Fall back to the top-level `/` directory in the source tree. + This is the layout an editable install sees (no copy step happens). + + Args: + kind: One of "memory", "templates", "presets", "skills", "commands", "agents". + + Returns: + Absolute Path to the directory. + + Raises: + ValueError: If `kind` is not a known asset directory. + FileNotFoundError: If neither candidate location exists. + """ + if kind not in _KNOWN_ASSETS: + raise ValueError(f"unknown asset kind: {kind}") + + package_files = resources.files("sqlmesh_specify") / "_assets" / kind + packaged = Path(str(package_files)) + if packaged.is_dir(): + return packaged + + # Editable install: walk up from the package's __init__.py to find the repo root, + # which contains the top-level asset directories. + repo_root = Path(str(resources.files("sqlmesh_specify"))).resolve().parents[1] + editable = repo_root / kind + if editable.is_dir(): + return editable + + raise FileNotFoundError( + f"asset directory '{kind}' not found at {packaged} or {editable}" + ) + + +def load_template(name: str) -> str: + """Read a top-level template file. + + Args: + name: Template name without extension (e.g., "spec" loads spec-template.md). + + Returns: + The template content as a string. + """ + candidates = [ + asset_dir("templates") / f"{name}-template.md", + asset_dir("templates") / f"{name}.md", + asset_dir("templates") / name, + ] + for candidate in candidates: + if candidate.exists(): + return candidate.read_text() + raise FileNotFoundError(f"no template found for name '{name}'") diff --git a/src/sqlmesh_specify/validate.py b/src/sqlmesh_specify/validate.py new file mode 100644 index 0000000..2564140 --- /dev/null +++ b/src/sqlmesh_specify/validate.py @@ -0,0 +1,97 @@ +"""Implementation of `sqlmesh-specify validate`.""" +from __future__ import annotations + +import re +from pathlib import Path + +import click + +from sqlmesh_specify.ears import classify_ears + +# Headers that mark the Acceptance Criteria section. Case-insensitive. +AC_SECTION_PATTERNS = [ + re.compile(r"^##+\s*acceptance\s+criteria\b", re.IGNORECASE), + re.compile(r"^##+\s*ACs?\b", re.IGNORECASE), +] + + +def validate_spec(spec_path: Path) -> int: + """Validate that a spec's Acceptance Criteria section is EARS-conformant. + + Args: + spec_path: Path to a spec.md file. + + Returns: + 0 if all ACs conform to an EARS pattern, 1 otherwise. + """ + text = spec_path.read_text() + ac_lines = _extract_ac_lines(text) + + if not ac_lines: + click.echo( + f"warning: no Acceptance Criteria section found in {spec_path.name}", + err=True, + ) + return 1 + + failures: list[tuple[int, str]] = [] + for line_no, line in ac_lines: + pattern = classify_ears(line) + if pattern is None: + failures.append((line_no, line)) + + if failures: + click.echo(f"\n{len(failures)} AC line(s) do not match an EARS pattern:") + for line_no, line in failures: + click.echo(f" line {line_no}: {line.strip()}") + click.echo( + "\nsee docs/ears-cheatsheet.md for the five EARS patterns and SQLMesh examples." + ) + return 1 + + click.echo(f"ok: {len(ac_lines)} AC line(s) all match an EARS pattern.") + return 0 + + +def _extract_ac_lines(text: str) -> list[tuple[int, str]]: + """Find lines that look like ACs inside the Acceptance Criteria section.""" + lines = text.splitlines() + in_ac_section = False + in_html_comment = False + next_section = re.compile(r"^##+\s+", re.IGNORECASE) + ac_lines: list[tuple[int, str]] = [] + + for i, line in enumerate(lines, start=1): + stripped = line.strip() + if any(p.match(stripped) for p in AC_SECTION_PATTERNS): + in_ac_section = True + continue + if in_ac_section and next_section.match(stripped): + # We hit the next ## section, so we're done with ACs. + break + if in_ac_section: + if in_html_comment: + if "-->" in stripped: + in_html_comment = False + continue + if stripped.startswith("" not in stripped + continue + # Treat any non-empty line that doesn't start with a markdown decoration + # as a potential AC line. Bullet markers and AC labels are stripped. + cleaned = _clean_ac_line(stripped) + if cleaned: + ac_lines.append((i, cleaned)) + + return ac_lines + + +def _clean_ac_line(line: str) -> str: + """Strip bullets, AC prefixes, and bold/italic markers.""" + # Strip leading list markers + line = re.sub(r"^[-*+]\s+", "", line) + # Strip AC labels like "AC1:", "**AC1**:", "AC1 —", etc. + line = re.sub(r"^\**AC\d*\**\s*[:\-—]\s*", "", line, flags=re.IGNORECASE) + # Strip EARS-pattern labels like "(Ubiquitous)" or "**(Event-driven)**" + line = re.sub(r"^\**\([^)]+\)\**\s*[—-]?\s*", "", line) + return line.strip() diff --git a/templates/CLAUDE.md.template b/templates/CLAUDE.md.template new file mode 100644 index 0000000..ee4278c --- /dev/null +++ b/templates/CLAUDE.md.template @@ -0,0 +1,26 @@ +# {{ project_name }} AI SDLC Context + +This is a SQLMesh project using sqlmesh-spec-kit. + +- Engine preset: `{{ engine }}` +- Durable memory lives in `.sqlmesh-specify/` +- Approved work lives in `specs/-/` + +## Rules for Agents + +1. Read `.sqlmesh-specify/constitution.md` before planning or editing. +2. Do not implement from a draft spec. +3. Do not create `tasks.md` until `plan.md` has exact status `approved`. +4. Implement only files listed in the approved plan. +5. Capture SQLMesh plan, audit, test, apply, or promote evidence before review. +6. Stop on unexpected backfill, restatement, forward-only behavior, or files outside the approved plan. + +## Common Commands + +```bash +sqlmesh-specify doctor --target . +sqlmesh-specify validate project --target . +sqlmesh-specify validate sqlmesh --target . +sqlmesh-specify report --target . --format markdown +sqlmesh plan dev +``` diff --git a/templates/ci/github-actions-sqlmesh-specify.yml b/templates/ci/github-actions-sqlmesh-specify.yml new file mode 100644 index 0000000..27853b3 --- /dev/null +++ b/templates/ci/github-actions-sqlmesh-specify.yml @@ -0,0 +1,14 @@ +name: sqlmesh-specify + +on: + pull_request: + push: + branches: [main] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v4 + - run: uvx --from sqlmesh-spec-kit sqlmesh-specify ci --target . diff --git a/templates/plan-template.md b/templates/plan-template.md new file mode 100644 index 0000000..272daa2 --- /dev/null +++ b/templates/plan-template.md @@ -0,0 +1,63 @@ +# Plan: + +**Author:** +**Date:** +**Status:** proposed +**Spec:** + +Status must be exactly one of: `proposed`, `approved`, `superseded`. + +## Architecture + + + +## SQLMesh Plan + +| Field | Decision | +|---|---| +| Target environment | | +| Change category | | +| Forward-only decision | | +| Restatement required | | +| Backfill window | | +| Snapshots affected | | +| Apply/promote path | | + +## Files to Add + +| Path | Purpose | ACs | +|---|---|---| +| | | AC1 | + +## Files to Modify + +| Path | Purpose | ACs | +|---|---|---| +| | | AC1 | + +## Files to Delete + +| Path | Reason | ACs | +|---|---|---| +| None | | | + +## Audits and Tests + +| Evidence | Command or artifact | ACs | +|---|---|---| +| SQLMesh plan | `sqlmesh plan dev` output attached to review | AC1 | +| SQLMesh tests | `sqlmesh test` or project-specific equivalent | AC1 | +| Audits | | AC2 | + +## Downstream Impact + + + +## SQLMesh Plan/Apply Evidence + +Paste or link: + +- `sqlmesh plan ` output summary +- unexpected backfill/restatement review, if any +- audit/test evidence +- apply/promote evidence before shipping diff --git a/templates/retro-template.md b/templates/retro-template.md new file mode 100644 index 0000000..0d7cbf1 --- /dev/null +++ b/templates/retro-template.md @@ -0,0 +1,19 @@ +# Retrospective: + +## Outcome + +- Shipped status: +- SQLMesh environment promoted: +- Plan/apply evidence: + +## What Changed + +- Models: +- Audits/tests: +- Downstream impact: + +## Lessons for Memory + +- Project convention to add: +- Engine-specific guardrail to add: +- Governance note to add: diff --git a/templates/spec-template.md b/templates/spec-template.md new file mode 100644 index 0000000..9015ddc --- /dev/null +++ b/templates/spec-template.md @@ -0,0 +1,45 @@ +# + +**Author:** +**Date:** +**Status:** draft + +Status must be exactly one of: `draft`, `approved`, `shipped`, `superseded`. + +## Problem + + + +## Users + +| User | Job to be done | +|---|---| +| | | + +## What this is + + + +## Acceptance Criteria + + + +- AC1: When , the system shall . +- AC2: If , then the system shall . +- AC3: Where , the system shall . + +## Out of Scope + +- + +## Constraints + +- SQLMesh environment: +- Affected models: +- Grain: +- Audits/tests expected: +- Downstream consumers: + +## Open Questions + +- [ ] diff --git a/templates/tasks-template.md b/templates/tasks-template.md new file mode 100644 index 0000000..63cf5ad --- /dev/null +++ b/templates/tasks-template.md @@ -0,0 +1,28 @@ +# Tasks: + +**Author:** +**Date:** +**Status:** in progress +**Plan:** + +Status must be exactly one of: `in progress`, `done`. + +## Task list + +- [ ] T-01 - Update SQLMesh models named in the approved plan. + - **Validates:** AC1 +- [ ] T-02 - Add or update SQLMesh audits and tests. + - **Validates:** AC2 +- [ ] T-03 - Run `sqlmesh plan dev` and capture plan output. + - **Validates:** AC1, AC2 +- [ ] T-04 - Run SQLMesh tests/audits and attach evidence. + - **Validates:** AC1, AC2 +- [ ] T-05 - Compare final diff to spec.md, plan.md, tasks.md, and SQLMesh evidence. + - **Validates:** AC1, AC2, AC3 + +## Stop Conditions + +- The SQLMesh plan shows unexpected backfill, restatement, or forward-only behavior. +- SQLMesh tests or audits fail. +- Implementation requires files outside the approved plan. +- Acceptance criteria are unclear or untraceable. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f2fe323 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import shutil +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture +def minimal_sqlmesh_project(tmp_path: Path) -> Path: + src = ROOT / "tests" / "fixtures" / "minimal_sqlmesh_project" + dst = tmp_path / "project" + shutil.copytree(src, dst) + return dst + + +def write_lifecycle( + root: Path, + *, + spec_status: str | None = "approved", + plan_status: str | None = "approved", + tasks_status: str | None = "in progress", +) -> Path: + spec_dir = root / "specs" / "001-demo" + spec_dir.mkdir(parents=True) + spec_status_line = f"**Status:** {spec_status}\n" if spec_status is not None else "" + plan_status_line = f"**Status:** {plan_status}\n" if plan_status is not None else "" + tasks_status_line = f"**Status:** {tasks_status}\n" if tasks_status is not None else "" + (spec_dir / "spec.md").write_text( + "# Demo\n\n" + f"{spec_status_line}\n" + "## Acceptance Criteria\n\n" + "- AC1: When the model is queried, the system shall return one row per customer.\n" + ) + (spec_dir / "plan.md").write_text( + "# Plan\n\n" + f"{plan_status_line}\n" + "## Architecture\nAC1\n" + "## Files to add\nAC1\n" + "## Files to modify\nAC1\n" + "## Files to delete\nAC1\n" + "## Audits and Tests\nAC1\n" + "## Downstream impact\nAC1\n" + ) + (spec_dir / "tasks.md").write_text( + "# Tasks\n\n" + f"{tasks_status_line}\n" + "## Task list\n\n" + "- [ ] T-01 - Implement model.\n - **Validates:** AC1\n" + ) + return spec_dir diff --git a/tests/fixtures/invalid_spec_non_ears.md b/tests/fixtures/invalid_spec_non_ears.md new file mode 100644 index 0000000..84c3062 --- /dev/null +++ b/tests/fixtures/invalid_spec_non_ears.md @@ -0,0 +1,7 @@ +# Sample invalid spec + +## Acceptance criteria + +- We want a CLI. +- It should probably work. +- AC3: The config.yaml is required. diff --git a/tests/fixtures/minimal_sqlmesh_project/audits/not_null.sql b/tests/fixtures/minimal_sqlmesh_project/audits/not_null.sql new file mode 100644 index 0000000..05447a3 --- /dev/null +++ b/tests/fixtures/minimal_sqlmesh_project/audits/not_null.sql @@ -0,0 +1,7 @@ +AUDIT ( + name not_null, +); + +SELECT * +FROM @this_model +WHERE @EACH(@columns, c -> c IS NULL); diff --git a/tests/fixtures/minimal_sqlmesh_project/config.yaml b/tests/fixtures/minimal_sqlmesh_project/config.yaml new file mode 100644 index 0000000..c0eb361 --- /dev/null +++ b/tests/fixtures/minimal_sqlmesh_project/config.yaml @@ -0,0 +1,8 @@ +gateways: + local: + connection: + type: duckdb + database: local.duckdb +default_gateway: local +model_defaults: + dialect: duckdb diff --git a/tests/fixtures/minimal_sqlmesh_project/models/example.sql b/tests/fixtures/minimal_sqlmesh_project/models/example.sql new file mode 100644 index 0000000..5ce0f32 --- /dev/null +++ b/tests/fixtures/minimal_sqlmesh_project/models/example.sql @@ -0,0 +1,7 @@ +MODEL ( + name sushi.example, + kind FULL, + audits (not_null(columns := (id))) +); + +SELECT 1 AS id; diff --git a/tests/fixtures/minimal_sqlmesh_project/tests/test_example.yaml b/tests/fixtures/minimal_sqlmesh_project/tests/test_example.yaml new file mode 100644 index 0000000..8141f5a --- /dev/null +++ b/tests/fixtures/minimal_sqlmesh_project/tests/test_example.yaml @@ -0,0 +1,7 @@ +test_example: + model: sushi.example + inputs: {} + outputs: + query: + rows: + - id: 1 diff --git a/tests/fixtures/valid_spec.md b/tests/fixtures/valid_spec.md new file mode 100644 index 0000000..5dbd8b8 --- /dev/null +++ b/tests/fixtures/valid_spec.md @@ -0,0 +1,7 @@ +# Sample valid spec + +## Acceptance criteria + +- AC1: The system shall ship a working CLI. +- AC2: When the user runs init, the system shall create the target directory. +- AC3: If the config.yaml is missing, then the system shall exit non-zero. diff --git a/tests/test_confluence.py b/tests/test_confluence.py new file mode 100644 index 0000000..3dcb641 --- /dev/null +++ b/tests/test_confluence.py @@ -0,0 +1,199 @@ +"""Tests for Confluence bridge helpers.""" +from __future__ import annotations + +from pathlib import Path + +import yaml +from click.testing import CliRunner +from pytest import MonkeyPatch + +from sqlmesh_specify.cli import main +from sqlmesh_specify.confluence import ( + ConfluencePage, + PublishedPage, + publish_spec_dir, + pull_page_to_context, + storage_to_markdown, + sync_spec_dir, +) + + +class FakeConfluenceClient: + def __init__(self, page: ConfluencePage | None = None) -> None: + self.page = page or _page() + self.created: list[dict[str, str | None]] = [] + self.updated: list[dict[str, str]] = [] + + def get_page(self, page_id: str) -> ConfluencePage: + assert page_id == self.page.page_id + return self.page + + def resolve_space_id(self, space_key: str) -> str: + assert space_key == "DATA" + return "space-123" + + def create_page( + self, + *, + space_id: str, + title: str, + storage: str, + parent_id: str | None = None, + ) -> PublishedPage: + self.created.append( + { + "space_id": space_id, + "title": title, + "storage": storage, + "parent_id": parent_id, + } + ) + return PublishedPage( + page_id="999", + title=title, + page_url="https://example.atlassian.net/wiki/spaces/DATA/pages/999", + created=True, + ) + + def update_page(self, *, page_id: str, title: str, storage: str) -> PublishedPage: + self.updated.append({"page_id": page_id, "title": title, "storage": storage}) + return PublishedPage( + page_id=page_id, + title=title, + page_url=f"https://example.atlassian.net/wiki/spaces/DATA/pages/{page_id}", + created=False, + ) + + +def test_cli_help_lists_confluence_group() -> None: + runner = CliRunner() + result = runner.invoke(main, ["--help"]) + assert result.exit_code == 0, result.output + assert "confluence" in result.output + + confluence_result = runner.invoke(main, ["confluence", "--help"]) + assert confluence_result.exit_code == 0, confluence_result.output + assert "pull-page" in confluence_result.output + assert "publish" in confluence_result.output + assert "sync" in confluence_result.output + + +def test_storage_to_markdown_converts_basic_storage() -> None: + markdown = storage_to_markdown( + "

Metric definitions

Owner: Finance

" + "
  • Net revenue excludes refunds
" + ) + + assert "# Metric definitions" in markdown + assert "**Owner:** Finance" in markdown + assert "- Net revenue excludes refunds" in markdown + + +def test_pull_page_to_context_records_manifest(tmp_path: Path) -> None: + spec_dir = tmp_path / "specs" / "001-player-journey" + output_path = spec_dir / "context" / "metric-definitions.md" + client = FakeConfluenceClient() + + page = pull_page_to_context( + client=client, + page_id="123", + output_path=output_path, + ) + + assert page.title == "Player metric definitions" + context = output_path.read_text() + assert "# Player metric definitions" in context + assert "Net revenue excludes refunds" in context + + manifest = yaml.safe_load((spec_dir / "confluence.yml").read_text()) + assert manifest["spec_dir"] == str(spec_dir) + assert manifest["source_pages"][0]["page_id"] == "123" + assert manifest["source_pages"][0]["context_file"] == str(output_path) + + +def test_publish_spec_dir_creates_page_and_manifest(tmp_path: Path) -> None: + spec_dir = tmp_path / "specs" / "001-player-journey" + spec_dir.mkdir(parents=True) + (spec_dir / "spec.md").write_text("# Spec\n\nApproved player journey spec.") + (spec_dir / "plan.md").write_text("# Plan\n\nUpdate fct_player_performance.") + client = FakeConfluenceClient() + + page = publish_spec_dir( + client=client, + spec_dir=spec_dir, + space_key="DATA", + parent_id="456", + ) + + assert page.created is True + assert client.created[0]["space_id"] == "space-123" + assert client.created[0]["parent_id"] == "456" + assert "Approved player journey spec." in str(client.created[0]["storage"]) + + manifest = yaml.safe_load((spec_dir / "confluence.yml").read_text()) + assert manifest["page_id"] == "999" + assert manifest["space_key"] == "DATA" + assert manifest["parent_id"] == "456" + + +def test_confluence_publish_dry_run_does_not_require_credentials( + tmp_path: Path, + monkeypatch: MonkeyPatch, +) -> None: + for name in ("CONFLUENCE_BASE_URL", "CONFLUENCE_EMAIL", "CONFLUENCE_API_TOKEN"): + monkeypatch.delenv(name, raising=False) + spec_dir = tmp_path / "specs" / "001-player-journey" + spec_dir.mkdir(parents=True) + (spec_dir / "spec.md").write_text("# Spec\n\nApproved spec.") + + result = CliRunner().invoke( + main, + [ + "confluence", + "publish", + "--spec-dir", + str(spec_dir), + "--space-key", + "DATA", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert "would create Confluence page " in result.output + assert not (spec_dir / "confluence.yml").exists() + + +def test_sync_spec_dir_updates_existing_page(tmp_path: Path) -> None: + spec_dir = tmp_path / "specs" / "001-player-journey" + spec_dir.mkdir(parents=True) + (spec_dir / "spec.md").write_text("# Spec\n\nApproved spec.") + (spec_dir / "confluence.yml").write_text( + yaml.safe_dump( + { + "page_id": "123", + "title": "Player journey summary", + "page_url": "https://example.atlassian.net/wiki/spaces/DATA/pages/123", + } + ) + ) + client = FakeConfluenceClient() + + page = sync_spec_dir(client=client, spec_dir=spec_dir) + + assert page.created is False + assert client.updated[0]["page_id"] == "123" + assert client.updated[0]["title"] == "Player journey summary" + assert "Approved spec." in client.updated[0]["storage"] + + +def _page() -> ConfluencePage: + return ConfluencePage( + page_id="123", + title="Player metric definitions", + space_id="space-123", + space_key="DATA", + version=7, + page_url="https://example.atlassian.net/wiki/spaces/DATA/pages/123", + storage="

Player metrics

Net revenue excludes refunds.

", + ) diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..f30ce59 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +import re +from pathlib import Path + +from sqlmesh_specify.init import SUPPORTED_ENGINES + +ROOT = Path(__file__).resolve().parents[1] + + +def test_readme_local_links_exist() -> None: + _assert_local_links_exist(ROOT / "README.md", ROOT) + + +def test_required_docs_exist() -> None: + for relative in [ + "docs/getting-started.md", + "docs/methodology.md", + "docs/enterprise-ci.md", + "docs/sqlmesh-ai-sdlc-walkthrough.md", + "docs/sqlmesh-memory-and-repo-hygiene.md", + "docs/brownfield-onboarding.md", + "docs/skills-and-sub-agents.md", + "docs/releasing.md", + "docs/integrations/jira.md", + "docs/integrations/confluence.md", + "docs/tutorials/README.md", + ".github/workflows/ci.yml", + ".github/workflows/release.yml", + ]: + assert (ROOT / relative).exists(), f"missing {relative}" + + +def test_docs_use_sqlmesh_public_interface() -> None: + readme = (ROOT / "README.md").read_text() + assert "sqlmesh-spec-kit" in readme + assert "sqlmesh-specify init analytics --engine duckdb" in readme + assert "sqlmesh-specify validate sqlmesh" in readme + assert "sqlmesh plan " in readme + + +def test_tutorials_exist() -> None: + for name in [ + "01-initialize-a-sqlmesh-repo.md", + "02-ship-a-sqlmesh-change.md", + "03-brownfield-enterprise-adoption.md", + "04-skills-and-agent-handoffs.md", + "05-jira-to-spec-workflow.md", + "06-confluence-context-workflow.md", + ]: + path = ROOT / "docs" / "tutorials" / name + assert path.exists(), f"missing tutorial {name}" + _assert_local_links_exist(path, path.parent) + + +def test_engine_guides_exist_for_supported_engines() -> None: + readme = (ROOT / "README.md").read_text() + for engine in SUPPORTED_ENGINES: + guide = ROOT / "docs" / "engine-guides" / f"{engine}.md" + assert guide.exists(), f"missing engine guide {engine}" + assert f"--engine {engine}" in guide.read_text() + assert engine in readme + + +def test_release_workflow_uses_trusted_publishing() -> None: + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text() + assert "id-token: write" in workflow + assert "pypa/gh-action-pypi-publish@release/v1" in workflow + assert "python -m build --sdist --wheel" in workflow + assert "python -m twine check dist/*" in workflow + + +def test_docs_do_not_use_old_public_names() -> None: + text = "\n".join( + path.read_text() for path in [ROOT / "README.md", ROOT / "docs" / "getting-started.md"] + ) + assert "dbt-spec" not in text.lower() + assert "--warehouse" not in text + + +def _markdown_links(text: str) -> list[str]: + return re.findall(r"(? bool: + return link.startswith(("http://", "https://", "mailto:", "#")) + + +def _assert_local_links_exist(path: Path, base: Path) -> None: + for link in _markdown_links(path.read_text()): + if _is_external_or_anchor(link): + continue + target = link.split("#", 1)[0] + if not target: + continue + assert (base / target).resolve().exists(), ( + f"{path.relative_to(ROOT)} link target does not exist: {link}" + ) diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 0000000..e256f2e --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from pathlib import Path + +from click.testing import CliRunner + +from sqlmesh_specify.cli import main +from sqlmesh_specify.doctor import doctor_project + + +def test_doctor_reports_adoption_gaps(minimal_sqlmesh_project: Path) -> None: + report = doctor_project(minimal_sqlmesh_project) + codes = {finding.code for finding in report.findings} + assert "SQLMESH_SPECIFY_MISSING" in codes + assert "MODEL_INVENTORY" in codes + + +def test_doctor_cli(minimal_sqlmesh_project: Path) -> None: + result = CliRunner().invoke(main, ["doctor", "--target", str(minimal_sqlmesh_project)]) + assert result.exit_code == 0 + assert "sqlmesh-specify doctor" in result.output diff --git a/tests/test_ears.py b/tests/test_ears.py new file mode 100644 index 0000000..fc67a28 --- /dev/null +++ b/tests/test_ears.py @@ -0,0 +1,34 @@ +"""Tests for EARS pattern classification.""" +from __future__ import annotations + +import pytest + +from sqlmesh_specify.ears import classify_ears + + +@pytest.mark.parametrize( + ("line", "expected"), + [ + ("The system shall ship a CLI.", "ubiquitous"), + ("the system must support snowflake", "ubiquitous"), + ("When init is invoked, the system shall create .sqlmesh-specify/.", "event_driven"), + ("While in plan mode, the system shall not write code.", "state_driven"), + ("If no config.yaml exists, then the system shall exit non-zero.", "unwanted"), + ("Where Snowflake is selected, the system shall add clustering hints.", "optional"), + ], +) +def test_classify_ears_positive(line: str, expected: str) -> None: + assert classify_ears(line) == expected + + +@pytest.mark.parametrize( + "line", + [ + "Hello world", + "We should probably build a CLI", + "Init creates .sqlmesh-specify/", + "The system would maybe support snowflake", + ], +) +def test_classify_ears_negative(line: str) -> None: + assert classify_ears(line) is None diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 0000000..75c5f3e --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from pathlib import Path + +from click.testing import CliRunner + +from sqlmesh_specify.cli import main +from sqlmesh_specify.init import SUPPORTED_ENGINES + + +def test_cli_help_lists_commands() -> None: + result = CliRunner().invoke(main, ["--help"]) + assert result.exit_code == 0 + for command in ["init", "doctor", "validate", "report", "ci", "jira", "confluence"]: + assert command in result.output + + +def test_init_succeeds_on_minimal_sqlmesh_project(minimal_sqlmesh_project: Path) -> None: + result = CliRunner().invoke( + main, + ["init", "demo", "--engine", "duckdb", "--target", str(minimal_sqlmesh_project)], + ) + assert result.exit_code == 0, result.output + assert (minimal_sqlmesh_project / ".sqlmesh-specify" / "constitution.md").exists() + assert (minimal_sqlmesh_project / ".sqlmesh-specify" / "commands" / "sqlmesh.plan.md").exists() + assert (minimal_sqlmesh_project / "CLAUDE.md").exists() + assert (minimal_sqlmesh_project / "specs" / ".gitkeep").exists() + + +def test_init_refuses_non_sqlmesh_target(tmp_path: Path) -> None: + result = CliRunner().invoke( + main, ["init", "demo", "--engine", "duckdb", "--target", str(tmp_path)] + ) + assert result.exit_code == 1 + assert "config.yaml" in result.output + + +def test_init_force_overwrites_existing(minimal_sqlmesh_project: Path) -> None: + runner = CliRunner() + assert runner.invoke( + main, ["init", "demo", "--engine", "duckdb", "--target", str(minimal_sqlmesh_project)] + ).exit_code == 0 + sentinel = minimal_sqlmesh_project / ".sqlmesh-specify" / "sentinel.txt" + sentinel.write_text("old") + result = runner.invoke( + main, + ["init", "demo", "--engine", "duckdb", "--target", str(minimal_sqlmesh_project), "--force"], + ) + assert result.exit_code == 0, result.output + assert not sentinel.exists() + + +def test_each_engine_preset_initializes(tmp_path: Path) -> None: + runner = CliRunner() + for engine in SUPPORTED_ENGINES: + project = tmp_path / engine + (project / "models").mkdir(parents=True) + (project / "models" / "example.sql").write_text( + "MODEL (name x.y, kind FULL); SELECT 1 AS id;" + ) + (project / "config.yaml").write_text("default_gateway: local\n") + result = runner.invoke(main, ["init", "demo", "--engine", engine, "--target", str(project)]) + assert result.exit_code == 0, result.output + constitution = (project / ".sqlmesh-specify" / "constitution.md").read_text() + plan = (project / ".sqlmesh-specify" / "templates" / "plan-template.md").read_text() + assert f"BEGIN {engine.upper()} ADDITIONS" in constitution + assert f"BEGIN {engine.upper()} PLAN ADDITIONS" in plan diff --git a/tests/test_jira.py b/tests/test_jira.py new file mode 100644 index 0000000..0ffcd49 --- /dev/null +++ b/tests/test_jira.py @@ -0,0 +1,203 @@ +"""Tests for Jira bridge helpers.""" +from __future__ import annotations + +from pathlib import Path + +import yaml +from click.testing import CliRunner + +from sqlmesh_specify.cli import main +from sqlmesh_specify.jira import ( + CreatedSubtask, + JiraIssue, + JiraSubtask, + attach_artifacts, + create_subtasks_from_tasks, + parse_tasks, + pull_issue_to_spec, +) + + +class FakeJiraClient: + def __init__(self, issue: JiraIssue) -> None: + self.issue = issue + self.attachments: list[Path] = [] + self.comments: list[str] = [] + self.created: list[dict[str, str]] = [] + + def get_issue(self, issue_key: str) -> JiraIssue: + assert issue_key == self.issue.key + return self.issue + + def attach_file(self, issue_key: str, path: Path) -> str: + assert issue_key == self.issue.key + self.attachments.append(path) + return path.name + + def add_comment(self, issue_key: str, text: str) -> None: + assert issue_key == self.issue.key + self.comments.append(text) + + def create_subtask( + self, + *, + parent_issue_key: str, + project_key: str, + issue_type_name: str, + summary: str, + description: str, + ) -> str: + assert parent_issue_key == self.issue.key + assert project_key == self.issue.project_key + assert issue_type_name == "Sub-task" + key = f"{self.issue.project_key}-{len(self.created) + 200}" + self.created.append({"summary": summary, "description": description, "key": key}) + return key + + +def test_cli_help_lists_jira_group() -> None: + runner = CliRunner() + result = runner.invoke(main, ["--help"]) + assert result.exit_code == 0, result.output + assert "jira" in result.output + + jira_result = runner.invoke(main, ["jira", "--help"]) + assert jira_result.exit_code == 0, jira_result.output + assert "pull" in jira_result.output + assert "attach" in jira_result.output + assert "create-tasks" in jira_result.output + assert "sync" in jira_result.output + + +def test_pull_issue_to_spec_creates_traceable_spec(tmp_path: Path) -> None: + (tmp_path / "config.yaml").write_text("name: demo\nversion: '1.0.0'\nprofile: demo\n") + client = FakeJiraClient(_issue()) + + spec_dir = pull_issue_to_spec( + client=client, # type: ignore[arg-type] + issue_key="NBA-123", + target_dir=tmp_path, + ) + + assert spec_dir.name == "001-nba-123-player-journey-mart" + spec = (spec_dir / "spec.md").read_text() + assert "# Player journey mart" in spec + assert "https://example.atlassian.net/browse/NBA-123" in spec + assert "AC1: The system shall implement `NBA-123` without unrelated SQLMesh changes." in spec + + jira_yml = yaml.safe_load((spec_dir / "jira.yml").read_text()) + assert jira_yml["issue_key"] == "NBA-123" + assert jira_yml["spec_dir"] == "specs/001-nba-123-player-journey-mart" + + +def test_parse_tasks_extracts_pending_items_and_validation() -> None: + tasks = parse_tasks( + """# Tasks + +## Task list + +- [ ] **T-01** — Add mart SQL. AC: AC1. + - **Done when:** model builds + - **Validates:** AC1, AC2 + +- [x] **T-02** — Add docs. AC: AC3. + - **Validates:** AC3 +""" + ) + + assert [task.task_id for task in tasks] == ["T-01", "T-02"] + assert tasks[0].description == "Add mart SQL. AC: AC1." + assert tasks[0].validates == "AC1, AC2" + assert tasks[1].checked is True + + +def test_attach_artifacts_uploads_files_and_comments(tmp_path: Path) -> None: + client = FakeJiraClient(_issue()) + spec = tmp_path / "spec.md" + plan = tmp_path / "plan.md" + spec.write_text("spec") + plan.write_text("plan") + + uploaded = attach_artifacts( + client=client, # type: ignore[arg-type] + issue_key="NBA-123", + files=[spec, plan], + ) + + assert uploaded == ["spec.md", "plan.md"] + assert client.attachments == [spec, plan] + assert "spec.md" in client.comments[0] + assert "plan.md" in client.comments[0] + + +def test_create_subtasks_is_idempotent(tmp_path: Path) -> None: + tasks_path = tmp_path / "tasks.md" + tasks_path.write_text( + """# Tasks + +## Task list + +- [ ] **T-01** — Add player journey mart. AC: AC1. + - **Validates:** AC1 +- [ ] **T-02** — Add tests and docs. AC: AC2. + - **Validates:** AC2 +""" + ) + issue = _issue( + subtasks=( + JiraSubtask( + key="NBA-199", + summary="[sqlmesh-specify T-01] Add player journey mart. AC: AC1.", + ), + ) + ) + client = FakeJiraClient(issue) + + results = create_subtasks_from_tasks( + client=client, # type: ignore[arg-type] + issue_key="NBA-123", + tasks_path=tasks_path, + ) + + assert results == [ + CreatedSubtask( + task_id="T-01", + summary="[sqlmesh-specify T-01] Add player journey mart. AC: AC1.", + key="NBA-199", + created=False, + ), + CreatedSubtask( + task_id="T-02", + summary="[sqlmesh-specify T-02] Add tests and docs. AC: AC2.", + key="NBA-200", + created=True, + ), + ] + assert client.created == [ + { + "summary": "[sqlmesh-specify T-02] Add tests and docs. AC: AC2.", + "description": ( + f"Created from {tasks_path}\n\n" + "Task: T-02\n" + "Description: Add tests and docs. AC: AC2.\n" + "Validates: AC2" + ), + "key": "NBA-200", + } + ] + + +def _issue(subtasks: tuple[JiraSubtask, ...] = ()) -> JiraIssue: + return JiraIssue( + key="NBA-123", + summary="Player journey mart", + description="Build a season-by-season player journey.", + project_key="NBA", + issue_url="https://example.atlassian.net/browse/NBA-123", + labels=("analytics",), + components=("SQLMesh",), + priority="Medium", + issue_type="Story", + status="Ready", + subtasks=subtasks, + ) diff --git a/tests/test_lifecycle.py b/tests/test_lifecycle.py new file mode 100644 index 0000000..9a40575 --- /dev/null +++ b/tests/test_lifecycle.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from pathlib import Path + +from sqlmesh_specify.lifecycle import validate_lifecycle +from tests.conftest import write_lifecycle + + +def _codes(root: Path) -> set[str]: + return {finding.code for finding in validate_lifecycle(root).findings} + + +def test_exact_valid_statuses_pass(tmp_path: Path) -> None: + write_lifecycle(tmp_path) + assert validate_lifecycle(tmp_path).error_count == 0 + + +def test_disapproved_spec_does_not_allow_plan(tmp_path: Path) -> None: + write_lifecycle(tmp_path, spec_status="disapproved") + codes = _codes(tmp_path) + assert "SPEC_STATUS_INVALID" in codes + assert "PLAN_BEFORE_SPEC_APPROVAL" in codes + + +def test_unapproved_plan_does_not_allow_tasks(tmp_path: Path) -> None: + write_lifecycle(tmp_path, plan_status="unapproved") + codes = _codes(tmp_path) + assert "PLAN_STATUS_INVALID" in codes + assert "TASKS_BEFORE_PLAN_APPROVAL" in codes + + +def test_not_approved_plan_does_not_allow_tasks(tmp_path: Path) -> None: + write_lifecycle(tmp_path, plan_status="not approved") + codes = _codes(tmp_path) + assert "PLAN_STATUS_INVALID" in codes + assert "TASKS_BEFORE_PLAN_APPROVAL" in codes + + +def test_missing_statuses_are_explicit(tmp_path: Path) -> None: + write_lifecycle(tmp_path, spec_status=None, plan_status=None, tasks_status=None) + codes = _codes(tmp_path) + assert {"SPEC_STATUS_MISSING", "PLAN_STATUS_MISSING", "TASKS_STATUS_MISSING"} <= codes + + +def test_missing_ac_traceability_fails(tmp_path: Path) -> None: + spec_dir = write_lifecycle(tmp_path) + (spec_dir / "tasks.md").write_text( + "**Status:** in progress\n\n## Task list\n\n- [ ] T-01 - Work.\n" + ) + assert "AC_NOT_TRACED" in _codes(tmp_path) diff --git a/tests/test_sqlmesh_artifacts.py b/tests/test_sqlmesh_artifacts.py new file mode 100644 index 0000000..fe9fda2 --- /dev/null +++ b/tests/test_sqlmesh_artifacts.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from pathlib import Path + +from click.testing import CliRunner + +from sqlmesh_specify.cli import main +from sqlmesh_specify.sqlmesh_artifacts import validate_sqlmesh_project + + +def test_validate_sqlmesh_reports_clean_minimal_project(minimal_sqlmesh_project: Path) -> None: + report = validate_sqlmesh_project(minimal_sqlmesh_project) + assert report.error_count == 0 + assert "AUDITS_MISSING" not in {finding.code for finding in report.findings} + + +def test_validate_sqlmesh_errors_without_config(tmp_path: Path) -> None: + (tmp_path / "models").mkdir() + report = validate_sqlmesh_project(tmp_path) + assert "SQLMESH_CONFIG_MISSING" in {finding.code for finding in report.findings} + + +def test_validate_sqlmesh_errors_without_models(tmp_path: Path) -> None: + (tmp_path / "config.yaml").write_text("default_gateway: local\n") + report = validate_sqlmesh_project(tmp_path) + assert "MODELS_DIR_MISSING" in {finding.code for finding in report.findings} + + +def test_validate_sqlmesh_warns_without_audits_or_tests(tmp_path: Path) -> None: + (tmp_path / "config.yaml").write_text("default_gateway: local\n") + (tmp_path / "models").mkdir() + (tmp_path / "models" / "example.sql").write_text("MODEL (name x.y, kind FULL); SELECT 1 AS id;") + report = validate_sqlmesh_project(tmp_path) + codes = {finding.code for finding in report.findings} + assert "AUDITS_MISSING" in codes + assert "TESTS_DIR_MISSING" in codes + + +def test_validate_sqlmesh_cli(minimal_sqlmesh_project: Path) -> None: + result = CliRunner().invoke( + main, ["validate", "sqlmesh", "--target", str(minimal_sqlmesh_project)] + ) + assert result.exit_code == 0 + assert "SQLMesh project validation" in result.output diff --git a/tests/test_validate.py b/tests/test_validate.py new file mode 100644 index 0000000..9d11b21 --- /dev/null +++ b/tests/test_validate.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from pathlib import Path + +from click.testing import CliRunner + +from sqlmesh_specify.cli import main +from sqlmesh_specify.validate import _extract_ac_lines + + +def test_html_comments_inside_acceptance_criteria_are_ignored() -> None: + text = """# Spec + +## Acceptance Criteria + + +- AC1: When the model is queried, the system shall return one row per customer. +""" + assert _extract_ac_lines(text) == [ + (6, "When the model is queried, the system shall return one row per customer.") + ] + + +def test_prose_inside_acceptance_criteria_fails(tmp_path: Path) -> None: + spec = tmp_path / "spec.md" + spec.write_text( + "# Spec\n\n## Acceptance Criteria\n\n" + "This paragraph should fail.\n" + "- AC1: When the model is queried, the system shall return rows.\n" + ) + result = CliRunner().invoke(main, ["validate", str(spec)]) + assert result.exit_code == 1 + assert "do not match" in result.output + + +def test_valid_spec_cli_passes(tmp_path: Path) -> None: + spec = tmp_path / "spec.md" + spec.write_text( + "# Spec\n\n## Acceptance Criteria\n\n" + "- AC1: When the model is queried, the system shall return rows.\n" + ) + result = CliRunner().invoke(main, ["validate", str(spec)]) + assert result.exit_code == 0