From c203445fceb284d1c0ee1a693280df8817a05143 Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:40:08 +0800 Subject: [PATCH 01/13] feat: Go project skeleton, CI and evidence files (D1) (#4) Starts Discreet's development for Milestone 2: - Go 1.27 module and the `discreet` command with serve, audit verify, audit export and version. Each describes itself until its issue builds it; run() takes its arguments and output streams so tests need no subprocess. 12 table-driven tests. - `make check` runs gofmt, go vet, go test -race, govulncheck (pinned v1.8.0) and a build, locally and in CI. CodeQL covers Go and the workflows. Every Action is pinned by commit SHA; Dependabot targets next. - Evidence files with honest "not measured yet" content: CONTROLS (v0, mappings pending D11), THREAT_MODEL, EVALS, RISKS, CHANGELOG and the build log, plus ADR 0001 (standard library only) and ADR 0002 (same integration-branch workflow as Vetted). Closes makoydev/vetted#30 Co-authored-by: Claude Opus 5.5 --- .github/CODEOWNERS | 2 + .github/dependabot.yml | 24 +++++++ .github/workflows/ci.yml | 34 ++++++++++ .github/workflows/codeql.yml | 55 ++++++++++++++++ CHANGELOG.md | 12 ++++ CLAUDE.md | 7 ++- CONTROLS.md | 11 ++++ EVALS.md | 14 +++++ Makefile | 21 +++++++ README.md | 15 ++++- RISKS.md | 13 ++++ THREAT_MODEL.md | 28 +++++++++ cmd/discreet/main.go | 93 ++++++++++++++++++++++++++++ cmd/discreet/main_test.go | 55 ++++++++++++++++ docs/HOW-THIS-WAS-BUILT.md | 32 ++++++++++ docs/adr/0001-go-standard-library.md | 24 +++++++ docs/adr/0002-integration-branch.md | 22 +++++++ go.mod | 3 + 18 files changed, 463 insertions(+), 2 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 CHANGELOG.md create mode 100644 CONTROLS.md create mode 100644 EVALS.md create mode 100644 Makefile create mode 100644 RISKS.md create mode 100644 THREAT_MODEL.md create mode 100644 cmd/discreet/main.go create mode 100644 cmd/discreet/main_test.go create mode 100644 docs/HOW-THIS-WAS-BUILT.md create mode 100644 docs/adr/0001-go-standard-library.md create mode 100644 docs/adr/0002-integration-branch.md create mode 100644 go.mod diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..f1a203e --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Every change needs review from the maintainer. +* @makoydev diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..91e35fa --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 +updates: + # Keeps SHA-pinned Actions current; Dependabot updates the SHA and the + # version comment together. + - package-ecosystem: github-actions + directory: / + target-branch: next + schedule: + interval: weekly + labels: + - dependencies + groups: + actions: + update-types: + - minor + - patch + + - package-ecosystem: gomod + directory: / + target-branch: next + schedule: + interval: weekly + labels: + - dependencies diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..81da0d9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI + +on: + pull_request: + branches: + - main + - next + push: + branches: + - main + - next + +permissions: + contents: read + +jobs: + test: + name: Lint and test + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + # gofmt, go vet, go test -race, govulncheck and a build (see Makefile) + - name: Run every check + run: make check diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..58efffe --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,55 @@ +name: CodeQL + +on: + pull_request: + branches: + - main + - next + push: + branches: + - main + - next + schedule: + - cron: '41 7 * * 3' + +permissions: + contents: read + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + include: + - language: go + build-mode: autobuild + - language: actions + build-mode: none + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Go + if: matrix.language == 'go' + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: false + + - name: Initialize CodeQL + uses: github/codeql-action/init@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + + - name: Perform CodeQL analysis + uses: github/codeql-action/analyze@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 + with: + category: /language:${{ matrix.language }} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b2ec1cc --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project uses [Semantic Versioning](https://semver.org/). + +## [Unreleased] + +### Added + +- Go 1.27 project skeleton: the `discreet` command with `serve`, `audit verify`, `audit export` and `version`, each describing itself until it is built (issue D1). +- CI on every pull request: `gofmt`, `go vet`, `go test -race`, `govulncheck` and a build (`make check`), plus CodeQL for Go and the workflows. All Actions pinned by commit SHA. +- Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. +- Vetted reviews every pull request (shadow mode until 2026-10-14). diff --git a/CLAUDE.md b/CLAUDE.md index 8eb9a13..c17910f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,4 +4,9 @@ Programme-wide rules are in `../CLAUDE.md` (loaded automatically) and the full b Discreet is a privacy-preserving LLM gateway written in Go. Development starts in Milestone 2 (due 22 Nov 2026): proxy with a mock and one real upstream, detectors from `sg-pii-rules` (vendored, checksummed), tokenise and rehydrate, a hash-chained audit log with `verify`, a four-pane demo UI, a benchmark in `EVALS.md`, and a public mock-mode deployment. -Until then this repository only holds the Vetted pilot: every pull request is reviewed by Vetted, in shadow mode until 2026-10-14 and opt-in afterwards. Work goes through pull requests into `next` (ADR 0009 in Vetted); only Michael merges `next` into `main`. +Development started on 2026-10-04 (issue D1). Every pull request is reviewed by Vetted, in shadow mode until 2026-10-14 and opt-in afterwards. Work goes through pull requests into `next` (ADR 0002); only Michael merges `next` into `main`. + +- Go 1.27, standard library only (ADR 0001); any new module needs an ADR and an exact version. +- Run `make check` (gofmt, vet, race tests, govulncheck, build) before every commit; never commit past a failure. +- Tests use synthetic data only; build secret- and NRIC-shaped strings at runtime. +- Issues live in `makoydev/vetted` under the "M2: Discreet v0.1" milestone (D1 = vetted#30). diff --git a/CONTROLS.md b/CONTROLS.md new file mode 100644 index 0000000..9a75569 --- /dev/null +++ b/CONTROLS.md @@ -0,0 +1,11 @@ +# Controls + +Each Discreet feature, the principle or obligation it supports, and the source. **Draft v0:** features are listed as they are planned, and each row is mapped to a primary source and graded (primary, secondary or unverified) in issue D11. No row claims compliance with anything. + +| Feature | Supports | Source | Status | +| --- | --- | --- | --- | +| Personal data replaced with placeholders before it reaches the model | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Planned (D3, D4); mapping pending | +| Purpose declared in a header; automated eligibility decisions refused | Human accountability for decisions about people | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | Planned (D5); mapping pending | +| Hash-chained audit log with `verify` | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | Planned (D7); mapping pending | +| Daily budget, access token, rate limit | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | Planned (D6); mapping pending | +| Pinned Actions, `govulncheck`, CodeQL, reviewed pull requests | Secure development | OWASP Top 10 for LLM Applications 2025 (supply chain) | In place (D1); mapping pending | diff --git a/EVALS.md b/EVALS.md new file mode 100644 index 0000000..35fdb1d --- /dev/null +++ b/EVALS.md @@ -0,0 +1,14 @@ +# Evaluations + +**Nothing is measured yet.** This file will only ever hold numbers that were actually measured, each with its method, sample size and limitations. + +| What | Planned in | Status | +| --- | --- | --- | +| Precision and recall per entity on about 5,000 synthetic samples from a generator written independently of the detectors | Issue D9 | Not started | +| Added latency (p50, p95) against the mock model | Issue D9 | Not started | +| Detector conformance: every sg-pii-rules fixture passes in Go | Issue D3 | Not started | +| Data-disclosure and adversarial-prompt run with Project Moonshot | Milestone 3 | Not started | + +Known limitation, stated in advance: synthetic data flatters rule-based detectors, because real text is messier than any generator. + +Vetted's review of this repository's pull requests is measured in Vetted's own `EVALS.md`. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..359eeac --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +# One command for every check, used locally and in CI: make check +GOVULNCHECK := golang.org/x/vuln/cmd/govulncheck@v1.8.0 + +.PHONY: check fmt vet test vuln build + +check: fmt vet test vuln build + +fmt: + @unformatted=$$(gofmt -l .); if [ -n "$$unformatted" ]; then echo "gofmt needed:"; echo "$$unformatted"; exit 1; fi + +vet: + go vet ./... + +test: + go test -race -count=1 ./... + +vuln: + go run $(GOVULNCHECK) ./... + +build: + go build -o bin/discreet ./cmd/discreet diff --git a/README.md b/README.md index caea506..b4e1b10 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,19 @@ A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`. -**Status: not started.** Development begins in Milestone 2 (due 22 Nov 2026). From its first pull request, every change here is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode for the first two weeks, then opt-in. +**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Only the command-line skeleton exists so far; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. + +## Build and check + +Needs Go 1.27. + +```sh +make check # gofmt, go vet, go test -race, govulncheck, build +./bin/discreet help +``` + +## Evidence + +[`CONTROLS.md`](CONTROLS.md) · [`THREAT_MODEL.md`](THREAT_MODEL.md) · [`EVALS.md`](EVALS.md) · [`RISKS.md`](RISKS.md) · [`CHANGELOG.md`](CHANGELOG.md) · [decision records](docs/adr/) · [how this was built](docs/HOW-THIS-WAS-BUILT.md) Part of the AI Governance Toolkit. Programme board: diff --git a/RISKS.md b/RISKS.md new file mode 100644 index 0000000..2f06095 --- /dev/null +++ b/RISKS.md @@ -0,0 +1,13 @@ +# Risks + +Reviewed at the start of each issue and at each milestone. Likelihood and impact are Low / Medium / High. + +| # | Risk | Likelihood | Impact | Mitigation | Owner | +| --- | --- | --- | --- | --- | --- | +| R1 | Go and TypeScript detectors disagree on edge cases | Low | Medium | Both use RE2 and must pass the same sg-pii-rules fixtures; any disagreement becomes a new fixture | Claude Code | +| R2 | Postal code and date-of-birth rules are noisy on real text | Medium | Medium | Context clues, hard negatives, false-positive rates published rather than hidden | Claude Code | +| R3 | The public demo is abused | Medium | Low | Mock-only without a token, per-IP rate limit, nothing kept but keyed hashes and counts | Claude Code | +| R4 | The hosting account is not ready by the week of 2 Nov | Low | Medium | Asked for early; fallback is "run locally in five minutes" with the slip recorded | Michael | +| R5 | Pilot evidence stays mock-only because `OPENAI_API_KEY` is not set | High | Medium | EVALS states reviews run vs. reviews with a real model; setting the key takes about 10 minutes | Michael | +| R6 | M2 slips past 22 Nov | Medium | High | Plan finishes about 10 days early; anything that doesn't fit moves to "Next" in the README with the reason | Claude Code (plan), Michael (review) | +| R7 | Michael can't explain AI-drafted Go code in an interview | Medium | High | Standard library only, small pull requests, an "If asked in an interview" section on each, ADRs he confirms | Michael | diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md new file mode 100644 index 0000000..5b21129 --- /dev/null +++ b/THREAT_MODEL.md @@ -0,0 +1,28 @@ +# Threat model + +**Draft.** Filled in as each feature lands; reviewed in issue D11. + +## What Discreet protects + +- Personal data in requests: NRIC/FIN numbers, phone numbers, emails, payment cards, addresses and dates of birth. +- The vault that maps placeholders back to real values. +- The audit log's integrity. +- The operator's model budget. + +## Trust boundaries + +1. **Calling application → Discreet.** Requests may contain anything, including prompt-injection text and real personal data. +2. **Discreet → model provider.** Everything crossing this line must already be tokenised. This is the boundary Discreet exists to enforce. +3. **Discreet → disk.** Only the audit log is written, holding keyed hashes and counts, never raw text. + +## Threats to address in Milestone 2 + +| Threat | Planned mitigation | Issue | +| --- | --- | --- | +| Personal data the detectors miss reaches the model | Shared conformance suite with hard negatives; measured recall published in EVALS | D3, D9 | +| The vault leaks values through logs or errors | Values encrypted in memory, short expiry, a test that scans logs for planted values | D4 | +| The model invents placeholders to extract other values | Only placeholders issued for that request are rehydrated | D4 | +| Someone edits the audit log to hide a request | SHA-256 hash chain; `audit verify` names the first broken record | D7 | +| Short identifiers are guessed from audit hashes | HMAC-SHA-256 with a server secret instead of plain hashes | D7 | +| The public demo is used to spend money | Mock model unless a valid access token is present; daily budget; rate limit | D6, D10 | +| A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 | diff --git a/cmd/discreet/main.go b/cmd/discreet/main.go new file mode 100644 index 0000000..b833033 --- /dev/null +++ b/cmd/discreet/main.go @@ -0,0 +1,93 @@ +// Command discreet is a privacy gateway between applications and large +// language models. It replaces personal data with placeholders before a +// request reaches the model, puts the data back in the answer, and keeps a +// tamper-evident audit log. +// +// This is the skeleton from issue D1: each subcommand describes itself, and +// later issues build them out (serve in D5, audit in D7). +package main + +import ( + "fmt" + "io" + "os" +) + +// version is overridden at release time with -ldflags "-X main.version=...". +var version = "0.0.0-dev" + +const usage = `discreet: a privacy gateway for large language model requests + +Usage: + discreet serve run the gateway + discreet audit verify check that the audit log has not been changed + discreet audit export export the audit log as CSV + discreet version print the version + discreet help show this help + +Run "discreet --help" for details of a command. +` + +const serveUsage = `Usage: discreet serve + +Runs the gateway: an OpenAI-compatible endpoint that swaps personal data for +placeholders before the request reaches the model. Not built yet (issue D5). +` + +const auditUsage = `Usage: discreet audit + + verify check every record's hash chain; fails if any byte was changed + export write the audit log as CSV + +Not built yet (issue D7). +` + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +// run executes one command and returns the process exit code: +// 0 for success, 1 for a command that failed, 2 for a usage error. +func run(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprint(stderr, usage) + return 2 + } + switch args[0] { + case "help", "-h", "--help": + fmt.Fprint(stdout, usage) + return 0 + case "version", "--version": + fmt.Fprintln(stdout, "discreet", version) + return 0 + case "serve": + if wantsHelp(args[1:]) { + fmt.Fprint(stdout, serveUsage) + return 0 + } + fmt.Fprintln(stderr, "discreet serve: not built yet (issue D5)") + return 1 + case "audit": + if wantsHelp(args[1:]) { + fmt.Fprint(stdout, auditUsage) + return 0 + } + if len(args) < 2 { + fmt.Fprint(stderr, auditUsage) + return 2 + } + switch args[1] { + case "verify", "export": + fmt.Fprintf(stderr, "discreet audit %s: not built yet (issue D7)\n", args[1]) + return 1 + } + fmt.Fprintf(stderr, "discreet audit: unknown subcommand %q\n\n%s", args[1], auditUsage) + return 2 + } + fmt.Fprintf(stderr, "discreet: unknown command %q\n\n%s", args[0], usage) + return 2 +} + +func wantsHelp(args []string) bool { + return len(args) > 0 && (args[0] == "-h" || args[0] == "--help" || args[0] == "help") +} diff --git a/cmd/discreet/main_test.go b/cmd/discreet/main_test.go new file mode 100644 index 0000000..a1a3422 --- /dev/null +++ b/cmd/discreet/main_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestRun(t *testing.T) { + tests := []struct { + name string + args []string + wantCode int + wantStdout string + wantStderr string + }{ + {"no arguments prints usage as an error", nil, 2, "", "Usage:"}, + {"help", []string{"help"}, 0, "discreet serve", ""}, + {"--help", []string{"--help"}, 0, "discreet audit verify", ""}, + {"version", []string{"version"}, 0, "discreet 0.0.0-dev", ""}, + {"serve help", []string{"serve", "--help"}, 0, "Usage: discreet serve", ""}, + {"serve is not built yet", []string{"serve"}, 1, "", "not built yet (issue D5)"}, + {"audit help", []string{"audit", "--help"}, 0, "verify", ""}, + {"audit without a subcommand", []string{"audit"}, 2, "", "Usage: discreet audit"}, + {"audit verify is not built yet", []string{"audit", "verify"}, 1, "", "audit verify: not built yet (issue D7)"}, + {"audit export is not built yet", []string{"audit", "export"}, 1, "", "audit export: not built yet (issue D7)"}, + {"unknown audit subcommand", []string{"audit", "delete"}, 2, "", `unknown subcommand "delete"`}, + {"unknown command", []string{"launch"}, 2, "", `unknown command "launch"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run(tt.args, &stdout, &stderr) + if code != tt.wantCode { + t.Errorf("exit code = %d, want %d", code, tt.wantCode) + } + check(t, "stdout", stdout.String(), tt.wantStdout) + check(t, "stderr", stderr.String(), tt.wantStderr) + }) + } +} + +// check asserts that got contains want, or is empty when want is empty. +func check(t *testing.T, stream, got, want string) { + t.Helper() + if want == "" { + if got != "" { + t.Errorf("%s = %q, want empty", stream, got) + } + return + } + if !strings.Contains(got, want) { + t.Errorf("%s = %q, want it to contain %q", stream, got, want) + } +} diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md new file mode 100644 index 0000000..0423d86 --- /dev/null +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -0,0 +1,32 @@ +# How this was built + +## Who does what + +- **Michael Mendoza** owns the project: he set the brief, approved the Milestone 2 plan on 2026-10-02, and reviews `next` → `main` at the end of each milestone. +- **Claude Code** (an AI coding assistant) drafts the code, tests, documents and decision records, opens a pull request for every change, and merges CI-green pull requests into `next` with the `ai-merged` label (ADR 0002). +- **Vetted** reviews every pull request with advisory comments. + +## Guardrails + +- A private brief and the programme's `CLAUDE.md` set the non-negotiables: no employer material, no real personal data, money capped, humans accountable. +- Every change goes through a pull request with required checks (`make check`, CodeQL) on both `next` and `main`, applied to admins. +- Code is broken deliberately to prove the tests catch it, then restored from a backup. +- Facts that may have changed since the AI's training (tool versions, prices, APIs) are checked against primary sources before use. + +## Decisions changed on human review + +| Date | Proposed by Claude Code | Changed by Michael to | Where | +| --- | --- | --- | --- | + +## How facts and code were checked + +| Date | Check | Result | +| --- | --- | --- | +| 2026-10-04 | Latest Go release (go.dev download feed) | Go 1.27.1; the module targets Go 1.27 | +| 2026-10-04 | Latest `actions/setup-go`, `actions/checkout`, CodeQL Action and `govulncheck` releases (GitHub API, Go module proxy) | Pinned v7.0.0, v7.0.1, v4.38.2 and v1.8.0 by commit or version | +| 2026-10-04 | Deliberate break: `discreet serve` made to exit 0 | The test failed (`exit code = 0, want 1`); restored from backup | + +## What the AI got wrong, and how it was caught + +| Date | What went wrong | How it was caught | Fix | +| --- | --- | --- | --- | diff --git a/docs/adr/0001-go-standard-library.md b/docs/adr/0001-go-standard-library.md new file mode 100644 index 0000000..cbaacde --- /dev/null +++ b/docs/adr/0001-go-standard-library.md @@ -0,0 +1,24 @@ +# 0001. Build on Go's standard library, without a web framework + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-04 + +## Context + +Discreet is a small HTTP proxy: one endpoint (`POST /v1/chat/completions`), a demo page, and a command-line tool for the audit log. Michael has to be able to read, explain and extend every line, and every dependency is something a security reviewer will ask about. + +## Options + +1. **Standard library only.** `net/http` has routed on method and path patterns (`POST /v1/chat/completions`) since Go 1.22; `crypto/aes`, `crypto/hmac`, `crypto/sha256`, `encoding/json` and `regexp` (which is RE2) cover the rest. +2. **A web framework such as Gin or Echo.** Adds middleware and binding helpers, plus their dependency trees and a second set of idioms to learn. +3. **A proxy toolkit such as Envoy or a plugin for an existing gateway.** Powerful, but the interesting logic (detection, tokenising, audit) would be buried in configuration and a plugin interface. + +## Decision + +Option 1. The only planned third-party module is a YAML parser for the policy file (issue D4), chosen and pinned when it is needed. Development tools such as `govulncheck` run at a pinned version and are not linked into the binary. + +## Consequences + +- One static binary with no runtime dependencies, easy to deploy and audit. +- `govulncheck` in CI only has the Go standard library to check, so its results are easy to act on. +- Some helpers a framework would provide (request binding, middleware chains) are written by hand. They are small, and writing them keeps the code explainable. diff --git a/docs/adr/0002-integration-branch.md b/docs/adr/0002-integration-branch.md new file mode 100644 index 0000000..88fc9d2 --- /dev/null +++ b/docs/adr/0002-integration-branch.md @@ -0,0 +1,22 @@ +# 0002. Same integration-branch workflow as Vetted + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-04 + +## Context + +Michael chose an integration branch for Vetted on 2026-09-30 (Vetted ADR 0009): Claude Code merges each CI-green pull request into a protected `next` branch, labelled `ai-merged`, and Michael reviews `next` → `main` at the end of the milestone. He approved the same approach for Discreet in the Milestone 2 plan on 2026-10-02. + +## Options + +1. **Reuse Vetted's workflow as is.** One way of working across the programme. +2. **Michael merges every Discreet pull request.** Strongest per-change evidence, but it makes him the bottleneck again, which is why Vetted moved away from it. + +## Decision + +Option 1. `next` and `main` carry the same protection: pull request required, the same required checks (`Lint and test`, `Analyze (go)`, `Analyze (actions)`, `CodeQL`), linear history, applied to admins, no force pushes. Releases are tagged from `main` only, after Michael's merge. + +## Consequences + +- Individual changes are reviewed by a human as a batch before `main`, not one by one. The `ai-merged` label and `docs/HOW-THIS-WAS-BUILT.md` say so. +- Vetted reviews every pull request into `next`, so its pilot numbers keep growing with Discreet's development. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f577e3c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/makoydev/discreet + +go 1.27 From f32278df6bd60ee5357c05322bcc20fdd7545914 Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:21:30 +0800 Subject: [PATCH 02/13] feat: Go detection engine passing all 369 shared cases (D3) (#5) - internal/detect implements the sg-pii-rules spec with Go's regexp (RE2): validators written from VALIDATORS.md (NRIC/FIN incl. M series, NRIC look-alikes, Luhn + card network prefix, postal sector, calendar date), value groups, and the overlap rule. Fails loudly on an unknown validator or a non-RE2 pattern. Offsets are bytes. - sg-pii-rules v0.2.0-rc.1 (9d40723) vendored by scripts/vendor-sg-pii-rules.sh into third_party/sgpiirules and embedded with go:embed; a test re-checks every file against SHA256SUMS and that nothing unlisted is vendored (ADR 0003). - Conformance: 369/369 shared cases pass, first run. - Deliberate breaks, all restored from backup: wrong M-series table (2 fail), Luhn always true (3), value groups ignored (52), hand-edited fixture (1). A reversed tie-break failed nothing: no shared case has two detectors on the same span. Added toy-detector overlap tests (3 now fail) and opened makoydev/vetted#44 for the shared suite. - About 3.9 MB/s on dense synthetic text (BenchmarkDetect). Closes makoydev/vetted#33 Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 2 + CLAUDE.md | 1 + CONTROLS.md | 2 +- EVALS.md | 19 +- README.md | 2 +- THREAT_MODEL.md | 3 +- docs/HOW-THIS-WAS-BUILT.md | 3 + .../adr/0003-vendor-and-embed-sg-pii-rules.md | 24 + internal/detect/detect.go | 138 +++++ internal/detect/detect_test.go | 172 ++++++ internal/detect/validators.go | 186 ++++++ scripts/vendor-sg-pii-rules.sh | 19 + third_party/sgpiirules/SHA256SUMS | 11 + third_party/sgpiirules/VERSION | 1 + third_party/sgpiirules/detectors.json | 87 +++ third_party/sgpiirules/embed.go | 13 + third_party/sgpiirules/embed_test.go | 59 ++ third_party/sgpiirules/fixtures/card.json | 465 ++++++++++++++ third_party/sgpiirules/fixtures/dob.json | 463 ++++++++++++++ third_party/sgpiirules/fixtures/email.json | 412 +++++++++++++ .../sgpiirules/fixtures/nric-like.json | 491 +++++++++++++++ third_party/sgpiirules/fixtures/nric.json | 574 ++++++++++++++++++ third_party/sgpiirules/fixtures/phone.json | 383 ++++++++++++ third_party/sgpiirules/fixtures/postal.json | 444 ++++++++++++++ third_party/sgpiirules/fixtures/unit.json | 444 ++++++++++++++ .../sgpiirules/schema/detectors.schema.json | 34 ++ .../sgpiirules/schema/fixtures.schema.json | 38 ++ 27 files changed, 4486 insertions(+), 4 deletions(-) create mode 100644 docs/adr/0003-vendor-and-embed-sg-pii-rules.md create mode 100644 internal/detect/detect.go create mode 100644 internal/detect/detect_test.go create mode 100644 internal/detect/validators.go create mode 100755 scripts/vendor-sg-pii-rules.sh create mode 100644 third_party/sgpiirules/SHA256SUMS create mode 100644 third_party/sgpiirules/VERSION create mode 100644 third_party/sgpiirules/detectors.json create mode 100644 third_party/sgpiirules/embed.go create mode 100644 third_party/sgpiirules/embed_test.go create mode 100644 third_party/sgpiirules/fixtures/card.json create mode 100644 third_party/sgpiirules/fixtures/dob.json create mode 100644 third_party/sgpiirules/fixtures/email.json create mode 100644 third_party/sgpiirules/fixtures/nric-like.json create mode 100644 third_party/sgpiirules/fixtures/nric.json create mode 100644 third_party/sgpiirules/fixtures/phone.json create mode 100644 third_party/sgpiirules/fixtures/postal.json create mode 100644 third_party/sgpiirules/fixtures/unit.json create mode 100644 third_party/sgpiirules/schema/detectors.schema.json create mode 100644 third_party/sgpiirules/schema/fixtures.schema.json diff --git a/CHANGELOG.md b/CHANGELOG.md index b2ec1cc..4550970 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,3 +10,5 @@ All notable changes to this project are documented here. The format follows [Kee - CI on every pull request: `gofmt`, `go vet`, `go test -race`, `govulncheck` and a build (`make check`), plus CodeQL for Go and the workflows. All Actions pinned by commit SHA. - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. - Vetted reviews every pull request (shadow mode until 2026-10-14). +- Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases. +- sg-pii-rules `v0.2.0-rc.1` vendored in `third_party/sgpiirules` and embedded in the binary; a test re-checks every file against `SHA256SUMS` (ADR 0003). `scripts/vendor-sg-pii-rules.sh ` updates it. diff --git a/CLAUDE.md b/CLAUDE.md index c17910f..1b0d71b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,4 +9,5 @@ Development started on 2026-10-04 (issue D1). Every pull request is reviewed by - Go 1.27, standard library only (ADR 0001); any new module needs an ADR and an exact version. - Run `make check` (gofmt, vet, race tests, govulncheck, build) before every commit; never commit past a failure. - Tests use synthetic data only; build secret- and NRIC-shaped strings at runtime. +- `third_party/sgpiirules` is a vendored sg-pii-rules release: never edit it by hand; update with `scripts/vendor-sg-pii-rules.sh ` (ADR 0003). - Issues live in `makoydev/vetted` under the "M2: Discreet v0.1" milestone (D1 = vetted#30). diff --git a/CONTROLS.md b/CONTROLS.md index 9a75569..76af4e9 100644 --- a/CONTROLS.md +++ b/CONTROLS.md @@ -4,7 +4,7 @@ Each Discreet feature, the principle or obligation it supports, and the source. | Feature | Supports | Source | Status | | --- | --- | --- | --- | -| Personal data replaced with placeholders before it reaches the model | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Planned (D3, D4); mapping pending | +| Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection in place (D3, 369/369 shared cases); replacement planned (D4); mapping pending | | Purpose declared in a header; automated eligibility decisions refused | Human accountability for decisions about people | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | Planned (D5); mapping pending | | Hash-chained audit log with `verify` | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | Planned (D7); mapping pending | | Daily budget, access token, rate limit | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | Planned (D6); mapping pending | diff --git a/EVALS.md b/EVALS.md index 35fdb1d..2f2e0df 100644 --- a/EVALS.md +++ b/EVALS.md @@ -6,9 +6,26 @@ | --- | --- | --- | | Precision and recall per entity on about 5,000 synthetic samples from a generator written independently of the detectors | Issue D9 | Not started | | Added latency (p50, p95) against the mock model | Issue D9 | Not started | -| Detector conformance: every sg-pii-rules fixture passes in Go | Issue D3 | Not started | +| Detector conformance: every sg-pii-rules fixture passes in Go | Issue D3 | **Done: 369 / 369** (see below) | | Data-disclosure and adversarial-prompt run with Project Moonshot | Milestone 3 | Not started | Known limitation, stated in advance: synthetic data flatters rule-based detectors, because real text is messier than any generator. Vetted's review of this repository's pull requests is measured in Vetted's own `EVALS.md`. + +## Detector conformance (issue D3, measured 2026-10-05) + +`go test ./internal/detect` runs every case in the vendored sg-pii-rules `v0.2.0-rc.1` fixtures through Discreet's Go engine: **369 of 369 pass**, across NRIC, NRIC look-alikes, phone, email, card, postal code, unit number and date of birth. These cases are a behaviour specification shared with Vetted, not a measure of real-world accuracy (that is the benchmark above). + +Deliberate breaks, each restored from a backup afterwards: + +| Break | Tests that failed | +| --- | --- | +| Wrong M-series check-letter table | 2 | +| Luhn check always passes | 3 | +| `value` groups ignored | 52 | +| Overlap tie-break reversed | 0 at first, because no shared case has two detectors matching the same span; 3 after adding toy-detector overlap tests (tracked for the shared suite in vetted#44) | +| A vendored fixture edited by hand | 1 (the checksum test) | + +Speed, for scale only: on a dense synthetic text with every kind of personal data in each sentence, the engine scans about **3.9 MB/s** on an Apple-silicon Mac (`go test -bench BenchmarkDetect ./internal/detect`), roughly half a millisecond for a 2 KB chat message. Latency through the whole gateway is measured in issue D9. + diff --git a/README.md b/README.md index b4e1b10..223d955 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`. -**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Only the command-line skeleton exists so far; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. +**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the command-line skeleton and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. ## Build and check diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 5b21129..d1918d5 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -19,7 +19,8 @@ | Threat | Planned mitigation | Issue | | --- | --- | --- | -| Personal data the detectors miss reaches the model | Shared conformance suite with hard negatives; measured recall published in EVALS | D3, D9 | +| Personal data the detectors miss reaches the model | Shared conformance suite with hard negatives (369/369 pass in Go); measured recall published in EVALS | D3 (done), D9 | +| Rules silently changed or swapped | Vendored release checked against `SHA256SUMS` on every test run; embedded in the binary (ADR 0003) | D3 (done) | | The vault leaks values through logs or errors | Values encrypted in memory, short expiry, a test that scans logs for planted values | D4 | | The model invents placeholders to extract other values | Only placeholders issued for that request are rehydrated | D4 | | Someone edits the audit log to hide a request | SHA-256 hash chain; `audit verify` names the first broken record | D7 | diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md index 0423d86..ad6c708 100644 --- a/docs/HOW-THIS-WAS-BUILT.md +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -25,6 +25,9 @@ | 2026-10-04 | Latest Go release (go.dev download feed) | Go 1.27.1; the module targets Go 1.27 | | 2026-10-04 | Latest `actions/setup-go`, `actions/checkout`, CodeQL Action and `govulncheck` releases (GitHub API, Go module proxy) | Pinned v7.0.0, v7.0.1, v4.38.2 and v1.8.0 by commit or version | | 2026-10-04 | Deliberate break: `discreet serve` made to exit 0 | The test failed (`exit code = 0, want 1`); restored from backup | +| 2026-10-05 | sg-pii-rules `v0.2.0-rc.1` vendored by script from the release tag | Every file matched the release's `SHA256SUMS` | +| 2026-10-05 | All 369 shared conformance cases through the Go engine | 369 / 369 pass, first run | +| 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite | ## What the AI got wrong, and how it was caught diff --git a/docs/adr/0003-vendor-and-embed-sg-pii-rules.md b/docs/adr/0003-vendor-and-embed-sg-pii-rules.md new file mode 100644 index 0000000..8d990ad --- /dev/null +++ b/docs/adr/0003-vendor-and-embed-sg-pii-rules.md @@ -0,0 +1,24 @@ +# 0003. Vendor sg-pii-rules releases and embed them in the binary + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-05 + +## Context + +Discreet and Vetted must detect the same personal data the same way. The rules live in sg-pii-rules as data (`detectors.json`) with a shared conformance suite, and consumers vendor a tagged release and verify it against `SHA256SUMS` (sg-pii-rules ADR 0004). Discreet also has to run as one self-contained binary on a small host (ADR 0001). + +## Options + +1. **Vendor a tagged release into `third_party/sgpiirules` and embed it with `go:embed`.** The rules, schemas and fixtures are compiled into the binary; a Go test re-checks every file's SHA-256 on every `go test`. +2. **Import sg-pii-rules as a Go module.** sg-pii-rules is a TypeScript and data repository with no Go module, and module versions would add a second way to say which rules are in use. +3. **Load `detectors.json` from disk at start-up.** Lets an operator change rules without a rebuild, but then the running rules are whatever file is on the machine, not a reviewed release. + +## Decision + +Option 1. `scripts/vendor-sg-pii-rules.sh ` downloads a release, verifies it against its `SHA256SUMS` and records the tag and commit in `VERSION`. The engine (`internal/detect`) implements `SPEC.md` with Go's `regexp`, which is RE2, the dialect the rules are written in. + +## Consequences + +- The binary always runs the exact rules of one reviewed release; `discreet` can report which (`VERSION`). +- A hand edit to a vendored file, or a file missing from `SHA256SUMS`, fails CI. +- Changing rules means vendoring a new release and passing its conformance suite: deliberate, reviewable upgrades. Currently vendored: `v0.2.0-rc.1` (`9d40723`); it becomes `v0.2.0`, byte for byte, after Michael's Milestone 2 review. diff --git a/internal/detect/detect.go b/internal/detect/detect.go new file mode 100644 index 0000000..268e480 --- /dev/null +++ b/internal/detect/detect.go @@ -0,0 +1,138 @@ +// Package detect finds Singapore personal data in text, implementing the +// sg-pii-rules specification (SPEC.md) with Go's regexp package, which is RE2. +// The rules themselves are data, vendored in third_party/sgpiirules. +package detect + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" + + "github.com/makoydev/discreet/third_party/sgpiirules" +) + +// Match is one piece of personal data found in a text. Start and End are +// byte offsets; they are not part of conformance (SPEC.md §5). +type Match struct { + Entity string + Value string + Start int + End int +} + +type rulesFile struct { + Version string `json:"version"` + Detectors []struct { + ID string `json:"id"` + Entity string `json:"entity"` + Pattern string `json:"pattern"` + Validator string `json:"validator"` + } `json:"detectors"` +} + +type detector struct { + id string + entity string + order int + pattern *regexp.Regexp + valueIdx int // index of the `value` group, or -1 to use the whole match + validate func(string) bool +} + +// Engine holds compiled detectors. It is safe for concurrent use. +type Engine struct { + Version string + detectors []detector +} + +// New compiles a detectors.json file. It fails on an invalid pattern or a +// validator this package doesn't implement, never silently (SPEC.md §2, §3). +func New(rulesJSON []byte) (*Engine, error) { + var rules rulesFile + if err := json.Unmarshal(rulesJSON, &rules); err != nil { + return nil, fmt.Errorf("detectors.json: %w", err) + } + e := &Engine{Version: rules.Version} + for i, r := range rules.Detectors { + re, err := regexp.Compile(r.Pattern) + if err != nil { + return nil, fmt.Errorf("detector %s: %w", r.ID, err) + } + d := detector{id: r.ID, entity: r.Entity, order: i, pattern: re, valueIdx: re.SubexpIndex("value")} + if r.Validator != "" { + if d.validate = validators[r.Validator]; d.validate == nil { + return nil, fmt.Errorf("detector %s names unknown validator %s", r.ID, r.Validator) + } + } + e.detectors = append(e.detectors, d) + } + return e, nil +} + +// Default compiles the vendored sg-pii-rules release. +func Default() (*Engine, error) { + data, err := sgpiirules.Files.ReadFile("detectors.json") + if err != nil { + return nil, err + } + return New(data) +} + +type candidate struct { + Match + order int +} + +// Detect runs the algorithm in SPEC.md §4 and returns matches in order of +// position, with no two overlapping. +func (e *Engine) Detect(text string) []Match { + var cands []candidate + for _, d := range e.detectors { + for _, loc := range d.pattern.FindAllStringSubmatchIndex(text, -1) { + start, end := loc[0], loc[1] + if d.valueIdx >= 0 { // report only the value group; the rest is context + start, end = loc[2*d.valueIdx], loc[2*d.valueIdx+1] + } + if start < 0 || start == end { + continue + } + value := text[start:end] + if d.validate != nil && !d.validate(value) { + continue + } + cands = append(cands, candidate{Match{d.entity, value, start, end}, d.order}) + } + } + + // Overlaps: sort by start, then longest first, then detector order. Kept + // matches never overlap, so a candidate can only clash with the last kept. + sort.SliceStable(cands, func(i, j int) bool { + a, b := cands[i], cands[j] + if a.Start != b.Start { + return a.Start < b.Start + } + if la, lb := a.End-a.Start, b.End-b.Start; la != lb { + return la > lb + } + return a.order < b.order + }) + var kept []candidate + for _, c := range cands { + if len(kept) == 0 || c.Start >= kept[len(kept)-1].End { + kept = append(kept, c) + continue + } + last := kept[len(kept)-1] + cl, ll := c.End-c.Start, last.End-last.Start + if cl > ll || (cl == ll && c.order < last.order) { + kept[len(kept)-1] = c + } + } + + out := make([]Match, len(kept)) + for i, c := range kept { + out[i] = c.Match + } + return out +} diff --git a/internal/detect/detect_test.go b/internal/detect/detect_test.go new file mode 100644 index 0000000..635de87 --- /dev/null +++ b/internal/detect/detect_test.go @@ -0,0 +1,172 @@ +package detect + +import ( + "encoding/json" + "io/fs" + "reflect" + "strings" + "testing" + + "github.com/makoydev/discreet/third_party/sgpiirules" +) + +type fixtureFile struct { + Entity string `json:"entity"` + Cases []struct { + ID string `json:"id"` + Kind string `json:"kind"` + Text string `json:"text"` + Expect []struct { + Entity string `json:"entity"` + Value string `json:"value"` + } `json:"expect"` + } `json:"cases"` +} + +// TestConformance runs every case in the vendored sg-pii-rules fixtures. +// Passing all of them is what conforming to SPEC.md means. +func TestConformance(t *testing.T) { + engine, err := Default() + if err != nil { + t.Fatal(err) + } + files, _ := fs.Glob(sgpiirules.Files, "fixtures/*.json") + total := 0 + for _, name := range files { + data, _ := sgpiirules.Files.ReadFile(name) + var f fixtureFile + if err := json.Unmarshal(data, &f); err != nil { + t.Fatalf("%s: %v", name, err) + } + for _, c := range f.Cases { + total++ + t.Run(c.ID, func(t *testing.T) { + got := []string{} + for _, m := range engine.Detect(c.Text) { + got = append(got, m.Entity+"="+m.Value) + } + want := []string{} + for _, e := range c.Expect { + want = append(want, e.Entity+"="+e.Value) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s (%s)\n text: %q\n got: %v\n want: %v", c.ID, c.Kind, c.Text, got, want) + } + }) + } + } + if total < 369 { + t.Errorf("ran %d cases; the vendored v0.2.0 release has 369", total) + } +} + +// The overlap rules in SPEC.md §4, with toy detectors. The shared fixtures +// never have two detectors matching the same span, so the tie-break is only +// tested here (as in the reference implementation's own detect tests). +func TestOverlapRules(t *testing.T) { + tests := []struct { + name, rules, text string + want []string + }{ + {"same span: the earlier detector wins", + `[{"id":"a","entity":"A","pattern":"ab"},{"id":"b","entity":"B","pattern":"ab"}]`, "ab", []string{"A=ab"}}, + {"same span, other order", + `[{"id":"b","entity":"B","pattern":"ab"},{"id":"a","entity":"A","pattern":"ab"}]`, "ab", []string{"B=ab"}}, + {"the longer match wins whatever the order", + `[{"id":"a","entity":"A","pattern":"a"},{"id":"b","entity":"B","pattern":"ab"}]`, "ab", []string{"B=ab"}}, + {"a later, longer overlap replaces the kept match", + `[{"id":"a","entity":"A","pattern":"ab"},{"id":"b","entity":"B","pattern":"bcd"}]`, "abcd", []string{"B=bcd"}}, + {"overlaps are judged on the value group, not the whole match", + `[{"id":"num","entity":"NUM","pattern":"\\d+"},{"id":"id","entity":"ID","pattern":"id:(?P\\d+)"}]`, "id:123", []string{"NUM=123"}}, + {"matches come back in order of position", + `[{"id":"b","entity":"B","pattern":"b+"},{"id":"a","entity":"A","pattern":"a+"}]`, "aa bb a", []string{"A=aa", "B=bb", "A=a"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + engine, err := New([]byte(`{"version":"0.0.0","detectors":` + tt.rules + `}`)) + if err != nil { + t.Fatal(err) + } + got := []string{} + for _, m := range engine.Detect(tt.text) { + got = append(got, m.Entity+"="+m.Value) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewRejectsUnknownValidator(t *testing.T) { + _, err := New([]byte(`{"version":"0.0.0","detectors":[{"id":"x","entity":"X","pattern":"x","validator":"nope"}]}`)) + if err == nil || !strings.Contains(err.Error(), "unknown validator nope") { + t.Errorf("err = %v, want an unknown-validator error", err) + } +} + +func TestNewRejectsNonRE2Pattern(t *testing.T) { + _, err := New([]byte(`{"version":"0.0.0","detectors":[{"id":"x","entity":"X","pattern":"(?<=a)b"}]}`)) + if err == nil { + t.Error("a lookbehind pattern compiled; RE2 must reject it") + } +} + +func TestValueGroupOffsetsAreBytes(t *testing.T) { + engine, _ := Default() + text := "地址 Singapore 520123" + got := engine.Detect(text) + if len(got) != 1 || got[0].Value != "520123" || text[got[0].Start:got[0].End] != "520123" { + t.Errorf("got %+v, want one POSTAL match whose byte offsets slice to 520123", got) + } +} + +func TestValidators(t *testing.T) { + cases := []struct { + name string + fn func(string) bool + value string + want bool + }{ + // Synthetic example from sg-pii-rules VALIDATORS.md, and published illustrative numbers that fail. + {"nric valid", isValidNRIC, "S1234567D", true}, + {"nric lowercase", isValidNRIC, "s1234567d", true}, + {"ICA example fails", isValidNRIC, "M1234567B", false}, + {"nric-like", hasNRICShapeButInvalidChecksum, "S1234567A", true}, + {"nric-like rejects valid", hasNRICShapeButInvalidChecksum, "S1234567D", false}, + // Published test card numbers, not issued cards. + {"visa", isPaymentCard, "4111 1111 1111 1111", true}, + {"visa bad luhn", isPaymentCard, "4111111111111112", false}, + {"amex", isPaymentCard, "378282246310005", true}, + {"amex wrong length", isPaymentCard, "3782822463100050", false}, + {"mastercard 2-series", isPaymentCard, "2221000000000009", true}, + {"no network", isPaymentCard, "9111111111111111", false}, + {"postal", isPostalCode, "520123", true}, + {"postal sector 74", isPostalCode, "740123", false}, + {"postal sector 83", isPostalCode, "830000", false}, + {"date day first", isPlausibleDate, "12/03/1988", true}, + {"date month first", isPlausibleDate, "03/31/1988", true}, + {"leap day", isPlausibleDate, "29/02/1992", true}, + {"not a leap year", isPlausibleDate, "29/02/1990", false}, + {"century not leap", isPlausibleDate, "29/02/1900", false}, + {"iso", isPlausibleDate, "1988-03-12", true}, + {"written", isPlausibleDate, "12 Sept. 1988", true}, + {"us written", isPlausibleDate, "March 12, 1988", true}, + {"31 April", isPlausibleDate, "31 April 1990", false}, + {"unknown month", isPlausibleDate, "12 Foo 1988", false}, + } + for _, c := range cases { + if got := c.fn(c.value); got != c.want { + t.Errorf("%s: %q = %v, want %v", c.name, c.value, got, c.want) + } + } +} + +func BenchmarkDetect(b *testing.B) { + engine, _ := Default() + text := strings.Repeat("Please call S1234567D on +65 9123 4567 about card 4111 1111 1111 1111, Blk 123 #05-123 Singapore 520123, DOB: 12/03/1988. ", 20) + b.SetBytes(int64(len(text))) + for b.Loop() { + engine.Detect(text) + } +} diff --git a/internal/detect/validators.go b/internal/detect/validators.go new file mode 100644 index 0000000..5b14731 --- /dev/null +++ b/internal/detect/validators.go @@ -0,0 +1,186 @@ +package detect + +import ( + "regexp" + "strconv" + "strings" +) + +// Validators named in detectors.json, implemented from sg-pii-rules' +// VALIDATORS.md. A detector naming a validator missing here fails to load. +var validators = map[string]func(string) bool{ + "sg_nric_fin_checksum": isValidNRIC, + "sg_nric_fin_checksum_invalid": hasNRICShapeButInvalidChecksum, + "payment_card_luhn_iin": isPaymentCard, + "sg_postal_sector": isPostalCode, + "calendar_date": isPlausibleDate, +} + +// ----- NRIC/FIN ----- + +var ( + nricShape = regexp.MustCompile(`^([STFGM])(\d{7})([A-Z])$`) + nricWeights = [7]int{2, 7, 6, 5, 4, 3, 2} + nricOffset = map[string]int{"S": 0, "T": 4, "F": 0, "G": 4, "M": 3} + nricLetters = map[string]string{ + "S": "JZIHGFEDCBA", "T": "JZIHGFEDCBA", + "F": "XWUTRQPNMLK", "G": "XWUTRQPNMLK", + "M": "XWUTRQPNJLK", + } +) + +// nricCheckLetter returns the expected check letter for a prefix and seven digits. +func nricCheckLetter(prefix, digits string) byte { + sum := nricOffset[prefix] + for i, w := range nricWeights { + sum += w * int(digits[i]-'0') + } + return nricLetters[prefix][sum%11] +} + +func isValidNRIC(value string) bool { + m := nricShape.FindStringSubmatch(strings.ToUpper(value)) + return m != nil && nricCheckLetter(m[1], m[2]) == m[3][0] +} + +func hasNRICShapeButInvalidChecksum(value string) bool { + return nricShape.MatchString(strings.ToUpper(value)) && !isValidNRIC(value) +} + +// ----- Payment cards ----- + +func passesLuhn(digits string) bool { + sum := 0 + for i := 0; i < len(digits); i++ { + d := int(digits[len(digits)-1-i] - '0') + if i%2 == 1 { + d *= 2 + if d > 9 { + d -= 9 + } + } + sum += d + } + return sum%10 == 0 +} + +type cardNetwork struct { + prefixes [][2]string // inclusive ranges of leading digits + lengths []int +} + +var cardNetworks = []cardNetwork{ + {[][2]string{{"4", "4"}}, []int{13, 16, 19}}, // Visa + {[][2]string{{"51", "55"}, {"2221", "2720"}}, []int{16}}, // Mastercard + {[][2]string{{"34", "34"}, {"37", "37"}}, []int{15}}, // American Express + {[][2]string{{"6011", "6011"}, {"644", "649"}, {"65", "65"}, {"622126", "622925"}}, span(16, 19)}, // Discover + {[][2]string{{"3528", "3589"}}, span(16, 19)}, // JCB + {[][2]string{{"62", "62"}}, span(16, 19)}, // UnionPay + {[][2]string{{"30", "30"}, {"36", "36"}, {"38", "39"}}, span(14, 19)}, // Diners Club +} + +func span(from, to int) []int { + var s []int + for n := from; n <= to; n++ { + s = append(s, n) + } + return s +} + +func hasCardNetworkPrefix(digits string) bool { + for _, n := range cardNetworks { + if !containsInt(n.lengths, len(digits)) { + continue + } + for _, r := range n.prefixes { + lead := digits[:len(r[0])] + if lead >= r[0] && lead <= r[1] { + return true + } + } + } + return false +} + +func containsInt(list []int, n int) bool { + for _, v := range list { + if v == n { + return true + } + } + return false +} + +var cardDigits = regexp.MustCompile(`^\d{13,19}$`) + +func isPaymentCard(value string) bool { + digits := strings.NewReplacer(" ", "", "-", "").Replace(value) + return cardDigits.MatchString(digits) && passesLuhn(digits) && hasCardNetworkPrefix(digits) +} + +// ----- Postal codes ----- + +var sixDigits = regexp.MustCompile(`^\d{6}$`) + +// isPostalCode accepts six digits whose sector (first two) is 01-82, except 74. +func isPostalCode(value string) bool { + if !sixDigits.MatchString(value) { + return false + } + sector, _ := strconv.Atoi(value[:2]) + return sector >= 1 && sector <= 82 && sector != 74 +} + +// ----- Dates ----- + +var ( + numericDate = regexp.MustCompile(`^(\d{1,2})[/.-](\d{1,2})[/.-](\d{4})$`) + isoDate = regexp.MustCompile(`^(\d{4})-(\d{1,2})-(\d{1,2})$`) + dayMonth = regexp.MustCompile(`(?i)^(\d{1,2}) ([a-z]+)\.?,? (\d{4})$`) + monthDay = regexp.MustCompile(`(?i)^([a-z]+)\.? (\d{1,2}),? (\d{4})$`) + months = "janfebmaraprmayjunjulaugsepoctnovdec" +) + +func isCalendarDate(year, month, day int) bool { + if month < 1 || month > 12 || day < 1 { + return false + } + leap := (year%4 == 0 && year%100 != 0) || year%400 == 0 + days := [12]int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31} + if leap { + days[1] = 29 + } + return day <= days[month-1] +} + +// monthNumber reads a month name by its first three letters; 0 if unknown. +func monthNumber(name string) int { + if len(name) < 3 { + return 0 + } + i := strings.Index(months, strings.ToLower(name[:3])) + if i < 0 || i%3 != 0 { + return 0 + } + return i/3 + 1 +} + +// isPlausibleDate accepts a real calendar date in the supported forms, +// reading 12/03/1988 day-first or month-first. It never compares with today. +func isPlausibleDate(value string) bool { + n := func(s string) int { v, _ := strconv.Atoi(s); return v } + if m := numericDate.FindStringSubmatch(value); m != nil { + a, b, y := n(m[1]), n(m[2]), n(m[3]) + return isCalendarDate(y, b, a) || isCalendarDate(y, a, b) + } + if m := isoDate.FindStringSubmatch(value); m != nil { + return isCalendarDate(n(m[1]), n(m[2]), n(m[3])) + } + if m := dayMonth.FindStringSubmatch(value); m != nil { + return isCalendarDate(n(m[3]), monthNumber(m[2]), n(m[1])) + } + if m := monthDay.FindStringSubmatch(value); m != nil { + return isCalendarDate(n(m[3]), monthNumber(m[1]), n(m[2])) + } + return false +} diff --git a/scripts/vendor-sg-pii-rules.sh b/scripts/vendor-sg-pii-rules.sh new file mode 100755 index 0000000..17a1e9f --- /dev/null +++ b/scripts/vendor-sg-pii-rules.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copies a tagged sg-pii-rules release into third_party/sgpiirules and +# verifies every file against the release's SHA256SUMS (sg-pii-rules ADR 0004). +# Usage: scripts/vendor-sg-pii-rules.sh v0.2.0-rc.1 +set -euo pipefail +tag=${1:?release tag, e.g. v0.2.0-rc.1} +dest=third_party/sgpiirules +base="https://raw.githubusercontent.com/makoydev/sg-pii-rules/$tag" + +curl -fsSL "$base/SHA256SUMS" -o "$dest/SHA256SUMS" +while read -r _ path; do + mkdir -p "$dest/$(dirname "$path")" + curl -fsSL "$base/$path" -o "$dest/$path" +done < "$dest/SHA256SUMS" +(cd "$dest" && if command -v sha256sum >/dev/null; then sha256sum -c SHA256SUMS; else shasum -a 256 -c SHA256SUMS; fi) + +commit=$(git ls-remote https://github.com/makoydev/sg-pii-rules.git "refs/tags/$tag^{}" "refs/tags/$tag" | head -1 | cut -c1-7) +echo "$tag $commit" > "$dest/VERSION" +echo "vendored sg-pii-rules $tag ($commit)" diff --git a/third_party/sgpiirules/SHA256SUMS b/third_party/sgpiirules/SHA256SUMS new file mode 100644 index 0000000..37ca86c --- /dev/null +++ b/third_party/sgpiirules/SHA256SUMS @@ -0,0 +1,11 @@ +620688185940f6c8f8c670236a66d4871871610b641a47ee532a1252a205cf22 detectors.json +82372d86f856a4a212c10ec841b229a883b8de3397fa26be5fb6972651644b1d schema/detectors.schema.json +1a71de6eb734b282c346b12f4d198e42c656a9f9e9a97e9eed2471a2a2f033a7 schema/fixtures.schema.json +bdff4a235813d62528b55c255cb8abd3e344428ccfb06a933a27e23416adac0d fixtures/card.json +f86ef8f133cf5f70443cf26430730c71e063a5b05aa949588436b63459cdc71a fixtures/dob.json +966898046e2b52715cb25fc37d7d4f71f6ca3fbcb81da7b953f8bd042a9ded71 fixtures/email.json +a55e63501ad92f8ba73259f38903200eab028cb6fc76d50df84bcc825c206472 fixtures/nric-like.json +f67c4b854ffb7b4c243bb6efa0944f0714f8cd1fbd9022b0281548c435b0b2d7 fixtures/nric.json +59624f085211d7ba73034802da633a8f50cce38513b78e103bcd72b0596d32d6 fixtures/phone.json +9fe1a4fdbddd764abe7583eaccda280d28f765f4f8b31e0715195c0fc1c3f5f1 fixtures/postal.json +caf4ceca2035a3601e999670c641077ba89277132dc5d643697d6b2b247fc86f fixtures/unit.json diff --git a/third_party/sgpiirules/VERSION b/third_party/sgpiirules/VERSION new file mode 100644 index 0000000..f0451ff --- /dev/null +++ b/third_party/sgpiirules/VERSION @@ -0,0 +1 @@ +v0.2.0-rc.1 9d40723 diff --git a/third_party/sgpiirules/detectors.json b/third_party/sgpiirules/detectors.json new file mode 100644 index 0000000..0174a9c --- /dev/null +++ b/third_party/sgpiirules/detectors.json @@ -0,0 +1,87 @@ +{ + "$schema": "./schema/detectors.schema.json", + "version": "0.2.0", + "detectors": [ + { + "id": "sg_nric_fin", + "entity": "NRIC", + "description": "Singapore NRIC and FIN numbers: prefix S, T, F, G or M, seven digits, and a check letter that must pass the checksum. Case-insensitive.", + "pattern": "(?i)\\b[STFGM]\\d{7}[A-Z]\\b", + "validator": "sg_nric_fin_checksum", + "sources": [ + "https://www.ica.gov.sg/news-and-publications/newsroom/media-release/new-m-fin-series-to-be-introduced-from-1-january-2022", + "https://www.pdpc.gov.sg/organisations/regulations-decisions/regulatory-guidance/advisory-guidelines-on-the-personal-data-protection-act-for-nric-and-other-national-identification-numbers", + "https://github.com/makoydev/sg-pii-rules/blob/main/VALIDATORS.md#sg_nric_fin_checksum" + ] + }, + { + "id": "sg_nric_fin_like", + "entity": "NRIC_LIKE", + "description": "NRIC/FIN shape with a check letter that fails the checksum: a mistyped NRIC or an NRIC-shaped look-alike such as an order number. Lower confidence than NRIC; each consumer decides how to treat it (ADR 0006).", + "pattern": "(?i)\\b[STFGM]\\d{7}[A-Z]\\b", + "validator": "sg_nric_fin_checksum_invalid", + "sources": [ + "https://github.com/makoydev/sg-pii-rules/blob/main/docs/adr/0006-nric-checksum-precision-over-recall.md" + ] + }, + { + "id": "sg_phone", + "entity": "PHONE", + "description": "Singapore phone numbers: eight digits starting 3 (internet telephony), 6 (fixed line), 8 or 9 (mobile) per the IMDA National Numbering Plan, optionally split 4-4 by a space or hyphen, optionally preceded by +65, (65), (+65), 0065 or 65.", + "pattern": "(?:(?:\\+65|\\(\\+?65\\)|\\b0065|\\b65)[ -]?|\\b)[3689]\\d{3}[ -]?\\d{4}\\b", + "sources": [ + "https://www.imda.gov.sg/regulations-and-licensing-listing/numbering/national-numbering-plan-and-allocation-process", + "https://www.imda.gov.sg/-/media/imda/files/regulation-licensing-and-consultations/frameworks-and-policies/numbering/national-numbering-plan-and-allocation-process/imda-national-numbering-plan.pdf" + ] + }, + { + "id": "email", + "entity": "EMAIL", + "description": "Email addresses: a local part of letters, digits and . _ % + -, an @, and a domain whose last label is two or more letters. Deliberately permissive: domains may start with a digit.", + "pattern": "\\b[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\\.[A-Za-z0-9-]+)*\\.[A-Za-z]{2,}\\b", + "sources": [ + "https://www.rfc-editor.org/rfc/rfc5322", + "https://www.rfc-editor.org/rfc/rfc2606" + ] + }, + { + "id": "payment_card", + "entity": "CARD", + "description": "Payment card numbers: 13 to 19 digits, optionally grouped by single spaces or hyphens, that pass the Luhn checksum and start with a prefix used by Visa, Mastercard, American Express, Discover, JCB, UnionPay or Diners Club, at a length that network issues.", + "pattern": "\\b\\d(?:[ -]?\\d){12,18}\\b", + "validator": "payment_card_luhn_iin", + "sources": [ + "https://www.mastercard.us/content/dam/mccom/en-us/documents/merchant-2-series-BIN-impact-checklist-aug-2016.pdf", + "https://en.wikipedia.org/wiki/Payment_card_number" + ] + }, + { + "id": "sg_postal_code", + "entity": "POSTAL", + "description": "Singapore postal codes: six digits with a valid postal sector (01 to 82, except 74), only after an address clue: \"Singapore\" (any case), \"S\" or \"S(\" directly before the digits, or a key such as postal code, postalCode, postcode or zip followed by an optional colon or equals sign and quotes. Only the six digits are reported; the clue is context (ADR 0010, ADR 0011).", + "pattern": "(?:(?i:\\bsingapore)[ ,]{1,2}|\\bS\\(?|(?i:\\b(?:postal(?:[ _-]?code)?|post[ _-]?code|zip(?:[ _-]?code)?))[\"']?[ \\t]*[:=]?[ \\t]*[\"']?)(?P\\d{6})\\b", + "validator": "sg_postal_sector", + "sources": [ + "https://en.wikipedia.org/wiki/Postal_codes_in_Singapore", + "https://www.99.co/singapore/insider/singapore-postal-codes/" + ] + }, + { + "id": "sg_unit_number", + "entity": "UNIT", + "description": "Singapore unit numbers in the #floor-unit form: #, a two-digit floor 01 to 99 or a basement B1 to B9, a hyphen, two to four digits and an optional capital letter, such as #05-123, #12-34 or #B1-23A. Shape only; written \"Unit 05-123\" without the # is not detected.", + "pattern": "#(?:B[1-9]|0[1-9]|[1-9]\\d)-\\d{2,4}[A-Z]?\\b", + "sources": ["https://en.wikipedia.org/wiki/Postal_codes_in_Singapore"] + }, + { + "id": "date_of_birth", + "entity": "DOB", + "description": "Dates of birth: a full date with a four-digit year (19xx or 20xx), only directly after a clue such as DOB, D.O.B., date of birth, birth date or born (on), or a key such as dob, dateOfBirth or birth_date, with an optional colon or equals sign and quotes. Formats: 12/03/1988 (day or month first), 12-03-1988, 12.03.1988, 1988-03-12, 12 Mar 1988 and March 12, 1988. Only the date is reported (ADR 0010, ADR 0012).", + "pattern": "(?i:\\b(?:d\\.?o\\.?b\\.?|date[ _-]?of[ _-]?birth|birth[ _-]?date|born(?:[ \\t]+on)?))[\"']?[ \\t]*[:=]?[ \\t]*[\"']?(?P\\d{1,2}[/.-]\\d{1,2}[/.-](?:19|20)\\d{2}|(?:19|20)\\d{2}-\\d{1,2}-\\d{1,2}|\\d{1,2} (?i:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\\.?,? (?:19|20)\\d{2}|(?i:jan(?:uary)?|feb(?:ruary)?|mar(?:ch)?|apr(?:il)?|may|june?|july?|aug(?:ust)?|sep(?:t(?:ember)?)?|oct(?:ober)?|nov(?:ember)?|dec(?:ember)?)\\.? \\d{1,2},? (?:19|20)\\d{2})\\b", + "validator": "calendar_date", + "sources": [ + "https://github.com/makoydev/sg-pii-rules/blob/main/docs/adr/0012-dates-of-birth-need-a-clue.md" + ] + } + ] +} diff --git a/third_party/sgpiirules/embed.go b/third_party/sgpiirules/embed.go new file mode 100644 index 0000000..0eef58a --- /dev/null +++ b/third_party/sgpiirules/embed.go @@ -0,0 +1,13 @@ +// Package sgpiirules embeds a vendored, checksummed release of sg-pii-rules +// (https://github.com/makoydev/sg-pii-rules), the detection rules Discreet +// shares with Vetted. Never edit these files by hand: run +// scripts/vendor-sg-pii-rules.sh . A test checks them against SHA256SUMS. +package sgpiirules + +import "embed" + +// Files holds detectors.json, the schemas, the conformance fixtures, +// SHA256SUMS and VERSION (" "). +// +//go:embed detectors.json SHA256SUMS VERSION schema/*.json fixtures/*.json +var Files embed.FS diff --git a/third_party/sgpiirules/embed_test.go b/third_party/sgpiirules/embed_test.go new file mode 100644 index 0000000..d728454 --- /dev/null +++ b/third_party/sgpiirules/embed_test.go @@ -0,0 +1,59 @@ +package sgpiirules + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io/fs" + "strings" + "testing" +) + +// The vendored files must be exactly the release's files (sg-pii-rules ADR +// 0004): a hand edit, or a file added without updating SHA256SUMS, fails here. +func TestFilesMatchSHA256SUMS(t *testing.T) { + sums, err := Files.ReadFile("SHA256SUMS") + if err != nil { + t.Fatal(err) + } + listed := map[string]bool{} + for _, line := range strings.Split(strings.TrimSpace(string(sums)), "\n") { + want, path, ok := strings.Cut(line, " ") + if !ok { + t.Fatalf("malformed SHA256SUMS line %q", line) + } + data, err := Files.ReadFile(path) + if err != nil { + t.Errorf("%s: listed in SHA256SUMS but not vendored: %v", path, err) + continue + } + got := sha256.Sum256(data) + if hex.EncodeToString(got[:]) != want { + t.Errorf("%s: checksum does not match SHA256SUMS; re-run scripts/vendor-sg-pii-rules.sh", path) + } + listed[path] = true + } + err = fs.WalkDir(Files, ".", func(path string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() && path != "SHA256SUMS" && path != "VERSION" && !listed[path] { + t.Errorf("%s: vendored but not listed in SHA256SUMS", path) + } + return err + }) + if err != nil { + t.Fatal(err) + } +} + +func TestVersionMatchesDetectors(t *testing.T) { + version, _ := Files.ReadFile("VERSION") + tag, _, _ := strings.Cut(strings.TrimSpace(string(version)), " ") + var rules struct{ Version string } + data, _ := Files.ReadFile("detectors.json") + if err := json.Unmarshal(data, &rules); err != nil { + t.Fatal(err) + } + release, _, _ := strings.Cut(strings.TrimPrefix(tag, "v"), "-") + if release != rules.Version { + t.Errorf("VERSION says %s but detectors.json says %s", tag, rules.Version) + } +} diff --git a/third_party/sgpiirules/fixtures/card.json b/third_party/sgpiirules/fixtures/card.json new file mode 100644 index 0000000..3505872 --- /dev/null +++ b/third_party/sgpiirules/fixtures/card.json @@ -0,0 +1,465 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "CARD", + "cases": [ + { + "id": "card-tp-001", + "kind": "true_positive", + "text": "Card number: 4521 9732 2083 2612", + "expect": [ + { + "entity": "CARD", + "value": "4521 9732 2083 2612" + } + ], + "note": "visa, 16 digits, spaces" + }, + { + "id": "card-tp-002", + "kind": "true_positive", + "text": "{\"card\": \"4221222513273297\"}", + "expect": [ + { + "entity": "CARD", + "value": "4221222513273297" + } + ], + "note": "visa, 16 digits, plain" + }, + { + "id": "card-tp-003", + "kind": "true_positive", + "text": "order_id,card\n1042,4055909888018", + "expect": [ + { + "entity": "CARD", + "value": "4055909888018" + } + ], + "note": "visa, 13 digits, plain" + }, + { + "id": "card-tp-004", + "kind": "true_positive", + "text": "Charged to 4787537453560242696 at checkout.", + "expect": [ + { + "entity": "CARD", + "value": "4787537453560242696" + } + ], + "note": "visa, 19 digits, plain" + }, + { + "id": "card-tp-005", + "kind": "true_positive", + "text": "const testCard = '4455-7914-0496-0169'", + "expect": [ + { + "entity": "CARD", + "value": "4455-7914-0496-0169" + } + ], + "note": "visa, 16 digits, hyphens" + }, + { + "id": "card-tp-006", + "kind": "true_positive", + "text": "PAN=5161 7060 2921 8423", + "expect": [ + { + "entity": "CARD", + "value": "5161 7060 2921 8423" + } + ], + "note": "mastercard, 16 digits, spaces" + }, + { + "id": "card-tp-007", + "kind": "true_positive", + "text": "Card number: 5594069146308164", + "expect": [ + { + "entity": "CARD", + "value": "5594069146308164" + } + ], + "note": "mastercard, 16 digits, plain" + }, + { + "id": "card-tp-008", + "kind": "true_positive", + "text": "{\"card\": \"2221-5629-4436-2092\"}", + "expect": [ + { + "entity": "CARD", + "value": "2221-5629-4436-2092" + } + ], + "note": "mastercard, 16 digits, hyphens" + }, + { + "id": "card-tp-009", + "kind": "true_positive", + "text": "order_id,card\n1042,2720 2613 2920 1753", + "expect": [ + { + "entity": "CARD", + "value": "2720 2613 2920 1753" + } + ], + "note": "mastercard, 16 digits, spaces" + }, + { + "id": "card-tp-010", + "kind": "true_positive", + "text": "Charged to 2500706103228337 at checkout.", + "expect": [ + { + "entity": "CARD", + "value": "2500706103228337" + } + ], + "note": "mastercard, 16 digits, plain" + }, + { + "id": "card-tp-011", + "kind": "true_positive", + "text": "const testCard = '3431 705981 79379'", + "expect": [ + { + "entity": "CARD", + "value": "3431 705981 79379" + } + ], + "note": "amex, 15 digits, amex" + }, + { + "id": "card-tp-012", + "kind": "true_positive", + "text": "PAN=376825985723655", + "expect": [ + { + "entity": "CARD", + "value": "376825985723655" + } + ], + "note": "amex, 15 digits, plain" + }, + { + "id": "card-tp-013", + "kind": "true_positive", + "text": "Card number: 6011 3002 8087 8123", + "expect": [ + { + "entity": "CARD", + "value": "6011 3002 8087 8123" + } + ], + "note": "discover, 16 digits, spaces" + }, + { + "id": "card-tp-014", + "kind": "true_positive", + "text": "{\"card\": \"6557337980557250\"}", + "expect": [ + { + "entity": "CARD", + "value": "6557337980557250" + } + ], + "note": "discover, 16 digits, plain" + }, + { + "id": "card-tp-015", + "kind": "true_positive", + "text": "order_id,card\n1042,6442-9333-8889-4847", + "expect": [ + { + "entity": "CARD", + "value": "6442-9333-8889-4847" + } + ], + "note": "discover, 16 digits, hyphens" + }, + { + "id": "card-tp-016", + "kind": "true_positive", + "text": "Charged to 6221267494177595252 at checkout.", + "expect": [ + { + "entity": "CARD", + "value": "6221267494177595252" + } + ], + "note": "discover, 19 digits, plain" + }, + { + "id": "card-tp-017", + "kind": "true_positive", + "text": "const testCard = '3528 6980 3140 3593'", + "expect": [ + { + "entity": "CARD", + "value": "3528 6980 3140 3593" + } + ], + "note": "jcb, 16 digits, spaces" + }, + { + "id": "card-tp-018", + "kind": "true_positive", + "text": "PAN=3589818871135921513", + "expect": [ + { + "entity": "CARD", + "value": "3589818871135921513" + } + ], + "note": "jcb, 19 digits, plain" + }, + { + "id": "card-tp-019", + "kind": "true_positive", + "text": "Card number: 6204 8814 8615 9647", + "expect": [ + { + "entity": "CARD", + "value": "6204 8814 8615 9647" + } + ], + "note": "unionpay, 16 digits, spaces" + }, + { + "id": "card-tp-020", + "kind": "true_positive", + "text": "{\"card\": \"6215517794825821160\"}", + "expect": [ + { + "entity": "CARD", + "value": "6215517794825821160" + } + ], + "note": "unionpay, 19 digits, plain" + }, + { + "id": "card-tp-021", + "kind": "true_positive", + "text": "order_id,card\n1042,36199076449535", + "expect": [ + { + "entity": "CARD", + "value": "36199076449535" + } + ], + "note": "diners, 14 digits, plain" + }, + { + "id": "card-tp-022", + "kind": "true_positive", + "text": "Charged to 30480268377817 at checkout.", + "expect": [ + { + "entity": "CARD", + "value": "30480268377817" + } + ], + "note": "diners, 14 digits, plain" + }, + { + "id": "card-tp-023", + "kind": "true_positive", + "text": "const testCard = '3814 5337 0326 1339'", + "expect": [ + { + "entity": "CARD", + "value": "3814 5337 0326 1339" + } + ], + "note": "diners, 16 digits, spaces" + }, + { + "id": "card-tp-contains-phone-shape", + "kind": "true_positive", + "text": "Card on file: 5500 9123 4567 8015", + "expect": [ + { + "entity": "CARD", + "value": "5500 9123 4567 8015" + } + ], + "note": "Two groups look like a phone number (9123 4567); the longer card match wins (SPEC.md §4, ADR 0007)." + }, + { + "id": "card-hn-luhn-fail-visa", + "kind": "hard_negative", + "text": "Card: 4295176483952250", + "expect": [], + "note": "Visa prefix and length, but the Luhn check fails. Written unspaced: spaced groups can look like a phone number (ADR 0007)." + }, + { + "id": "card-hn-luhn-fail-mastercard", + "kind": "hard_negative", + "text": "Card: 5388847869444741", + "expect": [], + "note": "Mastercard prefix and length, but the Luhn check fails." + }, + { + "id": "card-hn-luhn-fail-amex", + "kind": "hard_negative", + "text": "Card: 3743 233024 76184", + "expect": [], + "note": "Amex prefix and length, but the Luhn check fails." + }, + { + "id": "card-hn-no-network-1", + "kind": "hard_negative", + "text": "Reference 1758768013372523", + "expect": [], + "note": "Passes Luhn, but no card network uses prefix 1." + }, + { + "id": "card-hn-no-network-7", + "kind": "hard_negative", + "text": "Tracking 7028269788055998", + "expect": [], + "note": "Passes Luhn, but no card network uses prefix 7." + }, + { + "id": "card-hn-no-network-8", + "kind": "hard_negative", + "text": "Batch 8077898158352631", + "expect": [], + "note": "Passes Luhn, but no card network uses prefix 80." + }, + { + "id": "card-hn-no-network-9", + "kind": "hard_negative", + "text": "Order 9098884031420582", + "expect": [], + "note": "A Luhn-valid order number; no card network uses prefix 9." + }, + { + "id": "card-hn-amex-wrong-length", + "kind": "hard_negative", + "text": "Card: 3446942295644057", + "expect": [], + "note": "Amex prefix, but Amex numbers have 15 digits, not 16." + }, + { + "id": "card-hn-visa-14-digits", + "kind": "hard_negative", + "text": "Card: 47780944434507", + "expect": [], + "note": "Visa issues 13, 16 or 19 digits, not 14." + }, + { + "id": "card-hn-visa-15-digits", + "kind": "hard_negative", + "text": "Card: 452793099038153", + "expect": [], + "note": "Visa issues 13, 16 or 19 digits, not 15." + }, + { + "id": "card-hn-mastercard-15-digits", + "kind": "hard_negative", + "text": "Card: 514180904079828", + "expect": [], + "note": "Mastercard numbers have 16 digits." + }, + { + "id": "card-hn-twelve-digits", + "kind": "hard_negative", + "text": "Card: 489843866190", + "expect": [], + "note": "Twelve digits: shorter than any card number here." + }, + { + "id": "card-hn-twenty-digits", + "kind": "hard_negative", + "text": "Card: 44364601916986781381", + "expect": [], + "note": "Twenty digits: longer than any card number." + }, + { + "id": "card-hn-double-space", + "kind": "hard_negative", + "text": "Card: 4792 7871 2835 7775", + "expect": [], + "note": "Two separators in a row break the number into pieces too short to be a card." + }, + { + "id": "card-hn-dot-separated", + "kind": "hard_negative", + "text": "Card: 4337.0641.8176.0200", + "expect": [], + "note": "Dots are not card separators." + }, + { + "id": "card-hn-inside-word", + "kind": "hard_negative", + "text": "ID4502097034519968", + "expect": [], + "note": "No word boundary between the letters and the digits." + }, + { + "id": "card-hn-timestamp", + "kind": "hard_negative", + "text": "Created 1727654321000", + "expect": [], + "note": "A 13-digit millisecond timestamp (prefix 17 is not a card network)." + }, + { + "id": "card-hn-isbn", + "kind": "hard_negative", + "text": "ISBN 978-3-16-148410-0", + "expect": [], + "note": "An ISBN-13 with hyphens (prefix 97 is not a card network)." + }, + { + "id": "card-hn-iban-like", + "kind": "hard_negative", + "text": "IBAN SG12 0456 7450 1234 5178 90", + "expect": [], + "note": "Bank-account-shaped digits with no card prefix at a valid length." + }, + { + "id": "card-hn-masked", + "kind": "hard_negative", + "text": "Card ending **** **** **** 4242", + "expect": [], + "note": "Only the last four digits, which is not the card number." + }, + { + "id": "card-hn-amount", + "kind": "hard_negative", + "text": "Total 1,234,567,890,123.45", + "expect": [], + "note": "An amount with commas." + }, + { + "id": "card-hn-phone", + "kind": "hard_negative", + "text": "Call +65 9123 4567", + "expect": [ + { + "entity": "PHONE", + "value": "+65 9123 4567" + } + ], + "note": "A phone number, reported as PHONE, not CARD." + }, + { + "id": "card-mixed", + "kind": "mixed", + "text": "Card 4497 4440 6161 2653, order 9199016612525129", + "expect": [ + { + "entity": "CARD", + "value": "4497 4440 6161 2653" + } + ], + "note": "The order number passes Luhn but has no card prefix." + } + ] +} diff --git a/third_party/sgpiirules/fixtures/dob.json b/third_party/sgpiirules/fixtures/dob.json new file mode 100644 index 0000000..6552a5c --- /dev/null +++ b/third_party/sgpiirules/fixtures/dob.json @@ -0,0 +1,463 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "DOB", + "cases": [ + { + "id": "dob-tp-001", + "kind": "true_positive", + "text": "DOB: 01/06/2006", + "expect": [ + { + "entity": "DOB", + "value": "01/06/2006" + } + ], + "note": "dd/mm/yyyy" + }, + { + "id": "dob-tp-002", + "kind": "true_positive", + "text": "D.O.B. 3/9/1954", + "expect": [ + { + "entity": "DOB", + "value": "3/9/1954" + } + ], + "note": "d/m/yyyy" + }, + { + "id": "dob-tp-003", + "kind": "true_positive", + "text": "Date of birth: 26-05-1961", + "expect": [ + { + "entity": "DOB", + "value": "26-05-1961" + } + ], + "note": "dd-mm-yyyy" + }, + { + "id": "dob-tp-004", + "kind": "true_positive", + "text": "date_of_birth: \"10.04.2008\"", + "expect": [ + { + "entity": "DOB", + "value": "10.04.2008" + } + ], + "note": "dd.mm.yyyy" + }, + { + "id": "dob-tp-005", + "kind": "true_positive", + "text": "{\"dateOfBirth\":\"1985-10-04\"}", + "expect": [ + { + "entity": "DOB", + "value": "1985-10-04" + } + ], + "note": "yyyy-mm-dd" + }, + { + "id": "dob-tp-006", + "kind": "true_positive", + "text": "dob=17 May 1962", + "expect": [ + { + "entity": "DOB", + "value": "17 May 1962" + } + ], + "note": "d Mon yyyy" + }, + { + "id": "dob-tp-007", + "kind": "true_positive", + "text": "Birth date 2 September 1977", + "expect": [ + { + "entity": "DOB", + "value": "2 September 1977" + } + ], + "note": "d Month yyyy" + }, + { + "id": "dob-tp-008", + "kind": "true_positive", + "text": "birthDate: 'July 22, 1955'", + "expect": [ + { + "entity": "DOB", + "value": "July 22, 1955" + } + ], + "note": "Month d, yyyy" + }, + { + "id": "dob-tp-009", + "kind": "true_positive", + "text": "Born on 23/04/1984", + "expect": [ + { + "entity": "DOB", + "value": "23/04/1984" + } + ], + "note": "dd/mm/yyyy" + }, + { + "id": "dob-tp-010", + "kind": "true_positive", + "text": "Patient born 16/5/1963 in Singapore", + "expect": [ + { + "entity": "DOB", + "value": "16/5/1963" + } + ], + "note": "d/m/yyyy" + }, + { + "id": "dob-tp-011", + "kind": "true_positive", + "text": "Born: 14-06-1976", + "expect": [ + { + "entity": "DOB", + "value": "14-06-1976" + } + ], + "note": "dd-mm-yyyy" + }, + { + "id": "dob-tp-012", + "kind": "true_positive", + "text": "DATE OF BIRTH 06.05.1986", + "expect": [ + { + "entity": "DOB", + "value": "06.05.1986" + } + ], + "note": "dd.mm.yyyy" + }, + { + "id": "dob-tp-013", + "kind": "true_positive", + "text": "DOB: 1988-09-15", + "expect": [ + { + "entity": "DOB", + "value": "1988-09-15" + } + ], + "note": "yyyy-mm-dd" + }, + { + "id": "dob-tp-014", + "kind": "true_positive", + "text": "D.O.B. 21 May 1996", + "expect": [ + { + "entity": "DOB", + "value": "21 May 1996" + } + ], + "note": "d Mon yyyy" + }, + { + "id": "dob-tp-015", + "kind": "true_positive", + "text": "Date of birth: 26 November 1979", + "expect": [ + { + "entity": "DOB", + "value": "26 November 1979" + } + ], + "note": "d Month yyyy" + }, + { + "id": "dob-tp-016", + "kind": "true_positive", + "text": "date_of_birth: \"February 24, 1974\"", + "expect": [ + { + "entity": "DOB", + "value": "February 24, 1974" + } + ], + "note": "Month d, yyyy" + }, + { + "id": "dob-tp-017", + "kind": "true_positive", + "text": "{\"dateOfBirth\":\"26/02/1963\"}", + "expect": [ + { + "entity": "DOB", + "value": "26/02/1963" + } + ], + "note": "dd/mm/yyyy" + }, + { + "id": "dob-tp-018", + "kind": "true_positive", + "text": "dob=21/1/1982", + "expect": [ + { + "entity": "DOB", + "value": "21/1/1982" + } + ], + "note": "d/m/yyyy" + }, + { + "id": "dob-tp-019", + "kind": "true_positive", + "text": "Birth date 14-11-1972", + "expect": [ + { + "entity": "DOB", + "value": "14-11-1972" + } + ], + "note": "dd-mm-yyyy" + }, + { + "id": "dob-tp-020", + "kind": "true_positive", + "text": "birthDate: '25.11.1958'", + "expect": [ + { + "entity": "DOB", + "value": "25.11.1958" + } + ], + "note": "dd.mm.yyyy" + }, + { + "id": "dob-tp-021", + "kind": "true_positive", + "text": "Born on 2007-05-11", + "expect": [ + { + "entity": "DOB", + "value": "2007-05-11" + } + ], + "note": "yyyy-mm-dd" + }, + { + "id": "dob-tp-022", + "kind": "true_positive", + "text": "Patient born 26 Apr 1975 in Singapore", + "expect": [ + { + "entity": "DOB", + "value": "26 Apr 1975" + } + ], + "note": "d Mon yyyy" + }, + { + "id": "dob-tp-023", + "kind": "true_positive", + "text": "Born: 13 November 1961", + "expect": [ + { + "entity": "DOB", + "value": "13 November 1961" + } + ], + "note": "d Month yyyy" + }, + { + "id": "dob-tp-month-first", + "kind": "true_positive", + "text": "DOB: 09/26/1995", + "expect": [ + { + "entity": "DOB", + "value": "09/26/1995" + } + ], + "note": "Only valid month first (US style), which is accepted too." + }, + { + "id": "dob-hn-no-clue-release", + "kind": "hard_negative", + "text": "Released 12/03/1988", + "expect": [], + "note": "A date with no birth clue." + }, + { + "id": "dob-hn-no-clue-updated", + "kind": "hard_negative", + "text": "Updated: 2026-10-05", + "expect": [], + "note": "A date with no birth clue." + }, + { + "id": "dob-hn-no-clue-meeting", + "kind": "hard_negative", + "text": "Meeting on 12 March 1988", + "expect": [], + "note": "A date with no birth clue." + }, + { + "id": "dob-hn-clue-far-away", + "kind": "hard_negative", + "text": "DOB field added on 12/03/2024", + "expect": [], + "note": "Words between the clue and the date." + }, + { + "id": "dob-hn-born-again", + "kind": "hard_negative", + "text": "born-again 2020-01-01", + "expect": [], + "note": "A hyphen joins \"born\" to another word." + }, + { + "id": "dob-hn-doberman", + "kind": "hard_negative", + "text": "Doberman 12/03/1988", + "expect": [], + "note": "\"Dob\" at the start of a longer word is not the clue." + }, + { + "id": "dob-hn-feb-31", + "kind": "hard_negative", + "text": "DOB: 31/02/1990", + "expect": [], + "note": "February has no 31st, either way round." + }, + { + "id": "dob-hn-not-a-leap-year", + "kind": "hard_negative", + "text": "DOB: 29/02/1990", + "expect": [], + "note": "1990 is not a leap year." + }, + { + "id": "dob-hn-month-13", + "kind": "hard_negative", + "text": "DOB: 13/13/1990", + "expect": [], + "note": "Neither part is a month." + }, + { + "id": "dob-hn-day-zero", + "kind": "hard_negative", + "text": "DOB: 00/05/1990", + "expect": [], + "note": "There is no day 0 or month 0." + }, + { + "id": "dob-hn-iso-feb-30", + "kind": "hard_negative", + "text": "dob=1990-02-30", + "expect": [], + "note": "February has no 30th." + }, + { + "id": "dob-hn-april-31", + "kind": "hard_negative", + "text": "DOB: 31 April 1990", + "expect": [], + "note": "April has 30 days." + }, + { + "id": "dob-hn-year-1899", + "kind": "hard_negative", + "text": "DOB: 01/01/1899", + "expect": [], + "note": "Years before 1900 are out of range." + }, + { + "id": "dob-hn-year-2150", + "kind": "hard_negative", + "text": "DOB: 01/01/2150", + "expect": [], + "note": "Years after 2099 are out of range." + }, + { + "id": "dob-hn-year-only", + "kind": "hard_negative", + "text": "Born in 1988", + "expect": [], + "note": "A year alone is not a full date of birth." + }, + { + "id": "dob-hn-redacted", + "kind": "hard_negative", + "text": "DOB: [redacted]", + "expect": [], + "note": "No date." + }, + { + "id": "dob-hn-code", + "kind": "hard_negative", + "text": "if (isValidDob(value)) save()", + "expect": [], + "note": "Code that mentions dates of birth." + }, + { + "id": "dob-hn-time", + "kind": "hard_negative", + "text": "DOB: 12:30", + "expect": [], + "note": "A time, not a date." + }, + { + "id": "dob-hn-weekday", + "kind": "hard_negative", + "text": "DOB: Monday", + "expect": [], + "note": "A day name, not a date." + }, + { + "id": "dob-hn-nric", + "kind": "hard_negative", + "text": "Date of birth: S1234567D", + "expect": [ + { + "entity": "NRIC", + "value": "S1234567D" + } + ], + "note": "An NRIC after the clue (synthetic example), reported as NRIC." + }, + { + "id": "dob-hn-phone", + "kind": "hard_negative", + "text": "DOB: 9123 4567", + "expect": [ + { + "entity": "PHONE", + "value": "9123 4567" + } + ], + "note": "A phone number after the clue, reported as PHONE." + }, + { + "id": "dob-mixed", + "kind": "mixed", + "text": "DOB: 20/01/1979, joined 01/06/2020", + "expect": [ + { + "entity": "DOB", + "value": "20/01/1979" + } + ], + "note": "Only the date after the clue is a date of birth." + } + ] +} diff --git a/third_party/sgpiirules/fixtures/email.json b/third_party/sgpiirules/fixtures/email.json new file mode 100644 index 0000000..ae34efb --- /dev/null +++ b/third_party/sgpiirules/fixtures/email.json @@ -0,0 +1,412 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "EMAIL", + "cases": [ + { + "id": "email-tp-001", + "kind": "true_positive", + "text": "{\"email\": \"gadiboye40@example.org\"}", + "expect": [ + { + "entity": "EMAIL", + "value": "gadiboye40@example.org" + } + ] + }, + { + "id": "email-tp-002", + "kind": "true_positive", + "text": "row,email\n1,yefu_suli@example.net", + "expect": [ + { + "entity": "EMAIL", + "value": "yefu_suli@example.net" + } + ] + }, + { + "id": "email-tp-003", + "kind": "true_positive", + "text": "email us", + "expect": [ + { + "entity": "EMAIL", + "value": "ance+liga@mail.example.com" + } + ] + }, + { + "id": "email-tp-004", + "kind": "true_positive", + "text": "From: Support ", + "expect": [ + { + "entity": "EMAIL", + "value": "oror-peka.87@dept.example.org" + } + ] + }, + { + "id": "email-tp-005", + "kind": "true_positive", + "text": "const adminEmail = 'LIBO.taen@sub-team.example.net'", + "expect": [ + { + "entity": "EMAIL", + "value": "LIBO.taen@sub-team.example.net" + } + ] + }, + { + "id": "email-tp-006", + "kind": "true_positive", + "text": "Please cc gajo.anri@clinic.test, thanks", + "expect": [ + { + "entity": "EMAIL", + "value": "gajo.anri@clinic.test" + } + ] + }, + { + "id": "email-tp-007", + "kind": "true_positive", + "text": "Contact: jonaorce64@example.com.", + "expect": [ + { + "entity": "EMAIL", + "value": "jonaorce64@example.com" + } + ] + }, + { + "id": "email-tp-008", + "kind": "true_positive", + "text": "{\"email\": \"yeli_endi@example.org\"}", + "expect": [ + { + "entity": "EMAIL", + "value": "yeli_endi@example.org" + } + ] + }, + { + "id": "email-tp-009", + "kind": "true_positive", + "text": "row,email\n1,susu+pesu@example.net", + "expect": [ + { + "entity": "EMAIL", + "value": "susu+pesu@example.net" + } + ] + }, + { + "id": "email-tp-010", + "kind": "true_positive", + "text": "email us", + "expect": [ + { + "entity": "EMAIL", + "value": "yeri-wuwu.51@mail.example.com" + } + ] + }, + { + "id": "email-tp-011", + "kind": "true_positive", + "text": "From: Support ", + "expect": [ + { + "entity": "EMAIL", + "value": "HIAN.tadi@dept.example.org" + } + ] + }, + { + "id": "email-tp-012", + "kind": "true_positive", + "text": "const adminEmail = 'hiri.cemu@sub-team.example.net'", + "expect": [ + { + "entity": "EMAIL", + "value": "hiri.cemu@sub-team.example.net" + } + ] + }, + { + "id": "email-tp-013", + "kind": "true_positive", + "text": "Please cc ansuyeye58@clinic.test, thanks", + "expect": [ + { + "entity": "EMAIL", + "value": "ansuyeye58@clinic.test" + } + ] + }, + { + "id": "email-tp-014", + "kind": "true_positive", + "text": "Contact: lili_joga@example.com.", + "expect": [ + { + "entity": "EMAIL", + "value": "lili_joga@example.com" + } + ] + }, + { + "id": "email-tp-015", + "kind": "true_positive", + "text": "{\"email\": \"yedi+hisu@example.org\"}", + "expect": [ + { + "entity": "EMAIL", + "value": "yedi+hisu@example.org" + } + ] + }, + { + "id": "email-tp-016", + "kind": "true_positive", + "text": "row,email\n1,namu-boor.66@example.net", + "expect": [ + { + "entity": "EMAIL", + "value": "namu-boor.66@example.net" + } + ] + }, + { + "id": "email-tp-017", + "kind": "true_positive", + "text": "email us", + "expect": [ + { + "entity": "EMAIL", + "value": "WUAN.diwu@mail.example.com" + } + ] + }, + { + "id": "email-tp-018", + "kind": "true_positive", + "text": "From: Support ", + "expect": [ + { + "entity": "EMAIL", + "value": "anwu.anor@dept.example.org" + } + ] + }, + { + "id": "email-tp-019", + "kind": "true_positive", + "text": "const adminEmail = 'ritayean38@sub-team.example.net'", + "expect": [ + { + "entity": "EMAIL", + "value": "ritayean38@sub-team.example.net" + } + ] + }, + { + "id": "email-tp-020", + "kind": "true_positive", + "text": "Please cc gaor_anka@clinic.test, thanks", + "expect": [ + { + "entity": "EMAIL", + "value": "gaor_anka@clinic.test" + } + ] + }, + { + "id": "email-tp-021", + "kind": "true_positive", + "text": "Contact: muna+fuen@example.com.", + "expect": [ + { + "entity": "EMAIL", + "value": "muna+fuen@example.com" + } + ] + }, + { + "id": "email-hn-npm-version", + "kind": "hard_negative", + "text": "npm install left-pad@1.3.0", + "expect": [], + "note": "Package and version." + }, + { + "id": "email-hn-npm-scoped", + "kind": "hard_negative", + "text": "import from @scope/pkg@2.1.0", + "expect": [], + "note": "Scoped package and version." + }, + { + "id": "email-hn-npm-range", + "kind": "hard_negative", + "text": "\"react@^18.2.0\"", + "expect": [], + "note": "Version range." + }, + { + "id": "email-hn-no-tld", + "kind": "hard_negative", + "text": "ssh admin@localhost", + "expect": [], + "note": "No top-level domain." + }, + { + "id": "email-hn-bare-host", + "kind": "hard_negative", + "text": "user@intranet", + "expect": [], + "note": "No top-level domain." + }, + { + "id": "email-hn-one-letter-tld", + "kind": "hard_negative", + "text": "a@b.c", + "expect": [], + "note": "Top-level domain of one letter." + }, + { + "id": "email-hn-ip-domain", + "kind": "hard_negative", + "text": "x@192.0.2.10", + "expect": [], + "note": "Domain is an IP address (a documentation range, RFC 5737)." + }, + { + "id": "email-hn-double-at", + "kind": "hard_negative", + "text": "user@@example.com", + "expect": [], + "note": "Empty local part." + }, + { + "id": "email-hn-mention", + "kind": "hard_negative", + "text": "Thanks @reviewer for the catch", + "expect": [], + "note": "A mention." + }, + { + "id": "email-hn-decorator", + "kind": "hard_negative", + "text": "@Component({ selector: \"app\" })", + "expect": [], + "note": "A decorator." + }, + { + "id": "email-hn-css-at-rule", + "kind": "hard_negative", + "text": "@media screen and (max-width: 600px)", + "expect": [], + "note": "A CSS at-rule." + }, + { + "id": "email-hn-at-symbol-price", + "kind": "hard_negative", + "text": "Buy 3@$2.00 each", + "expect": [], + "note": "Price notation." + }, + { + "id": "email-hn-trailing-at", + "kind": "hard_negative", + "text": "Sent to team@", + "expect": [], + "note": "Nothing after the @." + }, + { + "id": "email-hn-leading-at", + "kind": "hard_negative", + "text": "@example.com", + "expect": [], + "note": "Nothing before the @." + }, + { + "id": "email-hn-space-inside", + "kind": "mixed", + "text": "jo an@example.com", + "expect": [ + { + "entity": "EMAIL", + "value": "an@example.com" + } + ], + "note": "The space splits it; only the part after it is an address." + }, + { + "id": "email-hn-numeric-tld", + "kind": "hard_negative", + "text": "build@host.123", + "expect": [], + "note": "Numeric top-level domain." + }, + { + "id": "email-hn-obfuscated", + "kind": "hard_negative", + "text": "jo [at] example [dot] com", + "expect": [], + "note": "Obfuscated address." + }, + { + "id": "email-hn-version-tag", + "kind": "hard_negative", + "text": "git tag v1.2@rc", + "expect": [], + "note": "Not an address." + }, + { + "id": "email-hn-docker-digest", + "kind": "hard_negative", + "text": "image@sha256:abc123", + "expect": [], + "note": "Container digest." + }, + { + "id": "email-hn-at-in-path", + "kind": "hard_negative", + "text": "https://registry.example.com/@types/node", + "expect": [], + "note": "A path segment." + }, + { + "id": "email-hn-underscore-domain", + "kind": "hard_negative", + "text": "x@exa_mple.com", + "expect": [], + "note": "Underscores are not allowed in host names." + }, + { + "id": "email-mixed", + "kind": "mixed", + "text": "Owner yeen.dibo@example.org; deps: lodash@4.17.21", + "expect": [ + { + "entity": "EMAIL", + "value": "yeen.dibo@example.org" + } + ] + }, + { + "id": "email-tp-numeric-local", + "kind": "true_positive", + "text": "SMS gateway 85880486@example.net", + "expect": [ + { + "entity": "EMAIL", + "value": "85880486@example.net" + } + ], + "note": "Reported once, as EMAIL, although the local part looks like a phone number." + } + ] +} diff --git a/third_party/sgpiirules/fixtures/nric-like.json b/third_party/sgpiirules/fixtures/nric-like.json new file mode 100644 index 0000000..c62764c --- /dev/null +++ b/third_party/sgpiirules/fixtures/nric-like.json @@ -0,0 +1,491 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "NRIC_LIKE", + "cases": [ + { + "id": "nric-like-tp-001", + "kind": "true_positive", + "text": "Applicant NRIC: S3722536D", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S3722536D" + } + ] + }, + { + "id": "nric-like-tp-002", + "kind": "true_positive", + "text": "{\"nric\": \"S1626954J\", \"plan\": \"basic\"}", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S1626954J" + } + ] + }, + { + "id": "nric-like-tp-003", + "kind": "true_positive", + "text": "row,id,status\n1,S2388876J,active", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S2388876J" + } + ] + }, + { + "id": "nric-like-tp-004", + "kind": "true_positive", + "text": "WARN lookup failed for member S1850281A at 10:42", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S1850281A" + } + ] + }, + { + "id": "nric-like-tp-005", + "kind": "true_positive", + "text": "const testMemberId = 'T8761135A'", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T8761135A" + } + ] + }, + { + "id": "nric-like-tp-006", + "kind": "true_positive", + "text": "Please update the record for T2415783H.", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T2415783H" + } + ] + }, + { + "id": "nric-like-tp-007", + "kind": "true_positive", + "text": "Holder (T9439209E) confirmed by phone", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T9439209E" + } + ] + }, + { + "id": "nric-like-tp-008", + "kind": "true_positive", + "text": "FIN T3357819F expires next month", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T3357819F" + } + ] + }, + { + "id": "nric-like-tp-009", + "kind": "true_positive", + "text": "NRIC：F1531465R", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F1531465R" + } + ] + }, + { + "id": "nric-like-tp-010", + "kind": "true_positive", + "text": "F1710584L", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F1710584L" + } + ] + }, + { + "id": "nric-like-tp-011", + "kind": "true_positive", + "text": "Applicant NRIC: F4076733V", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F4076733V" + } + ] + }, + { + "id": "nric-like-tp-012", + "kind": "true_positive", + "text": "{\"nric\": \"F3451782S\", \"plan\": \"basic\"}", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F3451782S" + } + ] + }, + { + "id": "nric-like-tp-013", + "kind": "true_positive", + "text": "row,id,status\n1,G4277574M,active", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "G4277574M" + } + ] + }, + { + "id": "nric-like-tp-014", + "kind": "true_positive", + "text": "WARN lookup failed for member G3174817O at 10:42", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "G3174817O" + } + ] + }, + { + "id": "nric-like-tp-015", + "kind": "true_positive", + "text": "const testMemberId = 'G0860760L'", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "G0860760L" + } + ] + }, + { + "id": "nric-like-tp-016", + "kind": "true_positive", + "text": "Please update the record for G8140776V.", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "G8140776V" + } + ] + }, + { + "id": "nric-like-tp-017", + "kind": "true_positive", + "text": "Holder (M9247298V) confirmed by phone", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M9247298V" + } + ] + }, + { + "id": "nric-like-tp-018", + "kind": "true_positive", + "text": "FIN M8639508Q expires next month", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M8639508Q" + } + ] + }, + { + "id": "nric-like-tp-019", + "kind": "true_positive", + "text": "NRIC：M5119669L", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M5119669L" + } + ] + }, + { + "id": "nric-like-tp-020", + "kind": "true_positive", + "text": "M6994053K", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M6994053K" + } + ] + }, + { + "id": "nric-like-tp-ica-example", + "kind": "true_positive", + "text": "Format example from ICA: M1234567B", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M1234567B" + } + ], + "note": "Illustrative example in the ICA media release of 12 Jul 2021; fails the checksum." + }, + { + "id": "nric-like-tp-pdpc-example", + "kind": "true_positive", + "text": "e.g. S1234567A", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S1234567A" + } + ], + "note": "Illustrative example in the PDPC NRIC guidelines; fails the checksum." + }, + { + "id": "nric-like-tp-order-number", + "kind": "true_positive", + "text": "Order number T1234567A shipped", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T1234567A" + } + ], + "note": "NRIC-shaped order number." + }, + { + "id": "nric-like-tp-m-with-fg-letter", + "kind": "true_positive", + "text": "M0000008M", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M0000008M" + } + ], + "note": "Remainder 8: the F/G table gives M but the M table gives J." + }, + { + "id": "nric-like-tp-f-with-m-letter", + "kind": "true_positive", + "text": "F0000004J", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F0000004J" + } + ], + "note": "Remainder 8: the M table gives J but the F/G table gives M." + }, + { + "id": "nric-like-hn-valid-01", + "kind": "hard_negative", + "text": "Applicant NRIC: S3065355F", + "expect": [ + { + "entity": "NRIC", + "value": "S3065355F" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-02", + "kind": "hard_negative", + "text": "{\"nric\": \"T1588608G\", \"plan\": \"basic\"}", + "expect": [ + { + "entity": "NRIC", + "value": "T1588608G" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-03", + "kind": "hard_negative", + "text": "row,id,status\n1,F8007665R,active", + "expect": [ + { + "entity": "NRIC", + "value": "F8007665R" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-04", + "kind": "hard_negative", + "text": "WARN lookup failed for member G4534663W at 10:42", + "expect": [ + { + "entity": "NRIC", + "value": "G4534663W" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-05", + "kind": "hard_negative", + "text": "const testMemberId = 'M1839003L'", + "expect": [ + { + "entity": "NRIC", + "value": "M1839003L" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-06", + "kind": "hard_negative", + "text": "Please update the record for S5973034G.", + "expect": [ + { + "entity": "NRIC", + "value": "S5973034G" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-07", + "kind": "hard_negative", + "text": "Holder (T0779753I) confirmed by phone", + "expect": [ + { + "entity": "NRIC", + "value": "T0779753I" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-08", + "kind": "hard_negative", + "text": "FIN F0664134L expires next month", + "expect": [ + { + "entity": "NRIC", + "value": "F0664134L" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-09", + "kind": "hard_negative", + "text": "NRIC：G9619016L", + "expect": [ + { + "entity": "NRIC", + "value": "G9619016L" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-valid-10", + "kind": "hard_negative", + "text": "M2791665U", + "expect": [ + { + "entity": "NRIC", + "value": "M2791665U" + } + ], + "note": "A valid NRIC is reported as NRIC, never NRIC_LIKE." + }, + { + "id": "nric-like-hn-prefix-a", + "kind": "hard_negative", + "text": "Ticket A1234567D", + "expect": [], + "note": "A is not an NRIC/FIN prefix." + }, + { + "id": "nric-like-hn-prefix-e", + "kind": "hard_negative", + "text": "Part E1234567D", + "expect": [], + "note": "E is not an NRIC/FIN prefix." + }, + { + "id": "nric-like-hn-too-short", + "kind": "hard_negative", + "text": "S123456D", + "expect": [], + "note": "Six digits." + }, + { + "id": "nric-like-hn-too-long", + "kind": "hard_negative", + "text": "S12345678D", + "expect": [], + "note": "Eight digits." + }, + { + "id": "nric-like-hn-embedded-left", + "kind": "hard_negative", + "text": "code ABS1234567D", + "expect": [], + "note": "No word boundary before the prefix." + }, + { + "id": "nric-like-hn-embedded-right", + "kind": "hard_negative", + "text": "token S1234567DZ", + "expect": [], + "note": "No word boundary after the check letter." + }, + { + "id": "nric-like-hn-trailing-digit", + "kind": "hard_negative", + "text": "S1234567D9", + "expect": [], + "note": "No word boundary after the check letter." + }, + { + "id": "nric-like-hn-masked", + "kind": "hard_negative", + "text": "Holder S****567D", + "expect": [], + "note": "Masked NRIC; not a full number." + }, + { + "id": "nric-like-hn-partial", + "kind": "hard_negative", + "text": "last four: 567D", + "expect": [], + "note": "Partial NRIC (PDPC guidelines §5.2); out of scope for these detectors." + }, + { + "id": "nric-like-hn-separator", + "kind": "hard_negative", + "text": "S1234567-D", + "expect": [], + "note": "Separator inside the number." + }, + { + "id": "nric-like-mixed", + "kind": "mixed", + "text": "Holder F3652570M and dependant M0745789Q", + "expect": [ + { + "entity": "NRIC", + "value": "F3652570M" + }, + { + "entity": "NRIC_LIKE", + "value": "M0745789Q" + } + ] + } + ] +} diff --git a/third_party/sgpiirules/fixtures/nric.json b/third_party/sgpiirules/fixtures/nric.json new file mode 100644 index 0000000..4284106 --- /dev/null +++ b/third_party/sgpiirules/fixtures/nric.json @@ -0,0 +1,574 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "NRIC", + "cases": [ + { + "id": "nric-tp-001", + "kind": "true_positive", + "text": "Applicant NRIC: S9263339Z", + "expect": [ + { + "entity": "NRIC", + "value": "S9263339Z" + } + ] + }, + { + "id": "nric-tp-002", + "kind": "true_positive", + "text": "{\"nric\": \"S5614969D\", \"plan\": \"basic\"}", + "expect": [ + { + "entity": "NRIC", + "value": "S5614969D" + } + ] + }, + { + "id": "nric-tp-003", + "kind": "true_positive", + "text": "row,id,status\n1,S0940338H,active", + "expect": [ + { + "entity": "NRIC", + "value": "S0940338H" + } + ] + }, + { + "id": "nric-tp-004", + "kind": "true_positive", + "text": "WARN lookup failed for member S9854593Z at 10:42", + "expect": [ + { + "entity": "NRIC", + "value": "S9854593Z" + } + ] + }, + { + "id": "nric-tp-005", + "kind": "true_positive", + "text": "const testMemberId = 'S8347622B'", + "expect": [ + { + "entity": "NRIC", + "value": "S8347622B" + } + ] + }, + { + "id": "nric-tp-006", + "kind": "true_positive", + "text": "Please update the record for T2422524G.", + "expect": [ + { + "entity": "NRIC", + "value": "T2422524G" + } + ] + }, + { + "id": "nric-tp-007", + "kind": "true_positive", + "text": "Holder (T7245354E) confirmed by phone", + "expect": [ + { + "entity": "NRIC", + "value": "T7245354E" + } + ] + }, + { + "id": "nric-tp-008", + "kind": "true_positive", + "text": "FIN T8496806J expires next month", + "expect": [ + { + "entity": "NRIC", + "value": "T8496806J" + } + ] + }, + { + "id": "nric-tp-009", + "kind": "true_positive", + "text": "NRIC：T8697778D", + "expect": [ + { + "entity": "NRIC", + "value": "T8697778D" + } + ] + }, + { + "id": "nric-tp-010", + "kind": "true_positive", + "text": "T4959040C", + "expect": [ + { + "entity": "NRIC", + "value": "T4959040C" + } + ] + }, + { + "id": "nric-tp-011", + "kind": "true_positive", + "text": "Applicant NRIC: F1552769L", + "expect": [ + { + "entity": "NRIC", + "value": "F1552769L" + } + ] + }, + { + "id": "nric-tp-012", + "kind": "true_positive", + "text": "{\"nric\": \"F4376852L\", \"plan\": \"basic\"}", + "expect": [ + { + "entity": "NRIC", + "value": "F4376852L" + } + ] + }, + { + "id": "nric-tp-013", + "kind": "true_positive", + "text": "row,id,status\n1,F0152662Q,active", + "expect": [ + { + "entity": "NRIC", + "value": "F0152662Q" + } + ] + }, + { + "id": "nric-tp-014", + "kind": "true_positive", + "text": "WARN lookup failed for member F7151769K at 10:42", + "expect": [ + { + "entity": "NRIC", + "value": "F7151769K" + } + ] + }, + { + "id": "nric-tp-015", + "kind": "true_positive", + "text": "const testMemberId = 'F6626475X'", + "expect": [ + { + "entity": "NRIC", + "value": "F6626475X" + } + ] + }, + { + "id": "nric-tp-016", + "kind": "true_positive", + "text": "Please update the record for G5107556R.", + "expect": [ + { + "entity": "NRIC", + "value": "G5107556R" + } + ] + }, + { + "id": "nric-tp-017", + "kind": "true_positive", + "text": "Holder (G9291093N) confirmed by phone", + "expect": [ + { + "entity": "NRIC", + "value": "G9291093N" + } + ] + }, + { + "id": "nric-tp-018", + "kind": "true_positive", + "text": "FIN G8527469X expires next month", + "expect": [ + { + "entity": "NRIC", + "value": "G8527469X" + } + ] + }, + { + "id": "nric-tp-019", + "kind": "true_positive", + "text": "NRIC：G3099261Q", + "expect": [ + { + "entity": "NRIC", + "value": "G3099261Q" + } + ] + }, + { + "id": "nric-tp-020", + "kind": "true_positive", + "text": "G5989903T", + "expect": [ + { + "entity": "NRIC", + "value": "G5989903T" + } + ] + }, + { + "id": "nric-tp-021", + "kind": "true_positive", + "text": "Applicant NRIC: M3960741N", + "expect": [ + { + "entity": "NRIC", + "value": "M3960741N" + } + ] + }, + { + "id": "nric-tp-022", + "kind": "true_positive", + "text": "{\"nric\": \"M0680454X\", \"plan\": \"basic\"}", + "expect": [ + { + "entity": "NRIC", + "value": "M0680454X" + } + ] + }, + { + "id": "nric-tp-023", + "kind": "true_positive", + "text": "row,id,status\n1,M5856672U,active", + "expect": [ + { + "entity": "NRIC", + "value": "M5856672U" + } + ] + }, + { + "id": "nric-tp-024", + "kind": "true_positive", + "text": "WARN lookup failed for member M5238863L at 10:42", + "expect": [ + { + "entity": "NRIC", + "value": "M5238863L" + } + ] + }, + { + "id": "nric-tp-025", + "kind": "true_positive", + "text": "const testMemberId = 'M0131471P'", + "expect": [ + { + "entity": "NRIC", + "value": "M0131471P" + } + ] + }, + { + "id": "nric-tp-lowercase", + "kind": "true_positive", + "text": "nric=t9790698f", + "expect": [ + { + "entity": "NRIC", + "value": "t9790698f" + } + ], + "note": "Lower case is still an NRIC; the value is reported as written." + }, + { + "id": "nric-hn-wrong-letter-01", + "kind": "hard_negative", + "text": "Order reference S9189805F dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S9189805F" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-02", + "kind": "hard_negative", + "text": "Order reference S7238445I dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S7238445I" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-03", + "kind": "hard_negative", + "text": "Order reference T9371181B dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T9371181B" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-04", + "kind": "hard_negative", + "text": "Order reference T0059291F dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T0059291F" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-05", + "kind": "hard_negative", + "text": "Order reference F9197193N dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F9197193N" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-06", + "kind": "hard_negative", + "text": "Order reference F1421846S dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F1421846S" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-07", + "kind": "hard_negative", + "text": "Order reference G8916540N dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "G8916540N" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-08", + "kind": "hard_negative", + "text": "Order reference G3918812S dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "G3918812S" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-09", + "kind": "hard_negative", + "text": "Order reference M6188872V dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M6188872V" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-wrong-letter-10", + "kind": "hard_negative", + "text": "Order reference M7300453M dispatched", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M7300453M" + } + ], + "note": "NRIC-shaped reference with an invalid check letter: NRIC_LIKE, not NRIC." + }, + { + "id": "nric-hn-ica-example", + "kind": "hard_negative", + "text": "Format example from ICA: M1234567B", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M1234567B" + } + ], + "note": "Illustrative example in the ICA media release of 12 Jul 2021; fails the checksum." + }, + { + "id": "nric-hn-pdpc-example", + "kind": "hard_negative", + "text": "e.g. S1234567A", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "S1234567A" + } + ], + "note": "Illustrative example in the PDPC NRIC guidelines; fails the checksum." + }, + { + "id": "nric-hn-order-number", + "kind": "hard_negative", + "text": "Order number T1234567A shipped", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "T1234567A" + } + ], + "note": "NRIC-shaped order number." + }, + { + "id": "nric-hn-m-with-fg-letter", + "kind": "hard_negative", + "text": "M0000008M", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "M0000008M" + } + ], + "note": "Remainder 8: the F/G table gives M but the M table gives J." + }, + { + "id": "nric-hn-f-with-m-letter", + "kind": "hard_negative", + "text": "F0000004J", + "expect": [ + { + "entity": "NRIC_LIKE", + "value": "F0000004J" + } + ], + "note": "Remainder 8: the M table gives J but the F/G table gives M." + }, + { + "id": "nric-hn-prefix-a", + "kind": "hard_negative", + "text": "Ticket A1234567D", + "expect": [], + "note": "A is not an NRIC/FIN prefix." + }, + { + "id": "nric-hn-prefix-e", + "kind": "hard_negative", + "text": "Part E1234567D", + "expect": [], + "note": "E is not an NRIC/FIN prefix." + }, + { + "id": "nric-hn-too-short", + "kind": "hard_negative", + "text": "S123456D", + "expect": [], + "note": "Six digits." + }, + { + "id": "nric-hn-too-long", + "kind": "hard_negative", + "text": "S12345678D", + "expect": [], + "note": "Eight digits." + }, + { + "id": "nric-hn-embedded-left", + "kind": "hard_negative", + "text": "code ABS1234567D", + "expect": [], + "note": "No word boundary before the prefix." + }, + { + "id": "nric-hn-embedded-right", + "kind": "hard_negative", + "text": "token S1234567DZ", + "expect": [], + "note": "No word boundary after the check letter." + }, + { + "id": "nric-hn-trailing-digit", + "kind": "hard_negative", + "text": "S1234567D9", + "expect": [], + "note": "No word boundary after the check letter." + }, + { + "id": "nric-hn-masked", + "kind": "hard_negative", + "text": "Holder S****567D", + "expect": [], + "note": "Masked NRIC; not a full number." + }, + { + "id": "nric-hn-partial", + "kind": "hard_negative", + "text": "last four: 567D", + "expect": [], + "note": "Partial NRIC (PDPC guidelines §5.2); out of scope for these detectors." + }, + { + "id": "nric-hn-separator", + "kind": "hard_negative", + "text": "S1234567-D", + "expect": [], + "note": "Separator inside the number." + }, + { + "id": "nric-mixed-two-valid", + "kind": "mixed", + "text": "Transfer from S2834031A to G4255606N approved", + "expect": [ + { + "entity": "NRIC", + "value": "S2834031A" + }, + { + "entity": "NRIC", + "value": "G4255606N" + } + ] + }, + { + "id": "nric-mixed-one-invalid", + "kind": "mixed", + "text": "Primary S2834031A, secondary T2995963A", + "expect": [ + { + "entity": "NRIC", + "value": "S2834031A" + }, + { + "entity": "NRIC_LIKE", + "value": "T2995963A" + } + ], + "note": "The second number has an invalid check letter." + } + ] +} diff --git a/third_party/sgpiirules/fixtures/phone.json b/third_party/sgpiirules/fixtures/phone.json new file mode 100644 index 0000000..3961da5 --- /dev/null +++ b/third_party/sgpiirules/fixtures/phone.json @@ -0,0 +1,383 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "PHONE", + "cases": [ + { + "id": "phone-tp-001", + "kind": "true_positive", + "text": "Contact number: 37988298", + "expect": [ + { + "entity": "PHONE", + "value": "37988298" + } + ] + }, + { + "id": "phone-tp-002", + "kind": "true_positive", + "text": "{\"mobile\": \"61387792\"}", + "expect": [ + { + "entity": "PHONE", + "value": "61387792" + } + ] + }, + { + "id": "phone-tp-003", + "kind": "true_positive", + "text": "row,phone\n1,8167 5247", + "expect": [ + { + "entity": "PHONE", + "value": "8167 5247" + } + ] + }, + { + "id": "phone-tp-004", + "kind": "true_positive", + "text": "Call 9844 3165 after 6pm.", + "expect": [ + { + "entity": "PHONE", + "value": "9844 3165" + } + ] + }, + { + "id": "phone-tp-005", + "kind": "true_positive", + "text": "tel:3416-3839", + "expect": [ + { + "entity": "PHONE", + "value": "3416-3839" + } + ] + }, + { + "id": "phone-tp-006", + "kind": "true_positive", + "text": "const fallbackPhone = '6273-9555'", + "expect": [ + { + "entity": "PHONE", + "value": "6273-9555" + } + ] + }, + { + "id": "phone-tp-007", + "kind": "true_positive", + "text": "Contact number: +65 8427 9814", + "expect": [ + { + "entity": "PHONE", + "value": "+65 8427 9814" + } + ] + }, + { + "id": "phone-tp-008", + "kind": "true_positive", + "text": "{\"mobile\": \"+65 9028 3087\"}", + "expect": [ + { + "entity": "PHONE", + "value": "+65 9028 3087" + } + ] + }, + { + "id": "phone-tp-009", + "kind": "true_positive", + "text": "row,phone\n1,+6532349943", + "expect": [ + { + "entity": "PHONE", + "value": "+6532349943" + } + ] + }, + { + "id": "phone-tp-010", + "kind": "true_positive", + "text": "Call +6560473510 after 6pm.", + "expect": [ + { + "entity": "PHONE", + "value": "+6560473510" + } + ] + }, + { + "id": "phone-tp-011", + "kind": "true_positive", + "text": "tel:+65-8935-4759", + "expect": [ + { + "entity": "PHONE", + "value": "+65-8935-4759" + } + ] + }, + { + "id": "phone-tp-012", + "kind": "true_positive", + "text": "const fallbackPhone = '+65-9476-5519'", + "expect": [ + { + "entity": "PHONE", + "value": "+65-9476-5519" + } + ] + }, + { + "id": "phone-tp-013", + "kind": "true_positive", + "text": "Contact number: (65) 3348 5062", + "expect": [ + { + "entity": "PHONE", + "value": "(65) 3348 5062" + } + ] + }, + { + "id": "phone-tp-014", + "kind": "true_positive", + "text": "{\"mobile\": \"(65) 6629 0650\"}", + "expect": [ + { + "entity": "PHONE", + "value": "(65) 6629 0650" + } + ] + }, + { + "id": "phone-tp-015", + "kind": "true_positive", + "text": "row,phone\n1,(+65)83254137", + "expect": [ + { + "entity": "PHONE", + "value": "(+65)83254137" + } + ] + }, + { + "id": "phone-tp-016", + "kind": "true_positive", + "text": "Call (+65)99407325 after 6pm.", + "expect": [ + { + "entity": "PHONE", + "value": "(+65)99407325" + } + ] + }, + { + "id": "phone-tp-017", + "kind": "true_positive", + "text": "tel:0065 3928 1916", + "expect": [ + { + "entity": "PHONE", + "value": "0065 3928 1916" + } + ] + }, + { + "id": "phone-tp-018", + "kind": "true_positive", + "text": "const fallbackPhone = '0065 6561 8922'", + "expect": [ + { + "entity": "PHONE", + "value": "0065 6561 8922" + } + ] + }, + { + "id": "phone-tp-019", + "kind": "true_positive", + "text": "Contact number: 6582106396", + "expect": [ + { + "entity": "PHONE", + "value": "6582106396" + } + ] + }, + { + "id": "phone-tp-020", + "kind": "true_positive", + "text": "{\"mobile\": \"6599861361\"}", + "expect": [ + { + "entity": "PHONE", + "value": "6599861361" + } + ] + }, + { + "id": "phone-hn-leading-0", + "kind": "hard_negative", + "text": "Reference 08868964", + "expect": [], + "note": "No eight-digit Singapore numbers start with 0 (IMDA NNP Table 2.1)." + }, + { + "id": "phone-hn-leading-1", + "kind": "hard_negative", + "text": "Reference 12883474", + "expect": [], + "note": "No eight-digit Singapore numbers start with 1 (IMDA NNP Table 2.1)." + }, + { + "id": "phone-hn-leading-2", + "kind": "hard_negative", + "text": "Reference 26231714", + "expect": [], + "note": "No eight-digit Singapore numbers start with 2 (IMDA NNP Table 2.1)." + }, + { + "id": "phone-hn-leading-4", + "kind": "hard_negative", + "text": "Reference 46738490", + "expect": [], + "note": "No eight-digit Singapore numbers start with 4 (IMDA NNP Table 2.1)." + }, + { + "id": "phone-hn-leading-5", + "kind": "hard_negative", + "text": "Reference 59780057", + "expect": [], + "note": "No eight-digit Singapore numbers start with 5 (IMDA NNP Table 2.1)." + }, + { + "id": "phone-hn-leading-7", + "kind": "hard_negative", + "text": "Reference 70134883", + "expect": [], + "note": "Leading 7 is for SMS short codes, not eight-digit numbers (IMDA NNP Table 2.1)." + }, + { + "id": "phone-hn-nine-digits", + "kind": "hard_negative", + "text": "ID 951764689", + "expect": [], + "note": "Nine digits." + }, + { + "id": "phone-hn-seven-digits", + "kind": "hard_negative", + "text": "ID 8559448", + "expect": [], + "note": "Seven digits." + }, + { + "id": "phone-hn-after-letter", + "kind": "hard_negative", + "text": "S98612291", + "expect": [], + "note": "No word boundary between a letter and the digits." + }, + { + "id": "phone-hn-country-code-too-long", + "kind": "hard_negative", + "text": "65912345678", + "expect": [], + "note": "Country code plus nine digits." + }, + { + "id": "phone-hn-toll-free", + "kind": "hard_negative", + "text": "Hotline 1800 123 4567", + "expect": [], + "note": "Toll-free number: 11 digits, and a business number, not a subscriber line (IMDA NNP §8.1)." + }, + { + "id": "phone-hn-premium", + "kind": "hard_negative", + "text": "Vote 1900 112 2334", + "expect": [], + "note": "Premium-rate number (IMDA NNP §9.1)." + }, + { + "id": "phone-hn-us-number", + "kind": "hard_negative", + "text": "US office +1 650 912 3456", + "expect": [], + "note": "United States number." + }, + { + "id": "phone-hn-uk-number", + "kind": "hard_negative", + "text": "UK mobile +44 7911 123456", + "expect": [], + "note": "United Kingdom number." + }, + { + "id": "phone-hn-split-4-5", + "kind": "hard_negative", + "text": "Order 8123-45678", + "expect": [], + "note": "Wrong grouping: four then five digits." + }, + { + "id": "phone-hn-double-space", + "kind": "hard_negative", + "text": "9123 4567", + "expect": [], + "note": "Two separators." + }, + { + "id": "phone-hn-price", + "kind": "hard_negative", + "text": "Total SGD 9,123.45", + "expect": [], + "note": "An amount." + }, + { + "id": "phone-hn-date", + "kind": "hard_negative", + "text": "Settled on 20260928", + "expect": [], + "note": "A date written as eight digits, starting with 2." + }, + { + "id": "phone-hn-emergency", + "kind": "hard_negative", + "text": "Dial 995 for emergencies", + "expect": [], + "note": "Three-digit emergency code." + }, + { + "id": "phone-hn-inside-email", + "kind": "hard_negative", + "text": "Reply to 94378773@example.com", + "expect": [ + { + "entity": "EMAIL", + "value": "94378773@example.com" + } + ], + "note": "The digits look like a mobile number, but the longer email match wins (SPEC.md §4)." + }, + { + "id": "phone-mixed", + "kind": "mixed", + "text": "Mobile +65 8055 8024, hotline 1800 123 4567", + "expect": [ + { + "entity": "PHONE", + "value": "+65 8055 8024" + } + ], + "note": "Only the mobile number is a personal phone number." + } + ] +} diff --git a/third_party/sgpiirules/fixtures/postal.json b/third_party/sgpiirules/fixtures/postal.json new file mode 100644 index 0000000..2a200c2 --- /dev/null +++ b/third_party/sgpiirules/fixtures/postal.json @@ -0,0 +1,444 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "POSTAL", + "cases": [ + { + "id": "postal-tp-001", + "kind": "true_positive", + "text": "Blk 123 Ang Mo Kio Avenue 3, Singapore 670507", + "expect": [ + { + "entity": "POSTAL", + "value": "670507" + } + ] + }, + { + "id": "postal-tp-002", + "kind": "true_positive", + "text": "10 Anson Road, SINGAPORE 799154", + "expect": [ + { + "entity": "POSTAL", + "value": "799154" + } + ] + }, + { + "id": "postal-tp-003", + "kind": "true_positive", + "text": "Address: Paya Lebar Road, Singapore, 092682", + "expect": [ + { + "entity": "POSTAL", + "value": "092682" + } + ] + }, + { + "id": "postal-tp-004", + "kind": "true_positive", + "text": "Ship to Tampines Street 81, Singapore 429912", + "expect": [ + { + "entity": "POSTAL", + "value": "429912" + } + ] + }, + { + "id": "postal-tp-005", + "kind": "true_positive", + "text": "Bedok North Avenue 1 S(126458)", + "expect": [ + { + "entity": "POSTAL", + "value": "126458" + } + ] + }, + { + "id": "postal-tp-006", + "kind": "true_positive", + "text": "Jurong West Street 52 S274870", + "expect": [ + { + "entity": "POSTAL", + "value": "274870" + } + ] + }, + { + "id": "postal-tp-007", + "kind": "true_positive", + "text": "Postal code: 729886", + "expect": [ + { + "entity": "POSTAL", + "value": "729886" + } + ] + }, + { + "id": "postal-tp-008", + "kind": "true_positive", + "text": "postal_code: \"453302\"", + "expect": [ + { + "entity": "POSTAL", + "value": "453302" + } + ] + }, + { + "id": "postal-tp-009", + "kind": "true_positive", + "text": "{\"postalCode\":\"203959\"}", + "expect": [ + { + "entity": "POSTAL", + "value": "203959" + } + ] + }, + { + "id": "postal-tp-010", + "kind": "true_positive", + "text": "postcode=159311", + "expect": [ + { + "entity": "POSTAL", + "value": "159311" + } + ] + }, + { + "id": "postal-tp-011", + "kind": "true_positive", + "text": "Zip: 197712", + "expect": [ + { + "entity": "POSTAL", + "value": "197712" + } + ] + }, + { + "id": "postal-tp-012", + "kind": "true_positive", + "text": "Postal Code 272481", + "expect": [ + { + "entity": "POSTAL", + "value": "272481" + } + ] + }, + { + "id": "postal-tp-013", + "kind": "true_positive", + "text": "Blk 123 Ang Mo Kio Avenue 3, Singapore 309030", + "expect": [ + { + "entity": "POSTAL", + "value": "309030" + } + ] + }, + { + "id": "postal-tp-014", + "kind": "true_positive", + "text": "10 Anson Road, SINGAPORE 527587", + "expect": [ + { + "entity": "POSTAL", + "value": "527587" + } + ] + }, + { + "id": "postal-tp-015", + "kind": "true_positive", + "text": "Address: Paya Lebar Road, Singapore, 530741", + "expect": [ + { + "entity": "POSTAL", + "value": "530741" + } + ] + }, + { + "id": "postal-tp-016", + "kind": "true_positive", + "text": "Ship to Tampines Street 81, Singapore 780824", + "expect": [ + { + "entity": "POSTAL", + "value": "780824" + } + ] + }, + { + "id": "postal-tp-017", + "kind": "true_positive", + "text": "Bedok North Avenue 1 S(308746)", + "expect": [ + { + "entity": "POSTAL", + "value": "308746" + } + ] + }, + { + "id": "postal-tp-018", + "kind": "true_positive", + "text": "Jurong West Street 52 S269743", + "expect": [ + { + "entity": "POSTAL", + "value": "269743" + } + ] + }, + { + "id": "postal-tp-019", + "kind": "true_positive", + "text": "Postal code: 134225", + "expect": [ + { + "entity": "POSTAL", + "value": "134225" + } + ] + }, + { + "id": "postal-tp-020", + "kind": "true_positive", + "text": "postal_code: \"219481\"", + "expect": [ + { + "entity": "POSTAL", + "value": "219481" + } + ] + }, + { + "id": "postal-tp-021", + "kind": "true_positive", + "text": "{\"postalCode\":\"465137\"}", + "expect": [ + { + "entity": "POSTAL", + "value": "465137" + } + ] + }, + { + "id": "postal-tp-022", + "kind": "true_positive", + "text": "postcode=201767", + "expect": [ + { + "entity": "POSTAL", + "value": "201767" + } + ] + }, + { + "id": "postal-tp-023", + "kind": "true_positive", + "text": "Zip: 622403", + "expect": [ + { + "entity": "POSTAL", + "value": "622403" + } + ] + }, + { + "id": "postal-tp-024", + "kind": "true_positive", + "text": "Postal Code 373815", + "expect": [ + { + "entity": "POSTAL", + "value": "373815" + } + ] + }, + { + "id": "postal-hn-sector-00", + "kind": "hard_negative", + "text": "Singapore 009046", + "expect": [], + "note": "Sector 00 does not exist." + }, + { + "id": "postal-hn-sector-74", + "kind": "hard_negative", + "text": "Singapore 744747", + "expect": [], + "note": "Sector 74 is not used." + }, + { + "id": "postal-hn-sector-83", + "kind": "hard_negative", + "text": "Singapore 833243", + "expect": [], + "note": "Sectors stop at 82." + }, + { + "id": "postal-hn-sector-99", + "kind": "hard_negative", + "text": "Postal code: 991711", + "expect": [], + "note": "Sectors stop at 82." + }, + { + "id": "postal-hn-no-clue-order", + "kind": "hard_negative", + "text": "Order 475104", + "expect": [], + "note": "Six digits with no address clue." + }, + { + "id": "postal-hn-no-clue-amount", + "kind": "hard_negative", + "text": "Amount: 817714", + "expect": [], + "note": "Six digits with no address clue." + }, + { + "id": "postal-hn-no-clue-code", + "kind": "hard_negative", + "text": "const timeoutMs = 120045", + "expect": [], + "note": "A numeric constant." + }, + { + "id": "postal-hn-seven-digits", + "kind": "hard_negative", + "text": "Singapore 5344437", + "expect": [], + "note": "Seven digits." + }, + { + "id": "postal-hn-five-digits", + "kind": "hard_negative", + "text": "Singapore 52012", + "expect": [], + "note": "Five digits." + }, + { + "id": "postal-hn-eight-digits", + "kind": "hard_negative", + "text": "Singapore 52012345", + "expect": [], + "note": "Eight digits, and not a phone number (leading 5)." + }, + { + "id": "postal-hn-glued-to-word", + "kind": "hard_negative", + "text": "Singapore587759", + "expect": [], + "note": "No space or comma after \"Singapore\"." + }, + { + "id": "postal-hn-year", + "kind": "hard_negative", + "text": "Singapore 2026 budget", + "expect": [], + "note": "A year after the clue." + }, + { + "id": "postal-hn-phone", + "kind": "hard_negative", + "text": "Singapore 6123 4567", + "expect": [ + { + "entity": "PHONE", + "value": "6123 4567" + } + ], + "note": "A phone number after the clue, reported as PHONE." + }, + { + "id": "postal-hn-phone-country-code", + "kind": "hard_negative", + "text": "Singapore +65 9123 4567", + "expect": [ + { + "entity": "PHONE", + "value": "+65 9123 4567" + } + ], + "note": "A phone number with country code." + }, + { + "id": "postal-hn-nric-like", + "kind": "hard_negative", + "text": "Ref S520123A", + "expect": [], + "note": "S, six digits and a letter: neither a postal code nor an NRIC." + }, + { + "id": "postal-hn-nric", + "kind": "hard_negative", + "text": "Member S1234567D", + "expect": [ + { + "entity": "NRIC", + "value": "S1234567D" + } + ], + "note": "An NRIC (synthetic example from VALIDATORS.md): S then seven digits." + }, + { + "id": "postal-hn-zipped", + "kind": "hard_negative", + "text": "zipped 159802 bytes", + "expect": [], + "note": "\"zipped\" is not the zip clue." + }, + { + "id": "postal-hn-blog-post", + "kind": "hard_negative", + "text": "Blog post 249717", + "expect": [], + "note": "\"post\" alone is not a clue." + }, + { + "id": "postal-hn-malaysia", + "kind": "hard_negative", + "text": "Kuala Lumpur 50088, Malaysia", + "expect": [], + "note": "A five-digit Malaysian postcode." + }, + { + "id": "postal-hn-us-zip", + "kind": "hard_negative", + "text": "ZIP 94043-1351", + "expect": [], + "note": "A US ZIP+4 code." + }, + { + "id": "postal-hn-lowercase-s", + "kind": "hard_negative", + "text": "s(366193)", + "expect": [], + "note": "Lowercase s( is a function call, not the S( address clue." + }, + { + "id": "postal-mixed", + "kind": "mixed", + "text": "Deliver to Singapore 275204, order 099955", + "expect": [ + { + "entity": "POSTAL", + "value": "275204" + } + ], + "note": "Only the digits after the clue are a postal code." + } + ] +} diff --git a/third_party/sgpiirules/fixtures/unit.json b/third_party/sgpiirules/fixtures/unit.json new file mode 100644 index 0000000..8f68bdd --- /dev/null +++ b/third_party/sgpiirules/fixtures/unit.json @@ -0,0 +1,444 @@ +{ + "$schema": "../schema/fixtures.schema.json", + "entity": "UNIT", + "cases": [ + { + "id": "unit-tp-001", + "kind": "true_positive", + "text": "Blk 123 Ang Mo Kio Avenue 3 #08-4052", + "expect": [ + { + "entity": "UNIT", + "value": "#08-4052" + } + ] + }, + { + "id": "unit-tp-002", + "kind": "true_positive", + "text": "10 Anson Road #10-61, International Plaza", + "expect": [ + { + "entity": "UNIT", + "value": "#10-61" + } + ] + }, + { + "id": "unit-tp-003", + "kind": "true_positive", + "text": "Unit: #38-429", + "expect": [ + { + "entity": "UNIT", + "value": "#38-429" + } + ] + }, + { + "id": "unit-tp-004", + "kind": "true_positive", + "text": "{\"unit\": \"#19-3440\"}", + "expect": [ + { + "entity": "UNIT", + "value": "#19-3440" + } + ] + }, + { + "id": "unit-tp-005", + "kind": "true_positive", + "text": "address_line_2,#32-14", + "expect": [ + { + "entity": "UNIT", + "value": "#32-14" + } + ] + }, + { + "id": "unit-tp-006", + "kind": "true_positive", + "text": "Deliver to #25-87A before noon.", + "expect": [ + { + "entity": "UNIT", + "value": "#25-87A" + } + ] + }, + { + "id": "unit-tp-007", + "kind": "true_positive", + "text": "Tampines Mall #35-873", + "expect": [ + { + "entity": "UNIT", + "value": "#35-873" + } + ] + }, + { + "id": "unit-tp-008", + "kind": "true_positive", + "text": "#33-436A Paya Lebar Square", + "expect": [ + { + "entity": "UNIT", + "value": "#33-436A" + } + ] + }, + { + "id": "unit-tp-009", + "kind": "true_positive", + "text": "Blk 123 Ang Mo Kio Avenue 3 #17-308", + "expect": [ + { + "entity": "UNIT", + "value": "#17-308" + } + ] + }, + { + "id": "unit-tp-010", + "kind": "true_positive", + "text": "10 Anson Road #09-0571, International Plaza", + "expect": [ + { + "entity": "UNIT", + "value": "#09-0571" + } + ] + }, + { + "id": "unit-tp-011", + "kind": "true_positive", + "text": "Unit: #07-4818", + "expect": [ + { + "entity": "UNIT", + "value": "#07-4818" + } + ] + }, + { + "id": "unit-tp-012", + "kind": "true_positive", + "text": "{\"unit\": \"#02-93\"}", + "expect": [ + { + "entity": "UNIT", + "value": "#02-93" + } + ] + }, + { + "id": "unit-tp-013", + "kind": "true_positive", + "text": "address_line_2,#30-3577", + "expect": [ + { + "entity": "UNIT", + "value": "#30-3577" + } + ] + }, + { + "id": "unit-tp-014", + "kind": "true_positive", + "text": "Deliver to #12-62 before noon.", + "expect": [ + { + "entity": "UNIT", + "value": "#12-62" + } + ] + }, + { + "id": "unit-tp-015", + "kind": "true_positive", + "text": "Tampines Mall #13-598", + "expect": [ + { + "entity": "UNIT", + "value": "#13-598" + } + ] + }, + { + "id": "unit-tp-016", + "kind": "true_positive", + "text": "#20-66 Paya Lebar Square", + "expect": [ + { + "entity": "UNIT", + "value": "#20-66" + } + ] + }, + { + "id": "unit-tp-017", + "kind": "true_positive", + "text": "Blk 123 Ang Mo Kio Avenue 3 #21-350C", + "expect": [ + { + "entity": "UNIT", + "value": "#21-350C" + } + ] + }, + { + "id": "unit-tp-018", + "kind": "true_positive", + "text": "10 Anson Road #19-4348, International Plaza", + "expect": [ + { + "entity": "UNIT", + "value": "#19-4348" + } + ] + }, + { + "id": "unit-tp-019", + "kind": "true_positive", + "text": "Unit: #B3-9861", + "expect": [ + { + "entity": "UNIT", + "value": "#B3-9861" + } + ] + }, + { + "id": "unit-tp-020", + "kind": "true_positive", + "text": "{\"unit\": \"#31-233\"}", + "expect": [ + { + "entity": "UNIT", + "value": "#31-233" + } + ] + }, + { + "id": "unit-tp-021", + "kind": "true_positive", + "text": "address_line_2,#38-3521", + "expect": [ + { + "entity": "UNIT", + "value": "#38-3521" + } + ] + }, + { + "id": "unit-tp-022", + "kind": "true_positive", + "text": "Deliver to #36-745 before noon.", + "expect": [ + { + "entity": "UNIT", + "value": "#36-745" + } + ] + }, + { + "id": "unit-tp-with-postal-1", + "kind": "true_positive", + "text": "Blk 456 Bedok North Street 1 #18-2376 Singapore 460296", + "expect": [ + { + "entity": "UNIT", + "value": "#18-2376" + }, + { + "entity": "POSTAL", + "value": "460296" + } + ], + "note": "A full address: the unit number and the postal code are both reported." + }, + { + "id": "unit-tp-with-postal-2", + "kind": "true_positive", + "text": "Blk 456 Bedok North Street 1 #40-5236C Singapore 567267", + "expect": [ + { + "entity": "UNIT", + "value": "#40-5236C" + }, + { + "entity": "POSTAL", + "value": "567267" + } + ], + "note": "A full address: the unit number and the postal code are both reported." + }, + { + "id": "unit-hn-issue-ref", + "kind": "hard_negative", + "text": "Fixed in #12", + "expect": [], + "note": "An issue reference: no hyphen and unit." + }, + { + "id": "unit-hn-four-digits", + "kind": "hard_negative", + "text": "See #1234", + "expect": [], + "note": "Four digits with no hyphen." + }, + { + "id": "unit-hn-one-digit-unit", + "kind": "hard_negative", + "text": "Step #12-3", + "expect": [], + "note": "Units have at least two digits." + }, + { + "id": "unit-hn-five-digit-unit", + "kind": "hard_negative", + "text": "Ticket #12-34567", + "expect": [], + "note": "Units have at most four digits." + }, + { + "id": "unit-hn-digit-run", + "kind": "hard_negative", + "text": "Build #12345", + "expect": [], + "note": "A run of digits." + }, + { + "id": "unit-hn-colour", + "kind": "hard_negative", + "text": "color: #ff0000;", + "expect": [], + "note": "A CSS colour." + }, + { + "id": "unit-hn-hex-pair", + "kind": "hard_negative", + "text": "Mask #1a-2b", + "expect": [], + "note": "Hex digits, not a floor and unit." + }, + { + "id": "unit-hn-slash", + "kind": "hard_negative", + "text": "Item #05/123", + "expect": [], + "note": "A slash instead of a hyphen." + }, + { + "id": "unit-hn-issue-range", + "kind": "hard_negative", + "text": "Closes #12-#15", + "expect": [], + "note": "A range of issue numbers." + }, + { + "id": "unit-hn-floor-00", + "kind": "hard_negative", + "text": "Room #00-12", + "expect": [], + "note": "There is no floor 00." + }, + { + "id": "unit-hn-single-digit-floor", + "kind": "hard_negative", + "text": "Room #5-123", + "expect": [], + "note": "Floors are written with two digits." + }, + { + "id": "unit-hn-basement-0", + "kind": "hard_negative", + "text": "Level #B0-12", + "expect": [], + "note": "Basements are B1 and below." + }, + { + "id": "unit-hn-letter-floor", + "kind": "hard_negative", + "text": "Grid #A1-23", + "expect": [], + "note": "A is not a basement or a floor." + }, + { + "id": "unit-hn-en-dash", + "kind": "hard_negative", + "text": "Range #12–345", + "expect": [], + "note": "An en dash, not a hyphen." + }, + { + "id": "unit-hn-underscore", + "kind": "hard_negative", + "text": "Key #12_345", + "expect": [], + "note": "An underscore, not a hyphen." + }, + { + "id": "unit-hn-spaced", + "kind": "hard_negative", + "text": "Score #12 - 345", + "expect": [], + "note": "Spaces around the hyphen." + }, + { + "id": "unit-hn-lowercase-suffix", + "kind": "hard_negative", + "text": "Tag #12-34ab", + "expect": [], + "note": "Letters after the unit are lowercase and run on." + }, + { + "id": "unit-hn-double-hash", + "kind": "hard_negative", + "text": "Heading ## 12-345", + "expect": [], + "note": "A Markdown heading." + }, + { + "id": "unit-hn-leading-hyphen", + "kind": "hard_negative", + "text": "Offset #-12-345", + "expect": [], + "note": "A hyphen before the floor." + }, + { + "id": "unit-hn-date", + "kind": "hard_negative", + "text": "Release #2026-10-05", + "expect": [], + "note": "A date after a hash." + }, + { + "id": "unit-hn-phone", + "kind": "hard_negative", + "text": "Hotline #9123-4567", + "expect": [ + { + "entity": "PHONE", + "value": "9123-4567" + } + ], + "note": "A phone number after a hash, reported as PHONE." + }, + { + "id": "unit-mixed", + "kind": "mixed", + "text": "Unit #35-0521, see ticket #1234", + "expect": [ + { + "entity": "UNIT", + "value": "#35-0521" + } + ], + "note": "Only the floor-unit number is a unit number." + } + ] +} diff --git a/third_party/sgpiirules/schema/detectors.schema.json b/third_party/sgpiirules/schema/detectors.schema.json new file mode 100644 index 0000000..b1d4b3c --- /dev/null +++ b/third_party/sgpiirules/schema/detectors.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/makoydev/sg-pii-rules/schema/detectors.schema.json", + "title": "sg-pii-rules detectors", + "type": "object", + "additionalProperties": false, + "required": ["version", "detectors"], + "properties": { + "$schema": { "type": "string" }, + "version": { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$" + }, + "detectors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "entity", "description", "pattern", "sources"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, + "entity": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" }, + "description": { "type": "string", "minLength": 1 }, + "pattern": { "type": "string", "minLength": 1 }, + "validator": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, + "sources": { + "type": "array", + "items": { "type": "string", "pattern": "^https://" } + } + } + } + } + } +} diff --git a/third_party/sgpiirules/schema/fixtures.schema.json b/third_party/sgpiirules/schema/fixtures.schema.json new file mode 100644 index 0000000..e796a7e --- /dev/null +++ b/third_party/sgpiirules/schema/fixtures.schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/makoydev/sg-pii-rules/schema/fixtures.schema.json", + "title": "sg-pii-rules conformance fixtures", + "type": "object", + "additionalProperties": false, + "required": ["entity", "cases"], + "properties": { + "$schema": { "type": "string" }, + "entity": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" }, + "cases": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "kind", "text", "expect"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9-]+$" }, + "kind": { "enum": ["true_positive", "hard_negative", "mixed"] }, + "text": { "type": "string" }, + "expect": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["entity", "value"], + "properties": { + "entity": { "type": "string" }, + "value": { "type": "string", "minLength": 1 } + } + } + }, + "note": { "type": "string" } + } + } + } + } +} From 71d0465f558c5468b4b1f947f0c156a88ec3a004 Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:32:22 +0800 Subject: [PATCH 03/13] feat: placeholders, encrypted vault and YAML policy (D4) (#6) - internal/protect: a session per request numbers placeholders per entity (), reuses them for the same value across messages, remembers values only by a keyed hash, redacts as [REDACTED_CARD] or refuses with BlockedError, following a strictly parsed YAML policy. Restore only replaces placeholders the same session issued. - internal/vault: AES-256-GCM in memory under a per-process random key, each value bound to its key, deleted on Close, expired after 10 minutes; prints as redacted. - Default policy: tokenise everything except CARD (redact); denied purposes listed for D5. - go.yaml.in/yaml/v3 v3.0.5 (maintained fork; gopkg.in/yaml.v3 is archived), the one module ADR 0001 allowed. ADR 0004. - Deliberate breaks: session-less vault keys, no placeholder reuse, redact restored; each failed a test; restored from backup. Closes makoydev/vetted#34 Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CONTROLS.md | 2 +- README.md | 2 +- THREAT_MODEL.md | 4 +- docs/HOW-THIS-WAS-BUILT.md | 2 + docs/adr/0004-policy-file-and-placeholders.md | 30 ++++ go.mod | 2 + go.sum | 2 + internal/protect/default-policy.yaml | 24 +++ internal/protect/policy.go | 76 ++++++++++ internal/protect/protect_test.go | 138 ++++++++++++++++++ internal/protect/session.go | 136 +++++++++++++++++ internal/vault/vault.go | 114 +++++++++++++++ internal/vault/vault_test.go | 84 +++++++++++ 14 files changed, 613 insertions(+), 4 deletions(-) create mode 100644 docs/adr/0004-policy-file-and-placeholders.md create mode 100644 go.sum create mode 100644 internal/protect/default-policy.yaml create mode 100644 internal/protect/policy.go create mode 100644 internal/protect/protect_test.go create mode 100644 internal/protect/session.go create mode 100644 internal/vault/vault.go create mode 100644 internal/vault/vault_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4550970..c4f2008 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,4 +11,5 @@ All notable changes to this project are documented here. The format follows [Kee - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. - Vetted reviews every pull request (shadow mode until 2026-10-14). - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases. +- Placeholders and the vault (`internal/protect`, `internal/vault`, issue D4): personal data is replaced per request with numbered placeholders such as `` (the same value gets the same placeholder), redacted as `[REDACTED_CARD]`, or the request is blocked, following a YAML policy (`internal/protect/default-policy.yaml`). Values are kept encrypted in memory with AES-256-GCM for at most 10 minutes and restored only into the same request's answer (ADR 0004). First third-party module: `go.yaml.in/yaml/v3` v3.0.5. - sg-pii-rules `v0.2.0-rc.1` vendored in `third_party/sgpiirules` and embedded in the binary; a test re-checks every file against `SHA256SUMS` (ADR 0003). `scripts/vendor-sg-pii-rules.sh ` updates it. diff --git a/CONTROLS.md b/CONTROLS.md index 76af4e9..f7c6dca 100644 --- a/CONTROLS.md +++ b/CONTROLS.md @@ -4,7 +4,7 @@ Each Discreet feature, the principle or obligation it supports, and the source. | Feature | Supports | Source | Status | | --- | --- | --- | --- | -| Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection in place (D3, 369/369 shared cases); replacement planned (D4); mapping pending | +| Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection (D3, 369/369 shared cases) and replacement with placeholders (D4) in place; mapping pending | | Purpose declared in a header; automated eligibility decisions refused | Human accountability for decisions about people | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | Planned (D5); mapping pending | | Hash-chained audit log with `verify` | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | Planned (D7); mapping pending | | Daily budget, access token, rate limit | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | Planned (D6); mapping pending | diff --git a/README.md b/README.md index 223d955..83e9624 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`. -**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the command-line skeleton and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. +**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the command-line skeleton, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. ## Build and check diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index d1918d5..b27e81a 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -21,8 +21,8 @@ | --- | --- | --- | | Personal data the detectors miss reaches the model | Shared conformance suite with hard negatives (369/369 pass in Go); measured recall published in EVALS | D3 (done), D9 | | Rules silently changed or swapped | Vendored release checked against `SHA256SUMS` on every test run; embedded in the binary (ADR 0003) | D3 (done) | -| The vault leaks values through logs or errors | Values encrypted in memory, short expiry, a test that scans logs for planted values | D4 | -| The model invents placeholders to extract other values | Only placeholders issued for that request are rehydrated | D4 | +| The vault leaks values through logs or errors | Values encrypted in memory (AES-256-GCM, per-process key), bound to their key, deleted when the request ends, expired after 10 minutes; vault and session print as redacted (tests) | D4 (done); log scan in D5 | +| The model invents placeholders to extract other values | Only placeholders issued for that request are rehydrated (test) | D4 (done) | | Someone edits the audit log to hide a request | SHA-256 hash chain; `audit verify` names the first broken record | D7 | | Short identifiers are guessed from audit hashes | HMAC-SHA-256 with a server secret instead of plain hashes | D7 | | The public demo is used to spend money | Mock model unless a valid access token is present; daily budget; rate limit | D6, D10 | diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md index ad6c708..2990d3d 100644 --- a/docs/HOW-THIS-WAS-BUILT.md +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -27,6 +27,8 @@ | 2026-10-04 | Deliberate break: `discreet serve` made to exit 0 | The test failed (`exit code = 0, want 1`); restored from backup | | 2026-10-05 | sg-pii-rules `v0.2.0-rc.1` vendored by script from the release tag | Every file matched the release's `SHA256SUMS` | | 2026-10-05 | All 369 shared conformance cases through the Go engine | 369 / 369 pass, first run | +| 2026-10-05 | Go YAML libraries (Go module proxy, GitHub) | `gopkg.in/yaml.v3` archived since April 2025; used the maintained fork `go.yaml.in/yaml/v3` v3.0.5 | +| 2026-10-05 | Three deliberate breaks of the placeholder session: vault keys without the session, no placeholder reuse, redact restored | Each failed at least one test; restored from backup | | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite | ## What the AI got wrong, and how it was caught diff --git a/docs/adr/0004-policy-file-and-placeholders.md b/docs/adr/0004-policy-file-and-placeholders.md new file mode 100644 index 0000000..5136adc --- /dev/null +++ b/docs/adr/0004-policy-file-and-placeholders.md @@ -0,0 +1,30 @@ +# 0004. Placeholders per request, an encrypted in-memory vault, and a YAML policy + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-05 + +## Context + +Discreet replaces personal data before a request reaches the model and puts it back in the answer (issue D4). Plain redaction ("call [REDACTED] about [REDACTED]") makes answers useless; but anything that can put values back must store them somewhere, and that store becomes the most sensitive thing in the system. The brief asks for policy as YAML, with `tokenise`, `redact` or `block` per entity. + +## Options + +1. **Numbered placeholders per request (``), values encrypted in memory for minutes.** The model can still tell two people apart and refer back to them. +2. **Format-preserving fake values** (a different valid-looking NRIC). Reads more naturally, but a fake that looks real can be mistaken for real data downstream, and mapping back is ambiguous if the model rewrites it. +3. **A persistent vault (SQLite).** Survives restarts, but stores encrypted personal data on disk for no benefit to non-streaming requests that last seconds. + +## Decision + +Option 1. + +- A **session** per request numbers placeholders per entity and reuses one for the same value anywhere in the request. It remembers values only by an HMAC-SHA-256 with a per-session random key, never in plain text. +- The **vault** keeps values encrypted with AES-256-GCM under a random key that exists only in this process, binds each value to its key (as additional data), deletes them when the request ends, and expires anything left after 10 minutes. Nothing in it formats or logs a value. +- **Restore** only replaces placeholders the same session issued, so a model that invents `` can't fish for another request's values. +- The **policy** is YAML, parsed strictly: an unknown key or action (`tokenize`) is an error, so a typo can't weaken protection. The parser is `go.yaml.in/yaml/v3` v3.0.5, the YAML organisation's maintained fork; the original `gopkg.in/yaml.v3` has been archived since April 2025 (checked 2026-10-05). It is the first third-party module (ADR 0001 allowed exactly this one). +- Default policy: everything `tokenise`, except `CARD: redact` (a model never needs a card number back). + +## Consequences + +- A restart loses in-flight values; for non-streaming requests that only fails requests already running. +- The model sees placeholders, which can make answers slightly stilted ("Dear "); the demo page shows this honestly. +- `block` refuses the whole request on the first blocked entity; values already placed in the vault for that request are deleted when the gateway closes the session. diff --git a/go.mod b/go.mod index f577e3c..7669bcd 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/makoydev/discreet go 1.27 + +require go.yaml.in/yaml/v3 v3.0.5 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8675e7f --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= diff --git a/internal/protect/default-policy.yaml b/internal/protect/default-policy.yaml new file mode 100644 index 0000000..a887ef7 --- /dev/null +++ b/internal/protect/default-policy.yaml @@ -0,0 +1,24 @@ +# Discreet's default policy: what happens to each kind of personal data +# before a request reaches the model. +# tokenise replace with a placeholder such as ; put back in the answer +# redact replace with [REDACTED_CARD]; never put back +# block refuse the whole request +version: 1 +default_action: tokenise +entities: + NRIC: tokenise # PDPC: don't disclose NRIC numbers you don't need to + NRIC_LIKE: tokenise # a mistyped NRIC is still personal data + PHONE: tokenise + EMAIL: tokenise + CARD: redact # a model never needs the card number back + POSTAL: tokenise + UNIT: tokenise + DOB: tokenise +# Purposes refused outright (checked in issue D5). Matching ignores case and +# punctuation. Discreet never takes part in decisions about people's +# eligibility for benefits, credit or services. +denied_purposes: + - automated eligibility decision + - eligibility determination + - benefits decision + - credit decision diff --git a/internal/protect/policy.go b/internal/protect/policy.go new file mode 100644 index 0000000..7409d50 --- /dev/null +++ b/internal/protect/policy.go @@ -0,0 +1,76 @@ +// Package protect replaces personal data with placeholders before a request +// reaches the model and puts it back in the answer, following a policy. +package protect + +import ( + "bytes" + _ "embed" + "fmt" + + "go.yaml.in/yaml/v3" +) + +// Action says what happens to one kind of personal data. +type Action string + +const ( + Tokenise Action = "tokenise" // placeholder, restored in the answer + Redact Action = "redact" // fixed marker, never restored + Block Action = "block" // the whole request is refused +) + +// Policy is the parsed policy file. +type Policy struct { + Version int `yaml:"version"` + DefaultAction Action `yaml:"default_action"` + Entities map[string]Action `yaml:"entities"` + DeniedPurposes []string `yaml:"denied_purposes"` +} + +//go:embed default-policy.yaml +var defaultPolicy []byte + +// DefaultPolicy returns the built-in policy. +func DefaultPolicy() *Policy { + p, err := ParsePolicy(defaultPolicy) + if err != nil { + panic("protect: built-in policy is invalid: " + err.Error()) + } + return p +} + +// ParsePolicy reads a policy file. Unknown keys and unknown actions are +// errors, so a typo can't silently weaken protection. +func ParsePolicy(data []byte) (*Policy, error) { + var p Policy + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&p); err != nil { + return nil, fmt.Errorf("policy: %w", err) + } + if p.Version != 1 { + return nil, fmt.Errorf("policy: unsupported version %d", p.Version) + } + if p.DefaultAction == "" { + p.DefaultAction = Tokenise + } + for entity, a := range p.Entities { + if !a.valid() { + return nil, fmt.Errorf("policy: entity %s has unknown action %q", entity, a) + } + } + if !p.DefaultAction.valid() { + return nil, fmt.Errorf("policy: unknown default_action %q", p.DefaultAction) + } + return &p, nil +} + +func (a Action) valid() bool { return a == Tokenise || a == Redact || a == Block } + +// ActionFor returns the action for an entity, or the default. +func (p *Policy) ActionFor(entity string) Action { + if a, ok := p.Entities[entity]; ok { + return a + } + return p.DefaultAction +} diff --git a/internal/protect/protect_test.go b/internal/protect/protect_test.go new file mode 100644 index 0000000..2b6f9a0 --- /dev/null +++ b/internal/protect/protect_test.go @@ -0,0 +1,138 @@ +package protect + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/makoydev/discreet/internal/detect" + "github.com/makoydev/discreet/internal/vault" +) + +// Synthetic values only: S1234567D is the worked example in sg-pii-rules' +// VALIDATORS.md; T0000001E is built from patterned digits. +func setup(t *testing.T, policy *Policy) (*detect.Engine, *Session, *vault.Vault) { + t.Helper() + engine, err := detect.Default() + if err != nil { + t.Fatal(err) + } + v, err := vault.New(time.Minute) + if err != nil { + t.Fatal(err) + } + return engine, NewSession(policy, v), v +} + +func protect(t *testing.T, e *detect.Engine, s *Session, text string) string { + t.Helper() + out, err := s.Protect(text, e.Detect(text)) + if err != nil { + t.Fatal(err) + } + return out +} + +func TestSameValueSamePlaceholder(t *testing.T) { + e, s, _ := setup(t, DefaultPolicy()) + nric2 := "T0000001" + "E" + got := protect(t, e, s, "Call S1234567D on 9123 4567. S1234567D and "+nric2+" are related.") + want := "Call on . and are related." + if got != want { + t.Errorf("got %q\nwant %q", got, want) + } + if c := s.Counts(); c["NRIC"] != 3 || c["PHONE"] != 1 { + t.Errorf("counts = %v", c) + } +} + +func TestPlaceholdersAreConsistentAcrossMessages(t *testing.T) { + e, s, _ := setup(t, DefaultPolicy()) + a := protect(t, e, s, "Member S1234567D") + b := protect(t, e, s, "Remind S1234567D tomorrow") + if a != "Member " || b != "Remind tomorrow" { + t.Errorf("got %q and %q", a, b) + } +} + +func TestRestore(t *testing.T) { + e, s, _ := setup(t, DefaultPolicy()) + protect(t, e, s, "Call S1234567D on 9123 4567") + got := s.Restore("Draft: Hi , we'll call . Ref , .") + want := "Draft: Hi S1234567D, we'll call 9123 4567. Ref , ." + if got != want { + t.Errorf("got %q\nwant %q", got, want) + } +} + +func TestRestoreIgnoresOtherSessions(t *testing.T) { + e, s1, v := setup(t, DefaultPolicy()) + protect(t, e, s1, "Member S1234567D") + s2 := NewSession(DefaultPolicy(), v) + if got := s2.Restore(""); got != "" { + t.Errorf("another session's value leaked: %q", got) + } +} + +func TestRedactIsNeverRestored(t *testing.T) { + e, s, _ := setup(t, DefaultPolicy()) + card := "4111 1111 1111 1111" // a published test card number + got := protect(t, e, s, "Card "+card+" was charged") + if got != "Card [REDACTED_CARD] was charged" { + t.Errorf("got %q", got) + } + if r := s.Restore(got); strings.Contains(r, "4111") { + t.Errorf("redacted value came back: %q", r) + } +} + +func TestBlock(t *testing.T) { + p, err := ParsePolicy([]byte("version: 1\nentities:\n NRIC: block\n")) + if err != nil { + t.Fatal(err) + } + e, s, v := setup(t, p) + _, err = s.Protect("Member S1234567D", e.Detect("Member S1234567D")) + var blocked *BlockedError + if !errors.As(err, &blocked) || blocked.Entity != "NRIC" { + t.Fatalf("err = %v, want a BlockedError for NRIC", err) + } + if v.Len() != 0 { + t.Error("a blocked request left values in the vault") + } +} + +func TestCloseClearsTheVault(t *testing.T) { + e, s, v := setup(t, DefaultPolicy()) + protect(t, e, s, "S1234567D, 9123 4567") + s.Close() + if v.Len() != 0 || s.Restore("") != "" { + t.Error("values still available after Close") + } +} + +func TestSessionNeverPrintsValues(t *testing.T) { + e, s, _ := setup(t, DefaultPolicy()) + protect(t, e, s, "Member S1234567D") + if out := fmt.Sprintf("%v %+v", s, s); strings.Contains(out, "S1234567D") { + t.Errorf("printing a session leaked a value: %s", out) + } +} + +func TestParsePolicyRejectsMistakes(t *testing.T) { + for _, bad := range []string{ + "version: 1\nentities:\n NRIC: tokenize\n", // American spelling isn't an action + "version: 1\nentitites:\n NRIC: block\n", // typo in a key + "version: 2\n", + "version: 1\ndefault_action: allow\n", + } { + if _, err := ParsePolicy([]byte(bad)); err == nil { + t.Errorf("accepted %q", bad) + } + } + if p := DefaultPolicy(); p.ActionFor("CARD") != Redact || p.ActionFor("SOMETHING_NEW") != Tokenise { + t.Error("default policy actions are not as documented") + } +} diff --git a/internal/protect/session.go b/internal/protect/session.go new file mode 100644 index 0000000..1a05a6a --- /dev/null +++ b/internal/protect/session.go @@ -0,0 +1,136 @@ +package protect + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "sort" + "strings" + + "github.com/makoydev/discreet/internal/detect" + "github.com/makoydev/discreet/internal/vault" +) + +// BlockedError means the policy refuses a request containing an entity. +type BlockedError struct{ Entity string } + +func (e *BlockedError) Error() string { + return fmt.Sprintf("request contains %s, which the policy blocks", e.Entity) +} + +// Session protects one request. Placeholders are numbered per entity and +// reused for the same value anywhere in the request, so the model can tell +// two people apart without seeing either. Values live only in the vault, +// encrypted; the session remembers them by a keyed hash. +type Session struct { + id string + policy *Policy + vault *vault.Vault + hashKey []byte + byHash map[string]string // keyed hash of entity+value → placeholder + next map[string]int // entity → last number issued + issued []string // placeholders this session created + found map[string]int // entity → matches seen, whatever the action +} + +// NewSession starts protecting one request. +func NewSession(p *Policy, v *vault.Vault) *Session { + id, key := make([]byte, 12), make([]byte, 32) + if _, err := rand.Read(id); err != nil { + panic(err) + } + if _, err := rand.Read(key); err != nil { + panic(err) + } + return &Session{ + id: hex.EncodeToString(id), policy: p, vault: v, hashKey: key, + byHash: map[string]string{}, next: map[string]int{}, found: map[string]int{}, + } +} + +func (s *Session) vaultKey(placeholder string) string { return s.id + "/" + placeholder } + +func (s *Session) hash(entity, value string) string { + m := hmac.New(sha256.New, s.hashKey) + m.Write([]byte(entity + "\x00" + value)) + return string(m.Sum(nil)) +} + +// Protect replaces each match in text according to the policy. Matches +// must come from detect.Engine.Detect on the same text. +func (s *Session) Protect(text string, matches []detect.Match) (string, error) { + var b strings.Builder + pos := 0 + for _, m := range matches { + s.found[m.Entity]++ + var replacement string + switch s.policy.ActionFor(m.Entity) { + case Block: + return "", &BlockedError{Entity: m.Entity} + case Redact: + replacement = "[REDACTED_" + m.Entity + "]" + default: + replacement = s.placeholder(m.Entity, m.Value) + } + b.WriteString(text[pos:m.Start]) + b.WriteString(replacement) + pos = m.End + } + b.WriteString(text[pos:]) + return b.String(), nil +} + +func (s *Session) placeholder(entity, value string) string { + h := s.hash(entity, value) + if p, ok := s.byHash[h]; ok { + return p + } + s.next[entity]++ + p := fmt.Sprintf("<%s_%d>", entity, s.next[entity]) + s.vault.Put(s.vaultKey(p), value) + s.byHash[h] = p + s.issued = append(s.issued, p) + return p +} + +var placeholderPattern = regexp.MustCompile(`<[A-Z][A-Z_]*_\d+>`) + +// Restore puts real values back in place of this session's placeholders. +// Placeholders it didn't issue, for example ones a model made up, are left +// as they are, so a model can't fish for values from other requests. +func (s *Session) Restore(text string) string { + return placeholderPattern.ReplaceAllStringFunc(text, func(p string) string { + if v, ok := s.vault.Get(s.vaultKey(p)); ok { + return v + } + return p + }) +} + +// Counts returns how many matches of each entity were seen. +func (s *Session) Counts() map[string]int { + out := make(map[string]int, len(s.found)) + for k, v := range s.found { + out[k] = v + } + return out +} + +// Placeholders lists the placeholders issued, in order. +func (s *Session) Placeholders() []string { + out := append([]string(nil), s.issued...) + sort.Strings(out) + return out +} + +// Close deletes this session's values from the vault straight away. +func (s *Session) Close() { + keys := make([]string, len(s.issued)) + for i, p := range s.issued { + keys[i] = s.vaultKey(p) + } + s.vault.Delete(keys...) +} diff --git a/internal/vault/vault.go b/internal/vault/vault.go new file mode 100644 index 0000000..37ec056 --- /dev/null +++ b/internal/vault/vault.go @@ -0,0 +1,114 @@ +// Package vault keeps the real values behind placeholders for a short time, +// encrypted in memory with AES-256-GCM under a key that exists only in this +// process. Nothing in this package logs or formats a value. +package vault + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "sync" + "time" +) + +// DefaultTTL is how long a value is kept: long enough for one request. +const DefaultTTL = 10 * time.Minute + +type entry struct { + nonce, sealed []byte + expires time.Time +} + +// Vault maps keys to encrypted values. It is safe for concurrent use. +type Vault struct { + mu sync.Mutex + aead cipher.AEAD + entries map[string]entry + ttl time.Duration + now func() time.Time +} + +// New creates a vault with a fresh random 256-bit key. +func New(ttl time.Duration) (*Vault, error) { + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + return nil, err + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, err + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, err + } + return &Vault{aead: aead, entries: map[string]entry{}, ttl: ttl, now: time.Now}, nil +} + +// Put encrypts value under key. The key is bound to the ciphertext as +// additional data, so a sealed value can't be moved to another key. +func (v *Vault) Put(key, value string) { + nonce := make([]byte, v.aead.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + panic("vault: no randomness: " + err.Error()) + } + sealed := v.aead.Seal(nil, nonce, []byte(value), []byte(key)) + v.mu.Lock() + defer v.mu.Unlock() + v.entries[key] = entry{nonce, sealed, v.now().Add(v.ttl)} +} + +// Get decrypts the value under key, if it exists and hasn't expired. +func (v *Vault) Get(key string) (string, bool) { + v.mu.Lock() + e, ok := v.entries[key] + if ok && !v.now().Before(e.expires) { + delete(v.entries, key) + ok = false + } + v.mu.Unlock() + if !ok { + return "", false + } + plain, err := v.aead.Open(nil, e.nonce, e.sealed, []byte(key)) + if err != nil { + return "", false + } + return string(plain), true +} + +// Delete removes keys immediately, for example when a request ends. +func (v *Vault) Delete(keys ...string) { + v.mu.Lock() + defer v.mu.Unlock() + for _, k := range keys { + delete(v.entries, k) + } +} + +// Sweep removes expired entries and returns how many it removed. +func (v *Vault) Sweep() int { + v.mu.Lock() + defer v.mu.Unlock() + n := 0 + for k, e := range v.entries { + if !v.now().Before(e.expires) { + delete(v.entries, k) + n++ + } + } + return n +} + +// Len reports how many entries are stored, expired or not. +func (v *Vault) Len() int { + v.mu.Lock() + defer v.mu.Unlock() + return len(v.entries) +} + +// String never shows contents, so a vault printed by mistake leaks nothing. +func (v *Vault) String() string { return "vault{redacted}" } + +// GoString does the same for %#v. +func (v *Vault) GoString() string { return v.String() } diff --git a/internal/vault/vault_test.go b/internal/vault/vault_test.go new file mode 100644 index 0000000..651812f --- /dev/null +++ b/internal/vault/vault_test.go @@ -0,0 +1,84 @@ +package vault + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func newTest(t *testing.T) *Vault { + t.Helper() + v, err := New(time.Minute) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestPutGet(t *testing.T) { + v := newTest(t) + v.Put("r1/", "S1234567D") // synthetic example NRIC + got, ok := v.Get("r1/") + if !ok || got != "S1234567D" { + t.Fatalf("Get = %q, %v", got, ok) + } + if _, ok := v.Get("r2/"); ok { + t.Error("a key that was never stored was found") + } +} + +func TestValuesAreEncryptedAndNeverPrinted(t *testing.T) { + v := newTest(t) + v.Put("k", "S1234567D") + for _, e := range v.entries { + if strings.Contains(string(e.sealed), "S1234567D") { + t.Error("value stored in plain text") + } + } + for _, s := range []string{fmt.Sprint(v), fmt.Sprintf("%v %+v %#v", v, v, v)} { + if strings.Contains(s, "S1234567D") { + t.Errorf("formatting leaked the value: %s", s) + } + } +} + +func TestSealedValueIsBoundToItsKey(t *testing.T) { + v := newTest(t) + v.Put("a", "secret-a") + v.entries["b"] = v.entries["a"] // move the ciphertext to another key + if _, ok := v.Get("b"); ok { + t.Error("a sealed value opened under a different key") + } +} + +func TestExpiry(t *testing.T) { + v := newTest(t) + now := time.Now() + v.now = func() time.Time { return now } + v.Put("k", "value") + now = now.Add(59 * time.Second) + if _, ok := v.Get("k"); !ok { + t.Fatal("expired too early") + } + now = now.Add(time.Second) + if _, ok := v.Get("k"); ok { + t.Error("value still readable after its TTL") + } + v.Put("x", "1") + v.Put("y", "2") + now = now.Add(time.Minute) + if n := v.Sweep(); n != 2 || v.Len() != 0 { + t.Errorf("Sweep removed %d, %d left", n, v.Len()) + } +} + +func TestDelete(t *testing.T) { + v := newTest(t) + v.Put("a", "1") + v.Put("b", "2") + v.Delete("a", "b") + if v.Len() != 0 { + t.Errorf("%d entries left", v.Len()) + } +} From 34c8018bc4bfd3519616408eb950d41471fb702e Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:37:04 +0800 Subject: [PATCH 04/13] feat: hash-chained audit log with verify and CSV export (D7) (#7) - internal/audit: append-only JSON lines; each record holds the SHA-256 of the previous one and HMAC-SHA-256 of prompt and response (key from DISCREET_HMAC_KEY, at least 32 characters), never the text. Writes are fsynced. Open verifies an existing log and refuses to append to a broken chain. Cost per UTC day is tracked for the D6 budget. - discreet audit verify names the first changed, missing or reordered record (no key needed); discreet audit export -csv exports a verified log with spreadsheet formulas neutralised. - Tests: changed byte, deleted record, swapped records, a rewritten record with its own hash recomputed, reopening, 50 concurrent appends, CSV. Deliberate breaks (no hash check, no link check, appending to a broken log) each failed tests; restored from backup. - ADR 0005, including the limits: a full chain rewrite or a truncated tail can't be detected without an external copy of the head hash. Closes makoydev/vetted#37 Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CONTROLS.md | 2 +- README.md | 4 +- THREAT_MODEL.md | 5 +- cmd/discreet/main.go | 54 +++++- cmd/discreet/main_test.go | 40 +++- docs/HOW-THIS-WAS-BUILT.md | 1 + docs/adr/0005-hash-chained-audit-log.md | 30 +++ internal/audit/audit.go | 240 ++++++++++++++++++++++++ internal/audit/audit_test.go | 179 ++++++++++++++++++ internal/audit/csv.go | 27 +++ internal/audit/helpers_test.go | 10 + 12 files changed, 581 insertions(+), 12 deletions(-) create mode 100644 docs/adr/0005-hash-chained-audit-log.md create mode 100644 internal/audit/audit.go create mode 100644 internal/audit/audit_test.go create mode 100644 internal/audit/csv.go create mode 100644 internal/audit/helpers_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c4f2008..8d9f58e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,5 +11,6 @@ All notable changes to this project are documented here. The format follows [Kee - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. - Vetted reviews every pull request (shadow mode until 2026-10-14). - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases. +- Tamper-evident audit log (`internal/audit`, issue D7): append-only JSON lines, each record holding the SHA-256 of the previous one and HMAC-SHA-256 hashes of prompt and response (never the text). `discreet audit verify` names the first changed, missing or reordered record; `discreet audit export -csv` exports a verified log. An existing broken log is refused at start-up (ADR 0005). - Placeholders and the vault (`internal/protect`, `internal/vault`, issue D4): personal data is replaced per request with numbered placeholders such as `` (the same value gets the same placeholder), redacted as `[REDACTED_CARD]`, or the request is blocked, following a YAML policy (`internal/protect/default-policy.yaml`). Values are kept encrypted in memory with AES-256-GCM for at most 10 minutes and restored only into the same request's answer (ADR 0004). First third-party module: `go.yaml.in/yaml/v3` v3.0.5. - sg-pii-rules `v0.2.0-rc.1` vendored in `third_party/sgpiirules` and embedded in the binary; a test re-checks every file against `SHA256SUMS` (ADR 0003). `scripts/vendor-sg-pii-rules.sh ` updates it. diff --git a/CONTROLS.md b/CONTROLS.md index f7c6dca..cc92f43 100644 --- a/CONTROLS.md +++ b/CONTROLS.md @@ -6,6 +6,6 @@ Each Discreet feature, the principle or obligation it supports, and the source. | --- | --- | --- | --- | | Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection (D3, 369/369 shared cases) and replacement with placeholders (D4) in place; mapping pending | | Purpose declared in a header; automated eligibility decisions refused | Human accountability for decisions about people | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | Planned (D5); mapping pending | -| Hash-chained audit log with `verify` | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | Planned (D7); mapping pending | +| Hash-chained audit log with `verify` and CSV export; prompts and responses kept only as keyed hashes | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | In place (D7); mapping pending | | Daily budget, access token, rate limit | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | Planned (D6); mapping pending | | Pinned Actions, `govulncheck`, CodeQL, reviewed pull requests | Secure development | OWASP Top 10 for LLM Applications 2025 (supply chain) | In place (D1); mapping pending | diff --git a/README.md b/README.md index 83e9624..f525128 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`. -**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the command-line skeleton, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. +**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the command-line skeleton, a tamper-evident audit log, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. ## Build and check @@ -11,6 +11,8 @@ Needs Go 1.27. ```sh make check # gofmt, go vet, go test -race, govulncheck, build ./bin/discreet help +./bin/discreet audit verify -log discreet-audit.jsonl # check the hash chain +./bin/discreet audit export -csv -log discreet-audit.jsonl > audit.csv ``` ## Evidence diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index b27e81a..a35bc52 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -23,7 +23,8 @@ | Rules silently changed or swapped | Vendored release checked against `SHA256SUMS` on every test run; embedded in the binary (ADR 0003) | D3 (done) | | The vault leaks values through logs or errors | Values encrypted in memory (AES-256-GCM, per-process key), bound to their key, deleted when the request ends, expired after 10 minutes; vault and session print as redacted (tests) | D4 (done); log scan in D5 | | The model invents placeholders to extract other values | Only placeholders issued for that request are rehydrated (test) | D4 (done) | -| Someone edits the audit log to hide a request | SHA-256 hash chain; `audit verify` names the first broken record | D7 | -| Short identifiers are guessed from audit hashes | HMAC-SHA-256 with a server secret instead of plain hashes | D7 | +| Someone edits the audit log to hide a request | SHA-256 hash chain; `audit verify` names the first broken record; a broken log is refused at start-up (tests for a changed byte, a deleted record, swapped records, a rewritten record) | D7 (done) | +| Someone with write access rewrites the whole chain, or cuts records off the end | Not prevented: `verify` prints the head hash to record elsewhere; automatic anchoring is Milestone 3 work (ADR 0005) | Accepted for v0.1 | +| Short identifiers are guessed from audit hashes | HMAC-SHA-256 with a server secret of at least 32 characters instead of plain hashes | D7 (done) | | The public demo is used to spend money | Mock model unless a valid access token is present; daily budget; rate limit | D6, D10 | | A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 | diff --git a/cmd/discreet/main.go b/cmd/discreet/main.go index b833033..efc3b43 100644 --- a/cmd/discreet/main.go +++ b/cmd/discreet/main.go @@ -8,9 +8,12 @@ package main import ( + "flag" "fmt" "io" "os" + + "github.com/makoydev/discreet/internal/audit" ) // version is overridden at release time with -ldflags "-X main.version=...". @@ -34,12 +37,13 @@ Runs the gateway: an OpenAI-compatible endpoint that swaps personal data for placeholders before the request reaches the model. Not built yet (issue D5). ` -const auditUsage = `Usage: discreet audit +const auditUsage = `Usage: discreet audit [-log path] - verify check every record's hash chain; fails if any byte was changed - export write the audit log as CSV + verify check every record's hash chain; fails, naming the record, + if any byte was changed, a record deleted or two swapped + export -csv write the verified log as CSV to standard output -Not built yet (issue D7). +The log defaults to $DISCREET_AUDIT_LOG, or discreet-audit.jsonl. ` func main() { @@ -78,8 +82,7 @@ func run(args []string, stdout, stderr io.Writer) int { } switch args[1] { case "verify", "export": - fmt.Fprintf(stderr, "discreet audit %s: not built yet (issue D7)\n", args[1]) - return 1 + return runAudit(args[1], args[2:], stdout, stderr) } fmt.Fprintf(stderr, "discreet audit: unknown subcommand %q\n\n%s", args[1], auditUsage) return 2 @@ -91,3 +94,42 @@ func run(args []string, stdout, stderr io.Writer) int { func wantsHelp(args []string) bool { return len(args) > 0 && (args[0] == "-h" || args[0] == "--help" || args[0] == "help") } + +func defaultLogPath() string { + if p := os.Getenv("DISCREET_AUDIT_LOG"); p != "" { + return p + } + return "discreet-audit.jsonl" +} + +// runAudit verifies or exports the audit log. Neither needs the HMAC key: +// the chain is plain SHA-256, so anyone holding the file can check it. +func runAudit(sub string, args []string, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("discreet audit "+sub, flag.ContinueOnError) + fs.SetOutput(stderr) + path := fs.String("log", defaultLogPath(), "audit log file") + fs.Bool("csv", true, "export as CSV (the only format)") + if err := fs.Parse(args); err != nil { + return 2 + } + f, err := os.Open(*path) + if err != nil { + fmt.Fprintf(stderr, "discreet audit %s: %v\n", sub, err) + return 1 + } + defer f.Close() + if sub == "export" { + if err := audit.ExportCSV(f, stdout); err != nil { + fmt.Fprintf(stderr, "discreet audit export: not exported, the log is broken: %v\n", err) + return 1 + } + return 0 + } + s, err := audit.Verify(f) + if err != nil { + fmt.Fprintf(stderr, "FAILED: %v\n", err) + return 1 + } + fmt.Fprintf(stdout, "OK: %d records, chain intact.\nHead hash: %s\n", s.Records, s.HeadHash) + return 0 +} diff --git a/cmd/discreet/main_test.go b/cmd/discreet/main_test.go index a1a3422..f8b19b4 100644 --- a/cmd/discreet/main_test.go +++ b/cmd/discreet/main_test.go @@ -2,8 +2,12 @@ package main import ( "bytes" + "os" + "path/filepath" "strings" "testing" + + "github.com/makoydev/discreet/internal/audit" ) func TestRun(t *testing.T) { @@ -22,8 +26,6 @@ func TestRun(t *testing.T) { {"serve is not built yet", []string{"serve"}, 1, "", "not built yet (issue D5)"}, {"audit help", []string{"audit", "--help"}, 0, "verify", ""}, {"audit without a subcommand", []string{"audit"}, 2, "", "Usage: discreet audit"}, - {"audit verify is not built yet", []string{"audit", "verify"}, 1, "", "audit verify: not built yet (issue D7)"}, - {"audit export is not built yet", []string{"audit", "export"}, 1, "", "audit export: not built yet (issue D7)"}, {"unknown audit subcommand", []string{"audit", "delete"}, 2, "", `unknown subcommand "delete"`}, {"unknown command", []string{"launch"}, 2, "", `unknown command "launch"`}, } @@ -53,3 +55,37 @@ func check(t *testing.T, stream, got, want string) { t.Errorf("%s = %q, want it to contain %q", stream, got, want) } } + +func TestAuditVerifyAndExport(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.jsonl") + l, err := audit.Open(path, []byte("test-key-0123456789abcdefghijklmnop")) + if err != nil { + t.Fatal(err) + } + l.Append(audit.Record{Tenant: "demo", Decision: audit.Allowed}) + l.Append(audit.Record{Tenant: "demo", Decision: audit.RefusedPurpose}) + l.Close() + + var out, errOut bytes.Buffer + if code := run([]string{"audit", "verify", "-log", path}, &out, &errOut); code != 0 || !strings.Contains(out.String(), "OK: 2 records") { + t.Fatalf("verify: code %d, out %q, err %q", code, out.String(), errOut.String()) + } + out.Reset() + if code := run([]string{"audit", "export", "-csv", "-log", path}, &out, &errOut); code != 0 || !strings.HasPrefix(out.String(), "seq,time,") { + t.Fatalf("export: code %d, out %q", code, out.String()) + } + + data, _ := os.ReadFile(path) + os.WriteFile(path, bytes.Replace(data, []byte("refused_purpose"), []byte("allowed"), 1), 0o600) + out.Reset() + errOut.Reset() + if code := run([]string{"audit", "verify", "-log", path}, &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "FAILED: record on line 2") { + t.Errorf("tampered verify: code %d, err %q", code, errOut.String()) + } + if code := run([]string{"audit", "export", "-log", path}, &out, &errOut); code != 1 { + t.Errorf("a broken log was exported (code %d)", code) + } + if code := run([]string{"audit", "verify", "-log", filepath.Join(t.TempDir(), "missing")}, &out, &errOut); code != 1 { + t.Errorf("missing log: code %d", code) + } +} diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md index 2990d3d..8e58874 100644 --- a/docs/HOW-THIS-WAS-BUILT.md +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -29,6 +29,7 @@ | 2026-10-05 | All 369 shared conformance cases through the Go engine | 369 / 369 pass, first run | | 2026-10-05 | Go YAML libraries (Go module proxy, GitHub) | `gopkg.in/yaml.v3` archived since April 2025; used the maintained fork `go.yaml.in/yaml/v3` v3.0.5 | | 2026-10-05 | Three deliberate breaks of the placeholder session: vault keys without the session, no placeholder reuse, redact restored | Each failed at least one test; restored from backup | +| 2026-10-05 | Three deliberate breaks of the audit log: no record-hash check, no chain-link check, appending to a broken log | Each failed at least one test; restored from backup | | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite | ## What the AI got wrong, and how it was caught diff --git a/docs/adr/0005-hash-chained-audit-log.md b/docs/adr/0005-hash-chained-audit-log.md new file mode 100644 index 0000000..e4c530b --- /dev/null +++ b/docs/adr/0005-hash-chained-audit-log.md @@ -0,0 +1,30 @@ +# 0005. A hash-chained JSON-lines audit log with keyed hashes of prompts and responses + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-05 + +## Context + +An operator must be able to show afterwards what Discreet did with each request (allowed, refused, blocked, what kinds of personal data, which model, what it cost) without the log itself becoming a store of personal data, and to prove the log hasn't been edited (issue D7). + +## Options + +1. **Append-only JSON lines, each record carrying the SHA-256 of the previous one.** Anyone can open the file, read it, edit a byte and watch `discreet audit verify` fail, which is the clearest tamper demonstration there is. +2. **A SQLite table** with the same chain. Better for queries, but harder for a reviewer to inspect, and the tamper demo needs a database tool. +3. **An external append-only service** (a transparency log, cloud object lock). Stronger against an attacker with full server access, but adds cost and an account, against the US$20/month cap. + +## Decision + +Option 1. + +- Fields: sequence number, UTC time, request ID, tenant, purpose, decision, reason, upstream, model, counts per entity, HMAC-SHA-256 of prompt and response, cost, latency, previous hash, hash. The hash covers the record's JSON with the hash field empty; field order is fixed, so it's deterministic. +- Prompts and responses are stored only as **HMAC-SHA-256 with a server secret** (`DISCREET_HMAC_KEY`, at least 32 characters; Discreet refuses to start without it). Plain hashes of short identifiers such as NRICs could be reversed by hashing every possible number. With the key, an investigator holding a disputed text can prove whether it was the one sent. +- `discreet audit verify` checks sequence, links and hashes and names the first broken record. It needs no key. Discreet also verifies an existing log at start-up and **refuses to append to a broken chain**. +- `discreet audit export -csv` exports only a verified log, with spreadsheet formulas neutralised. +- The daily budget (issue D6) is recomputed from the log at start-up, so a restart can't reset spending. + +## Consequences + +- **Limits, stated plainly:** someone with write access who rewrites the entire chain from some point on, recomputing every hash, produces a log that verifies. So does cutting records off the end. `verify` prints the head hash so it can be recorded somewhere else (a ticket, a daily email) and compared later; anchoring it automatically is Milestone 3 work. +- Writes are flushed to disk (`fsync`) before the response is sent, which costs a little latency per request (measured in issue D9). +- The purpose header is recorded as given; the gateway scrubs personal data from it first (issue D5). diff --git a/internal/audit/audit.go b/internal/audit/audit.go new file mode 100644 index 0000000..3a59dcf --- /dev/null +++ b/internal/audit/audit.go @@ -0,0 +1,240 @@ +// Package audit writes Discreet's tamper-evident audit log: an append-only +// file of JSON lines in which every record carries the SHA-256 of the one +// before it, so changing, deleting or reordering any record breaks the chain. +// Prompts and responses are stored only as HMAC-SHA-256 with a server secret, +// never as text: plain hashes of short identifiers such as NRICs could be +// reversed by trying every possible number. +package audit + +import ( + "bufio" + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + "time" +) + +// Genesis is the previous-hash of the first record. +const Genesis = "0000000000000000000000000000000000000000000000000000000000000000" + +// Decisions recorded for a request. +const ( + Allowed = "allowed" + RefusedPurpose = "refused_purpose" + BlockedEntity = "blocked_entity" + RefusedBudget = "refused_budget" + UpstreamError = "upstream_error" +) + +// Record is one line of the log. Field order is fixed, so the JSON +// encoding, and therefore the hash, is deterministic. +type Record struct { + Seq int64 `json:"seq"` + Time string `json:"time"` + RequestID string `json:"request_id"` + Tenant string `json:"tenant"` + Purpose string `json:"purpose"` + Decision string `json:"decision"` + Reason string `json:"reason,omitempty"` + Upstream string `json:"upstream"` + Model string `json:"model"` + Entities map[string]int `json:"entities"` + PromptHMAC string `json:"prompt_hmac,omitempty"` + ResponseHMAC string `json:"response_hmac,omitempty"` + CostUSD float64 `json:"cost_usd"` + LatencyMS int64 `json:"latency_ms"` + PrevHash string `json:"prev_hash"` + Hash string `json:"hash"` +} + +// computeHash is the SHA-256 of the record's JSON with the hash left empty. +func computeHash(r Record) string { + r.Hash = "" + data, _ := json.Marshal(r) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +// Log appends records. It is safe for concurrent use. +type Log struct { + mu sync.Mutex + f *os.File + key []byte + seq int64 + last string + spentDay map[string]float64 // UTC date → cost recorded that day + now func() time.Time +} + +// MinKeyLength is the shortest HMAC key accepted, in bytes. +const MinKeyLength = 32 + +// KeyFromEnv reads the HMAC key from DISCREET_HMAC_KEY. +func KeyFromEnv() ([]byte, error) { + k := os.Getenv("DISCREET_HMAC_KEY") + if len(k) < MinKeyLength { + return nil, fmt.Errorf("DISCREET_HMAC_KEY must be set to at least %d characters (for example: openssl rand -hex 32)", MinKeyLength) + } + return []byte(k), nil +} + +// Open opens or creates the log at path. An existing log is verified +// first: Discreet refuses to append to a broken chain. +func Open(path string, key []byte) (*Log, error) { + if len(key) < MinKeyLength { + return nil, fmt.Errorf("audit: HMAC key shorter than %d bytes", MinKeyLength) + } + l := &Log{key: key, last: Genesis, spentDay: map[string]float64{}, now: time.Now} + if existing, err := os.Open(path); err == nil { + err := scan(existing, func(r Record) { + l.seq, l.last = r.Seq, r.Hash + l.spentDay[r.Time[:10]] += r.CostUSD + }) + existing.Close() + if err != nil { + return nil, fmt.Errorf("audit: existing log %s is broken, refusing to append: %w", path, err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, err + } + l.f = f + return l, nil +} + +// HMAC returns the keyed hash of text, as stored for prompts and responses. +func (l *Log) HMAC(text string) string { + m := hmac.New(sha256.New, l.key) + m.Write([]byte(text)) + return hex.EncodeToString(m.Sum(nil)) +} + +// Append fills in the sequence number, time and hashes, writes the record +// and flushes it to disk before returning. +func (l *Log) Append(r Record) (Record, error) { + l.mu.Lock() + defer l.mu.Unlock() + r.Seq = l.seq + 1 + r.Time = l.now().UTC().Format(time.RFC3339Nano) + r.PrevHash = l.last + if r.Entities == nil { + r.Entities = map[string]int{} + } + r.Hash = computeHash(r) + line, err := json.Marshal(r) + if err != nil { + return Record{}, err + } + if _, err := l.f.Write(append(line, '\n')); err != nil { + return Record{}, err + } + if err := l.f.Sync(); err != nil { + return Record{}, err + } + l.seq, l.last = r.Seq, r.Hash + l.spentDay[r.Time[:10]] += r.CostUSD + return r, nil +} + +// SpentToday is the cost recorded since midnight UTC. +func (l *Log) SpentToday() float64 { + l.mu.Lock() + defer l.mu.Unlock() + return l.spentDay[l.now().UTC().Format("2006-01-02")] +} + +// Head returns the last record's hash and sequence number. +func (l *Log) Head() (string, int64) { + l.mu.Lock() + defer l.mu.Unlock() + return l.last, l.seq +} + +// Close closes the file. +func (l *Log) Close() error { return l.f.Close() } + +// VerifyError says which record broke the chain and how. +type VerifyError struct { + Line int + Problem string +} + +func (e *VerifyError) Error() string { + return fmt.Sprintf("record on line %d: %s", e.Line, e.Problem) +} + +// scan reads every record, checking the chain, and calls fn for each. +func scan(r io.Reader, fn func(Record)) error { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 1024*1024) + prev, seq, line := Genesis, int64(0), 0 + for sc.Scan() { + line++ + var rec Record + dec := json.NewDecoder(bytes.NewReader(sc.Bytes())) + dec.DisallowUnknownFields() + if err := dec.Decode(&rec); err != nil { + return &VerifyError{line, "not a valid record: " + err.Error()} + } + switch { + case rec.Seq != seq+1: + return &VerifyError{line, fmt.Sprintf("sequence %d, expected %d: a record is missing or out of order", rec.Seq, seq+1)} + case rec.PrevHash != prev: + return &VerifyError{line, "previous-hash does not match the record before it"} + case rec.Hash != computeHash(rec): + return &VerifyError{line, "hash does not match the record's contents: it was changed"} + case len(rec.Time) < 10: + return &VerifyError{line, "missing time"} + } + fn(rec) + prev, seq = rec.Hash, rec.Seq + } + return sc.Err() +} + +// Summary is the result of a successful verification. +type Summary struct { + Records int64 + HeadHash string +} + +// Verify checks a whole log. It needs no key: the chain uses plain SHA-256. +// It can't detect records cut off the end; compare the head hash with one +// recorded elsewhere for that. +func Verify(r io.Reader) (Summary, error) { + var s Summary + s.HeadHash = Genesis + err := scan(r, func(rec Record) { s.Records, s.HeadHash = rec.Seq, rec.Hash }) + return s, err +} + +// ExportCSV writes a verified log as CSV. A broken log is not exported. +func ExportCSV(r io.Reader, w io.Writer) error { + var rows []Record + if err := scan(r, func(rec Record) { rows = append(rows, rec) }); err != nil { + return err + } + cw := newCSV(w) + cw.row("seq", "time", "request_id", "tenant", "purpose", "decision", "reason", "upstream", "model", "entities", "prompt_hmac", "response_hmac", "cost_usd", "latency_ms", "prev_hash", "hash") + for _, r := range rows { + cw.row(fmt.Sprint(r.Seq), r.Time, r.RequestID, r.Tenant, r.Purpose, r.Decision, r.Reason, r.Upstream, r.Model, + entitiesText(r.Entities), r.PromptHMAC, r.ResponseHMAC, fmt.Sprintf("%.6f", r.CostUSD), fmt.Sprint(r.LatencyMS), r.PrevHash, r.Hash) + } + return cw.flush() +} + +func entitiesText(m map[string]int) string { + data, _ := json.Marshal(m) // keys sorted, so stable + return strings.Trim(string(data), "{}") +} diff --git a/internal/audit/audit_test.go b/internal/audit/audit_test.go new file mode 100644 index 0000000..5d664de --- /dev/null +++ b/internal/audit/audit_test.go @@ -0,0 +1,179 @@ +package audit + +import ( + "bytes" + "encoding/csv" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +var testKey = []byte("test-key-0123456789abcdefghijklmnop") + +func writeLog(t *testing.T, n int) string { + t.Helper() + path := filepath.Join(t.TempDir(), "audit.jsonl") + l, err := Open(path, testKey) + if err != nil { + t.Fatal(err) + } + for i := 0; i < n; i++ { + if _, err := l.Append(Record{RequestID: "r", Tenant: "demo", Purpose: "claims summary", Decision: Allowed, + Upstream: "mock", Model: "mock", Entities: map[string]int{"NRIC": i}, PromptHMAC: l.HMAC("p"), CostUSD: 0.001}); err != nil { + t.Fatal(err) + } + } + l.Close() + return path +} + +func lines(t *testing.T, path string) []string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return strings.Split(strings.TrimRight(string(data), "\n"), "\n") +} + +func verifyLines(ls []string) (Summary, error) { + return Verify(strings.NewReader(strings.Join(ls, "\n") + "\n")) +} + +func wantBrokenAt(t *testing.T, err error, line int) { + t.Helper() + var ve *VerifyError + if !errors.As(err, &ve) || ve.Line != line { + t.Fatalf("err = %v, want a VerifyError on line %d", err, line) + } +} + +func TestCleanLogVerifies(t *testing.T) { + path := writeLog(t, 5) + s, err := verifyLines(lines(t, path)) + if err != nil || s.Records != 5 || s.HeadHash == Genesis { + t.Fatalf("summary %+v, err %v", s, err) + } +} + +func TestOneChangedByteFails(t *testing.T) { + ls := lines(t, writeLog(t, 5)) + ls[2] = strings.Replace(ls[2], `"NRIC":2`, `"NRIC":3`, 1) + _, err := verifyLines(ls) + wantBrokenAt(t, err, 3) +} + +func TestDeletedRecordFails(t *testing.T) { + ls := lines(t, writeLog(t, 5)) + _, err := verifyLines(append(ls[:1:1], ls[2:]...)) + wantBrokenAt(t, err, 2) +} + +func TestSwappedRecordsFail(t *testing.T) { + ls := lines(t, writeLog(t, 5)) + ls[1], ls[2] = ls[2], ls[1] + _, err := verifyLines(ls) + wantBrokenAt(t, err, 2) +} + +func TestRewrittenRecordWithFixedSequenceFails(t *testing.T) { + // Recomputing a changed record's own hash still breaks the next link. + ls := lines(t, writeLog(t, 3)) + var r Record + if err := jsonUnmarshal(ls[0], &r); err != nil { + t.Fatal(err) + } + r.Tenant = "someone-else" + r.Hash = computeHash(r) + ls[0] = jsonMarshal(r) + _, err := verifyLines(ls) + wantBrokenAt(t, err, 2) +} + +func TestReopenContinuesTheChain(t *testing.T) { + path := writeLog(t, 2) + l, err := Open(path, testKey) + if err != nil { + t.Fatal(err) + } + r, _ := l.Append(Record{Decision: RefusedPurpose}) + l.Close() + if r.Seq != 3 { + t.Errorf("seq = %d, want 3", r.Seq) + } + if s, err := verifyLines(lines(t, path)); err != nil || s.Records != 3 { + t.Errorf("%+v %v", s, err) + } +} + +func TestOpenRefusesABrokenLog(t *testing.T) { + path := writeLog(t, 3) + ls := lines(t, path) + ls[1] = strings.Replace(ls[1], "demo", "dem0", 1) + os.WriteFile(path, []byte(strings.Join(ls, "\n")+"\n"), 0o600) + if _, err := Open(path, testKey); err == nil { + t.Error("opened a broken log for appending") + } +} + +func TestKeyRules(t *testing.T) { + if _, err := Open(filepath.Join(t.TempDir(), "a"), []byte("short")); err == nil { + t.Error("accepted a short key") + } + t.Setenv("DISCREET_HMAC_KEY", "") + if _, err := KeyFromEnv(); err == nil { + t.Error("accepted a missing key") + } + l, _ := Open(filepath.Join(t.TempDir(), "a"), testKey) + other, _ := Open(filepath.Join(t.TempDir(), "b"), []byte("another-key-0123456789abcdefghijkl")) + if l.HMAC("S1234567D") == other.HMAC("S1234567D") || l.HMAC("x") != l.HMAC("x") { + t.Error("HMAC must be deterministic per key and differ between keys") + } +} + +func TestConcurrentAppendsKeepTheChain(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.jsonl") + l, _ := Open(path, testKey) + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { defer wg.Done(); l.Append(Record{Decision: Allowed, CostUSD: 0.01}) }() + } + wg.Wait() + l.Close() + s, err := verifyLines(lines(t, path)) + if err != nil || s.Records != 50 { + t.Fatalf("%+v %v", s, err) + } + l2, _ := Open(path, testKey) + if got := l2.SpentToday(); got < 0.4999 || got > 0.5001 { + t.Errorf("SpentToday = %v, want 0.50", got) + } +} + +func TestExportCSV(t *testing.T) { + path := writeLog(t, 2) + var out bytes.Buffer + f, _ := os.Open(path) + defer f.Close() + if err := ExportCSV(f, &out); err != nil { + t.Fatal(err) + } + rows, err := csv.NewReader(&out).ReadAll() + if err != nil || len(rows) != 3 || rows[0][0] != "seq" || rows[2][0] != "2" || rows[2][9] != `"NRIC":1` { + t.Fatalf("rows %v err %v", rows, err) + } +} + +func TestCSVNeutralisesFormulas(t *testing.T) { + var out bytes.Buffer + c := newCSV(&out) + c.row("=HYPERLINK(1)", "ok") + c.flush() + if !strings.HasPrefix(out.String(), "'=HYPERLINK") { + t.Errorf("formula not neutralised: %q", out.String()) + } +} diff --git a/internal/audit/csv.go b/internal/audit/csv.go new file mode 100644 index 0000000..e6682b9 --- /dev/null +++ b/internal/audit/csv.go @@ -0,0 +1,27 @@ +package audit + +import ( + "encoding/csv" + "io" + "strings" +) + +type csvWriter struct{ w *csv.Writer } + +func newCSV(w io.Writer) *csvWriter { return &csvWriter{csv.NewWriter(w)} } + +// row writes one CSV row. Cells that a spreadsheet would run as a formula +// (starting =, +, -, @) get a leading apostrophe. +func (c *csvWriter) row(cells ...string) { + for i, cell := range cells { + if cell != "" && strings.ContainsRune("=+-@", rune(cell[0])) { + cells[i] = "'" + cell + } + } + _ = c.w.Write(cells) +} + +func (c *csvWriter) flush() error { + c.w.Flush() + return c.w.Error() +} diff --git a/internal/audit/helpers_test.go b/internal/audit/helpers_test.go new file mode 100644 index 0000000..78b6d0b --- /dev/null +++ b/internal/audit/helpers_test.go @@ -0,0 +1,10 @@ +package audit + +import "encoding/json" + +func jsonUnmarshal(s string, v any) error { return json.Unmarshal([]byte(s), v) } + +func jsonMarshal(v any) string { + data, _ := json.Marshal(v) + return string(data) +} From b066e9e98855177589a71c4a465f6d99fb61017e Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:44:21 +0800 Subject: [PATCH 05/13] feat: the gateway with mock model and purpose refusal (D5) (#8) - discreet serve: OpenAI-compatible POST /v1/chat/completions (non-streaming) and /healthz. Messages are protected in one session, sent to the model as placeholders, restored in the answer, which ends with a disclosure footer. Every request, allowed or not, is recorded; no answer without an audit record. Refuses to start without DISCREET_HMAC_KEY or on a broken audit log; graceful shutdown. - X-Discreet-Purpose required; automated eligibility / benefits decisions refused with 403 before the model is called (ADR 0006). Purpose stored scrubbed of personal data. - Only model, messages, token limit and temperature are forwarded; tools, images, tool roles and streaming are rejected. - Mock model (default, free) drafts a reply that reuses placeholders. - Checked with OpenAI's official Python library v3.24.0, changing only base_url. Tests include a scan of every log line for planted values. - Deliberate breaks (original text forwarded, purpose check skipped, prompt logged) each failed tests; restored from backup. - Two AI mistakes logged: misleading mock wording; a test that searched JSON-escaped text. Closes makoydev/vetted#35 Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CONTROLS.md | 2 +- README.md | 15 +- THREAT_MODEL.md | 3 + cmd/discreet/main.go | 128 +++++++- cmd/discreet/main_test.go | 45 ++- docs/HOW-THIS-WAS-BUILT.md | 4 + ...06-purpose-header-and-what-is-forwarded.md | 30 ++ internal/gateway/gateway.go | 291 +++++++++++++++++ internal/gateway/gateway_test.go | 293 ++++++++++++++++++ internal/gateway/purpose.go | 33 ++ internal/upstream/upstream.go | 83 +++++ 12 files changed, 920 insertions(+), 8 deletions(-) create mode 100644 docs/adr/0006-purpose-header-and-what-is-forwarded.md create mode 100644 internal/gateway/gateway.go create mode 100644 internal/gateway/gateway_test.go create mode 100644 internal/gateway/purpose.go create mode 100644 internal/upstream/upstream.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d9f58e..051c83a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project are documented here. The format follows [Kee - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. - Vetted reviews every pull request (shadow mode until 2026-10-14). - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases. +- The gateway (`internal/gateway`, issue D5): `discreet serve` runs an OpenAI-compatible `POST /v1/chat/completions` (non-streaming). Every message is protected in one session before the model sees it and restored afterwards; every answer ends with a disclosure footer; every request, allowed or not, is recorded in the audit log, and Discreet never answers a request it couldn't record. A free mock model is the default. Callers must send `X-Discreet-Purpose`; automated eligibility and benefits decisions are refused with HTTP 403 (ADR 0006). Only the model, messages, token limit and temperature are forwarded. Tested with OpenAI's official Python library by changing only `base_url`. - Tamper-evident audit log (`internal/audit`, issue D7): append-only JSON lines, each record holding the SHA-256 of the previous one and HMAC-SHA-256 hashes of prompt and response (never the text). `discreet audit verify` names the first changed, missing or reordered record; `discreet audit export -csv` exports a verified log. An existing broken log is refused at start-up (ADR 0005). - Placeholders and the vault (`internal/protect`, `internal/vault`, issue D4): personal data is replaced per request with numbered placeholders such as `` (the same value gets the same placeholder), redacted as `[REDACTED_CARD]`, or the request is blocked, following a YAML policy (`internal/protect/default-policy.yaml`). Values are kept encrypted in memory with AES-256-GCM for at most 10 minutes and restored only into the same request's answer (ADR 0004). First third-party module: `go.yaml.in/yaml/v3` v3.0.5. - sg-pii-rules `v0.2.0-rc.1` vendored in `third_party/sgpiirules` and embedded in the binary; a test re-checks every file against `SHA256SUMS` (ADR 0003). `scripts/vendor-sg-pii-rules.sh ` updates it. diff --git a/CONTROLS.md b/CONTROLS.md index cc92f43..78a8266 100644 --- a/CONTROLS.md +++ b/CONTROLS.md @@ -5,7 +5,7 @@ Each Discreet feature, the principle or obligation it supports, and the source. | Feature | Supports | Source | Status | | --- | --- | --- | --- | | Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection (D3, 369/369 shared cases) and replacement with placeholders (D4) in place; mapping pending | -| Purpose declared in a header; automated eligibility decisions refused | Human accountability for decisions about people | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | Planned (D5); mapping pending | +| Purpose declared in a header; automated eligibility decisions refused before the model is called; disclosure footer on every answer | Human accountability for decisions about people; transparency to end users | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | In place (D5); mapping pending | | Hash-chained audit log with `verify` and CSV export; prompts and responses kept only as keyed hashes | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | In place (D7); mapping pending | | Daily budget, access token, rate limit | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | Planned (D6); mapping pending | | Pinned Actions, `govulncheck`, CodeQL, reviewed pull requests | Secure development | OWASP Top 10 for LLM Applications 2025 (supply chain) | In place (D1); mapping pending | diff --git a/README.md b/README.md index f525128..0624f8d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`. -**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the command-line skeleton, a tamper-evident audit log, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. +**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the gateway (`discreet serve`, mock model), a tamper-evident audit log, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. ## Build and check @@ -15,6 +15,19 @@ make check # gofmt, go vet, go test -race, govulncheck, build ./bin/discreet audit export -csv -log discreet-audit.jsonl > audit.csv ``` +## Run it + +```sh +export DISCREET_HMAC_KEY=$(openssl rand -hex 32) # keys the audit log's hashes +./bin/discreet serve # listens on :8080, free mock model + +curl -s localhost:8080/v1/chat/completions \ + -H 'Content-Type: application/json' -H 'X-Discreet-Purpose: claims summary' \ + -d '{"model":"gpt-6-luna","messages":[{"role":"user","content":"Call S1234567D on 9123 4567"}]}' +``` + +The mock model shows what a real model would have received (placeholders only), and the answer comes back with the real values restored. Any OpenAI client works by pointing `base_url` at `http://localhost:8080/v1` and adding the `X-Discreet-Purpose` header. The example NRIC is synthetic. + ## Evidence [`CONTROLS.md`](CONTROLS.md) · [`THREAT_MODEL.md`](THREAT_MODEL.md) · [`EVALS.md`](EVALS.md) · [`RISKS.md`](RISKS.md) · [`CHANGELOG.md`](CHANGELOG.md) · [decision records](docs/adr/) · [how this was built](docs/HOW-THIS-WAS-BUILT.md) diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index a35bc52..d6b46c0 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -26,5 +26,8 @@ | Someone edits the audit log to hide a request | SHA-256 hash chain; `audit verify` names the first broken record; a broken log is refused at start-up (tests for a changed byte, a deleted record, swapped records, a rewritten record) | D7 (done) | | Someone with write access rewrites the whole chain, or cuts records off the end | Not prevented: `verify` prints the head hash to record elsewhere; automatic anchoring is Milestone 3 work (ADR 0005) | Accepted for v0.1 | | Short identifiers are guessed from audit hashes | HMAC-SHA-256 with a server secret of at least 32 characters instead of plain hashes | D7 (done) | +| Personal data travels in request fields the gateway doesn't inspect (`tools`, `user`, metadata) | Requests are rebuilt from model, messages, token limit and temperature only; tools, images and tool-role messages are rejected (ADR 0006, test) | D5 (done) | +| Personal data ends up in server logs | Logs carry counts and decisions, never text; a test captures every log line and searches it for planted values | D5 (done) | +| A caller uses Discreet for automated eligibility decisions | Required purpose header; denied purposes refused with 403 before the model is called, and recorded (ADR 0006) | D5 (done) | | The public demo is used to spend money | Mock model unless a valid access token is present; daily budget; rate limit | D6, D10 | | A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 | diff --git a/cmd/discreet/main.go b/cmd/discreet/main.go index efc3b43..594f2c8 100644 --- a/cmd/discreet/main.go +++ b/cmd/discreet/main.go @@ -8,12 +8,26 @@ package main import ( + "context" + "errors" "flag" "fmt" "io" + "log/slog" + "net" + "net/http" "os" + "os/signal" + "strings" + "syscall" + "time" "github.com/makoydev/discreet/internal/audit" + "github.com/makoydev/discreet/internal/detect" + "github.com/makoydev/discreet/internal/gateway" + "github.com/makoydev/discreet/internal/protect" + "github.com/makoydev/discreet/internal/vault" + "github.com/makoydev/discreet/third_party/sgpiirules" ) // version is overridden at release time with -ldflags "-X main.version=...". @@ -31,10 +45,17 @@ Usage: Run "discreet --help" for details of a command. ` -const serveUsage = `Usage: discreet serve +const serveUsage = `Usage: discreet serve [-addr :8080] [-audit-log path] [-policy file.yaml] -Runs the gateway: an OpenAI-compatible endpoint that swaps personal data for -placeholders before the request reaches the model. Not built yet (issue D5). +Runs the gateway: an OpenAI-compatible POST /v1/chat/completions that swaps +personal data for placeholders before the request reaches the model, puts +it back in the answer, and records every request in the audit log. + +Requires DISCREET_HMAC_KEY (at least 32 characters, e.g. openssl rand -hex 32). +Callers must send an X-Discreet-Purpose header. + +Environment: DISCREET_ADDR, DISCREET_AUDIT_LOG, DISCREET_POLICY, +DISCREET_HMAC_KEY. ` const auditUsage = `Usage: discreet audit [-log path] @@ -47,9 +68,15 @@ The log defaults to $DISCREET_AUDIT_LOG, or discreet-audit.jsonl. ` func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + serveContext = ctx os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } +// serveContext ends when the process is asked to stop; tests replace it. +var serveContext = context.Background() + // run executes one command and returns the process exit code: // 0 for success, 1 for a command that failed, 2 for a usage error. func run(args []string, stdout, stderr io.Writer) int { @@ -69,8 +96,7 @@ func run(args []string, stdout, stderr io.Writer) int { fmt.Fprint(stdout, serveUsage) return 0 } - fmt.Fprintln(stderr, "discreet serve: not built yet (issue D5)") - return 1 + return runServe(serveContext, args[1:], stderr) case "audit": if wantsHelp(args[1:]) { fmt.Fprint(stdout, auditUsage) @@ -133,3 +159,95 @@ func runAudit(sub string, args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "OK: %d records, chain intact.\nHead hash: %s\n", s.Records, s.HeadHash) return 0 } + +func envOr(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} + +// runServe starts the gateway and blocks until ctx ends. +func runServe(ctx context.Context, args []string, stderr io.Writer) int { + fs := flag.NewFlagSet("discreet serve", flag.ContinueOnError) + fs.SetOutput(stderr) + addr := fs.String("addr", envOr("DISCREET_ADDR", ":8080"), "listen address") + logPath := fs.String("audit-log", defaultLogPath(), "audit log file") + policyPath := fs.String("policy", os.Getenv("DISCREET_POLICY"), "policy YAML file (default: built-in policy)") + if err := fs.Parse(args); err != nil { + return 2 + } + fail := func(err error) int { fmt.Fprintf(stderr, "discreet serve: %v\n", err); return 1 } + + key, err := audit.KeyFromEnv() + if err != nil { + return fail(err) + } + policy := protect.DefaultPolicy() + if *policyPath != "" { + data, err := os.ReadFile(*policyPath) + if err != nil { + return fail(err) + } + if policy, err = protect.ParsePolicy(data); err != nil { + return fail(err) + } + } + engine, err := detect.Default() + if err != nil { + return fail(err) + } + v, err := vault.New(vault.DefaultTTL) + if err != nil { + return fail(err) + } + log, err := audit.Open(*logPath, key) + if err != nil { + return fail(err) + } + defer log.Close() + + logger := slog.New(slog.NewJSONHandler(stderr, nil)) + srv := &http.Server{ + Handler: gateway.New(gateway.Config{Engine: engine, Policy: policy, Vault: v, Audit: log, Logger: logger}).Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 120 * time.Second, + IdleTimeout: 60 * time.Second, + } + ln, err := net.Listen("tcp", *addr) + if err != nil { + return fail(err) + } + version, _ := sgpiirules.Files.ReadFile("VERSION") + head, seq := log.Head() + logger.Info("discreet listening", "addr", ln.Addr().String(), "upstream", "mock", + "rules", "sg-pii-rules "+strings.TrimSpace(string(version)), "audit_log", *logPath, "audit_records", seq, "audit_head", head) + + go func() { + t := time.NewTicker(time.Minute) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + v.Sweep() + } + } + }() + errc := make(chan error, 1) + go func() { errc <- srv.Serve(ln) }() + select { + case err := <-errc: + if !errors.Is(err, http.ErrServerClosed) { + return fail(err) + } + case <-ctx.Done(): + shutdown, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + srv.Shutdown(shutdown) + } + logger.Info("discreet stopped") + return 0 +} diff --git a/cmd/discreet/main_test.go b/cmd/discreet/main_test.go index f8b19b4..b99b892 100644 --- a/cmd/discreet/main_test.go +++ b/cmd/discreet/main_test.go @@ -2,10 +2,15 @@ package main import ( "bytes" + "context" + "io" + "net" + "net/http" "os" "path/filepath" "strings" "testing" + "time" "github.com/makoydev/discreet/internal/audit" ) @@ -23,7 +28,6 @@ func TestRun(t *testing.T) { {"--help", []string{"--help"}, 0, "discreet audit verify", ""}, {"version", []string{"version"}, 0, "discreet 0.0.0-dev", ""}, {"serve help", []string{"serve", "--help"}, 0, "Usage: discreet serve", ""}, - {"serve is not built yet", []string{"serve"}, 1, "", "not built yet (issue D5)"}, {"audit help", []string{"audit", "--help"}, 0, "verify", ""}, {"audit without a subcommand", []string{"audit"}, 2, "", "Usage: discreet audit"}, {"unknown audit subcommand", []string{"audit", "delete"}, 2, "", `unknown subcommand "delete"`}, @@ -89,3 +93,42 @@ func TestAuditVerifyAndExport(t *testing.T) { t.Errorf("missing log: code %d", code) } } + +func TestServeRefusesWithoutAKey(t *testing.T) { + t.Setenv("DISCREET_HMAC_KEY", "") + var errOut bytes.Buffer + if code := run([]string{"serve", "-audit-log", filepath.Join(t.TempDir(), "a.jsonl")}, io.Discard, &errOut); code != 1 || !strings.Contains(errOut.String(), "DISCREET_HMAC_KEY") { + t.Errorf("code %d, stderr %q", code, errOut.String()) + } +} + +func TestServeAnswersAndStops(t *testing.T) { + t.Setenv("DISCREET_HMAC_KEY", "test-key-0123456789abcdefghijklmnop") + ln, _ := net.Listen("tcp", "127.0.0.1:0") + addr := ln.Addr().String() + ln.Close() + ctx, cancel := context.WithCancel(context.Background()) + serveContext = ctx + t.Cleanup(func() { serveContext = context.Background() }) + done := make(chan int) + go func() { + done <- run([]string{"serve", "-addr", addr, "-audit-log", filepath.Join(t.TempDir(), "a.jsonl")}, io.Discard, io.Discard) + }() + + var resp *http.Response + var err error + for i := 0; i < 50; i++ { + if resp, err = http.Get("http://" + addr + "/healthz"); err == nil { + break + } + time.Sleep(20 * time.Millisecond) + } + if err != nil || resp.StatusCode != 200 { + t.Fatalf("healthz: %v %v", resp, err) + } + resp.Body.Close() + cancel() + if code := <-done; code != 0 { + t.Errorf("exit code %d", code) + } +} diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md index 8e58874..f06da79 100644 --- a/docs/HOW-THIS-WAS-BUILT.md +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -30,9 +30,13 @@ | 2026-10-05 | Go YAML libraries (Go module proxy, GitHub) | `gopkg.in/yaml.v3` archived since April 2025; used the maintained fork `go.yaml.in/yaml/v3` v3.0.5 | | 2026-10-05 | Three deliberate breaks of the placeholder session: vault keys without the session, no placeholder reuse, redact restored | Each failed at least one test; restored from backup | | 2026-10-05 | Three deliberate breaks of the audit log: no record-hash check, no chain-link check, appending to a broken log | Each failed at least one test; restored from backup | +| 2026-10-05 | OpenAI's official Python library (v3.24.0) against a local `discreet serve`, changing only `base_url` | Completion returned and parsed; eligibility purpose refused with 403 `purpose_refused`; audit log verified; no planted value in the server or audit log | +| 2026-10-05 | Three deliberate breaks of the gateway: original text sent to the model, purpose check skipped, prompt logged | Each failed a test; restored from backup | | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite | ## What the AI got wrong, and how it was caught | Date | What went wrong | How it was caught | Fix | | --- | --- | --- | --- | +| 2026-10-05 | The mock model's reply said "Here is what I received, exactly as a real model would see it", but readers see that text after Discreet restores the real values, so it showed them the opposite of what the model saw | Reading the reply returned to OpenAI's Python client during the compatibility check | Reworded: the mock says it only ever saw placeholders and that Discreet put the real details back | +| 2026-10-05 | A gateway test searched JSON-encoded messages for ``, but Go's encoder writes `<` as `\u003c`, so a correct gateway failed the test | The test failure itself, read before changing any gateway code | Fixed the test to read message text directly; the gateway now keeps `<` and `>` unescaped in responses so placeholders stay readable | diff --git a/docs/adr/0006-purpose-header-and-what-is-forwarded.md b/docs/adr/0006-purpose-header-and-what-is-forwarded.md new file mode 100644 index 0000000..82163bd --- /dev/null +++ b/docs/adr/0006-purpose-header-and-what-is-forwarded.md @@ -0,0 +1,30 @@ +# 0006. Callers declare a purpose; eligibility decisions are refused; only known fields are forwarded + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-05 + +## Context + +The brief's non-negotiable: Discreet never takes part in eligibility or benefits decisions; a person makes those. Discreet can't see what a caller does with an answer, but it can make the caller say what a request is for, refuse some purposes outright, and keep a record. Separately, OpenAI's request format has many fields beyond the messages (`tools`, `user`, `metadata`…), and any of them could carry personal data past a gateway that only inspects messages. + +## Options + +1. **A required `X-Discreet-Purpose` header**, checked against denied phrases in the policy plus a rule for variants ("eligibility" together with decide, determine, approve, reject or deny), refused with HTTP 403 and an audit record. +2. **Classify the prompt's intent with a model.** Catches undeclared misuse, but it's probabilistic, costs money per request, and sends the prompt to yet another model. +3. **No purpose at all**, relying on terms of use. + +For fields: **forward only the model, messages, token limit and temperature**, or forward everything and scrub every string. + +## Decision + +Option 1, and forward only the known fields. + +- Missing purpose: 400. Denied purpose: 403 with `purpose_refused`, recorded as `refused_purpose`; the model is never called. The purpose is stored in the audit log with any personal data replaced by the entity name and capped at 200 characters. +- Requests are rebuilt from model, messages (string or text parts), token limit and temperature. Other fields are ignored and never forwarded. Tools, images and tool-role messages are rejected (400), because v0.1 can't protect them. Streaming is rejected until it's supported. +- Every answer ends with a disclosure footer naming the Discreet request ID, so readers know personal data was replaced. + +## Consequences + +- A caller can lie about its purpose. The header makes the claim explicit and recorded, which is what accountability needs; it isn't a technical barrier. "Explain the eligibility criteria" is allowed; "decide eligibility" is not. +- Dropping fields means some OpenAI features (tools, JSON mode, images) don't work through Discreet v0.1. That's the price of not letting personal data slip through fields Discreet doesn't inspect. +- The footer changes the answer text, which can break callers that expect strict JSON output; v0.1 accepts that. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go new file mode 100644 index 0000000..253aa53 --- /dev/null +++ b/internal/gateway/gateway.go @@ -0,0 +1,291 @@ +// Package gateway is Discreet's HTTP server: an OpenAI-compatible +// /v1/chat/completions endpoint that protects personal data on the way to +// the model and restores it on the way back, recording every decision in +// the audit log. +package gateway + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "sort" + "strings" + "time" + + "github.com/makoydev/discreet/internal/audit" + "github.com/makoydev/discreet/internal/detect" + "github.com/makoydev/discreet/internal/protect" + "github.com/makoydev/discreet/internal/upstream" + "github.com/makoydev/discreet/internal/vault" +) + +// Footer is appended to every answer, so people reading it know. +const Footer = "\n\n---\nPersonal data was replaced before this request reached the AI model, and put back afterwards (Discreet request %s)." + +// Config wires the gateway together. +type Config struct { + Engine *detect.Engine + Policy *protect.Policy + Vault *vault.Vault + Audit *audit.Log + Upstream upstream.Client + Logger *slog.Logger + Tenant string + MaxBodyBytes int64 +} + +// Server handles requests. Create it with New. +type Server struct{ cfg Config } + +// New returns a server, filling in defaults. +func New(cfg Config) *Server { + if cfg.Upstream == nil { + cfg.Upstream = &upstream.Mock{} + } + if cfg.Logger == nil { + cfg.Logger = slog.Default() + } + if cfg.Tenant == "" { + cfg.Tenant = "default" + } + if cfg.MaxBodyBytes == 0 { + cfg.MaxBodyBytes = 64 << 10 + } + return &Server{cfg} +} + +// Handler returns the HTTP routes. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("POST /v1/chat/completions", s.chatCompletions) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok\n")) }) + return mux +} + +// chatRequest is the subset of OpenAI's request Discreet understands. +// Anything else in the body is ignored and never forwarded. +type chatRequest struct { + Model string `json:"model"` + Messages []chatMessageIn `json:"messages"` + MaxTokens *int `json:"max_tokens"` + MaxCompletionTokens *int `json:"max_completion_tokens"` + Temperature *float64 `json:"temperature"` + Stream bool `json:"stream"` +} + +type chatMessageIn struct { + Role string `json:"role"` + Content json.RawMessage `json:"content"` +} + +// text returns a message's text: a string, or an array of text parts. +func (m chatMessageIn) text() (string, error) { + var s string + if err := json.Unmarshal(m.Content, &s); err == nil { + return s, nil + } + var parts []struct { + Type string `json:"type"` + Text string `json:"text"` + } + if err := json.Unmarshal(m.Content, &parts); err != nil { + return "", errors.New("message content must be a string or an array of text parts") + } + var b strings.Builder + for _, p := range parts { + if p.Type != "text" { + return "", fmt.Errorf("content part type %q is not supported; only text", p.Type) + } + b.WriteString(p.Text) + } + return b.String(), nil +} + +var allowedRoles = map[string]bool{"system": true, "developer": true, "user": true, "assistant": true} + +type apiError struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + Code string `json:"code"` + } `json:"error"` +} + +func writeError(w http.ResponseWriter, status int, code, message string) { + var e apiError + e.Error.Message, e.Error.Type, e.Error.Code = message, "discreet_error", code + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(e) +} + +func newID() string { + b := make([]byte, 12) + rand.Read(b) + return hex.EncodeToString(b) +} + +func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { + start := time.Now() + id := newID() + w.Header().Set("X-Discreet-Request-Id", id) + log := s.cfg.Logger.With("request_id", id) + + var req chatRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, s.cfg.MaxBodyBytes)).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "request body is not valid JSON or is too large") + return + } + if req.Stream { + writeError(w, http.StatusBadRequest, "streaming_unsupported", "streaming is not supported in Discreet v0.1; send stream: false") + return + } + if len(req.Messages) == 0 { + writeError(w, http.StatusBadRequest, "invalid_request", "messages must not be empty") + return + } + texts := make([]string, len(req.Messages)) + for i, m := range req.Messages { + if !allowedRoles[m.Role] { + writeError(w, http.StatusBadRequest, "invalid_request", fmt.Sprintf("message role %q is not supported", m.Role)) + return + } + t, err := m.text() + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", err.Error()) + return + } + texts[i] = t + } + + // The purpose is required, recorded with any personal data removed, + // and refused outright for decisions about people's eligibility. + rawPurpose := strings.TrimSpace(r.Header.Get("X-Discreet-Purpose")) + if rawPurpose == "" { + writeError(w, http.StatusBadRequest, "missing_purpose", "set the X-Discreet-Purpose header to say what this request is for, for example \"claims summary\"") + return + } + purpose := s.scrub(rawPurpose) + rec := audit.Record{RequestID: id, Tenant: s.cfg.Tenant, Purpose: purpose, Upstream: s.cfg.Upstream.Name(), Model: req.Model} + if deniedPurpose(rawPurpose, s.cfg.Policy.DeniedPurposes) { + rec.Decision, rec.Reason = audit.RefusedPurpose, "purpose refused by policy" + if _, ok := s.finish(w, log, rec, start); !ok { + return + } + writeError(w, http.StatusForbidden, "purpose_refused", "Discreet does not take part in automated decisions about people's eligibility. A person must make that decision.") + return + } + + // Protect every message in one session, so placeholders are consistent. + session := protect.NewSession(s.cfg.Policy, s.cfg.Vault) + defer session.Close() + msgs := make([]upstream.Message, len(texts)) + for i, t := range texts { + protected, err := session.Protect(t, s.cfg.Engine.Detect(t)) + var blocked *protect.BlockedError + if errors.As(err, &blocked) { + rec.Decision, rec.Reason, rec.Entities = audit.BlockedEntity, blocked.Error(), session.Counts() + if _, ok := s.finish(w, log, rec, start); !ok { + return + } + writeError(w, http.StatusForbidden, "entity_blocked", blocked.Error()) + return + } + msgs[i] = upstream.Message{Role: req.Messages[i].Role, Content: protected} + } + rec.Entities = session.Counts() + promptJSON, _ := json.Marshal(texts) + rec.PromptHMAC = s.cfg.Audit.HMAC(string(promptJSON)) + + maxTokens := 0 + if req.MaxCompletionTokens != nil { + maxTokens = *req.MaxCompletionTokens + } else if req.MaxTokens != nil { + maxTokens = *req.MaxTokens + } + resp, err := s.cfg.Upstream.Complete(r.Context(), upstream.Request{Model: req.Model, Messages: msgs, MaxTokens: maxTokens, Temperature: req.Temperature}) + if err != nil { + rec.Decision, rec.Reason = audit.UpstreamError, "the model provider returned an error" + log.Warn("upstream error", "error", err.Error()) + if _, ok := s.finish(w, log, rec, start); !ok { + return + } + writeError(w, http.StatusBadGateway, "upstream_error", "the model provider returned an error") + return + } + + answer := session.Restore(resp.Content) + fmt.Sprintf(Footer, id) + rec.Decision, rec.Model, rec.ResponseHMAC = audit.Allowed, resp.Model, s.cfg.Audit.HMAC(answer) + saved, ok := s.finish(w, log, rec, start) + if !ok { + return + } + w.Header().Set("X-Discreet-Entities", entityHeader(rec.Entities)) + w.Header().Set("X-Discreet-Audit-Seq", fmt.Sprint(saved.Seq)) + w.Header().Set("Content-Type", "application/json") + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) // keep readable in raw output + enc.Encode(map[string]any{ + "id": "chatcmpl-" + id, + "object": "chat.completion", + "created": start.Unix(), + "model": resp.Model, + "choices": []map[string]any{{ + "index": 0, + "message": map[string]string{"role": "assistant", "content": answer}, + "finish_reason": resp.FinishReason, + }}, + "usage": map[string]int{ + "prompt_tokens": resp.PromptTokens, + "completion_tokens": resp.OutputTokens, + "total_tokens": resp.PromptTokens + resp.OutputTokens, + }, + }) +} + +// finish writes the audit record. If it can't, the request fails: Discreet +// never answers a request it couldn't record. +func (s *Server) finish(w http.ResponseWriter, log *slog.Logger, rec audit.Record, start time.Time) (audit.Record, bool) { + rec.LatencyMS = time.Since(start).Milliseconds() + saved, err := s.cfg.Audit.Append(rec) + if err != nil { + log.Error("audit write failed", "error", err.Error()) + writeError(w, http.StatusInternalServerError, "audit_failed", "the request could not be recorded, so it was not processed") + return saved, false + } + log.Info("request", "decision", rec.Decision, "entities", entityHeader(rec.Entities), "upstream", rec.Upstream, "latency_ms", rec.LatencyMS, "audit_seq", saved.Seq) + return saved, true +} + +// scrub replaces personal data in short metadata (the purpose) with the +// entity name, and caps its length. +func (s *Server) scrub(text string) string { + if len(text) > 200 { + text = text[:200] + } + var b strings.Builder + pos := 0 + for _, m := range s.cfg.Engine.Detect(text) { + b.WriteString(text[pos:m.Start] + "[" + m.Entity + "]") + pos = m.End + } + return b.String() + text[pos:] +} + +// entityHeader summarises counts, never values: "NRIC=2,PHONE=1". +func entityHeader(counts map[string]int) string { + keys := make([]string, 0, len(counts)) + for k := range counts { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, len(keys)) + for i, k := range keys { + parts[i] = fmt.Sprintf("%s=%d", k, counts[k]) + } + return strings.Join(parts, ",") +} diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go new file mode 100644 index 0000000..e35e1e2 --- /dev/null +++ b/internal/gateway/gateway_test.go @@ -0,0 +1,293 @@ +package gateway + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/makoydev/discreet/internal/audit" + "github.com/makoydev/discreet/internal/detect" + "github.com/makoydev/discreet/internal/protect" + "github.com/makoydev/discreet/internal/upstream" + "github.com/makoydev/discreet/internal/vault" +) + +// Synthetic values only. The NRIC is sg-pii-rules' worked example; the +// card is a published test number; the email uses a reserved domain. +const ( + nric = "S1234567D" + phone = "9123 4567" + email = "tan.ah.kow@example.com" + card = "4111 1111 1111 1111" +) + +var planted = []string{nric, phone, email, card, "4111111111111111"} + +type harness struct { + srv *httptest.Server + mock *upstream.Mock + logs *bytes.Buffer + logPath string + audit *audit.Log +} + +func newHarness(t *testing.T, policy *protect.Policy, up upstream.Client) *harness { + t.Helper() + engine, err := detect.Default() + if err != nil { + t.Fatal(err) + } + v, _ := vault.New(time.Minute) + h := &harness{logs: &bytes.Buffer{}, logPath: filepath.Join(t.TempDir(), "audit.jsonl")} + h.audit, err = audit.Open(h.logPath, []byte("test-key-0123456789abcdefghijklmnop")) + if err != nil { + t.Fatal(err) + } + if up == nil { + h.mock = &upstream.Mock{} + up = h.mock + } + logger := slog.New(slog.NewJSONHandler(h.logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + s := New(Config{Engine: engine, Policy: policy, Vault: v, Audit: h.audit, Upstream: up, Logger: logger}) + h.srv = httptest.NewServer(s.Handler()) + t.Cleanup(h.srv.Close) + return h +} + +func (h *harness) post(t *testing.T, purpose, body string) (*http.Response, map[string]any) { + t.Helper() + req, _ := http.NewRequest("POST", h.srv.URL+"/v1/chat/completions", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if purpose != "" { + req.Header.Set("X-Discreet-Purpose", purpose) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out map[string]any + json.NewDecoder(resp.Body).Decode(&out) + return resp, out +} + +func (h *harness) records(t *testing.T) []audit.Record { + t.Helper() + data, _ := os.ReadFile(h.logPath) + var out []audit.Record + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + var r audit.Record + json.Unmarshal([]byte(line), &r) + out = append(out, r) + } + return out +} + +func body(messages ...string) string { + type msg struct{ Role, Content string } + var ms []map[string]string + for i, m := range messages { + role := "user" + if i == 0 && len(messages) > 1 { + role = "system" + } + ms = append(ms, map[string]string{"role": role, "content": m}) + } + data, _ := json.Marshal(map[string]any{"model": "gpt-6-luna", "messages": ms}) + return string(data) +} + +func answer(t *testing.T, out map[string]any) string { + t.Helper() + choices, ok := out["choices"].([]any) + if !ok || len(choices) == 0 { + t.Fatalf("no choices in %v", out) + } + return choices[0].(map[string]any)["message"].(map[string]any)["content"].(string) +} + +func TestTheModelNeverSeesPersonalData(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + resp, out := h.post(t, "claims summary", body( + "You help with insurance claims.", + "Member "+nric+" (phone "+phone+", "+email+") disputes a charge on card "+card+". Draft a reply to "+nric+".")) + if resp.StatusCode != 200 { + t.Fatalf("status %d: %v", resp.StatusCode, out) + } + + var all []string + for _, m := range h.mock.Last.Messages { + all = append(all, m.Content) + } + seen := []byte(strings.Join(all, "\n")) + for _, v := range planted { + if strings.Contains(string(seen), v) { + t.Errorf("the model received %q", v) + } + } + for _, p := range []string{"", "", "", "[REDACTED_CARD]"} { + if !strings.Contains(string(seen), p) { + t.Errorf("the model did not receive %s", p) + } + } + if strings.Count(string(seen), "") != 2 { + t.Error("the same NRIC should be the same placeholder both times") + } + + got := answer(t, out) + for _, v := range []string{nric, phone, email} { + if !strings.Contains(got, v) { + t.Errorf("answer is missing restored %q: %s", v, got) + } + } + if strings.Contains(got, "4111") || !strings.Contains(got, "[REDACTED_CARD]") { + t.Error("a redacted card must never be restored") + } + if !strings.Contains(got, "Personal data was replaced") { + t.Error("disclosure footer missing") + } + if e := resp.Header.Get("X-Discreet-Entities"); e != "CARD=1,EMAIL=1,NRIC=2,PHONE=1" { + t.Errorf("X-Discreet-Entities = %q", e) + } + + recs := h.records(t) + if len(recs) != 1 || recs[0].Decision != audit.Allowed || recs[0].Entities["NRIC"] != 2 || recs[0].PromptHMAC == "" || recs[0].ResponseHMAC == "" { + t.Errorf("audit records %+v", recs) + } + logFile, _ := os.ReadFile(h.logPath) + for _, v := range planted { + if strings.Contains(h.logs.String(), v) { + t.Errorf("server logs contain %q", v) + } + if strings.Contains(string(logFile), v) { + t.Errorf("audit log contains %q", v) + } + } +} + +func TestPurposeIsRequired(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + resp, out := h.post(t, "", body("hi")) + if resp.StatusCode != 400 || !strings.Contains(out["error"].(map[string]any)["code"].(string), "missing_purpose") { + t.Errorf("status %d %v", resp.StatusCode, out) + } +} + +func TestEligibilityDecisionsAreRefused(t *testing.T) { + for _, purpose := range []string{ + "automated eligibility decision", + "Automated-Eligibility Decision!", + "decide eligibility for the subsidy", + "benefits decision", + "eligibility determination", + "approve or reject eligibility", + } { + t.Run(purpose, func(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + resp, _ := h.post(t, purpose, body("Is "+nric+" eligible?")) + if resp.StatusCode != 403 { + t.Fatalf("status %d", resp.StatusCode) + } + if h.mock.Last != nil { + t.Error("the model was called for a refused purpose") + } + if recs := h.records(t); len(recs) != 1 || recs[0].Decision != audit.RefusedPurpose { + t.Errorf("audit %+v", recs) + } + }) + } +} + +func TestOtherPurposesAreAllowed(t *testing.T) { + for _, purpose := range []string{"claims summary", "explain the eligibility criteria", "ticket triage"} { + h := newHarness(t, protect.DefaultPolicy(), nil) + if resp, out := h.post(t, purpose, body("hello")); resp.StatusCode != 200 { + t.Errorf("%q: status %d %v", purpose, resp.StatusCode, out) + } + } +} + +func TestPurposeIsScrubbedBeforeRecording(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + h.post(t, "follow up with "+nric, body("hello")) + if recs := h.records(t); recs[0].Purpose != "follow up with [NRIC]" { + t.Errorf("purpose recorded as %q", recs[0].Purpose) + } +} + +func TestBlockedEntityRefusesTheRequest(t *testing.T) { + p, _ := protect.ParsePolicy([]byte("version: 1\nentities:\n NRIC: block\n")) + h := newHarness(t, p, nil) + resp, out := h.post(t, "claims summary", body("Phone "+phone+", member "+nric)) + if resp.StatusCode != 403 || h.mock.Last != nil { + t.Fatalf("status %d, model called: %v (%v)", resp.StatusCode, h.mock.Last != nil, out) + } + if recs := h.records(t); recs[0].Decision != audit.BlockedEntity || recs[0].Entities["NRIC"] != 1 { + t.Errorf("audit %+v", recs) + } +} + +func TestOnlyKnownFieldsAreForwarded(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + resp, _ := h.post(t, "claims summary", `{"model":"m","user":"`+nric+`","tools":[{"type":"function","function":{"name":"lookup","description":"`+email+`"}}],"messages":[{"role":"user","content":[{"type":"text","text":"hi "},{"type":"text","text":"there"}]}]}`) + if resp.StatusCode != 200 || h.mock.Last.Messages[0].Content != "hi there" { + t.Fatalf("status %d, content %q", resp.StatusCode, h.mock.Last.Messages[0].Content) + } + if strings.Contains(h.logs.String(), nric) || strings.Contains(h.logs.String(), email) { + t.Error("unforwarded fields reached the logs") + } +} + +func TestBadRequests(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + for name, b := range map[string]string{ + "not json": "{", + "streaming": `{"model":"m","stream":true,"messages":[{"role":"user","content":"hi"}]}`, + "no message": `{"model":"m","messages":[]}`, + "tool role": `{"model":"m","messages":[{"role":"tool","content":"hi"}]}`, + "image part": `{"model":"m","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"x"}}]}]}`, + } { + if resp, _ := h.post(t, "claims summary", b); resp.StatusCode != 400 { + t.Errorf("%s: status %d", name, resp.StatusCode) + } + } +} + +type failing struct{} + +func (failing) Name() string { return "failing" } +func (failing) Complete(context.Context, upstream.Request) (upstream.Response, error) { + return upstream.Response{}, errors.New("boom") +} + +func TestUpstreamErrorIsRecorded(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), failing{}) + resp, _ := h.post(t, "claims summary", body("hi "+nric)) + if resp.StatusCode != 502 { + t.Fatalf("status %d", resp.StatusCode) + } + if recs := h.records(t); recs[0].Decision != audit.UpstreamError { + t.Errorf("audit %+v", recs) + } +} + +func TestNoAnswerWithoutAnAuditRecord(t *testing.T) { + h := newHarness(t, protect.DefaultPolicy(), nil) + h.audit.Close() + resp, _ := h.post(t, "claims summary", body("hi")) + if resp.StatusCode != 500 { + t.Errorf("status %d, want 500 when the audit log can't be written", resp.StatusCode) + } +} diff --git a/internal/gateway/purpose.go b/internal/gateway/purpose.go new file mode 100644 index 0000000..00cb941 --- /dev/null +++ b/internal/gateway/purpose.go @@ -0,0 +1,33 @@ +package gateway + +import ( + "regexp" + "strings" +) + +var nonWord = regexp.MustCompile(`[^a-z0-9]+`) + +func normalise(s string) string { + return strings.TrimSpace(nonWord.ReplaceAllString(strings.ToLower(s), " ")) +} + +// deniedPurpose reports whether a declared purpose is refused: it contains +// one of the policy's denied phrases, ignoring case and punctuation, or it +// pairs "eligibility" with deciding, determining, approving or rejecting. +// Discreet never takes part in decisions about people's eligibility. +func deniedPurpose(purpose string, denied []string) bool { + p := " " + normalise(purpose) + " " + for _, phrase := range denied { + if n := normalise(phrase); n != "" && strings.Contains(p, " "+n+" ") { + return true + } + } + if strings.Contains(p, "eligib") { + for _, verb := range []string{"decision", "decide", "determin", "approv", "reject", "deny", "denial"} { + if strings.Contains(p, verb) { + return true + } + } + } + return false +} diff --git a/internal/upstream/upstream.go b/internal/upstream/upstream.go new file mode 100644 index 0000000..f03c5a4 --- /dev/null +++ b/internal/upstream/upstream.go @@ -0,0 +1,83 @@ +// Package upstream sends protected requests to a model. Every client sees +// only placeholder text; restoring values happens in the gateway. +package upstream + +import ( + "context" + "regexp" + "strings" +) + +// Message is one chat message, already protected. +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +// Request is what the gateway forwards: only these fields, nothing else +// from the caller's request. +type Request struct { + Model string + Messages []Message + MaxTokens int + Temperature *float64 +} + +// Response is the model's answer. +type Response struct { + Content string + Model string + FinishReason string + PromptTokens int + OutputTokens int +} + +// Client is a model provider. +type Client interface { + Name() string + Complete(ctx context.Context, req Request) (Response, error) +} + +// Mock answers without calling any model, so it is free and works offline. +// It drafts a reply that repeats the placeholders it was given, which makes +// restoring them visible in the demo. +type Mock struct { + // Last holds the most recent request, for tests that check what a + // model would have seen. + Last *Request +} + +func (*Mock) Name() string { return "mock" } + +var mockPlaceholder = regexp.MustCompile(`<[A-Z][A-Z_]*_\d+>`) + +func (m *Mock) Complete(_ context.Context, req Request) (Response, error) { + m.Last = &req + var last string + for _, msg := range req.Messages { + if msg.Role == "user" { + last = msg.Content + } + } + reply := "This is Discreet's mock model: no AI was called. I only ever saw placeholders; Discreet put the real details back before you read this.\n\nYour message, as I received it:\n\n" + last + if ps := mockPlaceholder.FindAllString(last, -1); len(ps) > 0 { + reply += "\n\nDraft reply: Thanks. I've noted the details for " + strings.Join(unique(ps), ", ") + " and will follow up." + } + tokens := func(s string) int { return len(s)/4 + 1 } + prompt := 0 + for _, msg := range req.Messages { + prompt += tokens(msg.Content) + } + return Response{Content: reply, Model: "mock", FinishReason: "stop", PromptTokens: prompt, OutputTokens: tokens(reply)}, nil +} + +func unique(items []string) []string { + seen, out := map[string]bool{}, []string{} + for _, s := range items { + if !seen[s] { + seen[s] = true + out = append(out, s) + } + } + return out +} From e5574b86d16edcd59a538ec0fab15d28e164e85c Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:51:23 +0800 Subject: [PATCH 06/13] feat: real model behind access tokens, daily budget, rate limit (D6) (#9) - upstream.OpenAICompatible: OpenAI or Ollama /chat/completions, configurable token-limit parameter, optional reasoning effort; only the provider's HTTP status is ever reported, never its body. - Access tokens (tenant:token, 24+ chars) kept as SHA-256, compared in constant time. A missing or wrong token gets the free mock, so the public demo can't spend money; the caller's token is never forwarded. - One pinned model with checked prices (gpt-6-luna US$0.10/0.50 per M, OpenAI model page). Each real request reserves its worst case (at most US$0.0076) against a hard daily budget (default US$0.20) summed from the audit log, so restarts can't reset it; refusals are recorded. - 30 requests/minute per client address; proxy header only if named. - Tests use a fake OpenAI server: no paid API is called. Deliberate breaks of tokens (redone so it compiles), budget and rate limit each failed tests; restored from backup. ADR 0007. Closes makoydev/vetted#36 Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + CONTROLS.md | 2 +- EVALS.md | 4 + README.md | 11 + THREAT_MODEL.md | 3 +- cmd/discreet/main.go | 113 +++++++- cmd/discreet/main_test.go | 20 ++ docs/HOW-THIS-WAS-BUILT.md | 3 + .../0007-access-tokens-and-daily-budget.md | 26 ++ internal/gateway/gateway.go | 73 ++++- internal/gateway/gateway_test.go | 10 +- internal/gateway/limits.go | 181 ++++++++++++ internal/gateway/limits_test.go | 260 ++++++++++++++++++ internal/upstream/openai.go | 94 +++++++ 14 files changed, 787 insertions(+), 14 deletions(-) create mode 100644 docs/adr/0007-access-tokens-and-daily-budget.md create mode 100644 internal/gateway/limits.go create mode 100644 internal/gateway/limits_test.go create mode 100644 internal/upstream/openai.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 051c83a..4f926c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project are documented here. The format follows [Kee - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. - Vetted reviews every pull request (shadow mode until 2026-10-14). - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases. +- A real model behind access tokens (issue D6): an OpenAI-compatible client (OpenAI, or Ollama with `max_tokens`) used only for callers with a valid `DISCREET_ACCESS_TOKENS` token; everyone else gets the mock. One pinned model with checked prices (`gpt-6-luna`); each request reserves its worst-case cost (at most US$0.0076) against a hard daily budget (default US$0.20) recomputed from the audit log; 30 requests a minute per client address (ADR 0007). No test calls a paid API. - The gateway (`internal/gateway`, issue D5): `discreet serve` runs an OpenAI-compatible `POST /v1/chat/completions` (non-streaming). Every message is protected in one session before the model sees it and restored afterwards; every answer ends with a disclosure footer; every request, allowed or not, is recorded in the audit log, and Discreet never answers a request it couldn't record. A free mock model is the default. Callers must send `X-Discreet-Purpose`; automated eligibility and benefits decisions are refused with HTTP 403 (ADR 0006). Only the model, messages, token limit and temperature are forwarded. Tested with OpenAI's official Python library by changing only `base_url`. - Tamper-evident audit log (`internal/audit`, issue D7): append-only JSON lines, each record holding the SHA-256 of the previous one and HMAC-SHA-256 hashes of prompt and response (never the text). `discreet audit verify` names the first changed, missing or reordered record; `discreet audit export -csv` exports a verified log. An existing broken log is refused at start-up (ADR 0005). - Placeholders and the vault (`internal/protect`, `internal/vault`, issue D4): personal data is replaced per request with numbered placeholders such as `` (the same value gets the same placeholder), redacted as `[REDACTED_CARD]`, or the request is blocked, following a YAML policy (`internal/protect/default-policy.yaml`). Values are kept encrypted in memory with AES-256-GCM for at most 10 minutes and restored only into the same request's answer (ADR 0004). First third-party module: `go.yaml.in/yaml/v3` v3.0.5. diff --git a/CONTROLS.md b/CONTROLS.md index 78a8266..8243ad6 100644 --- a/CONTROLS.md +++ b/CONTROLS.md @@ -7,5 +7,5 @@ Each Discreet feature, the principle or obligation it supports, and the source. | Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection (D3, 369/369 shared cases) and replacement with placeholders (D4) in place; mapping pending | | Purpose declared in a header; automated eligibility decisions refused before the model is called; disclosure footer on every answer | Human accountability for decisions about people; transparency to end users | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | In place (D5); mapping pending | | Hash-chained audit log with `verify` and CSV export; prompts and responses kept only as keyed hashes | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | In place (D7); mapping pending | -| Daily budget, access token, rate limit | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | Planned (D6); mapping pending | +| Daily budget with a worst-case bound, access tokens, rate limit, one pinned model | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | In place (D6); mapping pending | | Pinned Actions, `govulncheck`, CodeQL, reviewed pull requests | Secure development | OWASP Top 10 for LLM Applications 2025 (supply chain) | In place (D1); mapping pending | diff --git a/EVALS.md b/EVALS.md index 2f2e0df..d1ecc99 100644 --- a/EVALS.md +++ b/EVALS.md @@ -13,6 +13,10 @@ Known limitation, stated in advance: synthetic data flatters rule-based detector Vetted's review of this repository's pull requests is measured in Vetted's own `EVALS.md`. +## Cost bound (issue D6, computed 2026-10-05) + +With `gpt-6-luna` at US$0.10 / US$0.50 per million input / output tokens (OpenAI's model page, primary source), the 64 KB request limit and the default 2,048-token output cap, one request can cost at most **US$0.0076**: (65,536 bytes + 16) × 0.10 / 10⁶ + 2,048 × 0.50 / 10⁶. The default daily budget of US$0.20 therefore allows at least 26 worst-case requests, and at most about US$6 a month. This is an upper bound by construction, not a measurement; no real request has been sent yet, because `OPENAI_API_KEY` is not configured. + ## Detector conformance (issue D3, measured 2026-10-05) `go test ./internal/detect` runs every case in the vendored sg-pii-rules `v0.2.0-rc.1` fixtures through Discreet's Go engine: **369 of 369 pass**, across NRIC, NRIC look-alikes, phone, email, card, postal code, unit number and date of birth. These cases are a behaviour specification shared with Vetted, not a measure of real-world accuracy (that is the benchmark above). diff --git a/README.md b/README.md index 0624f8d..97def2f 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,17 @@ curl -s localhost:8080/v1/chat/completions \ -d '{"model":"gpt-6-luna","messages":[{"role":"user","content":"Call S1234567D on 9123 4567"}]}' ``` +To use a real model, point Discreet at it and give callers tokens; anyone without a valid token still gets the mock: + +```sh +export DISCREET_UPSTREAM_URL=https://api.openai.com/v1 +export DISCREET_UPSTREAM_API_KEY=... # your OpenAI key, set it yourself +export DISCREET_ACCESS_TOKENS=claims-team:$(openssl rand -hex 16) +./bin/discreet serve # gpt-6-luna, US$0.20/day hard cap +``` + +Run `./bin/discreet serve --help` for every setting (budget, output cap, rate limit, Ollama). + The mock model shows what a real model would have received (placeholders only), and the answer comes back with the real values restored. Any OpenAI client works by pointing `base_url` at `http://localhost:8080/v1` and adding the `X-Discreet-Purpose` header. The example NRIC is synthetic. ## Evidence diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index d6b46c0..021ce98 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -29,5 +29,6 @@ | Personal data travels in request fields the gateway doesn't inspect (`tools`, `user`, metadata) | Requests are rebuilt from model, messages, token limit and temperature only; tools, images and tool-role messages are rejected (ADR 0006, test) | D5 (done) | | Personal data ends up in server logs | Logs carry counts and decisions, never text; a test captures every log line and searches it for planted values | D5 (done) | | A caller uses Discreet for automated eligibility decisions | Required purpose header; denied purposes refused with 403 before the model is called, and recorded (ADR 0006) | D5 (done) | -| The public demo is used to spend money | Mock model unless a valid access token is present; daily budget; rate limit | D6, D10 | +| The public demo is used to spend money | Mock model unless a valid access token is present (a wrong or missing token gets the mock, test); worst-case cost reserved against a hard daily budget that survives restarts (tests, including 100 parallel requests); 30 requests a minute per address (ADR 0007) | D6 (done), D10 | +| An access token leaks from memory or logs | Only SHA-256 hashes kept, compared in constant time; tokens and provider keys never logged; the caller's token is never forwarded | D6 (done) | | A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 | diff --git a/cmd/discreet/main.go b/cmd/discreet/main.go index 594f2c8..1a8f7f0 100644 --- a/cmd/discreet/main.go +++ b/cmd/discreet/main.go @@ -18,6 +18,7 @@ import ( "net/http" "os" "os/signal" + "strconv" "strings" "syscall" "time" @@ -26,6 +27,7 @@ import ( "github.com/makoydev/discreet/internal/detect" "github.com/makoydev/discreet/internal/gateway" "github.com/makoydev/discreet/internal/protect" + "github.com/makoydev/discreet/internal/upstream" "github.com/makoydev/discreet/internal/vault" "github.com/makoydev/discreet/third_party/sgpiirules" ) @@ -54,8 +56,25 @@ it back in the answer, and records every request in the audit log. Requires DISCREET_HMAC_KEY (at least 32 characters, e.g. openssl rand -hex 32). Callers must send an X-Discreet-Purpose header. -Environment: DISCREET_ADDR, DISCREET_AUDIT_LOG, DISCREET_POLICY, -DISCREET_HMAC_KEY. +By default every request goes to a free mock model. To reach a real model, +set DISCREET_UPSTREAM_URL and give callers access tokens; anyone without a +valid token still gets the mock. + + DISCREET_ADDR listen address (default :8080) + DISCREET_AUDIT_LOG audit log file (default discreet-audit.jsonl) + DISCREET_POLICY policy YAML (default: built in) + DISCREET_UPSTREAM_URL e.g. https://api.openai.com/v1 or http://localhost:11434/v1 + DISCREET_UPSTREAM_API_KEY the provider's key + DISCREET_UPSTREAM_MODEL default gpt-6-luna + DISCREET_UPSTREAM_TOKEN_PARAM max_completion_tokens (OpenAI) or max_tokens (Ollama) + DISCREET_PRICE_INPUT_PER_M US$ per million input tokens (known for gpt-6-luna) + DISCREET_PRICE_OUTPUT_PER_M US$ per million output tokens + DISCREET_ACCESS_TOKENS tenant:token,tenant2:token2 (tokens of 24+ characters) + DISCREET_DAILY_BUDGET_USD hard daily limit for the real model (default 0.20) + DISCREET_MAX_OUTPUT_TOKENS output cap per request (default 2048) + DISCREET_REASONING_EFFORT optional, e.g. low + DISCREET_RATE_LIMIT_PER_MINUTE per client address (default 30, bursts of 10) + DISCREET_CLIENT_IP_HEADER header set by a trusted proxy, e.g. Fly-Client-IP ` const auditUsage = `Usage: discreet audit [-log path] @@ -208,8 +227,25 @@ func runServe(ctx context.Context, args []string, stderr io.Writer) int { defer log.Close() logger := slog.New(slog.NewJSONHandler(stderr, nil)) + cfg := gateway.Config{Engine: engine, Policy: policy, Vault: v, Audit: log, Logger: logger} + rate, err := envInt("DISCREET_RATE_LIMIT_PER_MINUTE", 30) + if err != nil { + return fail(err) + } + cfg.RateLimit = gateway.NewRateLimiter(rate, 10, os.Getenv("DISCREET_CLIENT_IP_HEADER")) + upstreamInfo := "mock only" + if url := os.Getenv("DISCREET_UPSTREAM_URL"); url != "" { + if err := configureReal(&cfg, url, log.SpentToday); err != nil { + return fail(err) + } + upstreamInfo = fmt.Sprintf("%s for %d access token(s), otherwise mock; budget US$%.2f/day; worst case US$%.4f per request", + cfg.Model, cfg.Tokens.Len(), mustFloat("DISCREET_DAILY_BUDGET_USD", 0.20), cfg.Price.Ceiling(64<<10, 1, cfg.MaxOutputTokens)) + if cfg.Tokens.Len() == 0 { + logger.Warn("DISCREET_UPSTREAM_URL is set but there are no DISCREET_ACCESS_TOKENS, so every request gets the mock") + } + } srv := &http.Server{ - Handler: gateway.New(gateway.Config{Engine: engine, Policy: policy, Vault: v, Audit: log, Logger: logger}).Handler(), + Handler: gateway.New(cfg).Handler(), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 30 * time.Second, WriteTimeout: 120 * time.Second, @@ -221,7 +257,7 @@ func runServe(ctx context.Context, args []string, stderr io.Writer) int { } version, _ := sgpiirules.Files.ReadFile("VERSION") head, seq := log.Head() - logger.Info("discreet listening", "addr", ln.Addr().String(), "upstream", "mock", + logger.Info("discreet listening", "addr", ln.Addr().String(), "upstream", upstreamInfo, "rules", "sg-pii-rules "+strings.TrimSpace(string(version)), "audit_log", *logPath, "audit_records", seq, "audit_head", head) go func() { @@ -251,3 +287,72 @@ func runServe(ctx context.Context, args []string, stderr io.Writer) int { logger.Info("discreet stopped") return 0 } + +// configureReal sets up the real model from the environment. Secrets are +// read here and never logged. +func configureReal(cfg *gateway.Config, url string, spentToday func() float64) error { + tokens, err := gateway.ParseTokens(os.Getenv("DISCREET_ACCESS_TOKENS")) + if err != nil { + return err + } + model := envOr("DISCREET_UPSTREAM_MODEL", "gpt-6-luna") + price, known := gateway.KnownPrices[model] + in, inSet := os.LookupEnv("DISCREET_PRICE_INPUT_PER_M") + out, outSet := os.LookupEnv("DISCREET_PRICE_OUTPUT_PER_M") + if inSet || outSet { + if price.Input, err = strconv.ParseFloat(in, 64); err != nil { + return fmt.Errorf("DISCREET_PRICE_INPUT_PER_M: %w", err) + } + if price.Output, err = strconv.ParseFloat(out, 64); err != nil { + return fmt.Errorf("DISCREET_PRICE_OUTPUT_PER_M: %w", err) + } + } else if !known { + return fmt.Errorf("no known price for %s: set DISCREET_PRICE_INPUT_PER_M and DISCREET_PRICE_OUTPUT_PER_M (0 for a local model)", model) + } + budget, err := envFloat("DISCREET_DAILY_BUDGET_USD", 0.20) + if err != nil { + return err + } + maxOut, err := envInt("DISCREET_MAX_OUTPUT_TOKENS", 2048) + if err != nil { + return err + } + cfg.Real = &upstream.OpenAICompatible{ + BaseURL: url, + APIKey: os.Getenv("DISCREET_UPSTREAM_API_KEY"), + TokenParam: envOr("DISCREET_UPSTREAM_TOKEN_PARAM", "max_completion_tokens"), + ReasoningEffort: os.Getenv("DISCREET_REASONING_EFFORT"), + } + cfg.Tokens, cfg.Model, cfg.Price, cfg.MaxOutputTokens = tokens, model, price, maxOut + cfg.Budget = gateway.NewBudget(budget, spentToday) + return nil +} + +func envInt(name string, fallback int) (int, error) { + v := os.Getenv(name) + if v == "" { + return fallback, nil + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%s must be a positive whole number", name) + } + return n, nil +} + +func envFloat(name string, fallback float64) (float64, error) { + v := os.Getenv(name) + if v == "" { + return fallback, nil + } + f, err := strconv.ParseFloat(v, 64) + if err != nil || f < 0 { + return 0, fmt.Errorf("%s must be a number of US dollars", name) + } + return f, nil +} + +func mustFloat(name string, fallback float64) float64 { + f, _ := envFloat(name, fallback) + return f +} diff --git a/cmd/discreet/main_test.go b/cmd/discreet/main_test.go index b99b892..ba5282a 100644 --- a/cmd/discreet/main_test.go +++ b/cmd/discreet/main_test.go @@ -132,3 +132,23 @@ func TestServeAnswersAndStops(t *testing.T) { t.Errorf("exit code %d", code) } } + +func TestServeRefusesAnUnpricedModel(t *testing.T) { + t.Setenv("DISCREET_HMAC_KEY", "test-key-0123456789abcdefghijklmnop") + t.Setenv("DISCREET_UPSTREAM_URL", "http://127.0.0.1:1/v1") + t.Setenv("DISCREET_UPSTREAM_MODEL", "some-other-model") + var errOut bytes.Buffer + if code := run([]string{"serve", "-audit-log", filepath.Join(t.TempDir(), "a.jsonl")}, io.Discard, &errOut); code != 1 || !strings.Contains(errOut.String(), "no known price") { + t.Errorf("code %d, stderr %q", code, errOut.String()) + } +} + +func TestServeRefusesWeakAccessTokens(t *testing.T) { + t.Setenv("DISCREET_HMAC_KEY", "test-key-0123456789abcdefghijklmnop") + t.Setenv("DISCREET_UPSTREAM_URL", "http://127.0.0.1:1/v1") + t.Setenv("DISCREET_ACCESS_TOKENS", "acme:short") + var errOut bytes.Buffer + if code := run([]string{"serve", "-audit-log", filepath.Join(t.TempDir(), "a.jsonl")}, io.Discard, &errOut); code != 1 { + t.Errorf("code %d, stderr %q", code, errOut.String()) + } +} diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md index f06da79..a8b9bc5 100644 --- a/docs/HOW-THIS-WAS-BUILT.md +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -32,11 +32,14 @@ | 2026-10-05 | Three deliberate breaks of the audit log: no record-hash check, no chain-link check, appending to a broken log | Each failed at least one test; restored from backup | | 2026-10-05 | OpenAI's official Python library (v3.24.0) against a local `discreet serve`, changing only `base_url` | Completion returned and parsed; eligibility purpose refused with 403 `purpose_refused`; audit log verified; no planted value in the server or audit log | | 2026-10-05 | Three deliberate breaks of the gateway: original text sent to the model, purpose check skipped, prompt logged | Each failed a test; restored from backup | +| 2026-10-05 | `gpt-6-luna` on `/v1/chat/completions` and its price (OpenAI's model page, primary) | Supported; US$0.10 / US$0.50 per million input / output tokens | +| 2026-10-05 | Deliberate breaks of tokens, budget and rate limit | Each failed tests once the token break was redone so it compiled; restored from backup | | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite | ## What the AI got wrong, and how it was caught | Date | What went wrong | How it was caught | Fix | | --- | --- | --- | --- | +| 2026-10-05 | A deliberate break of the access-token check left two variables unused, so Go refused to compile, and the filter used to list failing tests showed nothing, which looked like an uncaught break. The same class of mistake as sg-pii-rules' unbalanced-bracket break earlier that day | Noticing that no test was listed at all, then checking the build | Redid the break so it compiles (any non-empty token matches): the token test fails. Breaks now run `go vet` first, so a build failure can't pass for a result | | 2026-10-05 | The mock model's reply said "Here is what I received, exactly as a real model would see it", but readers see that text after Discreet restores the real values, so it showed them the opposite of what the model saw | Reading the reply returned to OpenAI's Python client during the compatibility check | Reworded: the mock says it only ever saw placeholders and that Discreet put the real details back | | 2026-10-05 | A gateway test searched JSON-encoded messages for ``, but Go's encoder writes `<` as `\u003c`, so a correct gateway failed the test | The test failure itself, read before changing any gateway code | Fixed the test to read message text directly; the gateway now keeps `<` and `>` unescaped in responses so placeholders stay readable | diff --git a/docs/adr/0007-access-tokens-and-daily-budget.md b/docs/adr/0007-access-tokens-and-daily-budget.md new file mode 100644 index 0000000..52c8a42 --- /dev/null +++ b/docs/adr/0007-access-tokens-and-daily-budget.md @@ -0,0 +1,26 @@ +# 0007. The real model only for access tokens, under a hard daily budget + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-05 + +## Context + +The brief: anything that calls a paid model respects a hard daily budget, defaults to a mock in public demos, and public endpoints never relay to a paid model without an access token, with per-IP rate limits. The whole toolkit must stay under US$20 a month. Discreet's demo will be public (issue D10). + +## Options + +For access: **a valid token gets the real model; anyone else gets the mock**, or reject requests without a token (401). For the budget: **reserve each request's worst case before calling**, or count real costs after the fact. + +## Decision + +- **Tokens.** `DISCREET_ACCESS_TOKENS` holds `tenant:token` pairs (tokens of 24+ characters). Only SHA-256 hashes are kept in memory, compared in constant time. A missing or wrong token is not an error: the request goes to the free mock, and the `X-Discreet-Upstream` header says so. OpenAI's client libraries always send some key, so this keeps the public demo usable while making it impossible for it to spend money. The caller's token is never forwarded; the provider gets Discreet's own key. +- **One model.** The real upstream is pinned to one model (default `gpt-6-luna`, US$0.10 / US$0.50 per million input / output tokens, from OpenAI's model page on 2026-10-05). A request for another model is refused (400). An unpriced model needs explicit prices, or Discreet won't start. +- **Worst case first.** Before each real call, Discreet computes the most it could cost: the protected prompt's bytes (a token is at least one byte) plus 16 tokens per message, at the input price, plus the output cap (`DISCREET_MAX_OUTPUT_TOKENS`, default 2048) at the output price. With the 64 KB body limit that's at most **US$0.0076 per request**. The request is refused (429, recorded as `refused_budget`) unless today's spending plus all in-flight reservations plus this worst case fits the daily limit (`DISCREET_DAILY_BUDGET_USD`, default US$0.20, so at least 26 worst-case requests a day). The real cost from the provider's usage figures is recorded in the audit log. +- **Restarts.** Today's spending is summed from the audit log at start-up (UTC days), so a restart can't reset it. +- **Rate limit.** 30 requests a minute per client address, in bursts of up to 10. Behind a proxy, the client address comes only from a header the operator names (`DISCREET_CLIENT_IP_HEADER`, e.g. `Fly-Client-IP`), never a header a client could forge by default. + +## Consequences + +- US$0.20 a day is at most about US$6 a month for Discreet's real model, inside the toolkit's US$20 cap with Vetted's US$6 and hosting. +- The worst-case bound is deliberately pessimistic: real requests cost far less, so the cap may refuse earlier than strictly necessary. +- The OpenAI-compatible client supports Ollama through `DISCREET_UPSTREAM_TOKEN_PARAM=max_tokens` and zero prices. That path is tested only against a fake server, not a real Ollama. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 253aa53..2e17c78 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -32,10 +32,20 @@ type Config struct { Policy *protect.Policy Vault *vault.Vault Audit *audit.Log - Upstream upstream.Client + Upstream upstream.Client // the free mock everyone gets by default Logger *slog.Logger - Tenant string + Tenant string // tenant name for callers without a valid token MaxBodyBytes int64 + + // The real model, used only for callers with a valid access token + // (ADR 0007). Nil means everyone gets the mock. + Real upstream.Client + Tokens *Tokens + Model string // the one model Real may be asked for + Price Price + Budget *Budget + MaxOutputTokens int + RateLimit *RateLimiter } // Server handles requests. Create it with New. @@ -50,7 +60,10 @@ func New(cfg Config) *Server { cfg.Logger = slog.Default() } if cfg.Tenant == "" { - cfg.Tenant = "default" + cfg.Tenant = "public" + } + if cfg.MaxOutputTokens == 0 { + cfg.MaxOutputTokens = 2048 } if cfg.MaxBodyBytes == 0 { cfg.MaxBodyBytes = 64 << 10 @@ -134,6 +147,12 @@ func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { id := newID() w.Header().Set("X-Discreet-Request-Id", id) log := s.cfg.Logger.With("request_id", id) + if !s.cfg.RateLimit.Allow(r) { + log.Info("rate limited") + w.Header().Set("Retry-After", "10") + writeError(w, http.StatusTooManyRequests, "rate_limited", "too many requests from this address; try again shortly") + return + } var req chatRequest if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, s.cfg.MaxBodyBytes)).Decode(&req); err != nil { @@ -170,7 +189,26 @@ func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { return } purpose := s.scrub(rawPurpose) - rec := audit.Record{RequestID: id, Tenant: s.cfg.Tenant, Purpose: purpose, Upstream: s.cfg.Upstream.Name(), Model: req.Model} + + // Only a valid access token reaches the real model; anyone else gets + // the free mock, so the public endpoint can never spend money. + up, tenant, real := s.cfg.Upstream, s.cfg.Tenant, false + if t, ok := s.cfg.Tokens.Tenant(r); ok { + tenant = t + if s.cfg.Real != nil { + up, real = s.cfg.Real, true + } + } + w.Header().Set("X-Discreet-Upstream", up.Name()) + model := req.Model + if real { + if req.Model != "" && req.Model != s.cfg.Model { + writeError(w, http.StatusBadRequest, "model_not_allowed", fmt.Sprintf("this gateway only sends requests to %s", s.cfg.Model)) + return + } + model = s.cfg.Model + } + rec := audit.Record{RequestID: id, Tenant: tenant, Purpose: purpose, Upstream: up.Name(), Model: model} if deniedPurpose(rawPurpose, s.cfg.Policy.DeniedPurposes) { rec.Decision, rec.Reason = audit.RefusedPurpose, "purpose refused by policy" if _, ok := s.finish(w, log, rec, start); !ok { @@ -207,7 +245,29 @@ func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { } else if req.MaxTokens != nil { maxTokens = *req.MaxTokens } - resp, err := s.cfg.Upstream.Complete(r.Context(), upstream.Request{Model: req.Model, Messages: msgs, MaxTokens: maxTokens, Temperature: req.Temperature}) + if maxTokens <= 0 || maxTokens > s.cfg.MaxOutputTokens { + maxTokens = s.cfg.MaxOutputTokens + } + + // Real calls reserve their worst-case cost first (ADR 0007). + if real { + promptBytes := 0 + for _, m := range msgs { + promptBytes += len(m.Content) + } + ceiling := s.cfg.Price.Ceiling(promptBytes, len(msgs), maxTokens) + if !s.cfg.Budget.Reserve(ceiling) { + rec.Decision, rec.Reason = audit.RefusedBudget, fmt.Sprintf("daily budget reached (this request could cost up to US$%.4f)", ceiling) + if _, ok := s.finish(w, log, rec, start); !ok { + return + } + writeError(w, http.StatusTooManyRequests, "budget_exceeded", "today's spending limit for the AI model has been reached; requests resume tomorrow (UTC)") + return + } + defer s.cfg.Budget.Release(ceiling) // after the real cost is recorded below + } + + resp, err := up.Complete(r.Context(), upstream.Request{Model: model, Messages: msgs, MaxTokens: maxTokens, Temperature: req.Temperature}) if err != nil { rec.Decision, rec.Reason = audit.UpstreamError, "the model provider returned an error" log.Warn("upstream error", "error", err.Error()) @@ -220,6 +280,9 @@ func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { answer := session.Restore(resp.Content) + fmt.Sprintf(Footer, id) rec.Decision, rec.Model, rec.ResponseHMAC = audit.Allowed, resp.Model, s.cfg.Audit.HMAC(answer) + if real { + rec.CostUSD = s.cfg.Price.Cost(resp.PromptTokens, resp.OutputTokens) + } saved, ok := s.finish(w, log, rec, start) if !ok { return diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index e35e1e2..8b99be2 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "errors" + "io" "log/slog" "net/http" "net/http/httptest" @@ -33,9 +34,12 @@ const ( var planted = []string{nric, phone, email, card, "4111111111111111"} type harness struct { - srv *httptest.Server - mock *upstream.Mock - logs *bytes.Buffer + srv *httptest.Server + mock *upstream.Mock + logs interface { + io.Writer + String() string + } logPath string audit *audit.Log } diff --git a/internal/gateway/limits.go b/internal/gateway/limits.go new file mode 100644 index 0000000..4ee67ae --- /dev/null +++ b/internal/gateway/limits.go @@ -0,0 +1,181 @@ +package gateway + +import ( + "crypto/sha256" + "crypto/subtle" + "fmt" + "net" + "net/http" + "strings" + "sync" + "time" +) + +// ----- Access tokens ----- + +// Tokens maps a SHA-256 of each access token to its tenant name. Only the +// hashes are kept in memory, and comparison takes constant time. +type Tokens struct{ hashes map[[32]byte]string } + +// ParseTokens reads "tenant:token,tenant2:token2". Tokens must be at +// least 24 characters, so they can't be guessed. +func ParseTokens(spec string) (*Tokens, error) { + t := &Tokens{hashes: map[[32]byte]string{}} + for _, pair := range strings.Split(spec, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + tenant, token, ok := strings.Cut(pair, ":") + if !ok || tenant == "" || len(token) < 24 { + return nil, fmt.Errorf("access tokens must look like tenant:token, with tokens of at least 24 characters") + } + t.hashes[sha256.Sum256([]byte(token))] = tenant + } + return t, nil +} + +// Tenant returns the tenant for a request's bearer token, if it is valid. +func (t *Tokens) Tenant(r *http.Request) (string, bool) { + if t == nil { + return "", false + } + token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if !ok || token == "" { + return "", false + } + got := sha256.Sum256([]byte(token)) + var tenant string + found := 0 + for h, name := range t.hashes { + if subtle.ConstantTimeCompare(got[:], h[:]) == 1 { + tenant, found = name, 1 + } + } + return tenant, found == 1 +} + +// Len reports how many tokens are configured. +func (t *Tokens) Len() int { + if t == nil { + return 0 + } + return len(t.hashes) +} + +// ----- Prices and the daily budget ----- + +// Price is US dollars per million tokens. +type Price struct{ Input, Output float64 } + +// KnownPrices lists models whose standard prices were checked against the +// provider's own page (see EVALS.md and ADR 0007). +var KnownPrices = map[string]Price{ + "gpt-6-luna": {Input: 0.10, Output: 0.50}, // developers.openai.com, checked 2026-10-05 +} + +// Cost of a finished call. +func (p Price) Cost(inputTokens, outputTokens int) float64 { + return (float64(inputTokens)*p.Input + float64(outputTokens)*p.Output) / 1e6 +} + +// Ceiling is the most a request can cost. A token is at least one byte, so +// the protected prompt's size in bytes, plus a small allowance per message, +// bounds the input tokens; the output cap bounds the rest. +func (p Price) Ceiling(promptBytes, messages, maxOutputTokens int) float64 { + return p.Cost(promptBytes+16*messages, maxOutputTokens) +} + +// Budget enforces a hard daily limit. Spending already recorded comes from +// the audit log, so a restart can't reset it; requests in flight reserve +// their worst case, so parallel requests can't overspend together. +type Budget struct { + mu sync.Mutex + limit float64 + reserved float64 + spent func() float64 +} + +// NewBudget creates a budget of limit US dollars a day. +func NewBudget(limit float64, spentToday func() float64) *Budget { + return &Budget{limit: limit, spent: spentToday} +} + +// Reserve claims ceiling dollars, or reports that the day's budget would +// be exceeded. Every successful Reserve must be followed by Release. +func (b *Budget) Reserve(ceiling float64) bool { + b.mu.Lock() + defer b.mu.Unlock() + if b.spent()+b.reserved+ceiling > b.limit { + return false + } + b.reserved += ceiling + return true +} + +// Release returns a reservation once the real cost has been recorded. +func (b *Budget) Release(ceiling float64) { + b.mu.Lock() + defer b.mu.Unlock() + b.reserved -= ceiling +} + +// ----- Rate limit ----- + +// RateLimiter allows each client a burst of requests, refilled steadily. +type RateLimiter struct { + mu sync.Mutex + perMin float64 + burst float64 + clients map[string]*bucket + now func() time.Time + ipHeader string +} + +type bucket struct { + tokens float64 + last time.Time +} + +// NewRateLimiter allows perMinute requests a minute per client, up to burst +// at once. ipHeader names a header set by a trusted proxy (for example +// Fly-Client-IP); leave it empty to use the connection's address. +func NewRateLimiter(perMinute, burst int, ipHeader string) *RateLimiter { + return &RateLimiter{perMin: float64(perMinute), burst: float64(burst), clients: map[string]*bucket{}, now: time.Now, ipHeader: ipHeader} +} + +// Allow reports whether the request's client may make a request now. +func (l *RateLimiter) Allow(r *http.Request) bool { + if l == nil { + return true + } + ip := "" + if l.ipHeader != "" { + ip = strings.TrimSpace(r.Header.Get(l.ipHeader)) + } + if ip == "" { + ip, _, _ = net.SplitHostPort(r.RemoteAddr) + } + l.mu.Lock() + defer l.mu.Unlock() + now := l.now() + b, ok := l.clients[ip] + if !ok { + b = &bucket{tokens: l.burst, last: now} + l.clients[ip] = b + } + b.tokens = min(l.burst, b.tokens+now.Sub(b.last).Minutes()*l.perMin) + b.last = now + if len(l.clients) > 10000 { // forget idle clients now and then + for k, c := range l.clients { + if now.Sub(c.last) > 10*time.Minute { + delete(l.clients, k) + } + } + } + if b.tokens < 1 { + return false + } + b.tokens-- + return true +} diff --git a/internal/gateway/limits_test.go b/internal/gateway/limits_test.go new file mode 100644 index 0000000..4be0e42 --- /dev/null +++ b/internal/gateway/limits_test.go @@ -0,0 +1,260 @@ +package gateway + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/makoydev/discreet/internal/audit" + "github.com/makoydev/discreet/internal/detect" + "github.com/makoydev/discreet/internal/protect" + "github.com/makoydev/discreet/internal/upstream" + "github.com/makoydev/discreet/internal/vault" +) + +// A fake OpenAI server: no test ever calls a paid API. +type fakeOpenAI struct { + srv *httptest.Server + calls atomic.Int32 + mu sync.Mutex + bodies []map[string]any + auth []string + status int +} + +func newFakeOpenAI(t *testing.T) *fakeOpenAI { + f := &fakeOpenAI{status: 200} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.calls.Add(1) + var body map[string]any + json.NewDecoder(r.Body).Decode(&body) + f.mu.Lock() + f.bodies = append(f.bodies, body) + f.auth = append(f.auth, r.Header.Get("Authorization")) + f.mu.Unlock() + if f.status != 200 { + w.WriteHeader(f.status) + io.WriteString(w, `{"error":{"message":"nope"}}`) + return + } + msgs := body["messages"].([]any) + last := msgs[len(msgs)-1].(map[string]any)["content"].(string) + json.NewEncoder(w).Encode(map[string]any{ + "model": "gpt-6-luna-2026-09-01", + "choices": []any{map[string]any{"message": map[string]any{"role": "assistant", "content": "Reply about " + last}, "finish_reason": "stop"}}, + "usage": map[string]any{"prompt_tokens": 1000, "completion_tokens": 200}, + }) + })) + t.Cleanup(f.srv.Close) + return f +} + +const goodToken = "tenant-a-token-0123456789abcdef" + +type realHarness struct { + *harness + fake *fakeOpenAI +} + +func newRealHarness(t *testing.T, budget float64, rate *RateLimiter) *realHarness { + t.Helper() + fake := newFakeOpenAI(t) + engine, _ := detect.Default() + v, _ := vault.New(time.Minute) + h := &harness{logs: &strings.Builder{}, logPath: filepath.Join(t.TempDir(), "audit.jsonl")} + var err error + if h.audit, err = audit.Open(h.logPath, []byte("test-key-0123456789abcdefghijklmnop")); err != nil { + t.Fatal(err) + } + tokens, err := ParseTokens("acme:" + goodToken) + if err != nil { + t.Fatal(err) + } + h.mock = &upstream.Mock{} + s := New(Config{ + Engine: engine, Policy: protect.DefaultPolicy(), Vault: v, Audit: h.audit, Upstream: h.mock, + Logger: slog.New(slog.NewJSONHandler(h.logs, nil)), + Real: &upstream.OpenAICompatible{BaseURL: fake.srv.URL, APIKey: "provider-key"}, + Tokens: tokens, Model: "gpt-6-luna", Price: KnownPrices["gpt-6-luna"], + Budget: NewBudget(budget, h.audit.SpentToday), MaxOutputTokens: 1000, RateLimit: rate, + }) + h.srv = httptest.NewServer(s.Handler()) + t.Cleanup(h.srv.Close) + return &realHarness{h, fake} +} + +func (h *realHarness) postAs(t *testing.T, token, model string) *http.Response { + t.Helper() + b, _ := json.Marshal(map[string]any{"model": model, "messages": []map[string]string{{"role": "user", "content": "Call " + nric + " on " + phone}}}) + req, _ := http.NewRequest("POST", h.srv.URL+"/v1/chat/completions", strings.NewReader(string(b))) + req.Header.Set("X-Discreet-Purpose", "claims summary") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { resp.Body.Close() }) + return resp +} + +func TestValidTokenReachesTheRealModelWithPlaceholdersOnly(t *testing.T) { + h := newRealHarness(t, 1.0, nil) + resp := h.postAs(t, goodToken, "gpt-6-luna") + if resp.StatusCode != 200 || resp.Header.Get("X-Discreet-Upstream") != "openai-compatible" { + t.Fatalf("status %d, upstream %q", resp.StatusCode, resp.Header.Get("X-Discreet-Upstream")) + } + body := h.fake.bodies[0] + sent, _ := json.Marshal(body["messages"]) + if strings.Contains(string(sent), nric) || strings.Contains(string(sent), phone) { + t.Errorf("the provider received personal data: %s", sent) + } + if body["max_completion_tokens"] != float64(1000) || body["model"] != "gpt-6-luna" { + t.Errorf("body %v", body) + } + if h.fake.auth[0] != "Bearer provider-key" { + t.Errorf("provider got Authorization %q; the caller's token must never be forwarded", h.fake.auth[0]) + } + var out map[string]any + json.NewDecoder(resp.Body).Decode(&out) + if a := answer(t, out); !strings.Contains(a, nric) { + t.Errorf("answer not restored: %s", a) + } + rec := h.records(t)[0] + if rec.Tenant != "acme" || rec.Upstream != "openai-compatible" || rec.CostUSD < 0.000199 || rec.CostUSD > 0.000201 { + t.Errorf("record %+v (cost should be 1000×0.10 + 200×0.50 per million = US$0.0002)", rec) + } +} + +func TestWithoutAValidTokenEveryoneGetsTheMock(t *testing.T) { + h := newRealHarness(t, 1.0, nil) + for _, token := range []string{"", "not-needed-for-mock", goodToken + "x"} { + resp := h.postAs(t, token, "gpt-6-luna") + if resp.StatusCode != 200 || resp.Header.Get("X-Discreet-Upstream") != "mock" { + t.Errorf("token %q: status %d upstream %q", token, resp.StatusCode, resp.Header.Get("X-Discreet-Upstream")) + } + } + if n := h.fake.calls.Load(); n != 0 { + t.Errorf("the paid model was called %d times without a valid token", n) + } +} + +func TestOnlyTheConfiguredModel(t *testing.T) { + h := newRealHarness(t, 1.0, nil) + if resp := h.postAs(t, goodToken, "gpt-6-sol"); resp.StatusCode != 400 { + t.Errorf("status %d, want 400 for a model that isn't allowed", resp.StatusCode) + } +} + +func TestDailyBudgetIsEnforced(t *testing.T) { + // Worst case here: prompt ≤ ~50 bytes + 16, output 1000 tokens → about + // US$0.0005. A US$0.0006 budget allows one request, then refuses. + h := newRealHarness(t, 0.0006, nil) + if resp := h.postAs(t, goodToken, ""); resp.StatusCode != 200 { + t.Fatalf("first request: %d", resp.StatusCode) + } + resp := h.postAs(t, goodToken, "") + if resp.StatusCode != 429 { + t.Fatalf("second request: %d, want 429", resp.StatusCode) + } + if n := h.fake.calls.Load(); n != 1 { + t.Errorf("provider called %d times", n) + } + if recs := h.records(t); recs[1].Decision != audit.RefusedBudget { + t.Errorf("records %+v", recs) + } + // Mock requests are free and never refused for budget. + if resp := h.postAs(t, "", ""); resp.StatusCode != 200 { + t.Errorf("mock request: %d", resp.StatusCode) + } +} + +func TestBudgetSurvivesARestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "audit.jsonl") + key := []byte("test-key-0123456789abcdefghijklmnop") + l, _ := audit.Open(path, key) + l.Append(audit.Record{Decision: audit.Allowed, CostUSD: 0.19}) + l.Close() + l2, _ := audit.Open(path, key) + if b := NewBudget(0.20, l2.SpentToday); b.Reserve(0.02) { + t.Error("a restart reset today's spending") + } +} + +func TestParallelRequestsCantOverspend(t *testing.T) { + b := NewBudget(1.0, func() float64 { return 0 }) + var granted atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if b.Reserve(0.1) { + granted.Add(1) + } + }() + } + wg.Wait() + if granted.Load() != 10 { + t.Errorf("granted %d reservations of 0.1 within 1.0", granted.Load()) + } +} + +func TestProviderErrorsAreRecordedAndNotLeaked(t *testing.T) { + h := newRealHarness(t, 1.0, nil) + h.fake.status = 500 + if resp := h.postAs(t, goodToken, ""); resp.StatusCode != 502 { + t.Errorf("status %d", resp.StatusCode) + } + if recs := h.records(t); recs[0].Decision != audit.UpstreamError || recs[0].CostUSD != 0 { + t.Errorf("record %+v", recs[0]) + } +} + +func TestRateLimit(t *testing.T) { + h := newRealHarness(t, 1.0, NewRateLimiter(60, 2, "")) + codes := []int{} + for i := 0; i < 3; i++ { + codes = append(codes, h.postAs(t, "", "").StatusCode) + } + if codes[0] != 200 || codes[1] != 200 || codes[2] != 429 { + t.Errorf("codes %v, want [200 200 429]", codes) + } +} + +func TestRateLimiterRefills(t *testing.T) { + l := NewRateLimiter(60, 1, "Fly-Client-IP") + now := time.Now() + l.now = func() time.Time { return now } + r := httptest.NewRequest("POST", "/", nil) + r.Header.Set("Fly-Client-IP", "203.0.113.7") + if !l.Allow(r) || l.Allow(r) { + t.Fatal("burst of 1 not enforced") + } + other := httptest.NewRequest("POST", "/", nil) + other.Header.Set("Fly-Client-IP", "203.0.113.8") + if !l.Allow(other) { + t.Error("one client's limit affected another") + } + now = now.Add(time.Second) + if !l.Allow(r) { + t.Error("did not refill after a second at 60 a minute") + } +} + +func TestParseTokensRejectsWeakTokens(t *testing.T) { + for _, bad := range []string{"acme:short", "no-colon-0123456789abcdefghij", ":0123456789abcdefghijklmnop"} { + if _, err := ParseTokens(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} diff --git a/internal/upstream/openai.go b/internal/upstream/openai.go new file mode 100644 index 0000000..81bebd5 --- /dev/null +++ b/internal/upstream/openai.go @@ -0,0 +1,94 @@ +package upstream + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "time" +) + +// OpenAICompatible calls a /chat/completions endpoint: OpenAI itself, or a +// local Ollama (http://localhost:11434/v1). It sends only the protected +// messages and the few fields the gateway allows. +type OpenAICompatible struct { + BaseURL string // e.g. https://api.openai.com/v1 + APIKey string // the provider's key; never the caller's token + TokenParam string // "max_completion_tokens" (OpenAI) or "max_tokens" (Ollama) + ReasoningEffort string // optional, e.g. "low"; not sent when empty + HTTP *http.Client +} + +func (*OpenAICompatible) Name() string { return "openai-compatible" } + +// Complete sends one non-streaming chat completion. +func (o *OpenAICompatible) Complete(ctx context.Context, req Request) (Response, error) { + body := map[string]any{"model": req.Model, "messages": req.Messages} + if req.MaxTokens > 0 { + param := o.TokenParam + if param == "" { + param = "max_completion_tokens" + } + body[param] = req.MaxTokens + } + if req.Temperature != nil { + body["temperature"] = *req.Temperature + } + if o.ReasoningEffort != "" { + body["reasoning_effort"] = o.ReasoningEffort + } + data, err := json.Marshal(body) + if err != nil { + return Response{}, err + } + httpReq, err := http.NewRequestWithContext(ctx, "POST", strings.TrimRight(o.BaseURL, "/")+"/chat/completions", bytes.NewReader(data)) + if err != nil { + return Response{}, err + } + httpReq.Header.Set("Content-Type", "application/json") + if o.APIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+o.APIKey) + } + client := o.HTTP + if client == nil { + client = &http.Client{Timeout: 90 * time.Second} + } + resp, err := client.Do(httpReq) + if err != nil { + return Response{}, fmt.Errorf("upstream request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + // The provider's error body is not read or logged: only the status. + return Response{}, fmt.Errorf("upstream returned HTTP %d", resp.StatusCode) + } + var out struct { + Model string `json:"model"` + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return Response{}, fmt.Errorf("upstream response is not valid JSON: %w", err) + } + if len(out.Choices) == 0 { + return Response{}, errors.New("upstream response has no choices") + } + return Response{ + Content: out.Choices[0].Message.Content, + Model: out.Model, + FinishReason: out.Choices[0].FinishReason, + PromptTokens: out.Usage.PromptTokens, + OutputTokens: out.Usage.CompletionTokens, + }, nil +} From 66bd9731d228ea710d19b78f93db1dcbda9f0802 Mon Sep 17 00:00:00 2001 From: Michael Mendoza <62703135+makoydev@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:57:45 +0800 Subject: [PATCH 07/13] feat: four-pane demo page on the API's own pipeline (D8) (#10) - GET / serves an embedded page: what the app sent, what the AI saw, what it answered, what the app got back, and the audit record, with five made-up scenarios including a refused eligibility decision. - POST /demo/run runs the same pipeline as /v1/chat/completions: the gateway was refactored so both call one process() function, and every existing gateway test passed unchanged (ADR 0008). The demo always uses the mock model, even with a valid token (tested against a fake paid provider: zero calls). - Strict CSP (own scripts only, connect-src 'self'), nosniff, no referrer; user text inserted as text. Checked in headless Chrome at desktop and phone width, light and dark: no CSP violations. - Session.RestoreParts lets pane 4 highlight restored values; redacted markers stay highlighted as never restored. Closes makoydev/vetted#38 Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 1 + README.md | 4 + THREAT_MODEL.md | 1 + docs/HOW-THIS-WAS-BUILT.md | 1 + docs/adr/0008-demo-shares-the-api-pipeline.md | 24 +++ internal/gateway/demo.go | 93 +++++++++++ internal/gateway/demo_test.go | 106 ++++++++++++ internal/gateway/gateway.go | 158 +++++------------- internal/gateway/gateway_test.go | 3 + internal/gateway/pipeline.go | 123 ++++++++++++++ internal/gateway/web/demo.css | 79 +++++++++ internal/gateway/web/demo.js | 118 +++++++++++++ internal/gateway/web/index.html | 64 +++++++ internal/protect/protect_test.go | 13 ++ internal/protect/session.go | 38 +++++ 15 files changed, 707 insertions(+), 119 deletions(-) create mode 100644 docs/adr/0008-demo-shares-the-api-pipeline.md create mode 100644 internal/gateway/demo.go create mode 100644 internal/gateway/demo_test.go create mode 100644 internal/gateway/pipeline.go create mode 100644 internal/gateway/web/demo.css create mode 100644 internal/gateway/web/demo.js create mode 100644 internal/gateway/web/index.html diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f926c1..84ab315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ All notable changes to this project are documented here. The format follows [Kee - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records. - Vetted reviews every pull request (shadow mode until 2026-10-14). - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases. +- Four-pane demo page (issue D8) at `/`: what the app sent, what the AI saw, what it answered, and what the app got back, plus that request's audit record. Five made-up scenarios (claims, appointment, card dispute, mistyped NRIC, a refused eligibility decision). It runs through the same pipeline as the API, always on the mock model, even for token holders; strict Content-Security-Policy; works on phones, light and dark. - A real model behind access tokens (issue D6): an OpenAI-compatible client (OpenAI, or Ollama with `max_tokens`) used only for callers with a valid `DISCREET_ACCESS_TOKENS` token; everyone else gets the mock. One pinned model with checked prices (`gpt-6-luna`); each request reserves its worst-case cost (at most US$0.0076) against a hard daily budget (default US$0.20) recomputed from the audit log; 30 requests a minute per client address (ADR 0007). No test calls a paid API. - The gateway (`internal/gateway`, issue D5): `discreet serve` runs an OpenAI-compatible `POST /v1/chat/completions` (non-streaming). Every message is protected in one session before the model sees it and restored afterwards; every answer ends with a disclosure footer; every request, allowed or not, is recorded in the audit log, and Discreet never answers a request it couldn't record. A free mock model is the default. Callers must send `X-Discreet-Purpose`; automated eligibility and benefits decisions are refused with HTTP 403 (ADR 0006). Only the model, messages, token limit and temperature are forwarded. Tested with OpenAI's official Python library by changing only `base_url`. - Tamper-evident audit log (`internal/audit`, issue D7): append-only JSON lines, each record holding the SHA-256 of the previous one and HMAC-SHA-256 hashes of prompt and response (never the text). `discreet audit verify` names the first changed, missing or reordered record; `discreet audit export -csv` exports a verified log. An existing broken log is refused at start-up (ADR 0005). diff --git a/README.md b/README.md index 97def2f..92de874 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,10 @@ make check # gofmt, go vet, go test -race, govulncheck, build ./bin/discreet audit export -csv -log discreet-audit.jsonl > audit.csv ``` +## Try the demo + +Run `discreet serve` (below) and open : four panes show what the app sent, what the AI saw, what it answered and what came back, with the audit record. It always uses the free mock model. + ## Run it ```sh diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 021ce98..4c8fe5e 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -31,4 +31,5 @@ | A caller uses Discreet for automated eligibility decisions | Required purpose header; denied purposes refused with 403 before the model is called, and recorded (ADR 0006) | D5 (done) | | The public demo is used to spend money | Mock model unless a valid access token is present (a wrong or missing token gets the mock, test); worst-case cost reserved against a hard daily budget that survives restarts (tests, including 100 parallel requests); 30 requests a minute per address (ADR 0007) | D6 (done), D10 | | An access token leaks from memory or logs | Only SHA-256 hashes kept, compared in constant time; tokens and provider keys never logged; the caller's token is never forwarded | D6 (done) | +| The demo page is used to inject script or exfiltrate what people type | Strict Content-Security-Policy (own scripts only, connections only to this server), user text inserted as text never HTML, no third-party scripts; always the mock model (test) | D8 (done) | | A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 | diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md index a8b9bc5..709652a 100644 --- a/docs/HOW-THIS-WAS-BUILT.md +++ b/docs/HOW-THIS-WAS-BUILT.md @@ -34,6 +34,7 @@ | 2026-10-05 | Three deliberate breaks of the gateway: original text sent to the model, purpose check skipped, prompt logged | Each failed a test; restored from backup | | 2026-10-05 | `gpt-6-luna` on `/v1/chat/completions` and its price (OpenAI's model page, primary) | Supported; US$0.10 / US$0.50 per million input / output tokens | | 2026-10-05 | Deliberate breaks of tokens, budget and rate limit | Each failed tests once the token break was redone so it compiled; restored from backup | +| 2026-10-05 | Demo page in headless Chrome at 1280 px (light and dark) and 390 px, clicking through the examples | No Content-Security-Policy violations or script errors; one 404 for the browser's default icon request, fixed with an inline empty icon; audit log verified afterwards (7 records) | | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite | ## What the AI got wrong, and how it was caught diff --git a/docs/adr/0008-demo-shares-the-api-pipeline.md b/docs/adr/0008-demo-shares-the-api-pipeline.md new file mode 100644 index 0000000..c4b4333 --- /dev/null +++ b/docs/adr/0008-demo-shares-the-api-pipeline.md @@ -0,0 +1,24 @@ +# 0008. The demo page runs the API's own pipeline, always on the mock model + +Status: drafted by Claude Code, awaiting Michael's review +Date: 2026-10-05 + +## Context + +The brief asks for a demo page with four panes (original prompt, what the model saw, the model's answer, the rehydrated answer) plus the call's audit record. A demo that re-implements the gateway's steps can drift from what the API actually does, and then it demonstrates something that isn't true. + +## Options + +1. **One pipeline function** (`process`) used by both `/v1/chat/completions` and `/demo/run`, returning every stage; the demo endpoint just shows them. +2. **A separate demo handler** with its own simplified flow. Less refactoring, but two copies of the privacy logic. +3. **A static page calling the public API** and reconstructing the stages in the browser. The API (correctly) doesn't return what the model saw, so the browser couldn't show pane 2 honestly. + +## Decision + +Option 1. The demo always uses the mock model, even when the caller sends a valid access token, and records requests under the tenant `demo`. The page is plain HTML, CSS and JavaScript embedded in the binary, served with a strict Content-Security-Policy: its own scripts only, connections only back to the server, user text inserted as text, never HTML. + +## Consequences + +- What the demo shows is what the API does; the refactor kept every existing gateway test passing unchanged. +- Demo requests appear in the audit log like any other, which is the point: visitors can watch the record being created. +- The page needs no build step and no JavaScript framework (ADR 0001). diff --git a/internal/gateway/demo.go b/internal/gateway/demo.go new file mode 100644 index 0000000..f566c50 --- /dev/null +++ b/internal/gateway/demo.go @@ -0,0 +1,93 @@ +package gateway + +import ( + "embed" + "encoding/json" + "io/fs" + "net/http" + "strings" + "unicode/utf8" + + "github.com/makoydev/discreet/internal/protect" +) + +//go:embed web +var webFiles embed.FS + +// The demo page loads nothing but its own files and can only call back to +// this server. +const demoCSP = "default-src 'none'; script-src 'self'; style-src 'self' https://fonts.googleapis.com; " + + "font-src https://fonts.gstatic.com; connect-src 'self'; img-src 'self' data:; " + + "base-uri 'none'; form-action 'none'; frame-ancestors 'none'" + +func securityHeaders(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Security-Policy", demoCSP) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Referrer-Policy", "no-referrer") + next.ServeHTTP(w, r) + }) +} + +func (s *Server) demoRoutes(mux *http.ServeMux) { + static, _ := fs.Sub(webFiles, "web") + files := securityHeaders(http.FileServerFS(static)) + mux.Handle("GET /{$}", files) + mux.Handle("GET /demo.js", files) + mux.Handle("GET /demo.css", files) + mux.HandleFunc("POST /demo/run", s.demoRun) +} + +type demoPart struct { + Text string `json:"text"` + Placeholder string `json:"placeholder,omitempty"` +} + +// demoRun runs one message through exactly the same pipeline as the API, +// always with the mock model, and returns every stage for the four panes. +func (s *Server) demoRun(w http.ResponseWriter, r *http.Request) { + id := newID() + if !s.allow(w, r, id) { + return + } + var in struct { + Text string `json:"text"` + Purpose string `json:"purpose"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 16<<10)).Decode(&in); err != nil || strings.TrimSpace(in.Text) == "" { + writeError(w, http.StatusBadRequest, "invalid_request", "send some text") + return + } + if utf8.RuneCountInString(in.Text) > 4000 { + writeError(w, http.StatusBadRequest, "invalid_request", "the demo takes up to 4,000 characters") + return + } + o := s.process(r.Context(), call{ + id: id, tenant: "demo", purpose: strings.TrimSpace(in.Purpose), + roles: []string{"user"}, texts: []string{in.Text}, + up: s.cfg.Upstream, // the demo never uses the real model + }) + out := map[string]any{"request_id": id, "original": in.Text} + if o.status != http.StatusOK { + out["status"], out["code"], out["message"] = "refused", o.code, o.message + if o.record.Seq > 0 { + out["audit_record"] = o.record + } + } else { + parts := make([]demoPart, 0, len(o.parts)+1) + for _, p := range o.parts { + parts = append(parts, demoPart(p)) + } + parts = append(parts, demoPart{Text: strings.TrimPrefix(o.answer, protect.Join(o.parts))}) + out["status"] = "answered" + out["sent_to_model"] = o.sent[0].Content + out["model_answer"] = o.response.Content + out["returned"] = parts + out["audit_record"] = o.record + } + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + enc.Encode(out) +} diff --git a/internal/gateway/demo_test.go b/internal/gateway/demo_test.go new file mode 100644 index 0000000..090738c --- /dev/null +++ b/internal/gateway/demo_test.go @@ -0,0 +1,106 @@ +package gateway + +import ( + "encoding/json" + "io" + "net/http" + "strings" + "testing" +) + +func demoPost(t *testing.T, base, token string, in map[string]string) (int, map[string]any) { + t.Helper() + b, _ := json.Marshal(in) + req, _ := http.NewRequest("POST", base+"/demo/run", strings.NewReader(string(b))) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out map[string]any + json.NewDecoder(resp.Body).Decode(&out) + return resp.StatusCode, out +} + +func TestDemoPageIsServedWithAStrictPolicy(t *testing.T) { + h := newHarness(t, nil, nil) + for _, path := range []string{"/", "/demo.js", "/demo.css"} { + resp, err := http.Get(h.srv.URL + path) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 || len(body) == 0 { + t.Errorf("%s: status %d", path, resp.StatusCode) + } + csp := resp.Header.Get("Content-Security-Policy") + if !strings.Contains(csp, "default-src 'none'") || !strings.Contains(csp, "script-src 'self'") || strings.Contains(csp, "unsafe") { + t.Errorf("%s: CSP %q", path, csp) + } + } + if resp, _ := http.Get(h.srv.URL + "/nope"); resp.StatusCode != 404 { + t.Errorf("unknown path: %d", resp.StatusCode) + } +} + +func TestDemoShowsEveryStage(t *testing.T) { + h := newHarness(t, nil, nil) + code, out := demoPost(t, h.srv.URL, "", map[string]string{"purpose": "claims summary", "text": "Member " + nric + " on " + phone + ", card " + card}) + if code != 200 || out["status"] != "answered" { + t.Fatalf("%d %v", code, out) + } + sent := out["sent_to_model"].(string) + for _, v := range []string{nric, phone, "4111"} { + if strings.Contains(sent, v) || strings.Contains(out["model_answer"].(string), v) { + t.Errorf("the model saw %q", v) + } + } + var restored []string + for _, p := range out["returned"].([]any) { + part := p.(map[string]any) + if part["placeholder"] != nil { + restored = append(restored, part["text"].(string)) + } + } + // The mock repeats placeholders in its echo and its draft, so each value + // comes back twice; only the NRIC and phone may be restored, never the card. + if len(restored) != 4 || restored[0] != nric || restored[1] != phone || restored[2] != nric || restored[3] != phone { + t.Errorf("restored %v", restored) + } + rec := out["audit_record"].(map[string]any) + if rec["tenant"] != "demo" || rec["upstream"] != "mock" || rec["decision"] != "allowed" { + t.Errorf("record %v", rec) + } +} + +func TestDemoNeverUsesTheRealModel(t *testing.T) { + h := newRealHarness(t, 1.0, nil) + code, out := demoPost(t, h.srv.URL, goodToken, map[string]string{"purpose": "claims summary", "text": "hello " + nric}) + if code != 200 || h.fake.calls.Load() != 0 { + t.Errorf("status %d, paid model called %d times (%v)", code, h.fake.calls.Load(), out) + } +} + +func TestDemoRefusalIncludesTheAuditRecord(t *testing.T) { + h := newHarness(t, nil, nil) + code, out := demoPost(t, h.srv.URL, "", map[string]string{"purpose": "automated eligibility decision", "text": "Is " + nric + " eligible?"}) + if code != 200 || out["status"] != "refused" || out["code"] != "purpose_refused" { + t.Fatalf("%d %v", code, out) + } + if rec := out["audit_record"].(map[string]any); rec["decision"] != "refused_purpose" { + t.Errorf("record %v", rec) + } +} + +func TestDemoRejectsBadInput(t *testing.T) { + h := newHarness(t, nil, nil) + for name, text := range map[string]string{"empty": " ", "too long": strings.Repeat("a", 4001)} { + if code, _ := demoPost(t, h.srv.URL, "", map[string]string{"purpose": "x", "text": text}); code != 400 { + t.Errorf("%s: status %d", name, code) + } + } +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 2e17c78..11a082a 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -76,6 +76,7 @@ func (s *Server) Handler() http.Handler { mux := http.NewServeMux() mux.HandleFunc("POST /v1/chat/completions", s.chatCompletions) mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte("ok\n")) }) + s.demoRoutes(mux) return mux } @@ -143,17 +144,11 @@ func newID() string { } func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { - start := time.Now() id := newID() w.Header().Set("X-Discreet-Request-Id", id) - log := s.cfg.Logger.With("request_id", id) - if !s.cfg.RateLimit.Allow(r) { - log.Info("rate limited") - w.Header().Set("Retry-After", "10") - writeError(w, http.StatusTooManyRequests, "rate_limited", "too many requests from this address; try again shortly") + if !s.allow(w, r, id) { return } - var req chatRequest if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, s.cfg.MaxBodyBytes)).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "invalid_request", "request body is not valid JSON or is too large") @@ -167,8 +162,8 @@ func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "invalid_request", "messages must not be empty") return } - texts := make([]string, len(req.Messages)) - for i, m := range req.Messages { + c := call{id: id, purpose: strings.TrimSpace(r.Header.Get("X-Discreet-Purpose")), model: req.Model, temperature: req.Temperature} + for _, m := range req.Messages { if !allowedRoles[m.Role] { writeError(w, http.StatusBadRequest, "invalid_request", fmt.Sprintf("message role %q is not supported", m.Role)) return @@ -178,146 +173,71 @@ func (s *Server) chatCompletions(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, "invalid_request", err.Error()) return } - texts[i] = t + c.roles, c.texts = append(c.roles, m.Role), append(c.texts, t) } - - // The purpose is required, recorded with any personal data removed, - // and refused outright for decisions about people's eligibility. - rawPurpose := strings.TrimSpace(r.Header.Get("X-Discreet-Purpose")) - if rawPurpose == "" { - writeError(w, http.StatusBadRequest, "missing_purpose", "set the X-Discreet-Purpose header to say what this request is for, for example \"claims summary\"") - return + if req.MaxCompletionTokens != nil { + c.maxTokens = *req.MaxCompletionTokens + } else if req.MaxTokens != nil { + c.maxTokens = *req.MaxTokens } - purpose := s.scrub(rawPurpose) // Only a valid access token reaches the real model; anyone else gets // the free mock, so the public endpoint can never spend money. - up, tenant, real := s.cfg.Upstream, s.cfg.Tenant, false + c.up, c.tenant = s.cfg.Upstream, s.cfg.Tenant if t, ok := s.cfg.Tokens.Tenant(r); ok { - tenant = t + c.tenant = t if s.cfg.Real != nil { - up, real = s.cfg.Real, true - } - } - w.Header().Set("X-Discreet-Upstream", up.Name()) - model := req.Model - if real { - if req.Model != "" && req.Model != s.cfg.Model { - writeError(w, http.StatusBadRequest, "model_not_allowed", fmt.Sprintf("this gateway only sends requests to %s", s.cfg.Model)) - return - } - model = s.cfg.Model - } - rec := audit.Record{RequestID: id, Tenant: tenant, Purpose: purpose, Upstream: up.Name(), Model: model} - if deniedPurpose(rawPurpose, s.cfg.Policy.DeniedPurposes) { - rec.Decision, rec.Reason = audit.RefusedPurpose, "purpose refused by policy" - if _, ok := s.finish(w, log, rec, start); !ok { - return - } - writeError(w, http.StatusForbidden, "purpose_refused", "Discreet does not take part in automated decisions about people's eligibility. A person must make that decision.") - return - } - - // Protect every message in one session, so placeholders are consistent. - session := protect.NewSession(s.cfg.Policy, s.cfg.Vault) - defer session.Close() - msgs := make([]upstream.Message, len(texts)) - for i, t := range texts { - protected, err := session.Protect(t, s.cfg.Engine.Detect(t)) - var blocked *protect.BlockedError - if errors.As(err, &blocked) { - rec.Decision, rec.Reason, rec.Entities = audit.BlockedEntity, blocked.Error(), session.Counts() - if _, ok := s.finish(w, log, rec, start); !ok { - return - } - writeError(w, http.StatusForbidden, "entity_blocked", blocked.Error()) - return - } - msgs[i] = upstream.Message{Role: req.Messages[i].Role, Content: protected} - } - rec.Entities = session.Counts() - promptJSON, _ := json.Marshal(texts) - rec.PromptHMAC = s.cfg.Audit.HMAC(string(promptJSON)) - - maxTokens := 0 - if req.MaxCompletionTokens != nil { - maxTokens = *req.MaxCompletionTokens - } else if req.MaxTokens != nil { - maxTokens = *req.MaxTokens - } - if maxTokens <= 0 || maxTokens > s.cfg.MaxOutputTokens { - maxTokens = s.cfg.MaxOutputTokens - } - - // Real calls reserve their worst-case cost first (ADR 0007). - if real { - promptBytes := 0 - for _, m := range msgs { - promptBytes += len(m.Content) - } - ceiling := s.cfg.Price.Ceiling(promptBytes, len(msgs), maxTokens) - if !s.cfg.Budget.Reserve(ceiling) { - rec.Decision, rec.Reason = audit.RefusedBudget, fmt.Sprintf("daily budget reached (this request could cost up to US$%.4f)", ceiling) - if _, ok := s.finish(w, log, rec, start); !ok { - return - } - writeError(w, http.StatusTooManyRequests, "budget_exceeded", "today's spending limit for the AI model has been reached; requests resume tomorrow (UTC)") - return - } - defer s.cfg.Budget.Release(ceiling) // after the real cost is recorded below - } - - resp, err := up.Complete(r.Context(), upstream.Request{Model: model, Messages: msgs, MaxTokens: maxTokens, Temperature: req.Temperature}) - if err != nil { - rec.Decision, rec.Reason = audit.UpstreamError, "the model provider returned an error" - log.Warn("upstream error", "error", err.Error()) - if _, ok := s.finish(w, log, rec, start); !ok { - return + c.up, c.real = s.cfg.Real, true } - writeError(w, http.StatusBadGateway, "upstream_error", "the model provider returned an error") - return } + w.Header().Set("X-Discreet-Upstream", c.up.Name()) - answer := session.Restore(resp.Content) + fmt.Sprintf(Footer, id) - rec.Decision, rec.Model, rec.ResponseHMAC = audit.Allowed, resp.Model, s.cfg.Audit.HMAC(answer) - if real { - rec.CostUSD = s.cfg.Price.Cost(resp.PromptTokens, resp.OutputTokens) - } - saved, ok := s.finish(w, log, rec, start) - if !ok { + o := s.process(r.Context(), c) + if o.status != http.StatusOK { + writeError(w, o.status, o.code, o.message) return } - w.Header().Set("X-Discreet-Entities", entityHeader(rec.Entities)) - w.Header().Set("X-Discreet-Audit-Seq", fmt.Sprint(saved.Seq)) + w.Header().Set("X-Discreet-Entities", entityHeader(o.record.Entities)) + w.Header().Set("X-Discreet-Audit-Seq", fmt.Sprint(o.record.Seq)) w.Header().Set("Content-Type", "application/json") enc := json.NewEncoder(w) enc.SetEscapeHTML(false) // keep readable in raw output enc.Encode(map[string]any{ "id": "chatcmpl-" + id, "object": "chat.completion", - "created": start.Unix(), - "model": resp.Model, + "created": time.Now().Unix(), + "model": o.response.Model, "choices": []map[string]any{{ "index": 0, - "message": map[string]string{"role": "assistant", "content": answer}, - "finish_reason": resp.FinishReason, + "message": map[string]string{"role": "assistant", "content": o.answer}, + "finish_reason": o.response.FinishReason, }}, "usage": map[string]int{ - "prompt_tokens": resp.PromptTokens, - "completion_tokens": resp.OutputTokens, - "total_tokens": resp.PromptTokens + resp.OutputTokens, + "prompt_tokens": o.response.PromptTokens, + "completion_tokens": o.response.OutputTokens, + "total_tokens": o.response.PromptTokens + o.response.OutputTokens, }, }) } -// finish writes the audit record. If it can't, the request fails: Discreet -// never answers a request it couldn't record. -func (s *Server) finish(w http.ResponseWriter, log *slog.Logger, rec audit.Record, start time.Time) (audit.Record, bool) { +// allow applies the per-address rate limit. +func (s *Server) allow(w http.ResponseWriter, r *http.Request, id string) bool { + if s.cfg.RateLimit.Allow(r) { + return true + } + s.cfg.Logger.Info("rate limited", "request_id", id) + w.Header().Set("Retry-After", "10") + writeError(w, http.StatusTooManyRequests, "rate_limited", "too many requests from this address; try again shortly") + return false +} + +// finish writes the audit record. If it can't, the caller must fail the +// request: Discreet never answers a request it couldn't record. +func (s *Server) finish(log *slog.Logger, rec audit.Record, start time.Time) (audit.Record, bool) { rec.LatencyMS = time.Since(start).Milliseconds() saved, err := s.cfg.Audit.Append(rec) if err != nil { log.Error("audit write failed", "error", err.Error()) - writeError(w, http.StatusInternalServerError, "audit_failed", "the request could not be recorded, so it was not processed") return saved, false } log.Info("request", "decision", rec.Decision, "entities", entityHeader(rec.Entities), "upstream", rec.Upstream, "latency_ms", rec.LatencyMS, "audit_seq", saved.Seq) diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 8b99be2..2990ce2 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -46,6 +46,9 @@ type harness struct { func newHarness(t *testing.T, policy *protect.Policy, up upstream.Client) *harness { t.Helper() + if policy == nil { + policy = protect.DefaultPolicy() + } engine, err := detect.Default() if err != nil { t.Fatal(err) diff --git a/internal/gateway/pipeline.go b/internal/gateway/pipeline.go new file mode 100644 index 0000000..18d957d --- /dev/null +++ b/internal/gateway/pipeline.go @@ -0,0 +1,123 @@ +package gateway + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/makoydev/discreet/internal/audit" + "github.com/makoydev/discreet/internal/protect" + "github.com/makoydev/discreet/internal/upstream" +) + +// call is one request, already parsed, from the API or the demo page. +type call struct { + id, tenant, purpose, model string + roles, texts []string + maxTokens int + temperature *float64 + up upstream.Client + real bool +} + +// outcome is everything a request produced. status is 200 for an answer; +// otherwise code and message explain the refusal. +type outcome struct { + status int + code, message string + record audit.Record + sent []upstream.Message // what the model received + response upstream.Response // what the model answered, placeholders and all + parts []protect.Part // the restored answer, split for display + answer string // the restored answer with the footer +} + +// process is the whole pipeline, shared by the API and the demo page so +// the demo shows exactly what the API does: purpose check, protection, +// budget, model call, restore, footer and audit record. +func (s *Server) process(ctx context.Context, c call) outcome { + start := time.Now() + log := s.cfg.Logger.With("request_id", c.id) + refuse := func(rec audit.Record, status int, code, message string) outcome { + saved, ok := s.finish(log, rec, start) + if !ok { + return outcome{status: http.StatusInternalServerError, code: "audit_failed", message: "the request could not be recorded, so it was not processed"} + } + return outcome{status: status, code: code, message: message, record: saved} + } + + // The purpose is required, recorded with any personal data removed, + // and refused outright for decisions about people's eligibility. + if c.purpose == "" { + return outcome{status: http.StatusBadRequest, code: "missing_purpose", message: "set the X-Discreet-Purpose header to say what this request is for, for example \"claims summary\""} + } + model := c.model + if c.real { + if c.model != "" && c.model != s.cfg.Model { + return outcome{status: http.StatusBadRequest, code: "model_not_allowed", message: fmt.Sprintf("this gateway only sends requests to %s", s.cfg.Model)} + } + model = s.cfg.Model + } + rec := audit.Record{RequestID: c.id, Tenant: c.tenant, Purpose: s.scrub(c.purpose), Upstream: c.up.Name(), Model: model} + if deniedPurpose(c.purpose, s.cfg.Policy.DeniedPurposes) { + rec.Decision, rec.Reason = audit.RefusedPurpose, "purpose refused by policy" + return refuse(rec, http.StatusForbidden, "purpose_refused", "Discreet does not take part in automated decisions about people's eligibility. A person must make that decision.") + } + + // Protect every message in one session, so placeholders are consistent. + session := protect.NewSession(s.cfg.Policy, s.cfg.Vault) + defer session.Close() + msgs := make([]upstream.Message, len(c.texts)) + for i, t := range c.texts { + protected, err := session.Protect(t, s.cfg.Engine.Detect(t)) + var blocked *protect.BlockedError + if errors.As(err, &blocked) { + rec.Decision, rec.Reason, rec.Entities = audit.BlockedEntity, blocked.Error(), session.Counts() + return refuse(rec, http.StatusForbidden, "entity_blocked", blocked.Error()) + } + msgs[i] = upstream.Message{Role: c.roles[i], Content: protected} + } + rec.Entities = session.Counts() + promptJSON, _ := json.Marshal(c.texts) + rec.PromptHMAC = s.cfg.Audit.HMAC(string(promptJSON)) + + maxTokens := c.maxTokens + if maxTokens <= 0 || maxTokens > s.cfg.MaxOutputTokens { + maxTokens = s.cfg.MaxOutputTokens + } + // Real calls reserve their worst-case cost first (ADR 0007). + if c.real { + promptBytes := 0 + for _, m := range msgs { + promptBytes += len(m.Content) + } + ceiling := s.cfg.Price.Ceiling(promptBytes, len(msgs), maxTokens) + if !s.cfg.Budget.Reserve(ceiling) { + rec.Decision, rec.Reason = audit.RefusedBudget, fmt.Sprintf("daily budget reached (this request could cost up to US$%.4f)", ceiling) + return refuse(rec, http.StatusTooManyRequests, "budget_exceeded", "today's spending limit for the AI model has been reached; requests resume tomorrow (UTC)") + } + defer s.cfg.Budget.Release(ceiling) // after the real cost is recorded below + } + + resp, err := c.up.Complete(ctx, upstream.Request{Model: model, Messages: msgs, MaxTokens: maxTokens, Temperature: c.temperature}) + if err != nil { + log.Warn("upstream error", "error", err.Error()) + rec.Decision, rec.Reason = audit.UpstreamError, "the model provider returned an error" + return refuse(rec, http.StatusBadGateway, "upstream_error", "the model provider returned an error") + } + + parts := session.RestoreParts(resp.Content) + answer := protect.Join(parts) + fmt.Sprintf(Footer, c.id) + rec.Decision, rec.Model, rec.ResponseHMAC = audit.Allowed, resp.Model, s.cfg.Audit.HMAC(answer) + if c.real { + rec.CostUSD = s.cfg.Price.Cost(resp.PromptTokens, resp.OutputTokens) + } + saved, ok := s.finish(log, rec, start) + if !ok { + return outcome{status: http.StatusInternalServerError, code: "audit_failed", message: "the request could not be recorded, so it was not processed"} + } + return outcome{status: http.StatusOK, record: saved, sent: msgs, response: resp, parts: parts, answer: answer} +} diff --git a/internal/gateway/web/demo.css b/internal/gateway/web/demo.css new file mode 100644 index 0000000..72716e3 --- /dev/null +++ b/internal/gateway/web/demo.css @@ -0,0 +1,79 @@ +:root { + color-scheme: light; + --paper: #eef1f3; --card: #f9fbfc; --ink: #15212a; --muted: #56646e; --line: #cad3d9; --hairline: #e1e6e9; --code-bg: #ffffff; + --accent: #0b7a66; --accent-soft: #d5ece6; --on-accent: #ffffff; + --amber: #a86400; --amber-soft: #fbe9cc; --ai: #2d58cc; --ai-soft: #dde5fa; --no: #b3261e; --no-soft: #f7dcda; + --pii: #7a3fb0; --pii-soft: #efe3f8; --redact: #15212a; --redact-ink: #eef1f3; + --shadow: 0 1px 2px rgba(21,33,42,.06), 0 10px 30px rgba(21,33,42,.08); + --display: "Archivo", "Arial Narrow", Arial, sans-serif; + --body: "Public Sans", -apple-system, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + --mono: "JetBrains Mono", ui-monospace, "SF Mono", Menlo, Consolas, monospace; +} +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + color-scheme: dark; + --paper: #0e1418; --card: #152027; --ink: #e3e9ed; --muted: #96a4ad; --line: #283740; --hairline: #22303a; --code-bg: #10191e; + --accent: #45c4a8; --accent-soft: #123a33; --on-accent: #062019; + --amber: #eaa643; --amber-soft: #3a2a10; --ai: #88a4f7; --ai-soft: #1c2748; --no: #f08a82; --no-soft: #3d1b19; + --pii: #c9a0f0; --pii-soft: #2e1f3d; --redact: #c8d1d7; --redact-ink: #0e1418; + --shadow: 0 1px 2px rgba(0,0,0,.3), 0 10px 30px rgba(0,0,0,.35); + } +} +:root[data-theme="dark"] { + color-scheme: dark; + --paper: #0e1418; --card: #152027; --ink: #e3e9ed; --muted: #96a4ad; --line: #283740; --hairline: #22303a; --code-bg: #10191e; + --accent: #45c4a8; --accent-soft: #123a33; --on-accent: #062019; + --amber: #eaa643; --amber-soft: #3a2a10; --ai: #88a4f7; --ai-soft: #1c2748; --no: #f08a82; --no-soft: #3d1b19; + --pii: #c9a0f0; --pii-soft: #2e1f3d; --redact: #c8d1d7; --redact-ink: #0e1418; + --shadow: 0 1px 2px rgba(0,0,0,.3), 0 10px 30px rgba(0,0,0,.35); +} +* { box-sizing: border-box; } +body { margin: 0; background: var(--paper); color: var(--ink); font-family: var(--body); font-size: 16px; line-height: 1.55; -webkit-font-smoothing: antialiased; } +a { color: var(--accent); text-underline-offset: 3px; } +button, select, textarea { font: inherit; color: inherit; } +code, pre { font-family: var(--mono); } +.wrap { max-width: 1180px; margin: 0 auto; padding: 0 16px; } +@media (min-width: 720px) { .wrap { padding: 0 28px; } } +.sr { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); } +header { padding: 24px 0 8px; } +.top { display: flex; justify-content: space-between; align-items: center; gap: 10px; font-family: var(--mono); font-size: 12px; letter-spacing: .1em; text-transform: uppercase; color: var(--muted); } +.theme { font-family: var(--mono); font-size: 12px; border: 1px solid var(--line); border-radius: 999px; background: var(--card); color: var(--muted); padding: 3px 11px; cursor: pointer; } +h1 { font-family: var(--display); font-stretch: 85%; font-weight: 850; font-size: clamp(32px, 5.4vw, 58px); line-height: 1; letter-spacing: -.015em; margin: 14px 0 10px; } +h1 em { font-style: normal; color: var(--accent); } +.lede { font-size: clamp(16px, 1.9vw, 19px); max-width: 820px; margin: 0 0 12px; } +.notice { background: var(--amber-soft); border-radius: 12px; padding: 10px 14px; font-size: 14.5px; max-width: 820px; margin: 0 0 6px; } +.card { background: var(--card); border: 1px solid var(--hairline); border-radius: 16px; box-shadow: var(--shadow); padding: 16px; min-width: 0; } +.composer { margin-top: 14px; display: grid; gap: 10px; } +.row { display: flex; flex-wrap: wrap; gap: 8px 12px; align-items: center; } +.row label { font-weight: 700; font-size: 14px; } +.hint { color: var(--muted); font-size: 13.5px; font-weight: 400; } +select { border: 1px solid var(--line); background: var(--code-bg); border-radius: 10px; padding: 6px 10px; font-size: 14px; } +textarea { width: 100%; border: 1px solid var(--line); background: var(--code-bg); border-radius: 12px; padding: 12px; font-family: var(--mono); font-size: 14px; line-height: 1.6; resize: vertical; } +.buttons { justify-content: flex-end; } +.btn { border: 1px solid var(--line); background: var(--card); border-radius: 999px; padding: 9px 16px; font-weight: 650; cursor: pointer; } +.btn.primary { background: var(--accent); border-color: var(--accent); color: var(--on-accent); } +.btn:disabled { opacity: .55; cursor: default; } +:focus-visible { outline: 3px solid var(--ai); outline-offset: 2px; } +.status { margin: 0; font-size: 14px; color: var(--muted); min-height: 1.4em; } +.status.refused { color: var(--no); font-weight: 700; } +.panes { display: grid; gap: 14px; margin-top: 14px; grid-template-columns: minmax(0, 1fr); } +@media (min-width: 860px) { .panes { grid-template-columns: repeat(2, minmax(0, 1fr)); } } +.pane h2, .audit h2 { margin: 0 0 8px; font-size: 15px; font-weight: 800; display: flex; align-items: center; gap: 8px; } +.pane h2 .n { display: inline-grid; place-items: center; width: 24px; height: 24px; border-radius: 50%; background: var(--accent-soft); color: var(--accent); font-size: 13px; } +.pane.ai h2 .n { background: var(--ai-soft); color: var(--ai); } +.pane.ai { border-top: 4px solid var(--ai); } +.body { font-family: var(--mono); font-size: 13.5px; line-height: 1.7; white-space: pre-wrap; word-break: break-word; min-height: 110px; color: var(--ink); } +.body .empty { color: var(--muted); font-family: var(--body); } +.ph { background: var(--redact); color: var(--redact-ink); border-radius: 5px; padding: 0 5px; font-weight: 600; } +.ph.redacted { background: var(--no-soft); color: var(--no); } +.restored { background: var(--pii-soft); color: var(--pii); border-radius: 5px; padding: 0 4px; font-weight: 600; border-bottom: 2px solid var(--pii); } +.footer-note { color: var(--muted); } +.refusal { background: var(--no-soft); color: var(--no); border-radius: 10px; padding: 10px 12px; font-family: var(--body); font-weight: 650; } +.audit { margin-top: 14px; } +.chips { display: flex; flex-wrap: wrap; gap: 6px; margin: 6px 0 10px; } +.chip { font-size: 13px; font-weight: 700; border-radius: 999px; padding: 2px 10px; background: var(--accent-soft); color: var(--accent); } +.chip.refused { background: var(--no-soft); color: var(--no); } +pre { margin: 0; background: var(--code-bg); border: 1px solid var(--hairline); border-radius: 12px; padding: 12px; font-size: 12.5px; line-height: 1.6; overflow-x: auto; white-space: pre-wrap; word-break: break-all; } +footer { color: var(--muted); font-size: 14px; padding: 28px 0 48px; margin-top: 24px; border-top: 1px solid var(--line); } +footer p { margin: 4px 0; } +@media (prefers-reduced-motion: reduce) { * { transition: none !important; animation: none !important; } } diff --git a/internal/gateway/web/demo.js b/internal/gateway/web/demo.js new file mode 100644 index 0000000..4a74312 --- /dev/null +++ b/internal/gateway/web/demo.js @@ -0,0 +1,118 @@ +(function () { + 'use strict'; + var $ = function (id) { return document.getElementById(id); }; + var root = document.documentElement; + + // Theme toggle, remembered per browser. + try { var t = localStorage.getItem('discreet-theme'); if (t === 'light' || t === 'dark') root.setAttribute('data-theme', t); } catch (e) {} + function theme() { return root.getAttribute('data-theme') || (matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'); } + function label() { $('theme').textContent = theme() === 'dark' ? 'Light theme' : 'Dark theme'; } + label(); + $('theme').addEventListener('click', function () { + var next = theme() === 'dark' ? 'light' : 'dark'; + root.setAttribute('data-theme', next); + try { localStorage.setItem('discreet-theme', next); } catch (e) {} + label(); + }); + + // Made-up examples. S1234567D and T0000001E have valid check letters but + // are patterned, synthetic numbers; the card is a published test number. + var EXAMPLES = [ + { purpose: 'claims summary', text: 'Member S1234567D called from 9123 4567 about claim C-2041. Please draft a short reply to tan.ah.kow@example.com explaining the next steps. Copy her husband, T0000001E.' }, + { purpose: 'appointment reminder', text: 'Remind the patient (DOB: 12/03/1988) at Blk 123 Ang Mo Kio Avenue 3 #05-123, Singapore 560123, that her check-up is on Friday at 10am. Mobile +65 9876 5432.' }, + { purpose: 'card dispute reply', text: 'A customer disputes a charge of SGD 84.20 on card 4111 1111 1111 1111. Draft a reply to lim.mei@example.com saying we are investigating.' }, + { purpose: 'ticket triage', text: 'Ticket from S1234567A: "I typed my NRIC wrong on the form, my number is S1234567D, please fix it." Classify the ticket.' }, + { purpose: 'automated eligibility decision', text: 'Decide whether S1234567D is eligible for the household support scheme and reply yes or no.' } + ]; + var next = 0; + function load() { + var ex = EXAMPLES[next % EXAMPLES.length]; + next++; + $('purpose').value = ex.purpose; + $('text').value = ex.text; + } + load(); + $('example').addEventListener('click', function () { load(); run(); }); + + var TOKEN = /(<[A-Z][A-Z_]*_\d+>|\[REDACTED_[A-Z_]+\])/; + function placeholders(el, text, keep) { + if (!keep) el.textContent = ''; + text.split(TOKEN).forEach(function (piece) { + if (!piece) return; + if (TOKEN.test(piece) && piece.match(TOKEN)[0] === piece) { + var s = document.createElement('span'); + s.className = 'ph' + (piece.charAt(0) === '[' ? ' redacted' : ''); + s.textContent = piece; + el.appendChild(s); + } else { + el.appendChild(document.createTextNode(piece)); + } + }); + } + function empty(el, text) { + el.textContent = ''; + var s = document.createElement('span'); s.className = 'empty'; s.textContent = text; el.appendChild(s); + } + + function render(out) { + $('p1').textContent = out.original; + $('chips').textContent = ''; + var rec = out.audit_record; + if (out.status === 'refused') { + $('status').className = 'status refused'; + $('status').textContent = 'Refused: ' + out.message; + var box = document.createElement('div'); box.className = 'refusal'; box.textContent = out.message; + $('p2').textContent = ''; $('p2').appendChild(box); + empty($('p3'), 'The model was never called.'); + empty($('p4'), 'The app received an error (' + out.code + ') instead of an answer.'); + chip(rec ? rec.decision : out.code, true); + } else { + $('status').className = 'status'; + $('status').textContent = 'Done. Compare pane 1 with pane 2: the AI never saw the real details.'; + placeholders($('p2'), out.sent_to_model); + placeholders($('p3'), out.model_answer); + var p4 = $('p4'); p4.textContent = ''; + out.returned.forEach(function (part, i) { + if (part.placeholder) { + var s = document.createElement('span'); s.className = 'restored'; s.title = 'Put back in place of ' + part.placeholder; s.textContent = part.text; + p4.appendChild(s); + } else if (i === out.returned.length - 1 && part.text.indexOf('Personal data was replaced') !== -1) { + var f = document.createElement('span'); f.className = 'footer-note'; f.textContent = part.text; p4.appendChild(f); + } else { + placeholders(p4, part.text, true); // marks anything never restored, such as [REDACTED_CARD] + } + }); + chip(rec.decision, false); + } + if (rec) { + Object.keys(rec.entities || {}).sort().forEach(function (k) { chip(k + ' × ' + rec.entities[k], false); }); + $('record').textContent = JSON.stringify(rec, null, 2); + } else { + $('record').textContent = '(no record: the request was rejected before it was processed)'; + } + } + function chip(text, bad) { + var c = document.createElement('span'); c.className = 'chip' + (bad ? ' refused' : ''); c.textContent = text; $('chips').appendChild(c); + } + + function run() { + var text = $('text').value; + if (!text.trim()) { $('status').textContent = 'Type or load a message first.'; return; } + $('send').disabled = true; + $('status').className = 'status'; + $('status').textContent = 'Sending…'; + fetch('demo/run', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ text: text, purpose: $('purpose').value }) }) + .then(function (r) { return r.json().then(function (body) { return { ok: r.ok || r.status === 403, status: r.status, body: body }; }); }) + .then(function (res) { + if (res.body && res.body.original !== undefined) { render(res.body); return; } + var msg = res.body && res.body.error ? res.body.error.message : 'Something went wrong (' + res.status + ').'; + $('status').className = 'status refused'; $('status').textContent = msg; + }) + .catch(function () { $('status').className = 'status refused'; $('status').textContent = 'Could not reach Discreet.'; }) + .finally(function () { $('send').disabled = false; }); + } + $('send').addEventListener('click', run); + empty($('p1'), 'Load an example or type a message, then send it.'); + ['p2', 'p3', 'p4'].forEach(function (id) { empty($(id), 'Waiting for a message.'); }); + $('record').textContent = ''; +})(); diff --git a/internal/gateway/web/index.html b/internal/gateway/web/index.html new file mode 100644 index 0000000..723d09c --- /dev/null +++ b/internal/gateway/web/index.html @@ -0,0 +1,64 @@ + + + + + +Discreet Demo + + + + + + + + +
+
AI Governance Toolkit · Discreet
+

Send personal data to an AI. Without sending it.

+

Discreet sits between an app and an AI model. It swaps NRICs, phone numbers, cards and other personal data for placeholders on the way out, puts them back on the way in, and records every request in a tamper-evident log.

+

Please don't type real personal data. Use the made-up examples. This demo runs on a mock model: no AI is called and nothing costs money. Discreet keeps only counts and keyed fingerprints of what you send, never the text.

+
+ +
+
+
+ + +
+ + +
+ + +
+

+
+ +
+

1 What the app sent

+

2 What the AI saw

+

3 What the AI answered

+

4 What the app got back

+
+ +
+

The audit record for this request

+

One line in an append-only log. It holds counts and keyed fingerprints, never the text, and the hash of the line before it: change any byte and discreet audit verify fails, naming the record.

+
+ 

+  

+

+
+
+
+
+
diff --git a/internal/protect/protect_test.go b/internal/protect/protect_test.go
index 2b6f9a0..fb861a2 100644
--- a/internal/protect/protect_test.go
+++ b/internal/protect/protect_test.go
@@ -136,3 +136,16 @@ func TestParsePolicyRejectsMistakes(t *testing.T) {
 		t.Error("default policy actions are not as documented")
 	}
 }
+
+func TestRestorePartsMatchesRestore(t *testing.T) {
+	e, s, _ := setup(t, DefaultPolicy())
+	protect(t, e, s, "Call S1234567D on 9123 4567")
+	text := "Hi , call .  stays."
+	parts := s.RestoreParts(text)
+	if Join(parts) != s.Restore(text) {
+		t.Errorf("Join(RestoreParts) = %q, Restore = %q", Join(parts), s.Restore(text))
+	}
+	if len(parts) != 5 || parts[1].Placeholder != "" || parts[1].Text != "S1234567D" || parts[3].Placeholder != "" {
+		t.Errorf("parts %+v", parts)
+	}
+}
diff --git a/internal/protect/session.go b/internal/protect/session.go
index 1a05a6a..60f9795 100644
--- a/internal/protect/session.go
+++ b/internal/protect/session.go
@@ -110,6 +110,44 @@ func (s *Session) Restore(text string) string {
 	})
 }
 
+// Part is a piece of a restored answer: plain text, or a value put back in
+// place of Placeholder. The demo page uses it to highlight restored values.
+type Part struct {
+	Text        string `json:"text"`
+	Placeholder string `json:"placeholder,omitempty"`
+}
+
+// RestoreParts is Restore, split into parts.
+func (s *Session) RestoreParts(text string) []Part {
+	var parts []Part
+	pos := 0
+	for _, loc := range placeholderPattern.FindAllStringIndex(text, -1) {
+		p := text[loc[0]:loc[1]]
+		v, ok := s.vault.Get(s.vaultKey(p))
+		if !ok {
+			continue
+		}
+		if loc[0] > pos {
+			parts = append(parts, Part{Text: text[pos:loc[0]]})
+		}
+		parts = append(parts, Part{Text: v, Placeholder: p})
+		pos = loc[1]
+	}
+	if pos < len(text) {
+		parts = append(parts, Part{Text: text[pos:]})
+	}
+	return parts
+}
+
+// Join puts parts back together as plain text.
+func Join(parts []Part) string {
+	var b strings.Builder
+	for _, p := range parts {
+		b.WriteString(p.Text)
+	}
+	return b.String()
+}
+
 // Counts returns how many matches of each entity were seen.
 func (s *Session) Counts() map[string]int {
 	out := make(map[string]int, len(s.found))

From e2df83ea8a7c987f6edeeef8c6a3990ed2eaef13 Mon Sep 17 00:00:00 2001
From: Michael Mendoza <62703135+makoydev@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:10:59 +0800
Subject: [PATCH 08/13] feat: benchmark and latency numbers in EVALS (D9) (#11)

- internal/bench: a seeded generator written independently of the rules
  (own NRIC checksum and Luhn, own templates) producing 5,000 samples
  with 0-3 pieces of personal data in realistic styles, some the rules
  don't support, plus look-alike decoys. Scored per entity and per
  writing style; "caught" means fully covered by the same kind.
- cmd/benchmark (make bench) prints Markdown: 76.5% recall overall, 100%
  on every supported style, 92.3% precision (482 false alarms, all bare
  8-digit numbers as PHONE, the trade-off in sg-pii-rules ADR 0007).
- Latency vs a baseline without Discreet: about 3 ms added at p50, of
  which 2.9 ms is the audit record flushed to disk and 0.08 ms detection.
- Deliberate break (seed ignored) failed the determinism test; restored.

Closes makoydev/vetted#39

Co-authored-by: Claude Opus 5.5 
---
 CHANGELOG.md                 |   1 +
 EVALS.md                     |  89 ++++++++++-
 Makefile                     |   6 +-
 README.md                    |   4 +
 cmd/benchmark/main.go        | 176 +++++++++++++++++++++
 docs/HOW-THIS-WAS-BUILT.md   |   2 +
 internal/bench/bench_test.go |  57 +++++++
 internal/bench/evaluate.go   | 104 +++++++++++++
 internal/bench/generate.go   | 285 +++++++++++++++++++++++++++++++++++
 9 files changed, 720 insertions(+), 4 deletions(-)
 create mode 100644 cmd/benchmark/main.go
 create mode 100644 internal/bench/bench_test.go
 create mode 100644 internal/bench/evaluate.go
 create mode 100644 internal/bench/generate.go

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 84ab315..150616c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ All notable changes to this project are documented here. The format follows [Kee
 - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records.
 - Vetted reviews every pull request (shadow mode until 2026-10-14).
 - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases.
+- Benchmark (issue D9, `make bench`): 5,000 synthetic samples from a generator written independently of the rules, scored per entity and per writing style, plus latency against a baseline. Results in `EVALS.md`: 76.5% recall overall (100% on every supported style), 92.3% precision (all false alarms are bare eight-digit numbers as `PHONE`), about 3 ms added per request, almost all of it the audit write flushed to disk.
 - Four-pane demo page (issue D8) at `/`: what the app sent, what the AI saw, what it answered, and what the app got back, plus that request's audit record. Five made-up scenarios (claims, appointment, card dispute, mistyped NRIC, a refused eligibility decision). It runs through the same pipeline as the API, always on the mock model, even for token holders; strict Content-Security-Policy; works on phones, light and dark.
 - A real model behind access tokens (issue D6): an OpenAI-compatible client (OpenAI, or Ollama with `max_tokens`) used only for callers with a valid `DISCREET_ACCESS_TOKENS` token; everyone else gets the mock. One pinned model with checked prices (`gpt-6-luna`); each request reserves its worst-case cost (at most US$0.0076) against a hard daily budget (default US$0.20) recomputed from the audit log; 30 requests a minute per client address (ADR 0007). No test calls a paid API.
 - The gateway (`internal/gateway`, issue D5): `discreet serve` runs an OpenAI-compatible `POST /v1/chat/completions` (non-streaming). Every message is protected in one session before the model sees it and restored afterwards; every answer ends with a disclosure footer; every request, allowed or not, is recorded in the audit log, and Discreet never answers a request it couldn't record. A free mock model is the default. Callers must send `X-Discreet-Purpose`; automated eligibility and benefits decisions are refused with HTTP 403 (ADR 0006). Only the model, messages, token limit and temperature are forwarded. Tested with OpenAI's official Python library by changing only `base_url`.
diff --git a/EVALS.md b/EVALS.md
index d1ecc99..89b5a05 100644
--- a/EVALS.md
+++ b/EVALS.md
@@ -1,11 +1,11 @@
 # Evaluations
 
-**Nothing is measured yet.** This file will only ever hold numbers that were actually measured, each with its method, sample size and limitations.
+This file only holds numbers that were actually measured, each with its method, sample size and limitations.
 
 | What | Planned in | Status |
 | --- | --- | --- |
-| Precision and recall per entity on about 5,000 synthetic samples from a generator written independently of the detectors | Issue D9 | Not started |
-| Added latency (p50, p95) against the mock model | Issue D9 | Not started |
+| Precision and recall per entity on about 5,000 synthetic samples from a generator written independently of the detectors | Issue D9 | **Done** (below) |
+| Added latency (p50, p95) against the mock model | Issue D9 | **Done** (below) |
 | Detector conformance: every sg-pii-rules fixture passes in Go | Issue D3 | **Done: 369 / 369** (see below) |
 | Data-disclosure and adversarial-prompt run with Project Moonshot | Milestone 3 | Not started |
 
@@ -13,6 +13,89 @@ Known limitation, stated in advance: synthetic data flatters rule-based detector
 
 Vetted's review of this repository's pull requests is measured in Vetted's own `EVALS.md`.
 
+## Benchmark: detection on 5,000 synthetic samples (issue D9, measured 2026-10-05)
+
+Reproduce with `make bench` (`go run ./cmd/benchmark`). The generator (`internal/bench`) is written independently of the detection rules: its own NRIC checksum and Luhn code, its own sentence templates, and on purpose it writes personal data in styles a rule-based detector may not handle (an NRIC with spaces, a phone in 2-2-2-2 groups, a date of birth after "Birthday"). Every sample mixes 0 to 3 pieces of personal data with 0 to 2 look-alike decoys (order numbers, OTPs, amounts, timestamps, tracking numbers). A value counts as **caught** only if a detection of the same kind covers it completely; partly covered still leaks, so it counts as missed.
+
+| Entity | True values | Caught as the right kind (recall) | Protected by any detection | Detections | False alarms | Precision |
+| --- | --- | --- | --- | --- | --- | --- |
+| `CARD` | 1075 | 933 (86.8%) | 933 (86.8%) | 933 | 0 | 100.0% |
+| `DOB` | 1044 | 614 (58.8%) | 614 (58.8%) | 614 | 0 | 100.0% |
+| `EMAIL` | 1083 | 996 (92.0%) | 996 (92.0%) | 996 | 0 | 100.0% |
+| `NRIC` | 1105 | 880 (79.6%) | 880 (79.6%) | 880 | 0 | 100.0% |
+| `PHONE` | 1112 | 856 (77.0%) | 856 (77.0%) | 1338 | 482 | 64.0% |
+| `POSTAL` | 1058 | 779 (73.6%) | 779 (73.6%) | 779 | 0 | 100.0% |
+| `UNIT` | 1054 | 702 (66.6%) | 702 (66.6%) | 702 | 0 | 100.0% |
+| **All** | 7531 | 5760 (76.5%) | 5760 (76.5%) | 6242 | 482 | 92.3% |
+
+Recall by writing style:
+
+| Entity | Style | Caught / total |
+| --- | --- | --- |
+| `CARD` | double spaces | 0 / 142 (0%) |
+| `CARD` | hyphens | 124 / 124 (100%) |
+| `CARD` | plain | 141 / 141 (100%) |
+| `CARD` | spaced | 668 / 668 (100%) |
+| `DOB` | DOB dd/mm/yyyy | 155 / 155 (100%) |
+| `DOB` | JSON ISO | 171 / 171 (100%) |
+| `DOB` | birthday clue | 0 / 147 (0%) |
+| `DOB` | born on Month d, yyyy | 141 / 141 (100%) |
+| `DOB` | date of birth d Month yyyy | 147 / 147 (100%) |
+| `DOB` | two-digit year | 0 / 124 (0%) |
+| `DOB` | words between clue and date | 0 / 159 (0%) |
+| `EMAIL` | obfuscated | 0 / 87 (0%) |
+| `EMAIL` | plus tag | 127 / 127 (100%) |
+| `EMAIL` | standard | 764 / 764 (100%) |
+| `EMAIL` | uppercase | 105 / 105 (100%) |
+| `NRIC` | hyphenated | 0 / 123 (0%) |
+| `NRIC` | lowercase | 104 / 104 (100%) |
+| `NRIC` | mistyped check letter | 102 / 102 (100%) |
+| `NRIC` | spaced | 0 / 102 (0%) |
+| `NRIC` | standard | 674 / 674 (100%) |
+| `PHONE` | (65) | 138 / 138 (100%) |
+| `PHONE` | +65 joined | 160 / 160 (100%) |
+| `PHONE` | +65 space | 145 / 145 (100%) |
+| `PHONE` | 2-2-2-2 | 0 / 124 (0%) |
+| `PHONE` | 3-5 | 0 / 132 (0%) |
+| `PHONE` | 4-4 hyphen | 152 / 152 (100%) |
+| `PHONE` | 4-4 space | 123 / 123 (100%) |
+| `PHONE` | plain | 138 / 138 (100%) |
+| `POSTAL` | S'pore abbreviation | 0 / 134 (0%) |
+| `POSTAL` | S(…) | 118 / 118 (100%) |
+| `POSTAL` | after Singapore | 535 / 535 (100%) |
+| `POSTAL` | no clue | 0 / 145 (0%) |
+| `POSTAL` | postal code | 126 / 126 (100%) |
+| `UNIT` | # with space | 0 / 174 (0%) |
+| `UNIT` | #floor-unit | 702 / 702 (100%) |
+| `UNIT` | Unit without # | 0 / 178 (0%) |
+
+False-alarm examples (synthetic text):
+- `PHONE`: "92121752"
+- `PHONE`: "60130067"
+- `PHONE`: "65478180"
+
+**Reading this honestly.**
+
+- **Every miss is a writing style the rules don't support yet**, and every supported style scored 100%: NRICs with spaces or hyphens, phones in 2-2-2-2 or 3-5 groups, cards with double spaces, obfuscated emails ("name at example dot com"), postal codes with no clue or after "S'pore", unit numbers without `#` or with a space after it, and dates of birth after "Birthday", with words between clue and date, or with a two-digit year. The generator's mix of styles is a choice, so the overall 76.5% depends on it; the per-style table is the more useful result. These gaps are candidates for sg-pii-rules in Milestone 3.
+- **False alarms: 482, all `PHONE`**, all bare eight-digit decoys starting 6, 8 or 9 (order and batch numbers). This is the trade-off sg-pii-rules accepted in its ADR 0007: a false alarm replaces a number with a placeholder; a miss leaks a phone number. Every other entity had no false alarms.
+- **Synthetic data flatters rule-based detectors.** Real text is messier than any generator, and real personal data can't be used to check. Treat these numbers as an upper bound on recall for the styles covered.
+
+## Latency (issue D9, measured 2026-10-05)
+
+Latency over 2000 sequential requests on loopback, each with one benchmark sample:
+
+| | p50 | p95 | p99 |
+| --- | --- | --- | --- |
+| Baseline: same HTTP and JSON work, mock model, no Discreet | 0.04 ms | 0.11 ms | 0.17 ms |
+| Through Discreet (detect, protect, restore, audit write with fsync) | 3.05 ms | 4.90 ms | 11.74 ms |
+| **Added by Discreet** | **3.00 ms** | **4.79 ms** | 11.57 ms |
+| of which: detection alone | 0.079 ms | 0.230 ms | 0.578 ms |
+| of which: one audit write, flushed to disk | 2.884 ms | 3.628 ms | 4.828 ms |
+
+Measured on an Apple-silicon Mac over loopback, sequential requests, each carrying one benchmark sample, with the mock model. The baseline server does the same HTTP and JSON work and the same mock call without Discreet, so the difference is what Discreet adds.
+
+**Where the time goes:** almost all of the roughly 3 ms is the audit record being flushed to disk before the answer is sent (ADR 0005); detection itself takes under 0.1 ms. That flush is the price of "no answer without a record". Tail latency (p99) varied between runs (4.7 ms and 11.7 ms on the same machine), so treat it as indicative. Against a real model, which takes hundreds of milliseconds to seconds, the added time is small. Disk flush speed differs by machine and hosting; this will be re-measured on the deployment (issue D10).
+
 ## Cost bound (issue D6, computed 2026-10-05)
 
 With `gpt-6-luna` at US$0.10 / US$0.50 per million input / output tokens (OpenAI's model page, primary source), the 64 KB request limit and the default 2,048-token output cap, one request can cost at most **US$0.0076**: (65,536 bytes + 16) × 0.10 / 10⁶ + 2,048 × 0.50 / 10⁶. The default daily budget of US$0.20 therefore allows at least 26 worst-case requests, and at most about US$6 a month. This is an upper bound by construction, not a measurement; no real request has been sent yet, because `OPENAI_API_KEY` is not configured.
diff --git a/Makefile b/Makefile
index 359eeac..9845eda 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
 # One command for every check, used locally and in CI: make check
 GOVULNCHECK := golang.org/x/vuln/cmd/govulncheck@v1.8.0
 
-.PHONY: check fmt vet test vuln build
+.PHONY: check fmt vet test vuln build bench
 
 check: fmt vet test vuln build
 
@@ -19,3 +19,7 @@ vuln:
 
 build:
 	go build -o bin/discreet ./cmd/discreet
+
+# Detection benchmark and latency, printed as Markdown for EVALS.md
+bench:
+	go run ./cmd/benchmark
diff --git a/README.md b/README.md
index 92de874..a11c412 100644
--- a/README.md
+++ b/README.md
@@ -43,6 +43,10 @@ Run `./bin/discreet serve --help` for every setting (budget, output cap, rate li
 
 The mock model shows what a real model would have received (placeholders only), and the answer comes back with the real values restored. Any OpenAI client works by pointing `base_url` at `http://localhost:8080/v1` and adding the `X-Discreet-Purpose` header. The example NRIC is synthetic.
 
+## Measured
+
+On 5,000 synthetic samples from an independent generator: **76.5% recall** overall, 100% on every supported writing style, **92.3% precision** (false alarms are only bare eight-digit numbers read as phones). Discreet adds about **3 ms** per request, almost all of it writing the audit record safely to disk. Details and limits: [`EVALS.md`](EVALS.md).
+
 ## Evidence
 
 [`CONTROLS.md`](CONTROLS.md) · [`THREAT_MODEL.md`](THREAT_MODEL.md) · [`EVALS.md`](EVALS.md) · [`RISKS.md`](RISKS.md) · [`CHANGELOG.md`](CHANGELOG.md) · [decision records](docs/adr/) · [how this was built](docs/HOW-THIS-WAS-BUILT.md)
diff --git a/cmd/benchmark/main.go b/cmd/benchmark/main.go
new file mode 100644
index 0000000..84b9375
--- /dev/null
+++ b/cmd/benchmark/main.go
@@ -0,0 +1,176 @@
+// Command benchmark measures Discreet's detection on synthetic text from an
+// independent generator, and the latency the gateway adds. It prints
+// Markdown for EVALS.md. Usage: go run ./cmd/benchmark [-n 5000] [-seed N]
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/makoydev/discreet/internal/audit"
+	"github.com/makoydev/discreet/internal/bench"
+	"github.com/makoydev/discreet/internal/detect"
+	"github.com/makoydev/discreet/internal/gateway"
+	"github.com/makoydev/discreet/internal/protect"
+	"github.com/makoydev/discreet/internal/upstream"
+	"github.com/makoydev/discreet/internal/vault"
+	"github.com/makoydev/discreet/third_party/sgpiirules"
+)
+
+func main() {
+	n := flag.Int("n", 5000, "samples")
+	seed := flag.Uint64("seed", 20261005, "random seed")
+	requests := flag.Int("requests", 2000, "requests per latency run")
+	flag.Parse()
+
+	engine, err := detect.Default()
+	if err != nil {
+		panic(err)
+	}
+	samples := bench.Generate(*n, *seed)
+	version, _ := sgpiirules.Files.ReadFile("VERSION")
+	fmt.Printf("Benchmark: %d samples, seed %d, rules sg-pii-rules %s, %s %s/%s\n\n", *n, *seed, strings.TrimSpace(string(version)), runtime.Version(), runtime.GOOS, runtime.GOARCH)
+
+	results := bench.Evaluate(samples, engine)
+	fmt.Println("| Entity | True values | Caught as the right kind (recall) | Protected by any detection | Detections | False alarms | Precision |")
+	fmt.Println("| --- | --- | --- | --- | --- | --- | --- |")
+	var keys []string
+	for k := range results {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	tot := bench.Result{}
+	for _, k := range keys {
+		r := results[k]
+		fmt.Printf("| `%s` | %d | %d (%.1f%%) | %d (%.1f%%) | %d | %d | %.1f%% |\n", k, r.Total, r.Caught, 100*r.Recall(), r.Protected, pct(r.Protected, r.Total), r.Detected, r.FalsePositives, 100*r.Precision())
+		tot.Total += r.Total
+		tot.Caught += r.Caught
+		tot.Protected += r.Protected
+		tot.Detected += r.Detected
+		tot.FalsePositives += r.FalsePositives
+	}
+	fmt.Printf("| **All** | %d | %d (%.1f%%) | %d (%.1f%%) | %d | %d | %.1f%% |\n\n", tot.Total, tot.Caught, 100*tot.Recall(), tot.Protected, pct(tot.Protected, tot.Total), tot.Detected, tot.FalsePositives, 100*tot.Precision())
+
+	fmt.Println("Recall by writing style:")
+	fmt.Println()
+	fmt.Println("| Entity | Style | Caught / total |")
+	fmt.Println("| --- | --- | --- |")
+	for _, k := range keys {
+		var styles []string
+		for v := range results[k].Variants {
+			styles = append(styles, v)
+		}
+		sort.Strings(styles)
+		for _, v := range styles {
+			c := results[k].Variants[v]
+			fmt.Printf("| `%s` | %s | %d / %d (%.0f%%) |\n", k, v, c[0], c[1], pct(c[0], c[1]))
+		}
+	}
+	fmt.Println()
+	fmt.Println("False-alarm examples (synthetic text):")
+	for _, k := range keys {
+		for _, ex := range results[k].FPExamples {
+			fmt.Printf("- `%s`: %q\n", k, ex)
+		}
+	}
+
+	fmt.Println()
+	latency(samples, engine, *requests)
+}
+
+func pct(a, b int) float64 {
+	if b == 0 {
+		return 0
+	}
+	return 100 * float64(a) / float64(b)
+}
+
+// latency compares the full gateway (mock model, audit log on disk) with a
+// baseline server that does the same JSON work and mock call without
+// Discreet's processing. The difference is what Discreet adds.
+func latency(samples []bench.Sample, engine *detect.Engine, requests int) {
+	dir, _ := os.MkdirTemp("", "discreet-bench")
+	defer os.RemoveAll(dir)
+	log, err := audit.Open(filepath.Join(dir, "audit.jsonl"), []byte("benchmark-key-0123456789abcdefghijk"))
+	if err != nil {
+		panic(err)
+	}
+	defer log.Close()
+	v, _ := vault.New(vault.DefaultTTL)
+	gw := httptest.NewServer(gateway.New(gateway.Config{
+		Engine: engine, Policy: protect.DefaultPolicy(), Vault: v, Audit: log,
+		Logger: slog.New(slog.NewJSONHandler(io.Discard, nil)),
+	}).Handler())
+	defer gw.Close()
+	mock := &upstream.Mock{}
+	base := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var req struct {
+			Model    string             `json:"model"`
+			Messages []upstream.Message `json:"messages"`
+		}
+		json.NewDecoder(r.Body).Decode(&req)
+		resp, _ := mock.Complete(context.Background(), upstream.Request{Model: req.Model, Messages: req.Messages})
+		json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"message": map[string]string{"role": "assistant", "content": resp.Content}}}})
+	}))
+	defer base.Close()
+
+	run := func(url string) []time.Duration {
+		var ds []time.Duration
+		for i := 0; i < requests; i++ {
+			body, _ := json.Marshal(map[string]any{"model": "mock", "messages": []map[string]string{{"role": "user", "content": samples[i%len(samples)].Text}}})
+			req, _ := http.NewRequest("POST", url+"/v1/chat/completions", bytes.NewReader(body))
+			req.Header.Set("Content-Type", "application/json")
+			req.Header.Set("X-Discreet-Purpose", "benchmark")
+			start := time.Now()
+			resp, err := http.DefaultClient.Do(req)
+			if err != nil {
+				panic(err)
+			}
+			io.Copy(io.Discard, resp.Body)
+			resp.Body.Close()
+			ds = append(ds, time.Since(start))
+		}
+		sort.Slice(ds, func(i, j int) bool { return ds[i] < ds[j] })
+		return ds
+	}
+	run(gw.URL) // warm up
+	run(base.URL)
+	g, b := run(gw.URL), run(base.URL)
+	q := func(ds []time.Duration, p float64) time.Duration { return ds[int(float64(len(ds)-1)*p)] }
+	ms := func(d time.Duration) string { return fmt.Sprintf("%.2f ms", float64(d.Microseconds())/1000) }
+	fmt.Printf("Latency over %d sequential requests on loopback, each with one benchmark sample:\n\n", requests)
+	fmt.Println("| | p50 | p95 | p99 |")
+	fmt.Println("| --- | --- | --- | --- |")
+	fmt.Printf("| Baseline: same HTTP and JSON work, mock model, no Discreet | %s | %s | %s |\n", ms(q(b, .5)), ms(q(b, .95)), ms(q(b, .99)))
+	fmt.Printf("| Through Discreet (detect, protect, restore, audit write with fsync) | %s | %s | %s |\n", ms(q(g, .5)), ms(q(g, .95)), ms(q(g, .99)))
+	fmt.Printf("| **Added by Discreet** | **%s** | **%s** | %s |\n", ms(q(g, .5)-q(b, .5)), ms(q(g, .95)-q(b, .95)), ms(q(g, .99)-q(b, .99)))
+
+	// Where the time goes: detection alone, and one durable audit write.
+	var det, wr []time.Duration
+	for i := 0; i < requests; i++ {
+		start := time.Now()
+		engine.Detect(samples[i%len(samples)].Text)
+		det = append(det, time.Since(start))
+		start = time.Now()
+		log.Append(audit.Record{Decision: audit.Allowed})
+		wr = append(wr, time.Since(start))
+	}
+	sort.Slice(det, func(i, j int) bool { return det[i] < det[j] })
+	sort.Slice(wr, func(i, j int) bool { return wr[i] < wr[j] })
+	us := func(d time.Duration) string { return fmt.Sprintf("%.3f ms", float64(d.Nanoseconds())/1e6) }
+	fmt.Printf("| of which: detection alone | %s | %s | %s |\n", us(q(det, .5)), us(q(det, .95)), us(q(det, .99)))
+	fmt.Printf("| of which: one audit write, flushed to disk | %s | %s | %s |\n", us(q(wr, .5)), us(q(wr, .95)), us(q(wr, .99)))
+}
diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md
index 709652a..c18148b 100644
--- a/docs/HOW-THIS-WAS-BUILT.md
+++ b/docs/HOW-THIS-WAS-BUILT.md
@@ -35,6 +35,8 @@
 | 2026-10-05 | `gpt-6-luna` on `/v1/chat/completions` and its price (OpenAI's model page, primary) | Supported; US$0.10 / US$0.50 per million input / output tokens |
 | 2026-10-05 | Deliberate breaks of tokens, budget and rate limit | Each failed tests once the token break was redone so it compiled; restored from backup |
 | 2026-10-05 | Demo page in headless Chrome at 1280 px (light and dark) and 390 px, clicking through the examples | No Content-Security-Policy violations or script errors; one 404 for the browser's default icon request, fixed with an inline empty icon; audit log verified afterwards (7 records) |
+| 2026-10-05 | Benchmark (5,000 samples) and latency (2,000 requests per run) | Results in EVALS.md. A suspicion that the audit flush dominated latency was measured rather than assumed: 2.9 ms of the 3.0 ms |
+| 2026-10-05 | Deliberate break: the generator ignoring its seed | The determinism test failed; restored from backup |
 | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite |
 
 ## What the AI got wrong, and how it was caught
diff --git a/internal/bench/bench_test.go b/internal/bench/bench_test.go
new file mode 100644
index 0000000..66f3be1
--- /dev/null
+++ b/internal/bench/bench_test.go
@@ -0,0 +1,57 @@
+package bench
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/makoydev/discreet/internal/detect"
+)
+
+func TestGenerateIsDeterministicAndSpansAreExact(t *testing.T) {
+	a, b := Generate(300, 7), Generate(300, 7)
+	if !reflect.DeepEqual(a, b) {
+		t.Fatal("the same seed gave different samples")
+	}
+	if reflect.DeepEqual(a, Generate(300, 8)) {
+		t.Fatal("different seeds gave the same samples")
+	}
+	entities := map[string]int{}
+	for _, s := range a {
+		for _, sp := range s.Spans {
+			if sp.Start < 0 || sp.End > len(s.Text) || sp.Start >= sp.End {
+				t.Fatalf("bad span %+v in %q", sp, s.Text)
+			}
+			entities[sp.Entity]++
+		}
+	}
+	for _, e := range entityOrder {
+		if entities[e] == 0 {
+			t.Errorf("no %s in 300 samples", e)
+		}
+	}
+}
+
+func TestGeneratedValuesAreValidWhereTheyShouldBe(t *testing.T) {
+	engine, _ := detect.Default()
+	// Standard NRICs from the independent generator must pass the engine's checksum.
+	r := Generate(2000, 3)
+	for _, s := range r {
+		for _, sp := range s.Spans {
+			if sp.Entity == "NRIC" && sp.Variant == "standard" {
+				found := engine.Detect(s.Text[sp.Start:sp.End])
+				if len(found) != 1 || found[0].Entity != "NRIC" {
+					t.Fatalf("independently generated NRIC %q not valid to the engine: %v", s.Text[sp.Start:sp.End], found)
+				}
+			}
+		}
+	}
+}
+
+func TestEvaluateScoresCoverage(t *testing.T) {
+	engine, _ := detect.Default()
+	s := []Sample{{Text: "call 9123 4567 or 91 23 45 67", Spans: []Span{{"PHONE", 5, 14, "4-4"}, {"PHONE", 18, 29, "2-2-2-2"}}}}
+	r := Evaluate(s, engine)["PHONE"]
+	if r.Total != 2 || r.Caught != 1 || r.Detected != 1 || r.FalsePositives != 0 {
+		t.Errorf("%+v", r)
+	}
+}
diff --git a/internal/bench/evaluate.go b/internal/bench/evaluate.go
new file mode 100644
index 0000000..39d0f28
--- /dev/null
+++ b/internal/bench/evaluate.go
@@ -0,0 +1,104 @@
+package bench
+
+import (
+	"sort"
+
+	"github.com/makoydev/discreet/internal/detect"
+)
+
+// sameKind maps a detected entity to the true entity it counts for: a
+// mistyped NRIC found as NRIC_LIKE still protects an NRIC.
+func sameKind(detected string) string {
+	if detected == "NRIC_LIKE" {
+		return "NRIC"
+	}
+	return detected
+}
+
+// Result counts one entity's outcomes.
+type Result struct {
+	Entity string
+	// Recall side: true values of this entity.
+	Total     int
+	Caught    int // fully covered by a detection of the same kind
+	Protected int // fully covered by any detection(s), whatever the kind
+	// Precision side: detections reported as this entity.
+	Detected       int
+	FalsePositives int // overlap no true value of the same kind
+	Variants       map[string][2]int
+	FPExamples     []string
+}
+
+func (r *Result) Recall() float64    { return ratio(r.Caught, r.Total) }
+func (r *Result) Precision() float64 { return ratio(r.Detected-r.FalsePositives, r.Detected) }
+
+func ratio(a, b int) float64 {
+	if b == 0 {
+		return 0
+	}
+	return float64(a) / float64(b)
+}
+
+// Evaluate runs the engine over every sample and scores it.
+func Evaluate(samples []Sample, engine *detect.Engine) map[string]*Result {
+	results := map[string]*Result{}
+	get := func(e string) *Result {
+		if results[e] == nil {
+			results[e] = &Result{Entity: e, Variants: map[string][2]int{}}
+		}
+		return results[e]
+	}
+	for _, s := range samples {
+		found := engine.Detect(s.Text)
+		for _, span := range s.Spans {
+			r := get(span.Entity)
+			r.Total++
+			v := r.Variants[span.Variant]
+			v[1]++
+			caught := false
+			for _, d := range found {
+				if sameKind(d.Entity) == span.Entity && d.Start <= span.Start && d.End >= span.End {
+					caught = true
+				}
+			}
+			if caught {
+				r.Caught++
+				v[0]++
+			}
+			if covered(found, span) {
+				r.Protected++
+			}
+			r.Variants[span.Variant] = v
+		}
+		for _, d := range found {
+			r := get(sameKind(d.Entity))
+			r.Detected++
+			hit := false
+			for _, span := range s.Spans {
+				if span.Entity == sameKind(d.Entity) && d.Start < span.End && span.Start < d.End {
+					hit = true
+				}
+			}
+			if !hit {
+				r.FalsePositives++
+				if len(r.FPExamples) < 3 {
+					r.FPExamples = append(r.FPExamples, s.Text[d.Start:d.End])
+				}
+			}
+		}
+	}
+	return results
+}
+
+// covered reports whether every byte of span is inside some detection.
+func covered(found []detect.Match, span Span) bool {
+	pos := span.Start
+	sorted := append([]detect.Match(nil), found...)
+	sort.Slice(sorted, func(i, j int) bool { return sorted[i].Start < sorted[j].Start })
+	for _, d := range sorted {
+		if d.Start <= pos && d.End > pos {
+			pos = d.End
+		}
+	}
+	return pos >= span.End
+}
diff --git a/internal/bench/generate.go b/internal/bench/generate.go
new file mode 100644
index 0000000..ff32d89
--- /dev/null
+++ b/internal/bench/generate.go
@@ -0,0 +1,285 @@
+// Package bench measures how well Discreet finds personal data, using
+// synthetic text from a generator written independently of the detection
+// rules: its own NRIC checksum and Luhn code, its own templates, and
+// deliberately realistic variants the rules may not handle. Synthetic data
+// still flatters rule-based detectors; EVALS.md says so.
+package bench
+
+import (
+	"fmt"
+	"math/rand/v2"
+	"strings"
+)
+
+// Span is one piece of real personal data in a sample, by byte offset.
+type Span struct {
+	Entity     string
+	Start, End int
+	Variant    string // which writing style produced it
+}
+
+// Sample is one generated text with its true personal data.
+type Sample struct {
+	Text  string
+	Spans []Span
+}
+
+type builder struct {
+	b     strings.Builder
+	spans []Span
+}
+
+func (b *builder) text(s string) { b.b.WriteString(s) }
+
+func (b *builder) value(entity, variant, s string) {
+	start := b.b.Len()
+	b.b.WriteString(s)
+	b.spans = append(b.spans, Span{entity, start, b.b.Len(), variant})
+}
+
+// gen draws values and their written forms.
+type gen struct{ r *rand.Rand }
+
+func (g gen) pick(items ...string) string { return items[g.r.IntN(len(items))] }
+func (g gen) digits(n int) string {
+	var b strings.Builder
+	for i := 0; i < n; i++ {
+		b.WriteByte(byte('0' + g.r.IntN(10)))
+	}
+	return b.String()
+}
+
+// nric builds a synthetic NRIC/FIN with a correct check letter, computed
+// here independently of internal/detect.
+func (g gen) nric() string {
+	prefix := g.pick("S", "T", "F", "G", "M")
+	d := g.digits(7)
+	weights := []int{2, 7, 6, 5, 4, 3, 2}
+	sum := map[string]int{"S": 0, "T": 4, "F": 0, "G": 4, "M": 3}[prefix]
+	for i, w := range weights {
+		sum += w * int(d[i]-'0')
+	}
+	table := map[string]string{"S": "JZIHGFEDCBA", "T": "JZIHGFEDCBA", "F": "XWUTRQPNMLK", "G": "XWUTRQPNMLK", "M": "XWUTRQPNJLK"}[prefix]
+	return prefix + d + string(table[sum%11])
+}
+
+func (g gen) luhnComplete(partial string) string {
+	sum := 0
+	for i := len(partial) - 1; i >= 0; i-- {
+		d := int(partial[i] - '0')
+		if (len(partial)-i)%2 == 1 {
+			if d *= 2; d > 9 {
+				d -= 9
+			}
+		}
+		sum += d
+	}
+	return partial + string(byte('0'+(10-sum%10)%10))
+}
+
+func (g gen) card() (string, int) {
+	switch g.r.IntN(3) {
+	case 0:
+		return g.luhnComplete("4" + g.digits(14)), 16
+	case 1:
+		return g.luhnComplete(g.pick("51", "52", "53", "54", "55") + g.digits(13)), 16
+	default:
+		return g.luhnComplete(g.pick("34", "37") + g.digits(12)), 15
+	}
+}
+
+func groups(s string, sizes []int, sep string) string {
+	var parts []string
+	for _, n := range sizes {
+		parts = append(parts, s[:n])
+		s = s[n:]
+	}
+	return strings.Join(parts, sep)
+}
+
+var firstNames = []string{"tan", "lim", "lee", "ng", "wong", "goh", "chua", "ong", "koh", "teo", "raj", "siti", "ahmad", "mei", "wei"}
+var streets = []string{"Ang Mo Kio Avenue 3", "Tampines Street 81", "Jurong West Street 52", "Bedok North Road", "Yishun Ring Road", "Toa Payoh Lorong 1", "Clementi Avenue 2"}
+var months = []string{"January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"}
+
+// One writer per entity: it writes a value in one of several styles,
+// including ones a rule-based detector may miss.
+type writer func(g gen, b *builder)
+
+var writers = map[string]writer{
+	"NRIC": func(g gen, b *builder) {
+		n := g.nric()
+		switch g.r.IntN(10) {
+		case 0:
+			b.value("NRIC", "lowercase", strings.ToLower(n))
+		case 1:
+			b.value("NRIC", "spaced", n[:1]+" "+n[1:8]+" "+n[8:])
+		case 2:
+			b.value("NRIC", "hyphenated", n[:1]+"-"+n[1:8]+"-"+n[8:])
+		case 3:
+			wrong := n[:8] + string(byte('A'+(int(n[8]-'A')+1+g.r.IntN(24))%26))
+			b.value("NRIC", "mistyped check letter", wrong)
+		default:
+			b.value("NRIC", "standard", n)
+		}
+	},
+	"PHONE": func(g gen, b *builder) {
+		n := g.pick("9", "8", "6", "9", "8") + g.digits(7)
+		forms := [][2]string{
+			{"plain", n}, {"4-4 space", n[:4] + " " + n[4:]}, {"4-4 hyphen", n[:4] + "-" + n[4:]},
+			{"+65 space", "+65 " + n[:4] + " " + n[4:]}, {"+65 joined", "+65" + n}, {"(65)", "(65) " + n[:4] + " " + n[4:]},
+			{"2-2-2-2", n[:2] + " " + n[2:4] + " " + n[4:6] + " " + n[6:]}, {"3-5", n[:3] + " " + n[3:]},
+		}
+		f := forms[g.r.IntN(len(forms))]
+		b.value("PHONE", f[0], f[1])
+	},
+	"EMAIL": func(g gen, b *builder) {
+		user := g.pick(firstNames...) + g.pick(".", "_", "", ".") + g.pick(firstNames...) + g.pick("", "", g.digits(2))
+		domain := g.pick("example.com", "example.org", "mail.example.net", "clinic.example.com", "example.test")
+		switch g.r.IntN(10) {
+		case 0:
+			b.value("EMAIL", "obfuscated", user+" at "+strings.ReplaceAll(domain, ".", " dot "))
+		case 1:
+			b.value("EMAIL", "uppercase", strings.ToUpper(user+"@"+domain))
+		case 2:
+			b.value("EMAIL", "plus tag", user+"+claims@"+domain)
+		default:
+			b.value("EMAIL", "standard", user+"@"+domain)
+		}
+	},
+	"CARD": func(g gen, b *builder) {
+		c, n := g.card()
+		sizes := []int{4, 4, 4, 4}
+		if n == 15 {
+			sizes = []int{4, 6, 5}
+		}
+		switch g.r.IntN(8) {
+		case 0:
+			b.value("CARD", "plain", c)
+		case 1:
+			b.value("CARD", "hyphens", groups(c, sizes, "-"))
+		case 2:
+			b.value("CARD", "double spaces", groups(c, sizes, "  "))
+		default:
+			b.value("CARD", "spaced", groups(c, sizes, " "))
+		}
+	},
+	"POSTAL": func(g gen, b *builder) {
+		sector := 1 + g.r.IntN(82)
+		if sector == 74 {
+			sector = 73
+		}
+		code := fmt.Sprintf("%02d%s", sector, g.digits(4))
+		switch g.r.IntN(8) {
+		case 0:
+			b.text("S(")
+			b.value("POSTAL", "S(…)", code)
+			b.text(")")
+		case 1:
+			b.text("postal code ")
+			b.value("POSTAL", "postal code", code)
+		case 2:
+			b.text("S'pore ")
+			b.value("POSTAL", "S'pore abbreviation", code)
+		case 3:
+			b.text(g.pick(streets...) + ", ")
+			b.value("POSTAL", "no clue", code)
+		default:
+			b.text("Singapore ")
+			b.value("POSTAL", "after Singapore", code)
+		}
+	},
+	"UNIT": func(g gen, b *builder) {
+		u := fmt.Sprintf("%02d-%0*d", 1+g.r.IntN(30), 2+g.r.IntN(2), g.r.IntN(1000))
+		switch g.r.IntN(6) {
+		case 0:
+			b.text("Unit ")
+			b.value("UNIT", "Unit without #", u)
+		case 1:
+			b.value("UNIT", "# with space", "# "+u)
+		default:
+			b.value("UNIT", "#floor-unit", "#"+u)
+		}
+	},
+	"DOB": func(g gen, b *builder) {
+		y, m, d := 1945+g.r.IntN(60), 1+g.r.IntN(12), 1+g.r.IntN(28)
+		forms := []struct{ clue, variant, value string }{
+			{"DOB: ", "DOB dd/mm/yyyy", fmt.Sprintf("%02d/%02d/%d", d, m, y)},
+			{"Date of birth ", "date of birth d Month yyyy", fmt.Sprintf("%d %s %d", d, months[m-1], y)},
+			{"born on ", "born on Month d, yyyy", fmt.Sprintf("%s %d, %d", months[m-1], d, y)},
+			{"\"dateOfBirth\": \"", "JSON ISO", fmt.Sprintf("%d-%02d-%02d", y, m, d)},
+			{"born in Singapore on ", "words between clue and date", fmt.Sprintf("%d/%d/%d", d, m, y)},
+			{"DOB ", "two-digit year", fmt.Sprintf("%02d/%02d/%02d", d, m, y%100)},
+			{"Birthday: ", "birthday clue", fmt.Sprintf("%02d.%02d.%d", d, m, y)},
+		}
+		f := forms[g.r.IntN(len(forms))]
+		b.text(f.clue)
+		b.value("DOB", f.variant, f.value)
+		if strings.HasPrefix(f.clue, "\"") {
+			b.text("\"")
+		}
+	},
+}
+
+// Decoys look like personal data but aren't; they measure false alarms.
+var decoys = []func(g gen) string{
+	func(g gen) string { return "order #" + g.digits(8) },
+	func(g gen) string { return "invoice INV-2026" + g.digits(4) + "-" + g.digits(4) },
+	func(g gen) string { return "amount SGD " + g.digits(3) + "," + g.digits(3) + ".00" },
+	func(g gen) string { return "OTP " + g.digits(6) },
+	func(g gen) string { return "ticket " + g.digits(6) },
+	func(g gen) string { return "timestamp 17" + g.digits(11) },
+	func(g gen) string { return "tracking " + g.pick("SG", "RR") + g.digits(9) + "SG" },
+	func(g gen) string { return "version 2." + g.digits(1) + "." + g.digits(2) },
+	func(g gen) string { return "ref S$" + g.digits(7) },
+	func(g gen) string { return "meeting on " + g.digits(1) + " " + months[g.r.IntN(12)] + " 2026" },
+	func(g gen) string { return "colour #" + g.pick("ff", "0a", "c3") + g.digits(4) },
+	func(g gen) string { return "batch " + g.pick("8", "9", "6") + g.digits(7) },
+	func(g gen) string { return "account 0" + g.digits(2) + "-" + g.digits(6) + "-" + g.digits(1) },
+	func(g gen) string { return "issue #" + g.digits(2) + "-#" + g.digits(2) },
+}
+
+var openers = []string{
+	"Hi team, ", "Customer note: ", "Please follow up. ", "From the claims queue: ", "Clinic intake. ",
+	"Re: your enquiry. ", "Summary of call: ", "", "URGENT ", "log: ", "Fwd: ", "As discussed, ",
+}
+var joiners = []string{" and ", ", ", ". Also ", "; ", " - ", ". ", " (", "\n"}
+var labels = map[string][]string{
+	"NRIC":   {"NRIC ", "IC no. ", "FIN: ", "member ", "", "id="},
+	"PHONE":  {"call ", "mobile ", "tel: ", "contact ", "", "hp "},
+	"EMAIL":  {"email ", "reply to ", "", "cc ", "from: "},
+	"CARD":   {"card ", "charged to ", "PAN ", "", "cc no "},
+	"POSTAL": {"address ", "deliver to Blk 12, ", "", "home: "},
+	"UNIT":   {"Blk 123 ", "office ", "", "flat "},
+	"DOB":    {"", "patient ", "applicant "},
+}
+
+var entityOrder = []string{"NRIC", "PHONE", "EMAIL", "CARD", "POSTAL", "UNIT", "DOB"}
+
+// Generate returns n samples from a fixed seed: the same seed always gives
+// the same samples.
+func Generate(n int, seed uint64) []Sample {
+	g := gen{rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))}
+	out := make([]Sample, 0, n)
+	for i := 0; i < n; i++ {
+		var b builder
+		b.text(g.pick(openers...))
+		items := g.r.IntN(4) // 0-3 pieces of personal data
+		decoyCount := g.r.IntN(3)
+		order := g.r.Perm(items + decoyCount)
+		for k, slot := range order {
+			if k > 0 {
+				b.text(g.pick(joiners...))
+			}
+			if slot < items {
+				entity := entityOrder[g.r.IntN(len(entityOrder))]
+				b.text(g.pick(labels[entity]...))
+				writers[entity](g, &b)
+			} else {
+				b.text(decoys[g.r.IntN(len(decoys))](g))
+			}
+		}
+		b.text(g.pick(".", "", " thanks", ". Regards"))
+		out = append(out, Sample{Text: b.b.String(), Spans: b.spans})
+	}
+	return out
+}

From c7c999568d59d940bee74bbd1ae655879d60c6b5 Mon Sep 17 00:00:00 2001
From: Michael Mendoza <62703135+makoydev@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:17:42 +0800
Subject: [PATCH 09/13] feat: container image, Fly.io config and deploy
 workflow (D10) (#12)

- Dockerfile: static Go binary on distroless static-debian13:nonroot,
  non-root, /data owned by that user; both base images pinned by digest.
  Built and run locally the way Fly will run it: 9.97 MB, placeholders
  only to the model, audit verify OK on the volume, no values in logs.
- CI job "Container image" builds and smoke-tests it on every PR.
- fly.toml: Singapore, 256 MB shared machine that stops when idle, 1 GB
  volume, /healthz check, Fly-Client-IP for rate limits. Mock only: no
  upstream is ever configured (ADR 0009).
- deploy.yml: from main only, skips without FLY_API_TOKEN, pinned
  flyctl action, smoke test against the live URL.
- docs/DEPLOY.md: the one-time steps only Michael can do. README: live
  link (once deployed), three-minute demo path, five-minute quickstart.
- Build log: merge-helper race (refused safely) and a zsh echo slip.

Part of makoydev/vetted#40 (deployment waits for Michael's Fly account)

Co-authored-by: Claude Opus 5.5 
---
 .dockerignore                       |  5 +++
 .github/workflows/ci.yml            | 26 +++++++++++++
 .github/workflows/deploy.yml        | 59 +++++++++++++++++++++++++++++
 CHANGELOG.md                        |  1 +
 Dockerfile                          | 19 ++++++++++
 README.md                           | 17 +++++++++
 THREAT_MODEL.md                     |  2 +
 docs/DEPLOY.md                      | 49 ++++++++++++++++++++++++
 docs/HOW-THIS-WAS-BUILT.md          |  4 ++
 docs/adr/0009-public-demo-on-fly.md | 21 ++++++++++
 fly.toml                            | 35 +++++++++++++++++
 11 files changed, 238 insertions(+)
 create mode 100644 .dockerignore
 create mode 100644 .github/workflows/deploy.yml
 create mode 100644 Dockerfile
 create mode 100644 docs/DEPLOY.md
 create mode 100644 docs/adr/0009-public-demo-on-fly.md
 create mode 100644 fly.toml

diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..ec05d56
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,5 @@
+bin/
+.git/
+*.jsonl
+*.test
+coverage.*
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 81da0d9..f458768 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -32,3 +32,29 @@ jobs:
       # gofmt, go vet, go test -race, govulncheck and a build (see Makefile)
       - name: Run every check
         run: make check
+
+  container:
+    name: Container image
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      # Builds the deployment image and runs it the way Fly.io will: as the
+      # non-root user, writing the audit log to a volume at /data.
+      - name: Build and smoke-test the image
+        run: |
+          docker build --build-arg VERSION=ci -t discreet:ci .
+          test "$(docker inspect --format '{{.Config.User}}' discreet:ci)" = "nonroot:nonroot"
+          docker volume create discreet-data
+          docker run -d --name discreet -p 8080:8080 -v discreet-data:/data \
+            -e DISCREET_HMAC_KEY="$(openssl rand -hex 32)" discreet:ci
+          curl -fsS --retry 10 --retry-delay 1 --retry-all-errors http://localhost:8080/healthz
+          out=$(curl -fsS http://localhost:8080/demo/run -H 'Content-Type: application/json' \
+            -d '{"purpose":"claims summary","text":"Member S1234567D on 9123 4567"}')
+          printf '%s' "$out" | grep -q '"sent_to_model":"Member  on "'
+          docker exec discreet /discreet audit verify -log /data/discreet-audit.jsonl
+          docker logs discreet 2>&1 | grep -vq 'S1234567D'
+
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
new file mode 100644
index 0000000..c7e16e5
--- /dev/null
+++ b/.github/workflows/deploy.yml
@@ -0,0 +1,59 @@
+name: Deploy demo
+
+# Deploys the public mock-mode demo to Fly.io from main only, after
+# Michael's merge. Does nothing until the FLY_API_TOKEN secret exists.
+on:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: deploy
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    name: Deploy to Fly.io
+    runs-on: ubuntu-latest
+    env:
+      FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+    steps:
+      - name: Skip without a Fly.io token
+        id: gate
+        run: |
+          if [ -z "$FLY_API_TOKEN" ]; then
+            echo "::notice::FLY_API_TOKEN is not set; skipping deployment (see docs/DEPLOY.md)."
+            echo "run=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "run=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Checkout
+        if: steps.gate.outputs.run == 'true'
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up flyctl
+        if: steps.gate.outputs.run == 'true'
+        uses: superfly/flyctl-actions/setup-flyctl@fc53c09e1bc3be6f54706524e3b82c4f462f77be # 1.5
+
+      - name: Deploy
+        if: steps.gate.outputs.run == 'true'
+        run: flyctl deploy --remote-only --build-arg VERSION="${GITHUB_SHA::7}"
+
+      - name: Smoke test the live demo
+        if: steps.gate.outputs.run == 'true'
+        run: |
+          app=$(sed -n 's/^app = "\(.*\)"/\1/p' fly.toml)
+          url="https://$app.fly.dev"
+          curl -fsS --retry 5 --retry-delay 5 --retry-all-errors "$url/healthz"
+          out=$(curl -fsS "$url/demo/run" -H 'Content-Type: application/json' \
+            -d '{"purpose":"claims summary","text":"Member S1234567D on 9123 4567"}')
+          printf '%s' "$out" | grep -q '"sent_to_model":"Member  on "'
+          printf '%s' "$out" | grep -q '"upstream":"mock"'
+          echo "Live demo OK: $url"
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 150616c..715ae11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ All notable changes to this project are documented here. The format follows [Kee
 - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records.
 - Vetted reviews every pull request (shadow mode until 2026-10-14).
 - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases.
+- Deployment (issue D10): a distroless, non-root container image pinned by digest and smoke-tested in CI on every pull request; `fly.toml` for a mock-only public demo in Singapore that sleeps when idle; a deploy workflow from `main` that waits for the `FLY_API_TOKEN` secret and smoke-tests the live URL; `docs/DEPLOY.md` with the one-time steps only Michael can do (ADR 0009).
 - Benchmark (issue D9, `make bench`): 5,000 synthetic samples from a generator written independently of the rules, scored per entity and per writing style, plus latency against a baseline. Results in `EVALS.md`: 76.5% recall overall (100% on every supported style), 92.3% precision (all false alarms are bare eight-digit numbers as `PHONE`), about 3 ms added per request, almost all of it the audit write flushed to disk.
 - Four-pane demo page (issue D8) at `/`: what the app sent, what the AI saw, what it answered, and what the app got back, plus that request's audit record. Five made-up scenarios (claims, appointment, card dispute, mistyped NRIC, a refused eligibility decision). It runs through the same pipeline as the API, always on the mock model, even for token holders; strict Content-Security-Policy; works on phones, light and dark.
 - A real model behind access tokens (issue D6): an OpenAI-compatible client (OpenAI, or Ollama with `max_tokens`) used only for callers with a valid `DISCREET_ACCESS_TOKENS` token; everyone else gets the mock. One pinned model with checked prices (`gpt-6-luna`); each request reserves its worst-case cost (at most US$0.0076) against a hard daily budget (default US$0.20) recomputed from the audit log; 30 requests a minute per client address (ADR 0007). No test calls a paid API.
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..95158cc
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,19 @@
+# Discreet's container: a static Go binary on Google's distroless image,
+# running as a non-root user. Base images are pinned by digest.
+FROM golang:1.27.1-trixie@sha256:3b77fc618ec235a1ab412de7737f120dd507c57e8d87de4cbb7994fb94275ed5 AS build
+WORKDIR /src
+COPY go.mod go.sum ./
+RUN go mod download
+COPY . .
+ARG VERSION=0.0.0-dev
+RUN CGO_ENABLED=0 go build -trimpath -ldflags "-s -w -X main.version=${VERSION}" -o /out/discreet ./cmd/discreet \
+ && mkdir -p /out/data
+
+FROM gcr.io/distroless/static-debian13:nonroot@sha256:e2e927ec666bae08560abb3c55d0659eceabb657f56b6782ab500a9fc7f555e3
+COPY --from=build /out/discreet /discreet
+# /data holds the audit log; owned by the non-root user so it can write.
+COPY --from=build --chown=nonroot:nonroot /out/data /data
+USER nonroot:nonroot
+ENV DISCREET_ADDR=:8080 DISCREET_AUDIT_LOG=/data/discreet-audit.jsonl
+EXPOSE 8080
+ENTRYPOINT ["/discreet", "serve"]
diff --git a/README.md b/README.md
index a11c412..b8ed6f8 100644
--- a/README.md
+++ b/README.md
@@ -17,6 +17,23 @@ make check            # gofmt, go vet, go test -race, govulncheck, build
 
 ## Try the demo
 
+**Live (once deployed):** . Mock model only; see [`docs/DEPLOY.md`](docs/DEPLOY.md).
+
+**Three-minute demo path:** open the page, press *Load a made-up example* to cycle through five scenarios, and compare pane 1 (what the app sent) with pane 2 (what the AI saw). The fifth scenario asks for an automated eligibility decision: it's refused before any model is called, and the audit record shows why.
+
+**Five-minute quickstart, on a clean machine with Docker:**
+
+```sh
+git clone https://github.com/makoydev/discreet && cd discreet
+docker build -t discreet .
+docker run -p 8080:8080 -e DISCREET_HMAC_KEY="$(openssl rand -hex 32)" discreet
+# open http://localhost:8080/
+```
+
+Or with Go 1.27: `make check && ./bin/discreet serve` after setting `DISCREET_HMAC_KEY`.
+
+### Running from source
+
 Run `discreet serve` (below) and open : four panes show what the app sent, what the AI saw, what it answered and what came back, with the audit record. It always uses the free mock model.
 
 ## Run it
diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md
index 4c8fe5e..f3880f2 100644
--- a/THREAT_MODEL.md
+++ b/THREAT_MODEL.md
@@ -32,4 +32,6 @@
 | The public demo is used to spend money | Mock model unless a valid access token is present (a wrong or missing token gets the mock, test); worst-case cost reserved against a hard daily budget that survives restarts (tests, including 100 parallel requests); 30 requests a minute per address (ADR 0007) | D6 (done), D10 |
 | An access token leaks from memory or logs | Only SHA-256 hashes kept, compared in constant time; tokens and provider keys never logged; the caller's token is never forwarded | D6 (done) |
 | The demo page is used to inject script or exfiltrate what people type | Strict Content-Security-Policy (own scripts only, connections only to this server), user text inserted as text never HTML, no third-party scripts; always the mock model (test) | D8 (done) |
+| The public demo is reconfigured to reach a paid model | `fly.toml` never sets an upstream; `docs/DEPLOY.md` forbids setting one as a secret; without an upstream every request gets the mock (ADR 0007, ADR 0009) | D10 (done) |
+| The container is used to escalate on the host | Distroless image (no shell), non-root user, static binary, images pinned by digest, built and smoke-tested in CI | D10 (done) |
 | A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 |
diff --git a/docs/DEPLOY.md b/docs/DEPLOY.md
new file mode 100644
index 0000000..4be4405
--- /dev/null
+++ b/docs/DEPLOY.md
@@ -0,0 +1,49 @@
+# Deploying the public demo (Fly.io)
+
+The public demo runs **mock-only**: `fly.toml` never sets `DISCREET_UPSTREAM_URL`, so nothing anyone sends can reach a paid model (ADR 0007, ADR 0009). It sleeps when idle, so it costs well under US$1 a month (one 256 MB shared machine plus a 1 GB volume).
+
+Deployment happens from `main` only, by `.github/workflows/deploy.yml`, after Michael merges `next` → `main`. Until the `FLY_API_TOKEN` secret exists, the workflow skips itself.
+
+## One-time setup (Michael, about 10 minutes)
+
+These steps need your account and card, so only you can do them. Run them from the `discreet` folder.
+
+1. Install the Fly.io command-line tool and sign in. Sign-up asks for a card.
+   ```sh
+   brew install flyctl
+   fly auth signup        # or: fly auth login
+   ```
+2. In the Fly.io dashboard (Billing), turn on a usage alert so you're emailed before any surprise.
+3. Create the app. If the name `discreet-demo` is taken, choose another and change `app` in `fly.toml` to match.
+   ```sh
+   fly apps create discreet-demo
+   ```
+4. Create the 1 GB volume for the audit log, in Singapore:
+   ```sh
+   fly volumes create discreet_data --region sin --size 1 -a discreet-demo
+   ```
+5. Set the audit log's HMAC key. It's generated and sent straight to Fly; it never appears on screen or in the repository.
+   ```sh
+   fly secrets set DISCREET_HMAC_KEY="$(openssl rand -hex 32)" -a discreet-demo
+   ```
+6. Create a deploy token for this app only, and store it as a GitHub secret. Paste the token when `gh` asks for it.
+   ```sh
+   fly tokens create deploy -a discreet-demo --expiry 2160h   # 90 days
+   gh secret set FLY_API_TOKEN -R makoydev/discreet
+   ```
+7. Deploy: either merge `next` → `main`, or run the workflow by hand:
+   ```sh
+   gh workflow run deploy.yml -R makoydev/discreet
+   ```
+   The workflow finishes with a smoke test against `https://discreet-demo.fly.dev`: the page answers, the model saw only placeholders, and the model was the mock.
+
+## Never do this on the public app
+
+- Don't set `DISCREET_UPSTREAM_URL` or `DISCREET_UPSTREAM_API_KEY` with `fly secrets set`. The public demo must never reach a paid model.
+- Don't share `DISCREET_HMAC_KEY`. Without it, nobody can test whether a guessed NRIC matches a fingerprint in the audit log.
+
+## Checking the audit log on the server
+
+```sh
+fly ssh console -a discreet-demo -C "/discreet audit verify -log /data/discreet-audit.jsonl"
+```
diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md
index c18148b..b02a0d4 100644
--- a/docs/HOW-THIS-WAS-BUILT.md
+++ b/docs/HOW-THIS-WAS-BUILT.md
@@ -37,12 +37,16 @@
 | 2026-10-05 | Demo page in headless Chrome at 1280 px (light and dark) and 390 px, clicking through the examples | No Content-Security-Policy violations or script errors; one 404 for the browser's default icon request, fixed with an inline empty icon; audit log verified afterwards (7 records) |
 | 2026-10-05 | Benchmark (5,000 samples) and latency (2,000 requests per run) | Results in EVALS.md. A suspicion that the audit flush dominated latency was measured rather than assumed: 2.9 ms of the 3.0 ms |
 | 2026-10-05 | Deliberate break: the generator ignoring its seed | The determinism test failed; restored from backup |
+| 2026-10-05 | Fly.io `fly.toml` keys (docs.fly.io, primary); volume ownership for non-root images (Fly community forum, secondary); base image digests from Docker Hub and gcr.io; flyctl action v1.5 | Config written to the documented keys; ownership to be confirmed by the first deploy's smoke test |
+| 2026-10-05 | The container image built and run locally the way Fly runs it (Docker started by Michael): non-root, audit log on a volume | 9.97 MB image; demo answered with placeholders only; `audit verify` passed inside the container; no planted value in its logs |
 | 2026-10-05 | Five deliberate breaks of the engine and vendored rules (EVALS.md) | Four caught at once. The reversed overlap tie-break was not: no shared case covers it. Added toy-detector overlap tests (now 3 fail on that break) and opened vetted#44 for the shared suite |
 
 ## What the AI got wrong, and how it was caught
 
 | Date | What went wrong | How it was caught | Fix |
 | --- | --- | --- | --- |
+| 2026-10-05 | The merge helper treated "no check pending" as "done" while CI hadn't started yet, saw only the fast Vetted check, and stopped waiting | It then refused to merge because a required check hadn't passed, the safe outcome | The helper now waits until every check required on `next` has appeared and finished |
+| 2026-10-05 | A local smoke test piped JSON through zsh's `echo`, which turns `\\n` escapes into real line breaks, so a correct response looked like invalid JSON | The parse error, checked against the raw response before suspecting Discreet | Saved the response to a file instead; the CI and deploy workflows use `printf '%s'` so their checks don't depend on the shell's `echo` |
 | 2026-10-05 | A deliberate break of the access-token check left two variables unused, so Go refused to compile, and the filter used to list failing tests showed nothing, which looked like an uncaught break. The same class of mistake as sg-pii-rules' unbalanced-bracket break earlier that day | Noticing that no test was listed at all, then checking the build | Redid the break so it compiles (any non-empty token matches): the token test fails. Breaks now run `go vet` first, so a build failure can't pass for a result |
 | 2026-10-05 | The mock model's reply said "Here is what I received, exactly as a real model would see it", but readers see that text after Discreet restores the real values, so it showed them the opposite of what the model saw | Reading the reply returned to OpenAI's Python client during the compatibility check | Reworded: the mock says it only ever saw placeholders and that Discreet put the real details back |
 | 2026-10-05 | A gateway test searched JSON-encoded messages for ``, but Go's encoder writes `<` as `\u003c`, so a correct gateway failed the test | The test failure itself, read before changing any gateway code | Fixed the test to read message text directly; the gateway now keeps `<` and `>` unescaped in responses so placeholders stay readable |
diff --git a/docs/adr/0009-public-demo-on-fly.md b/docs/adr/0009-public-demo-on-fly.md
new file mode 100644
index 0000000..2efb070
--- /dev/null
+++ b/docs/adr/0009-public-demo-on-fly.md
@@ -0,0 +1,21 @@
+# 0009. The public demo runs mock-only on one small Fly.io machine
+
+Status: drafted by Claude Code, awaiting Michael's review
+Date: 2026-10-05
+
+## Context
+
+The Milestone 2 plan (approved 2026-10-02) chose Fly.io: pay-as-you-go, with machines that stop when idle (about US$0.50 a month for the smallest, per Fly's pricing page). The brief requires a public deployment in mock mode, and public endpoints never relay to a paid model.
+
+## Decision
+
+- **Image:** a static Go binary (`CGO_ENABLED=0`) on Google's distroless `static-debian13:nonroot`, no shell or package manager, running as a non-root user. Both base images are pinned by digest. A CI job builds the image on every pull request and smoke-tests it the way Fly runs it: non-root, audit log on a volume at `/data`, `audit verify` passing, no planted value in the logs.
+- **Fly:** region `sin` (Singapore), one shared-CPU 256 MB machine, `auto_stop_machines = "stop"` and `min_machines_running = 0`, a 1 GB volume for the audit log, a health check on `/healthz`, and the client address taken from `Fly-Client-IP` for rate limiting.
+- **Mock only:** `fly.toml` never sets `DISCREET_UPSTREAM_URL`; `docs/DEPLOY.md` says never to set it as a secret on the public app.
+- **Deploys** run from `main` only, through a pinned workflow that skips itself without `FLY_API_TOKEN` and ends with a smoke test against the live URL.
+
+## Consequences
+
+- The first request after idling waits a few seconds while the machine starts.
+- Fly gives the volume to the image's user, according to Fly staff on the community forum (secondary source); the first deployment's smoke test confirms it, because the demo can't answer without writing its audit record.
+- Fly can only be set up by Michael (account, card, token); until then the demo runs locally or in Docker.
diff --git a/fly.toml b/fly.toml
new file mode 100644
index 0000000..8e898e9
--- /dev/null
+++ b/fly.toml
@@ -0,0 +1,35 @@
+# Public demo of Discreet on Fly.io (issue D10, ADR 0009). Mock model only:
+# DISCREET_UPSTREAM_URL is deliberately never set here, so the public app
+# cannot reach a paid model whatever anyone sends it.
+app = "discreet-demo"
+primary_region = "sin"
+
+[env]
+  DISCREET_ADDR = ":8080"
+  DISCREET_AUDIT_LOG = "/data/discreet-audit.jsonl"
+  DISCREET_CLIENT_IP_HEADER = "Fly-Client-IP"
+  DISCREET_RATE_LIMIT_PER_MINUTE = "30"
+
+[http_service]
+  internal_port = 8080
+  force_https = true
+  auto_stop_machines = "stop"
+  auto_start_machines = true
+  min_machines_running = 0
+
+[[http_service.checks]]
+  grace_period = "10s"
+  interval = "30s"
+  method = "GET"
+  path = "/healthz"
+  timeout = "5s"
+
+[mounts]
+  source = "discreet_data"
+  destination = "/data"
+  initial_size = "1gb"
+
+[[vm]]
+  memory = "256mb"
+  cpu_kind = "shared"
+  cpus = 1

From 518b62cb18d035b20bc01310c6ffc80d2040a0b7 Mon Sep 17 00:00:00 2001
From: Michael Mendoza <62703135+makoydev@users.noreply.github.com>
Date: Mon, 5 Oct 2026 23:26:13 +0800
Subject: [PATCH 10/13] docs: evidence pack for v0.1 (D11) (#14)

- CONTROLS.md v0.1: eleven controls mapped to section-level primary
  sources (IMDA MGF for GenAI and Agentic AI, PDPA ss.24 and 26, PDPC AI
  and NRIC guidelines, OWASP LLM Top 10 2025, CSA, NIST SSDF), reusing
  the citations read for Vetted's matrix and saying so; gaps listed.
- THREAT_MODEL.md reviewed for v0.1 with accepted residual risks,
  including that Vetted's pilot still uses sg-pii-rules v0.1.0.
- RISKS.md updated; docs/CV-NUMBERS.md; build log summary for M2.

Part of makoydev/vetted#41

Co-authored-by: Claude Opus 5.5 
---
 CHANGELOG.md               |  1 +
 CONTROLS.md                | 33 ++++++++++++++++++++++++---------
 RISKS.md                   | 10 ++++++----
 THREAT_MODEL.md            | 11 ++++++++++-
 docs/CV-NUMBERS.md         | 13 +++++++++++++
 docs/HOW-THIS-WAS-BUILT.md |  6 ++++++
 6 files changed, 60 insertions(+), 14 deletions(-)
 create mode 100644 docs/CV-NUMBERS.md

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 715ae11..1b1aa6f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@ All notable changes to this project are documented here. The format follows [Kee
 - Evidence files: `CONTROLS.md`, `THREAT_MODEL.md`, `EVALS.md`, `RISKS.md`, `docs/HOW-THIS-WAS-BUILT.md` and the first two decision records.
 - Vetted reviews every pull request (shadow mode until 2026-10-14).
 - Detection engine (`internal/detect`, issue D3): implements the sg-pii-rules specification with Go's `regexp` (RE2), including the five named validators (NRIC/FIN, NRIC look-alikes, payment cards, postal sectors, calendar dates), `value` groups and the overlap rule. Passes all 369 shared conformance cases.
+- Evidence pack (issue D11): `CONTROLS.md` v0.1 with section-level primary sources, the threat model reviewed with residual risks, `RISKS.md` updated, `docs/CV-NUMBERS.md`.
 - Deployment (issue D10): a distroless, non-root container image pinned by digest and smoke-tested in CI on every pull request; `fly.toml` for a mock-only public demo in Singapore that sleeps when idle; a deploy workflow from `main` that waits for the `FLY_API_TOKEN` secret and smoke-tests the live URL; `docs/DEPLOY.md` with the one-time steps only Michael can do (ADR 0009).
 - Benchmark (issue D9, `make bench`): 5,000 synthetic samples from a generator written independently of the rules, scored per entity and per writing style, plus latency against a baseline. Results in `EVALS.md`: 76.5% recall overall (100% on every supported style), 92.3% precision (all false alarms are bare eight-digit numbers as `PHONE`), about 3 ms added per request, almost all of it the audit write flushed to disk.
 - Four-pane demo page (issue D8) at `/`: what the app sent, what the AI saw, what it answered, and what the app got back, plus that request's audit record. Five made-up scenarios (claims, appointment, card dispute, mistyped NRIC, a refused eligibility decision). It runs through the same pipeline as the API, always on the mock model, even for token holders; strict Content-Security-Policy; works on phones, light and dark.
diff --git a/CONTROLS.md b/CONTROLS.md
index 8243ad6..4633ca2 100644
--- a/CONTROLS.md
+++ b/CONTROLS.md
@@ -1,11 +1,26 @@
-# Controls
+# Controls matrix (v0.1)
 
-Each Discreet feature, the principle or obligation it supports, and the source. **Draft v0:** features are listed as they are planned, and each row is mapped to a primary source and graded (primary, secondary or unverified) in issue D11. No row claims compliance with anything.
+What each part of Discreet supports, with source and section. **P** = primary source, **S** = secondary. The citations below were read from the primary sources for [Vetted's controls matrix](https://github.com/makoydev/vetted/blob/next/CONTROLS.md#sources) on 2026-09-30 and are reused here where Discreet implements the same principle; "by analogy" marks guidance written for a neighbouring case. No row claims compliance with a law or framework: each says which principle a feature supports.
 
-| Feature | Supports | Source | Status |
-| --- | --- | --- | --- |
-| Personal data detected with the shared, checksummed sg-pii-rules release (NRIC/FIN, phone, email, card, postal code, unit number, date of birth) | Data minimisation; protecting national identification numbers | PDPA; PDPC Advisory Guidelines on NRIC and other national identification numbers | Detection (D3, 369/369 shared cases) and replacement with placeholders (D4) in place; mapping pending |
-| Purpose declared in a header; automated eligibility decisions refused before the model is called; disclosure footer on every answer | Human accountability for decisions about people; transparency to end users | IMDA Model AI Governance Framework for Agentic AI; PDPC guidelines on AI recommendation and decision systems | In place (D5); mapping pending |
-| Hash-chained audit log with `verify` and CSV export; prompts and responses kept only as keyed hashes | Accountability, traceability, incident investigation | IMDA Model AI Governance Framework for Generative AI | In place (D7); mapping pending |
-| Daily budget with a worst-case bound, access tokens, rate limit, one pinned model | Bounding risk and cost | OWASP Top 10 for LLM Applications 2025 (unbounded consumption) | In place (D6); mapping pending |
-| Pinned Actions, `govulncheck`, CodeQL, reviewed pull requests | Secure development | OWASP Top 10 for LLM Applications 2025 (supply chain) | In place (D1); mapping pending |
+| #   | Feature | Supports | Source (grade) | Evidence |
+| --- | --- | --- | --- | --- |
+| 1   | Personal data replaced with placeholders before the model sees it: NRIC/FIN (including mistyped), phone, email, card, postal code, unit number, date of birth; values restored only into the same request's answer | Protecting personal data; not disclosing NRIC numbers that aren't needed; pseudonymising before use in AI | PDPA 2012 s.24 Protection of personal data (P) · PDPC _Advisory Guidelines on Use of Personal Data in AI Recommendation and Decision Systems_ ¶7.2, pseudonymise as a basic control (P, by analogy) · PDPC _Advisory Guidelines for NRIC and other National Identification Numbers_ §3.1 (P) · OWASP LLM02:2025 Sensitive Information Disclosure (P) · IMDA _Starter Kit for Testing LLM-Based Applications_ §3.5 Data Leakage, which names NRIC (P) | 369/369 shared conformance cases; gateway test that the model never receives a planted value; EVALS benchmark |
+| 2   | Values kept only encrypted in memory (AES-256-GCM, per-process key), deleted when the request ends, expired after 10 minutes, never logged | Protection of personal data in the processor itself | PDPA s.24 (P) · CSA _Guidelines on Securing AI Systems_ (Oct 2024) 2.3, protect data and logs (P) | `internal/vault` tests; log scan test (ADR 0004) |
+| 3   | Only the model, messages, token limit and temperature are forwarded; tools, images and other fields are dropped or refused | No personal data in fields the gateway doesn't inspect; data minimisation towards the provider | OWASP LLM02:2025 (P) · PDPA s.26 transfer outside Singapore, and Personal Data Protection Regulations 2021 regs 9–12 (P) | Gateway tests (ADR 0006) |
+| 4   | Callers must declare a purpose; automated eligibility and benefits decisions are refused before any model call and recorded | People stay accountable for decisions about people | IMDA _Model AI Governance Framework for Generative AI_ (2024), Accountability, p.6 (P) · IMDA _Model AI Governance Framework for Agentic AI_ v1.5 §2.2 human accountability, p.25 (P, by analogy) · PDPC AI Advisory Guidelines (P, by analogy: Discreet refuses rather than supports the decision) | Six refusal phrasings tested; refusals in the audit log (ADR 0006) |
+| 5   | Disclosure footer on every answer naming the Discreet request | Transparency to the people reading AI output | MGF for GenAI, Content Provenance, p.23, and Trusted Development and Deployment ("food label"), p.12 (P) | Gateway test |
+| 6   | Hash-chained audit log: every request, allowed or refused, with counts, model, cost and HMAC-SHA-256 fingerprints, never text; `audit verify`; no answer without a record | Traceability and incident investigation | CSA Guidelines 4.1 and 2.3, which name prompts and logs (P) · MGF for GenAI, Incident Reporting, p.16 (P) · ISO/IEC 42001:2023 A.6.2.8 AI system event logs (S) | Tamper tests: changed byte, deleted record, swapped records, rewritten record (ADR 0005) |
+| 7   | Real model only for access tokens; one pinned, priced model; worst-case cost reserved against a hard daily budget that survives restarts; per-address rate limit | Bounding cost and abuse | OWASP LLM10:2025 Unbounded Consumption (P) · CSA _Companion Guide on Securing AI Systems_ 4.1.2 (P) | Tests against a fake provider, including 100 parallel requests (ADR 0007) |
+| 8   | Shared conformance suite and an independent 5,000-sample benchmark with published false positives and misses | Testing and assurance; disclosing limitations | MGF for GenAI, Testing and Assurance, p.19 (P) · Starter Kit §3.5 and §3.7.3 continuous re-testing (P) · NIST SSDF PW.8.2 (P) | EVALS.md |
+| 9   | SHA-pinned Actions, images pinned by digest, one third-party module, `govulncheck`, CodeQL, vendored rules verified by SHA-256, all required checks | Secure supply chain | NIST SSDF PW.4.1, PW.4.4, PS.2.1 (P) · OWASP LLM03:2025 Supply Chain (P) · CSA Guidelines 2.1 (P) | CI; `third_party/sgpiirules` checksum test (ADR 0003) |
+| 10  | Synthetic data only, in tests, fixtures, the benchmark and the demo | No real personal data in testing | NIST AI 600-1 MG-2.2-009 (P) · brief non-negotiable 2 | Seeded generators; demo notice |
+| 11  | Public demo is mock-only by construction, in a distroless non-root container | Public endpoints never reach a paid model | Brief non-negotiable 5 · OWASP LLM10:2025 (P) | `fly.toml`, CI container job (ADR 0009) |
+
+## Gaps (planned for Milestone 3 or later)
+
+- **Hold-for-review queue** with reviewer decisions and override rates (IMDA MGF for Agentic AI asks for monitoring human override rates and response times).
+- **Policy presets** `sg-healthcare` and `sg-finance` (MAS FEAT, MOH AI in Healthcare Guidelines, HIPAA Safe Harbor identifiers).
+- **Project Moonshot** data-disclosure and adversarial-prompt run (IMDA Starter Kit).
+- **Person names** (needs NER) and the writing styles the benchmark shows the rules miss.
+- **Anchoring the audit log's head hash** outside the server (ADR 0005).
+- Each row above should be re-read against the current edition of its source before v1.0.
diff --git a/RISKS.md b/RISKS.md
index 2f06095..e932bad 100644
--- a/RISKS.md
+++ b/RISKS.md
@@ -4,10 +4,12 @@ Reviewed at the start of each issue and at each milestone. Likelihood and impact
 
 | # | Risk | Likelihood | Impact | Mitigation | Owner |
 | --- | --- | --- | --- | --- | --- |
-| R1 | Go and TypeScript detectors disagree on edge cases | Low | Medium | Both use RE2 and must pass the same sg-pii-rules fixtures; any disagreement becomes a new fixture | Claude Code |
-| R2 | Postal code and date-of-birth rules are noisy on real text | Medium | Medium | Context clues, hard negatives, false-positive rates published rather than hidden | Claude Code |
+| R1 | Go and TypeScript detectors disagree on edge cases | Low | Medium | Both use RE2 and pass the same 369 fixtures (Go: 369/369). The tie-break rule had no shared case; tested in each implementation, shared cases tracked in vetted#44 | Claude Code |
+| R2 | Postal code and date-of-birth rules are noisy on real text | Low | Medium | Clue-gated; 0 false alarms for both in the benchmark and on 36 MB of real code; misses published (EVALS) | Claude Code |
 | R3 | The public demo is abused | Medium | Low | Mock-only without a token, per-IP rate limit, nothing kept but keyed hashes and counts | Claude Code |
-| R4 | The hosting account is not ready by the week of 2 Nov | Low | Medium | Asked for early; fallback is "run locally in five minutes" with the slip recorded | Michael |
+| R4 | The hosting account is not ready | Medium | Medium | Everything is ready except the account (docs/DEPLOY.md, about 10 minutes); fallback is the Docker quickstart | Michael |
 | R5 | Pilot evidence stays mock-only because `OPENAI_API_KEY` is not set | High | Medium | EVALS states reviews run vs. reviews with a real model; setting the key takes about 10 minutes | Michael |
-| R6 | M2 slips past 22 Nov | Medium | High | Plan finishes about 10 days early; anything that doesn't fit moves to "Next" in the README with the reason | Claude Code (plan), Michael (review) |
+| R6 | M2 slips past 22 Nov | Low | High | All build issues merged into `next` by 2026-10-05; remaining: Michael's review, the Fly.io account, and the pilot switch on 14 Oct | Michael |
 | R7 | Michael can't explain AI-drafted Go code in an interview | Medium | High | Standard library only, small pull requests, an "If asked in an interview" section on each, ADRs he confirms | Michael |
+| R8 | Benchmark recall is read as real-world accuracy | Medium | Medium | EVALS states that synthetic data flatters rules and gives recall per writing style, not just the total | Michael (in interviews) |
+| R9 | Vetted's pilot misses the new entities because it vendors sg-pii-rules v0.1.0 | High until M3 | Low (synthetic test data only) | Upgrade Vetted to v0.2.0 in Milestone 3; recorded in the threat model | Claude Code |
diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md
index f3880f2..5821f08 100644
--- a/THREAT_MODEL.md
+++ b/THREAT_MODEL.md
@@ -1,6 +1,6 @@
 # Threat model
 
-**Draft.** Filled in as each feature lands; reviewed in issue D11.
+**Reviewed for v0.1 on 2026-10-05** (issue D11). Each threat lists its mitigation and where it's tested; accepted risks are listed at the end.
 
 ## What Discreet protects
 
@@ -35,3 +35,12 @@
 | The public demo is reconfigured to reach a paid model | `fly.toml` never sets an upstream; `docs/DEPLOY.md` forbids setting one as a secret; without an upstream every request gets the mock (ADR 0007, ADR 0009) | D10 (done) |
 | The container is used to escalate on the host | Distroless image (no shell), non-root user, static binary, images pinned by digest, built and smoke-tested in CI | D10 (done) |
 | A compromised dependency or Action | Standard library only (ADR 0001), SHA-pinned Actions, Dependabot, `govulncheck`, CodeQL | D1 |
+
+## Accepted for v0.1 (residual risks)
+
+- **Missed personal data.** The benchmark measured 76.5% recall over a deliberately varied mix of writing styles (EVALS.md). Anything the rules miss reaches the model. Names are never detected.
+- **A dishonest purpose.** A caller can declare a harmless purpose and still use the answer for an eligibility decision. The header makes the claim explicit and recorded; it isn't a technical barrier.
+- **Full audit-chain rewrite or truncation** by someone with write access to the server (ADR 0005).
+- **The model provider** sees placeholders and the surrounding text; whatever context remains (a claim number, a diagnosis) is still sent.
+- **Vetted's own pilot ran with sg-pii-rules v0.1.0**, so Discreet's synthetic cards, postal codes, unit numbers and dates of birth in test files reached Vetted's (mock) model unscrubbed during Milestone 2. Upgrading Vetted to v0.2.0 is Milestone 3 work.
+
diff --git a/docs/CV-NUMBERS.md b/docs/CV-NUMBERS.md
new file mode 100644
index 0000000..6887065
--- /dev/null
+++ b/docs/CV-NUMBERS.md
@@ -0,0 +1,13 @@
+# Numbers for the CV (after Milestone 2)
+
+Each number says how it was measured and on what sample, so it survives "how do you know?". All data is synthetic.
+
+| Number | How it was measured | Sample |
+| --- | --- | --- |
+| **0 personal data values reached the model** in gateway tests, including a scan of every log line and the audit log for the planted values | `go test ./internal/gateway`; the model stub records exactly what it received | NRIC, phone, email and card values in every request shape tested |
+| **369 / 369 shared conformance cases** pass in Go, from a Go engine written independently of the TypeScript reference | `go test ./internal/detect` against vendored sg-pii-rules v0.2.0-rc.1 | 8 entity types; five deliberate breaks each caught |
+| **76.5% recall, 92.3% precision** on synthetic text; **100% on every supported writing style**; all false alarms are bare 8-digit numbers read as phones | `make bench`, generator written independently of the rules, fixed seed | 5,000 samples, 7,531 pieces of personal data |
+| **About 3 ms added per request**, of which 2.9 ms is writing the tamper-evident audit record to disk | `make bench`, against a baseline server without Discreet | 2,000 sequential requests, Apple-silicon Mac |
+| **At most US$0.0076 per request and US$0.20 a day** for the real model, by construction; US$0 in public (mock-only) | Worst case: (64 KB + 16) × input price + 2,048 × output price (`gpt-6-luna`, OpenAI's model page) | Upper bound, not a measurement; no paid call made yet |
+
+Delivery: 13 issues planned on 2026-10-02; 11 build and evidence pull requests merged into `next` by Claude Code (`ai-merged`) by 2026-10-05, each with CI (tests with the race detector, `govulncheck`, CodeQL, a container smoke test) and reviewed in shadow mode by Vetted; 458 Go tests; 9 decision records.
diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md
index b02a0d4..8c5ac72 100644
--- a/docs/HOW-THIS-WAS-BUILT.md
+++ b/docs/HOW-THIS-WAS-BUILT.md
@@ -13,6 +13,12 @@
 - Code is broken deliberately to prove the tests catch it, then restored from a backup.
 - Facts that may have changed since the AI's training (tool versions, prices, APIs) are checked against primary sources before use.
 
+## Milestone 2 in numbers (2026-10-05)
+
+- Plan approved by Michael on 2026-10-02: 13 issues. By 2026-10-05, 11 pull requests were merged into `next` by Claude Code (labelled `ai-merged`), each green on CI and reviewed by Vetted in shadow mode. Two items wait for things only Michael or the calendar can provide: the Fly.io account (D10) and the pilot switch on 14 Oct (V1).
+- 458 Go tests; every issue's code was broken deliberately and the tests caught it, with two breaks first written wrongly (they failed to compile) and redone.
+- AI mistakes this milestone are listed below, with how each was caught. None reached `next` uncaught: they were caught by tests, the gate, the merge helper's refusal, or reading output.
+
 ## Decisions changed on human review
 
 | Date | Proposed by Claude Code | Changed by Michael to | Where |

From 77b824a390bc179a13e496bc6ec4e2f58709dc3e Mon Sep 17 00:00:00 2001
From: Michael Mendoza <62703135+makoydev@users.noreply.github.com>
Date: Tue, 6 Oct 2026 00:23:11 +0800
Subject: [PATCH 11/13] docs: record Michael's Milestone 2 approval and the
 v0.1.0 changelog (#16)

Michael reviewed the Milestone 2 batch on 2026-10-06 and accepted all
nine ADRs without changes.

Co-authored-by: Claude Opus 5.5 
---
 CHANGELOG.md                                          | 4 ++++
 README.md                                             | 2 +-
 docs/HOW-THIS-WAS-BUILT.md                            | 4 ++++
 docs/adr/0001-go-standard-library.md                  | 2 +-
 docs/adr/0002-integration-branch.md                   | 2 +-
 docs/adr/0003-vendor-and-embed-sg-pii-rules.md        | 2 +-
 docs/adr/0004-policy-file-and-placeholders.md         | 2 +-
 docs/adr/0005-hash-chained-audit-log.md               | 2 +-
 docs/adr/0006-purpose-header-and-what-is-forwarded.md | 2 +-
 docs/adr/0007-access-tokens-and-daily-budget.md       | 2 +-
 docs/adr/0008-demo-shares-the-api-pipeline.md         | 2 +-
 docs/adr/0009-public-demo-on-fly.md                   | 2 +-
 12 files changed, 18 insertions(+), 10 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1b1aa6f..04c2b11 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,10 @@ All notable changes to this project are documented here. The format follows [Kee
 
 ## [Unreleased]
 
+## [0.1.0] - 2026-10-06
+
+First release: Discreet, a privacy gateway that replaces Singapore personal data with placeholders before a request reaches an AI model and restores it in the answer, refuses automated eligibility decisions, and keeps a tamper-evident audit log. Reviewed and approved by Michael Mendoza on 2026-10-06.
+
 ### Added
 
 - Go 1.27 project skeleton: the `discreet` command with `serve`, `audit verify`, `audit export` and `version`, each describing itself until it is built (issue D1).
diff --git a/README.md b/README.md
index b8ed6f8..a850b40 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`.
 
-**Status: in development** for v0.1 (Milestone 2, due 22 Nov 2026). Built so far: the gateway (`discreet serve`, mock model), a tamper-evident audit log, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in.
+**Status: v0.1.0, released 2026-10-06** (Milestone 2). Built so far: the gateway (`discreet serve`, mock model), a tamper-evident audit log, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in.
 
 ## Build and check
 
diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md
index 8c5ac72..4bd400e 100644
--- a/docs/HOW-THIS-WAS-BUILT.md
+++ b/docs/HOW-THIS-WAS-BUILT.md
@@ -19,6 +19,10 @@
 - 458 Go tests; every issue's code was broken deliberately and the tests caught it, with two breaks first written wrongly (they failed to compile) and redone.
 - AI mistakes this milestone are listed below, with how each was caught. None reached `next` uncaught: they were caught by tests, the gate, the merge helper's refusal, or reading output.
 
+## Milestone 2 review
+
+On 2026-10-06 Michael reviewed the batch in `next` with the Milestone 2 decision report (https://makoydev.github.io/discreet/m2-review/), accepted all nine ADRs without changes, and approved the release. Claude Code then rebase-merged `next` into `main` and tagged `v0.1.0` on his behalf; the merge appears under his account but was performed by Claude Code after his approval.
+
 ## Decisions changed on human review
 
 | Date | Proposed by Claude Code | Changed by Michael to | Where |
diff --git a/docs/adr/0001-go-standard-library.md b/docs/adr/0001-go-standard-library.md
index cbaacde..ed63d0a 100644
--- a/docs/adr/0001-go-standard-library.md
+++ b/docs/adr/0001-go-standard-library.md
@@ -1,6 +1,6 @@
 # 0001. Build on Go's standard library, without a web framework
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-04
 
 ## Context
diff --git a/docs/adr/0002-integration-branch.md b/docs/adr/0002-integration-branch.md
index 88fc9d2..244e8a3 100644
--- a/docs/adr/0002-integration-branch.md
+++ b/docs/adr/0002-integration-branch.md
@@ -1,6 +1,6 @@
 # 0002. Same integration-branch workflow as Vetted
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-04
 
 ## Context
diff --git a/docs/adr/0003-vendor-and-embed-sg-pii-rules.md b/docs/adr/0003-vendor-and-embed-sg-pii-rules.md
index 8d990ad..8549cdc 100644
--- a/docs/adr/0003-vendor-and-embed-sg-pii-rules.md
+++ b/docs/adr/0003-vendor-and-embed-sg-pii-rules.md
@@ -1,6 +1,6 @@
 # 0003. Vendor sg-pii-rules releases and embed them in the binary
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context
diff --git a/docs/adr/0004-policy-file-and-placeholders.md b/docs/adr/0004-policy-file-and-placeholders.md
index 5136adc..4236d6d 100644
--- a/docs/adr/0004-policy-file-and-placeholders.md
+++ b/docs/adr/0004-policy-file-and-placeholders.md
@@ -1,6 +1,6 @@
 # 0004. Placeholders per request, an encrypted in-memory vault, and a YAML policy
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context
diff --git a/docs/adr/0005-hash-chained-audit-log.md b/docs/adr/0005-hash-chained-audit-log.md
index e4c530b..20d7e06 100644
--- a/docs/adr/0005-hash-chained-audit-log.md
+++ b/docs/adr/0005-hash-chained-audit-log.md
@@ -1,6 +1,6 @@
 # 0005. A hash-chained JSON-lines audit log with keyed hashes of prompts and responses
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context
diff --git a/docs/adr/0006-purpose-header-and-what-is-forwarded.md b/docs/adr/0006-purpose-header-and-what-is-forwarded.md
index 82163bd..8bc6381 100644
--- a/docs/adr/0006-purpose-header-and-what-is-forwarded.md
+++ b/docs/adr/0006-purpose-header-and-what-is-forwarded.md
@@ -1,6 +1,6 @@
 # 0006. Callers declare a purpose; eligibility decisions are refused; only known fields are forwarded
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context
diff --git a/docs/adr/0007-access-tokens-and-daily-budget.md b/docs/adr/0007-access-tokens-and-daily-budget.md
index 52c8a42..b646b15 100644
--- a/docs/adr/0007-access-tokens-and-daily-budget.md
+++ b/docs/adr/0007-access-tokens-and-daily-budget.md
@@ -1,6 +1,6 @@
 # 0007. The real model only for access tokens, under a hard daily budget
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context
diff --git a/docs/adr/0008-demo-shares-the-api-pipeline.md b/docs/adr/0008-demo-shares-the-api-pipeline.md
index c4b4333..8ed8076 100644
--- a/docs/adr/0008-demo-shares-the-api-pipeline.md
+++ b/docs/adr/0008-demo-shares-the-api-pipeline.md
@@ -1,6 +1,6 @@
 # 0008. The demo page runs the API's own pipeline, always on the mock model
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context
diff --git a/docs/adr/0009-public-demo-on-fly.md b/docs/adr/0009-public-demo-on-fly.md
index 2efb070..538c298 100644
--- a/docs/adr/0009-public-demo-on-fly.md
+++ b/docs/adr/0009-public-demo-on-fly.md
@@ -1,6 +1,6 @@
 # 0009. The public demo runs mock-only on one small Fly.io machine
 
-Status: drafted by Claude Code, awaiting Michael's review
+Status: accepted. Drafted by Claude Code; reviewed and accepted by Michael Mendoza on 2026-10-06.
 Date: 2026-10-05
 
 ## Context

From c8b3081340906afb5e5ff7dcd7354ed893af64be Mon Sep 17 00:00:00 2001
From: Michael Mendoza <62703135+makoydev@users.noreply.github.com>
Date: Tue, 6 Oct 2026 00:26:03 +0800
Subject: [PATCH 12/13] docs: fix the README status paragraph for v0.1.0 (#17)

Co-authored-by: Claude Opus 5.5 
---
 README.md                  | 2 +-
 docs/HOW-THIS-WAS-BUILT.md | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index a850b40..51eef0d 100644
--- a/README.md
+++ b/README.md
@@ -2,7 +2,7 @@
 
 A privacy-preserving gateway for large language models: an OpenAI-compatible proxy that detects and tokenises Singapore personal data on the way in, puts it back on the way out, enforces per-tenant policy, and keeps a tamper-evident audit log. Any client uses it by changing `base_url`.
 
-**Status: v0.1.0, released 2026-10-06** (Milestone 2). Built so far: the gateway (`discreet serve`, mock model), a tamper-evident audit log, placeholders with an encrypted in-memory vault, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). The gateway itself comes next; each command says which issue builds it. Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in.
+**Status: v0.1.0, released 2026-10-06** (Milestone 2): the gateway (`discreet serve`, OpenAI-compatible, mock model by default), placeholders with an encrypted in-memory vault, the tamper-evident audit log, and the detection engine, which passes all 369 shared test cases for Singapore personal data (NRIC/FIN, phone, email, card, postal code, unit number, date of birth). Every change is reviewed by [Vetted](https://github.com/makoydev/vetted): in shadow mode until 14 Oct 2026, then opt-in. Next (Milestone 3): a hold-for-review queue, policy presets for healthcare and finance, and a Project Moonshot test run.
 
 ## Build and check
 
diff --git a/docs/HOW-THIS-WAS-BUILT.md b/docs/HOW-THIS-WAS-BUILT.md
index 4bd400e..a61489e 100644
--- a/docs/HOW-THIS-WAS-BUILT.md
+++ b/docs/HOW-THIS-WAS-BUILT.md
@@ -55,6 +55,7 @@ On 2026-10-06 Michael reviewed the batch in `next` with the Milestone 2 decision
 
 | Date | What went wrong | How it was caught | Fix |
 | --- | --- | --- | --- |
+| 2026-10-06 | When recording the release, the README's status line was updated but kept an outdated sentence ("the gateway itself comes next"), and was merged into `next` | Rereading the merged diff before the `next` → `main` merge | Rewrote the status paragraph before anything reached `main` |
 | 2026-10-05 | The merge helper treated "no check pending" as "done" while CI hadn't started yet, saw only the fast Vetted check, and stopped waiting | It then refused to merge because a required check hadn't passed, the safe outcome | The helper now waits until every check required on `next` has appeared and finished |
 | 2026-10-05 | A local smoke test piped JSON through zsh's `echo`, which turns `\\n` escapes into real line breaks, so a correct response looked like invalid JSON | The parse error, checked against the raw response before suspecting Discreet | Saved the response to a file instead; the CI and deploy workflows use `printf '%s'` so their checks don't depend on the shell's `echo` |
 | 2026-10-05 | A deliberate break of the access-token check left two variables unused, so Go refused to compile, and the filter used to list failing tests showed nothing, which looked like an uncaught break. The same class of mistake as sg-pii-rules' unbalanced-bracket break earlier that day | Noticing that no test was listed at all, then checking the build | Redid the break so it compiles (any non-empty token matches): the token test fails. Breaks now run `go vet` first, so a build failure can't pass for a result |

From b464e538efa58939f3cfc2300c628e3de49db858 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 5 Oct 2026 16:30:21 +0000
Subject: [PATCH 13/13] chore(deps): Bump superfly/flyctl-actions/setup-flyctl

Bumps the actions group with 1 update: [superfly/flyctl-actions/setup-flyctl](https://github.com/superfly/flyctl-actions).


Updates `superfly/flyctl-actions/setup-flyctl` from 1.5 to 1.6
- [Release notes](https://github.com/superfly/flyctl-actions/releases)
- [Commits](https://github.com/superfly/flyctl-actions/compare/fc53c09e1bc3be6f54706524e3b82c4f462f77be...ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1)

---
updated-dependencies:
- dependency-name: superfly/flyctl-actions/setup-flyctl
  dependency-version: '1.6'
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: actions
...

Signed-off-by: dependabot[bot] 
---
 .github/workflows/deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
index c7e16e5..17f0bb6 100644
--- a/.github/workflows/deploy.yml
+++ b/.github/workflows/deploy.yml
@@ -40,7 +40,7 @@ jobs:
 
       - name: Set up flyctl
         if: steps.gate.outputs.run == 'true'
-        uses: superfly/flyctl-actions/setup-flyctl@fc53c09e1bc3be6f54706524e3b82c4f462f77be # 1.5
+        uses: superfly/flyctl-actions/setup-flyctl@ed8efb33836e8b2096c7fd3ba1c8afe303ebbff1 # 1.6
 
       - name: Deploy
         if: steps.gate.outputs.run == 'true'