Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
b1aae66
test(cms): refuse to run the test suites on a non-test database
snowopsdev Nov 10, 2026
2a0dbc5
test(cms): make four weak assertions able to fail
snowopsdev Nov 10, 2026
05e03c6
ci: run the Playwright suite, use a test database, cancel only pull r…
snowopsdev Nov 10, 2026
398bae6
test(cms): assert redirects themselves and stop counting the run bar …
snowopsdev Nov 10, 2026
5092f6e
docs: say what the Playwright suite needs to run
snowopsdev Nov 10, 2026
6be4b8a
fix(ci): seed without dev push, guard every E2E run, wait for hydration
snowopsdev Nov 10, 2026
dbfec33
chore: remove the vendored Cursor plugin
snowopsdev Nov 10, 2026
6cbb589
chore: remove finished one-off audit artefacts and plans
snowopsdev Nov 10, 2026
339a2de
chore: build the code graph locally instead of tracking it
snowopsdev Nov 10, 2026
6b237b5
docs: scope DESIGN.md to the public reader
snowopsdev Nov 10, 2026
b1a1289
chore(cms): point VS Code at the hoisted next and typescript
snowopsdev Nov 10, 2026
842dc48
fix(cms): accept raster images only in the media collection
snowopsdev Nov 10, 2026
b09b2e5
docs: configFingerprint is a provenance stamp, not a staleness check
snowopsdev Nov 10, 2026
1d454c2
refactor(cms): tidy ops.css duplicates, dead fallbacks and stale comm…
snowopsdev Nov 10, 2026
a5ef934
perf(cms): count site pages in SQL for the setup hub
snowopsdev Nov 10, 2026
621b922
perf(cms): select only the compared fields when scanning published ar…
snowopsdev Nov 10, 2026
a41817e
perf(cms): keep only the status of the newest run in the setup data
snowopsdev Nov 10, 2026
1420b6c
fix(cms): drop the retired board wording from topic discovery
snowopsdev Nov 10, 2026
e4c1655
docs: warn off graphify's hook installer and fix two misplaced comments
snowopsdev Nov 10, 2026
37ab7ad
test(cms): wait for hydration before toggling the nav
snowopsdev Nov 10, 2026
4bb9be9
test(cms): actually wait for hydration, and refuse a missing database…
snowopsdev Nov 10, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions .claude/settings.json

This file was deleted.

1 change: 0 additions & 1 deletion .gitattributes

This file was deleted.

74 changes: 69 additions & 5 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,11 @@ on:
permissions:
contents: read

# A newer push supersedes an older run on the same pull request. Runs on main
# are never cancelled: two merges landing close together each keep their check.
concurrency:
group: ci-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
static:
Expand Down Expand Up @@ -46,17 +48,18 @@ jobs:
env:
POSTGRES_USER: datum
POSTGRES_PASSWORD: datum
POSTGRES_DB: datum
POSTGRES_DB: datum_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U datum -d datum"
--health-cmd "pg_isready -U datum -d datum_test"
--health-interval 5s
--health-timeout 5s
--health-retries 10

env:
DATABASE_URL: postgresql://datum:datum@127.0.0.1:5432/datum
# The suite refuses a database whose name has no "test" in it.
DATABASE_URL: postgresql://datum:datum@127.0.0.1:5432/datum_test
PAYLOAD_SECRET: ci-payload-secret-not-for-production
MOCK_MODE: 'true'
NODE_OPTIONS: --no-deprecation
Expand All @@ -76,6 +79,65 @@ jobs:
- name: Test cms integration
run: npm run test:int --workspace cms

e2e:
runs-on: ubuntu-latest
timeout-minutes: 45
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: datum
POSTGRES_PASSWORD: datum
POSTGRES_DB: datum_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U datum -d datum_test"
--health-interval 5s
--health-timeout 5s
--health-retries 10

env:
# Playwright starts its own dev server from this environment, and refuses
# a database whose name has no "test" in it.
DATABASE_URL: postgresql://datum:datum@127.0.0.1:5432/datum_test
PAYLOAD_SECRET: ci-payload-secret-not-for-production
MOCK_MODE: 'true'
NODE_OPTIONS: --no-deprecation
# The seed boots Payload too; on a migration-built database it must not
# run the dev schema push (which can also stop at a prompt and exit 0).
DISABLE_DEV_PUSH: 'true'

steps:
- name: Checkout
uses: actions/checkout@v7
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: '22'
cache: npm
- name: Install dependencies
run: npm ci
- name: Migrate cms database
run: npm run payload --workspace cms -- migrate
# The specs need the templates (several of them), the demo workspace
# assets and the admin user that the seed installs.
- name: Seed cms database
run: npm run seed --workspace cms
- name: Install Playwright browser
run: npx playwright install --with-deps chromium
- name: Test cms end to end
run: npm run test:e2e --workspace cms
- name: Upload Playwright report
if: failure()
uses: actions/upload-artifact@v7
with:
name: playwright-report
path: |
cms/playwright-report/
cms/test-results/
retention-days: 7

build:
runs-on: ubuntu-latest
timeout-minutes: 30
Expand Down Expand Up @@ -118,15 +180,17 @@ jobs:
ci:
name: ci
runs-on: ubuntu-latest
needs: [static, integration, build]
needs: [static, integration, e2e, build]
if: always()
steps:
- name: Require all checks to succeed
env:
STATIC_RESULT: ${{ needs.static.result }}
INTEGRATION_RESULT: ${{ needs.integration.result }}
E2E_RESULT: ${{ needs.e2e.result }}
BUILD_RESULT: ${{ needs.build.result }}
run: |
test "$STATIC_RESULT" = success
test "$INTEGRATION_RESULT" = success
test "$E2E_RESULT" = success
test "$BUILD_RESULT" = success
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,5 @@ yarn-error.log*
.omc/
.playwright-mcp/

# Graphify caches and machine-local outputs. Shared products are force-added.
# Graphify output is built locally with `graphify update .` and never committed.
graphify-out/
4 changes: 0 additions & 4 deletions .graphifyignore
Original file line number Diff line number Diff line change
@@ -1,15 +1,11 @@
# Keep the Datum graph focused on maintained project code.
vendor/
.codex/
.agents/
graphify-out/
.claude/
.impeccable/
.omc/

# One-off audit artefacts (96 screenshots and a capture log).
docs/audits/

# Dependency lockfiles and generated database snapshots duplicate source data.
package-lock.json
cms/src/migrations/*.json
5 changes: 3 additions & 2 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ npm run jobs:run --workspace cms # production content worker
- Next loads `cms/.env`; the pipeline loads `cms/.env` then root `.env`. Shell variables win.
- Use a per-worktree database for schema work. Decline dev-push prompts that would drop another branch's tables.
- Integration tests need a migration-built database because Payload dev push is disabled under Vitest.
- The CMS test suites refuse to start unless the `DATABASE_URL` database name contains `test` (e.g. `datum_test`), or `DATUM_TEST_DATABASE_OK=true` is set. They rewrite workspace state and add append-only rows, and `cms/.env` is loaded by default. Never opt in on a database you care about.
- Preserve append-only audit and information-gain records and the review gates in `articleReviewGate.ts`.
- Mock mode is the local default. Do not trigger paid APIs unless the task explicitly requires live mode.

Expand All @@ -59,13 +60,13 @@ Co-Authored-By: (the agent's name and attribution byline)

## graphify

This project has a knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships.
This project can build a local knowledge graph at graphify-out/ with god nodes, community structure, and cross-file relationships. The graph is not tracked: build it with `graphify update .` after cloning (see `CONTRIBUTING.md`). A missing or dirty graphify-out/ is normal and is never committed.

When the user types `/graphify`, use the installed graphify skill or instructions before doing anything else.

Rules:
- For codebase questions, first run `graphify query "<question>"` when graphify-out/graph.json exists. Use `graphify path "<A>" "<B>"` for relationships and `graphify explain "<concept>"` for focused concepts. These return a scoped subgraph, usually much smaller than GRAPH_REPORT.md or raw grep output.
- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it.
- Dirty graphify-out/ files are expected after hooks or incremental updates; dirty graph files are not a reason to skip graphify. If graph.json is missing, run `graphify update .` first. Only skip graphify if the task is about stale or incorrect graph output, or the user explicitly says not to use it.
- If graphify-out/wiki/index.md exists, use it for broad navigation instead of raw source browsing.
- Read graphify-out/GRAPH_REPORT.md only for broad architecture review or when query/path/explain do not surface enough context.
- After modifying code, run `graphify update .` to keep the graph current (AST-only, no API cost).
66 changes: 32 additions & 34 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,23 +8,25 @@ Thanks for contributing. This guide covers local setup, checks, and how we revie
2. Follow the [Quick start](README.md#quick-start) in the root README (`cp` env files, `docker compose up -d`, `npm install`, `npm run seed`).
3. Use `MOCK_MODE=true` unless you intentionally need live Anthropic/Ahrefs calls.

## Shared code graph
## Local code graph

Datum uses [Graphify's team setup](https://github.com/Graphify-Labs/graphify#team-setup).
Datum uses [Graphify](https://github.com/Graphify-Labs/graphify) to answer code
questions from a queryable graph. The graph is built on your machine and is not
committed: `graphify-out/` is git-ignored, and a missing or dirty graph is normal.
The Codex skill and query-first instructions are checked into `.codex/` and
`AGENTS.md`. Install the CLI and Git hooks once per clone from the repo root
(Python 3.10+ and [uv](https://docs.astral.sh/uv/) are required):
`AGENTS.md`. Install the CLI once (Python 3.10+ and
[uv](https://docs.astral.sh/uv/) are required), then build the graph after cloning:

```bash
uv tool install graphifyy==0.9.77
graphify hook install
graphify hook status
graphify update . # local AST pass, no API cost
```

The hooks rebuild the code graph in the background after commits and branch
switches, and register a merge driver for `graph.json`. Re-run `graphify hook
install` after upgrading or reinstalling the CLI to refresh its interpreter
path. If `graphify` is not found, run `uv tool update-shell` and open a new terminal.
If `graphify` is not found, run `uv tool update-shell` and open a new terminal.
Re-run `graphify update .` after code changes, a `git pull`, or a merge. Do not
run `graphify hook install` in this repository: it also registers a git merge
driver and writes a `.gitattributes` line for `graphify-out/graph.json`, which
only made sense while the graph was tracked.

The checked-in `.codex/skills/graphify/SKILL.md` bootstrap only detects an
existing installation; it never installs or upgrades packages automatically.
Expand All @@ -35,35 +37,19 @@ review the generated diff before committing it. Do not restore `uv tool run
--from graphifyy`, automatic install/upgrade commands, or
`--break-system-packages` fallbacks.

Teammates can query the shared graph immediately after cloning:
Query the graph before searching raw source:

```bash
graphify query "articleReviewGate brief approval"
graphify explain "completeJSONLogged"
graphify update . # after code changes, git pull, or a merge; local AST only
```

The graph covers code relationships without paid API calls. Documentation and
media need a separate semantic pass through `$graphify --update` in Codex;
`graphify update .` does not perform that pass. `.graphifyignore` excludes
vendored plugins, agent state, dependency lockfiles, and generated migration
JSON snapshots; migration TypeScript remains indexed. The AST parser may omit
unsupported files or files without symbols, so use source files to verify details.
Initial validation found no missing or dangling edge endpoints. It flagged three
self-links: `sameValue` and `upsertDomain` recurse, while the `create` test helper's
call to `payload.create` was misidentified as recursion. Treat graph edges as
navigation hints and confirm ambiguous calls in source.

Share only the queryable graph, report, and portable manifest. The rest of
`graphify-out/` stays ignored, including caches, local HTML, and machine paths:

```bash
git add -f graphify-out/graph.json graphify-out/GRAPH_REPORT.md graphify-out/manifest.json
```

The hooks finish asynchronously, so run `graphify update .` before staging these
files when the latest code changes must be included. A new clone must install
its own hooks; `.git/hooks/` and the merge-driver Git configuration are local.
The graph covers code relationships. Documentation and media need a separate
semantic pass through `$graphify --update` in Codex; `graphify update .` does not
perform that pass. `.graphifyignore` excludes agent state, dependency lockfiles,
and generated migration JSON snapshots; migration TypeScript remains indexed.
The AST parser may omit unsupported files or files without symbols, so treat
graph edges as navigation hints and confirm ambiguous calls in source.

## Commands

Expand All @@ -90,7 +76,19 @@ npm test --workspace pipeline
npx tsx --test pipeline/test/structuralChecks.test.ts
```

Use **npm** for scripts and CI. `npm test --workspace cms` runs both CMS suites. Playwright starts its own dev server on `TEST_PORT` (default 3101), so do not start `npm run dev` in `cms/` first: Next refuses a second dev server in the same directory. To drive a server you already have running, pass its address: `TEST_BASE_URL=http://127.0.0.1:3000 npm run test:e2e --workspace cms`.
The CMS integration suite and the Playwright suite rewrite workspace state (audiences, profile, brand voice, webhook settings) and add rows that are never removed, so they refuse to start unless the `DATABASE_URL` database name contains `test`. Create a separate database for them, migrate it, and export its URL in the shell that runs the tests:

```bash
createdb -O datum datum_test
export DATABASE_URL=postgresql://datum:datum@127.0.0.1:5432/datum_test
npm run payload --workspace cms -- migrate
```

`cms/.env` is loaded too, so a plain `npm run test:int` otherwise uses your working database and stops with an error. To run against a database whose name has no `test` in it anyway, set `DATUM_TEST_DATABASE_OK=true`; that database will be modified. Playwright applies the same rule, also with `TEST_BASE_URL`: the specs write to `DATABASE_URL` through Payload whichever server they drive.

Use **npm** for scripts and CI. `npm test --workspace cms` runs both CMS suites. Playwright starts its own dev server on `TEST_PORT` (default 3101), so do not start `npm run dev` in `cms/` first: Next refuses a second dev server in the same directory. To drive a server you already have running, pass its address: `TEST_BASE_URL=http://127.0.0.1:3000 npm run test:e2e --workspace cms`; start that server with `SITE_URL` set to its own address, `DISABLE_DEV_PUSH=true` and the same `DATABASE_URL` as the tests.

The Playwright suite needs a migrated **and seeded** test database (`DISABLE_DEV_PUSH=true npm run seed --workspace cms` installs the templates, demo workspace and admin user the specs use) and a browser: `npx playwright install chromium` once. CI runs it as the `e2e` job on the same steps.

## Schema / types

Expand Down
5 changes: 5 additions & 0 deletions DESIGN.md
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,11 @@ components:

---

> **Scope.** This is the visual reference for the public reader pages
> (`cms/src/app/(frontend)`), which still follow it. It was written as an analysis
> of Notion's design language and does not describe the admin. The admin's tokens
> live in `cms/src/components/ops/ops.css` and are checked by
> `cms/tests/int/dsContrast.int.spec.ts`.

## Overview

Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,9 @@ For running the test suites. Set these in your shell, not in a committed `.env`.

| Variable | Purpose |
| --- | --- |
| `DATUM_TEST_DATABASE_OK` | `true` to let the CMS integration suite (and Playwright, when it starts its own server) run against a database whose name does not contain `test`. Without it they refuse to start, because they rewrite workspace state and add rows that are never removed. Prefer a separate database such as `datum_test`; see [CONTRIBUTING.md](CONTRIBUTING.md). |
| `DISABLE_DEV_PUSH` | `true` to turn off Payload's dev schema push, which `next dev` otherwise runs on boot. Playwright's `test:e2e` script sets it automatically, because E2E runs on a migration-built database and the processes that boot Payload for seeding must not push over it; set it yourself in the shell that runs `npm run dev` when you point Playwright at that server via `TEST_BASE_URL` instead. |
| `TEST_BASE_URL` | Base URL Playwright drives instead of starting its own dev server — e.g. `http://127.0.0.1:3000` to test against a server you already started with `npm run dev`. Unset, Playwright starts its own on `TEST_PORT`. |
| `TEST_BASE_URL` | Base URL Playwright drives instead of starting its own dev server — e.g. `http://127.0.0.1:3000` to test against a server you already started with `npm run dev`. Unset, Playwright starts its own on `TEST_PORT` (and sets `SITE_URL` for it). A server you start yourself needs `SITE_URL` set to its own address, or the structured-data spec fails. |
| `TEST_PORT` | Port for the dev server Playwright starts when `TEST_BASE_URL` is unset (default `3101`). |
| `DEV_ALLOWED_ORIGINS` (cms/.env only) | Comma-separated extra hosts allowed to load `next dev` resources, for reaching a dev server by a name other than `localhost` or `127.0.0.1` (a LAN or tailnet hostname). Development only. |

Expand All @@ -180,9 +181,8 @@ For running the test suites. Set these in your shell, not in a committed `.env`.
- [docs/tenant-context.md](docs/tenant-context.md) explains the workspace profile, audiences, positioning and evidence bank, how they gate a run, and how they reach the prompts.
- [docs/information-gain.md](docs/information-gain.md) explains claim scoring, the corpus, and the source review queue.
- [docs/operations.md](docs/operations.md) documents the job queues, webhooks, scheduled publishing, caching, and the fixed limits.
- [docs/open-source-checklist.md](docs/open-source-checklist.md) lists the maintainer steps for making the repository public.

Agent helpers live under `.claude/` and `vendor/claude-plugins/`. Datum runs without them.
Agent helpers live under `.claude/`. Datum runs without them. The pstack plugin is not vendored; install it per user from [github.com/cursor/plugins](https://github.com/cursor/plugins) if you want it.

## License

Expand Down
2 changes: 1 addition & 1 deletion cms/.vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"name": "Next.js: debug full stack",
"type": "node",
"request": "launch",
"program": "${workspaceFolder}/node_modules/next/dist/bin/next",
"program": "${workspaceFolder}/../node_modules/next/dist/bin/next",
"runtimeArgs": ["--inspect"],
"skipFiles": ["<node_internals>/**"],
"serverReadyAction": {
Expand Down
2 changes: 1 addition & 1 deletion cms/.vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"editor.formatOnSave": true
},
"editor.formatOnSaveMode": "file",
"js/ts.tsdk.path": "node_modules/typescript/lib",
"js/ts.tsdk.path": "../node_modules/typescript/lib",
"js/ts.tsdk.promptToUseWorkspaceVersion": true,
"[javascript][typescript][typescriptreact]": {
"editor.codeActionsOnSave": {
Expand Down
14 changes: 13 additions & 1 deletion cms/playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,16 @@ import { defineConfig, devices } from '@playwright/test'
*/
import 'dotenv/config'

import { assertTestDatabase } from './tests/helpers/testDatabaseGuard'

const testPort = process.env.TEST_PORT || '3101'
const testBaseURL = process.env.TEST_BASE_URL || `http://127.0.0.1:${testPort}`

// The specs seed users, create and archive articles and rewrite globals through
// Payload against `DATABASE_URL`, whether Playwright starts the server or
// `TEST_BASE_URL` points at one you started, so the guard always applies.
assertTestDatabase()

/**
* See https://playwright.dev/docs/test-configuration.
*/
Expand All @@ -23,7 +30,7 @@ export default defineConfig({
// CI so dashboard redirect expectations cannot race another spec.
workers: 1,
/* Reporter to use. See https://playwright.dev/docs/test-reporters */
reporter: 'html',
reporter: [['list'], ['html', { open: 'never' }]],
/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
use: {
/* Base URL to use in actions like `await page.goto('/')`. */
Expand All @@ -42,7 +49,12 @@ export default defineConfig({
? undefined
: {
command: `npm run dev -- --hostname 127.0.0.1 --port ${testPort}`,
// Structured data only carries absolute URLs when SITE_URL is set
// (see `getConfiguredSiteUrl`), and frontend.e2e.spec.ts reads them.
env: { SITE_URL: testBaseURL },
reuseExistingServer: false,
// A cold dev server compiles the first page on request; CI is slower than a laptop.
timeout: 180_000,
url: testBaseURL,
},
})
6 changes: 5 additions & 1 deletion cms/src/collections/Media.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,9 @@ export const Media: CollectionConfig = {
required: true,
},
],
upload: true,
upload: {
// Media is world-readable and served from the app origin, so accept raster
// images only: an SVG or HTML upload would run in that origin.
mimeTypes: ['image/png', 'image/jpeg', 'image/webp', 'image/gif', 'image/avif'],
},
}
Loading
Loading