From 6c796a2f55e067a7ff1d437399f1dbda9560bbd0 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 00:50:46 +0000 Subject: [PATCH 1/6] feat(verify): the dashboard date-window family (#15953 slice 5, WIP) deriveDashboardWindowCases / runDashboardWindowProofs / formatDashboardWindowReport: every widget a dashboard's dateRange binds counts, under each preset of DATE_RANGE_PRESETS, the fixture rows that window selects, asked of POST /analytics/dataset/query as the signed-in principal. The binding is read through DashboardSchema / DatasetSchema, the window from DATE_RANGE_PRESET_MACRO_WINDOWS (handed as tokens), the ground truth's day boundaries from core's resolveFilterTokens. Fixture suite with two composed negative controls and an attribution control. Claude-Session: https://claude.ai/code/session_01VoSxBQujKLZKPwK2u5ehQ6 Co-authored-by: Claude --- packages/verify/src/dashboard-window.test.ts | 347 ++++++ packages/verify/src/dashboard-window.ts | 1029 ++++++++++++++++++ packages/verify/src/index.ts | 28 +- 3 files changed, 1403 insertions(+), 1 deletion(-) create mode 100644 packages/verify/src/dashboard-window.test.ts create mode 100644 packages/verify/src/dashboard-window.ts diff --git a/packages/verify/src/dashboard-window.test.ts b/packages/verify/src/dashboard-window.test.ts new file mode 100644 index 00000000000..04ec04bc550 --- /dev/null +++ b/packages/verify/src/dashboard-window.test.ts @@ -0,0 +1,347 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// The dashboard date-window family — what it reads from the metadata, what it +// proves on a real stack, and the controls that keep its green honest: +// +// • two NEGATIVE controls, each a deployment whose analytics bridge does not +// honour the window — built with `AnalyticsServicePlugin`'s published +// `executeAggregate` option, no engine change: one drops the window from +// the query that reaches storage (a floor that matches every row), one +// answers nothing whenever a window is present (a window that zeroes the +// widget). Both are found, and the first is named inert; +// • the ATTRIBUTION control: an app hook that rewrites the window's field on +// write makes the probe not run, never contradicted — the family judges a +// window only over rows that read back as written. + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; + +import { defineStack } from '@objectstack/spec'; +import { DATE_RANGE_PRESETS } from '@objectstack/spec/data'; +import { AnalyticsServicePlugin } from '@objectstack/service-analytics'; + +import { bootStack, type VerifyStack } from './harness.js'; +import { + deriveDashboardWindowCases, + runDashboardWindowProofs, + formatDashboardWindowReport, + type DashboardWindowReport, +} from './dashboard-window.js'; + +// Booting the full in-process stack runs well past vitest's 5s default. +const BOOT_TIMEOUT = 120_000; +const RUN_TIMEOUT = 240_000; + +const ticket = { + name: 'wnd_ticket', + label: 'Ticket', + fields: { + subject: { type: 'text', label: 'Subject', required: true }, + status: { type: 'select', label: 'Status', options: [{ value: 'open', label: 'Open' }, { value: 'closed', label: 'Closed' }] }, + priority: { type: 'select', label: 'Priority', options: [{ value: 'high', label: 'High' }, { value: 'low', label: 'Low' }] }, + is_closed: { type: 'boolean', label: 'Closed' }, + effort: { type: 'number', label: 'Effort' }, + opened_at: { type: 'datetime', label: 'Opened at' }, + due_on: { type: 'date', label: 'Due on' }, + assignee: { type: 'lookup', label: 'Assignee', reference: 'wnd_agent' }, + }, +}; + +const source = { + manifest: { id: 'com.example.dashboard-window', namespace: 'wnd', version: '0.0.1', type: 'app', name: 'Dashboard Window Fixture' }, + objects: [ + { name: 'wnd_agent', label: 'Agent', fields: { name: { type: 'text', label: 'Name', required: true } } }, + ticket, + ], + datasets: [ + { + name: 'ticket_metrics', + label: 'Ticket metrics', + object: 'wnd_ticket', + dimensions: [ + { name: 'status', label: 'Status', field: 'status', type: 'string' }, + { name: 'priority', label: 'Priority', field: 'priority', type: 'string' }, + ], + measures: [ + { name: 'ticket_count', label: 'Tickets', aggregate: 'count' }, + { name: 'closed_count', label: 'Closed', aggregate: 'count', filter: { is_closed: true } }, + { name: 'assigned_count', label: 'Assigned', aggregate: 'count', field: 'assignee' }, + { name: 'avg_effort', label: 'Effort', aggregate: 'avg', field: 'effort' }, + { name: 'closed_rate', label: 'Closed rate', derived: { op: 'ratio', of: ['closed_count', 'ticket_count'] } }, + ], + }, + { + name: 'user_metrics', + label: 'User metrics', + object: 'sys_user', + dimensions: [], + measures: [{ name: 'user_count', label: 'Users', aggregate: 'count' }], + }, + ], + dashboards: [ + { + name: 'wnd_desk', + label: 'Desk', + dateRange: { field: 'opened_at', defaultRange: 'last_30_days' }, + widgets: [ + { id: 'open_tickets', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { status: 'open' } }, + { id: 'by_status', type: 'bar', dataset: 'ticket_metrics', dimensions: ['status'], values: ['ticket_count'] }, + { id: 'due_window', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: 'due_on' } }, + { id: 'mine', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { owner_id: '{current_user_id}' } }, + { id: 'closed_tickets', type: 'metric', dataset: 'ticket_metrics', values: ['closed_count'] }, + { id: 'with_assignee', type: 'metric', dataset: 'ticket_metrics', values: ['assigned_count'] }, + { id: 'high_or_low', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { priority: { $in: ['high', 'low'] } } }, + { id: 'mixed', type: 'table', dataset: 'ticket_metrics', dimensions: ['priority'], values: ['ticket_count', 'avg_effort'] }, + // Outside the population, each with its reason. + { id: 'trend', type: 'line', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: false } }, + { id: 'effort', type: 'metric', dataset: 'ticket_metrics', values: ['avg_effort'] }, + { id: 'rate', type: 'metric', dataset: 'ticket_metrics', values: ['closed_rate'] }, + { id: 'recent', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { opened_at: { $gte: '{30_days_ago}' } } }, + { id: 'either', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { $or: [{ status: 'open' }, { is_closed: true }] } }, + { id: 'by_subject', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: 'subject' } }, + { id: 'ghost', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: 'closed_on' } }, + { id: 'users', type: 'metric', dataset: 'user_metrics', values: ['user_count'] }, + ], + }, + { + // No `field`: the window lands on the filter's own default, `created_at`. + name: 'wnd_overview', + label: 'Overview', + dateRange: { defaultRange: 'this_month' }, + widgets: [{ id: 'all_tickets', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'] }], + }, + { name: 'wnd_plain', label: 'Plain', widgets: [{ id: 'plain', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'] }] }, + ], +}; + +const PROVEN = ['open_tickets', 'by_status', 'due_window', 'mine', 'closed_tickets', 'with_assignee', 'high_or_low', 'mixed', 'all_tickets']; + +const widgetOf = (report: DashboardWindowReport, widget: string) => { + const r = report.results.find((x) => x.widget === widget); + if (!r) throw new Error(`no result for ${widget}`); + return r; +}; + +describe('deriveDashboardWindowCases — what the family reads from the metadata', () => { + it("reads the binding through the spec's parse and names every widget it cannot compare", () => { + const cases = new Map(deriveDashboardWindowCases(source).map((c) => [`${c.dashboard}/${c.widget}`, c])); + // The plain dashboard declares no window, so none of its widgets is a case. + expect([...cases.keys()].filter((k) => k.startsWith('wnd_plain/'))).toEqual([]); + expect(cases.get('wnd_desk/open_tickets')).toMatchObject({ object: 'wnd_ticket', field: 'opened_at', binding: 'default', filter: { status: 'open' } }); + expect(cases.get('wnd_desk/due_window')).toMatchObject({ field: 'due_on', binding: 'explicit' }); + // An unauthored `field` is the filter's own default. + expect(cases.get('wnd_overview/all_tickets')).toMatchObject({ field: 'created_at', binding: 'default' }); + expect(cases.get('wnd_desk/trend')).toMatchObject({ unbound: true }); + expect(cases.get('wnd_desk/with_assignee')!.measures).toEqual([{ measure: 'assigned_count', countField: 'assignee' }]); + expect(cases.get('wnd_desk/closed_tickets')!.measures).toEqual([{ measure: 'closed_count', filter: { is_closed: true } }]); + expect(cases.get('wnd_desk/mixed')!.measures.map((m) => m.excluded?.kind ?? 'probed')).toEqual(['probed', 'not-a-count']); + expect(cases.get('wnd_desk/effort')!.excluded).toMatchObject({ kind: 'no-count-measure' }); + expect(cases.get('wnd_desk/rate')!.excluded!.reason).toMatch(/"closed_rate" is derived \(ratio of closed_count, ticket_count\)/); + expect(cases.get('wnd_desk/users')!.excluded).toMatchObject({ kind: 'object-undeclared' }); + }); + + it('names a dataset the app does not declare, and a dashboard that does not parse', () => { + const config = { + ...source, + dashboards: [ + { name: 'wnd_lost', label: 'Lost', dateRange: { field: 'opened_at' }, widgets: [{ id: 'lost', type: 'metric', dataset: 'nope_metrics', values: ['n'] }] }, + { name: 'wnd_bad', label: 'Bad', dateRange: { field: 'opened_at', defaultRange: 'last_60_days' }, widgets: [] }, + ], + }; + const cases = deriveDashboardWindowCases(config); + expect(cases.find((c) => c.widget === 'lost')!.excluded).toMatchObject({ kind: 'dataset-undeclared' }); + expect(cases.find((c) => c.dashboard === 'wnd_bad')!.excluded!.reason).toMatch(/^the dashboard does not parse under the spec's DashboardSchema \(dateRange\.defaultRange/); + }); +}); + +describe('runDashboardWindowProofs — the fixture app', () => { + let stack: VerifyStack; + let report: DashboardWindowReport; + const app = defineStack(source as never) as unknown as typeof source; + + beforeAll(async () => { + stack = await bootStack(app as never); + report = await runDashboardWindowProofs(stack, await stack.signIn(), app); + }, BOOT_TIMEOUT + RUN_TIMEOUT); + + afterAll(async () => { + await stack?.stop(); + }); + + it("hands each preset the spec's macro window and resolves it with the platform's resolver", () => { + expect(report.door).toBe('rest'); + expect(report.timezone).toBe('UTC'); + expect(report.presets.map((p) => p.preset)).toEqual([...DATE_RANGE_PRESETS]); + const byPreset = new Map(report.presets.map((p) => [p.preset, p])); + expect(byPreset.get('last_7_days')!.condition).toEqual({ $gte: '{7_days_ago}' }); + expect(byPreset.get('this_month')!.condition).toEqual({ $gte: '{month_start}', $lte: '{month_end}' }); + for (const p of report.presets) expect(p.resolved.$gte).toMatch(/^\d{4}-\d{2}-\d{2}$/); + }); + + it('proves every bound count widget under every preset, on a datetime and a date field', () => { + for (const widget of PROVEN) { + const r = widgetOf(report, widget); + expect(r.status, `${widget}\n${formatDashboardWindowReport(report)}`).toBe('proven'); + for (const m of r.measures.filter((x) => x.status === 'probed')) { + expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'held')); + // The control leg held: with no window the widget counted every fixture row. + expect(m.unwindowed!.answered).toBe(m.days!.length); + expect(m.inert).toBe(false); + expect(m.narrowing).toEqual([]); + // Each preset selects some fixture rows, and leaves the day before its first out. + for (const p of m.presets) { + expect(p.expected).toBeGreaterThan(0); + expect(p.expected).toBeLessThan(m.days!.length); + } + } + } + expect(widgetOf(report, 'open_tickets').fieldType).toBe('datetime'); + expect(widgetOf(report, 'due_window').fieldType).toBe('date'); + expect(widgetOf(report, 'all_tickets').fieldType).toBe('datetime'); + }); + + it('names every widget outside the population, with its reason', () => { + expect(widgetOf(report, 'trend').status).toBe('unbound'); + const excluded = (w: string) => [widgetOf(report, w).status, widgetOf(report, w).excludedKind]; + expect(excluded('effort')).toEqual(['excluded', 'no-count-measure']); + expect(excluded('rate')).toEqual(['excluded', 'no-count-measure']); + expect(excluded('recent')).toEqual(['excluded', 'filter-unmodelled']); + expect(widgetOf(report, 'recent').reason).toMatch(/a condition on opened_at, the field the window lands on/); + expect(excluded('either')).toEqual(['excluded', 'filter-unmodelled']); + expect(widgetOf(report, 'either').reason).toMatch(/the operator \$or/); + expect(excluded('by_subject')).toEqual(['excluded', 'field-type']); + expect(excluded('ghost')).toEqual(['excluded', 'field-missing']); + expect(excluded('users')).toEqual(['excluded', 'object-undeclared']); + // The non-count measure beside a probed one is counted, not probed. + expect(widgetOf(report, 'mixed').measures.map((m) => `${m.measure} ${m.status}`)).toEqual(['ticket_count probed', 'avg_effort excluded']); + }); + + it('counts every dashboard, widget, measure and window', () => { + expect(report.summary).toEqual({ + dashboards: { declared: 3, windowed: 2, notWindowed: 1, excluded: 0 }, + widgets: { total: 17, unbound: 1, excluded: 7, probed: 9, proven: 9, contradicted: 0, failed: 0, inconclusive: 0 }, + excludedBy: { 'no-count-measure': 2, 'filter-unmodelled': 2, 'field-type': 1, 'field-missing': 1, 'object-undeclared': 1 }, + measures: { probed: 9, excluded: 1, excludedBy: { 'not-a-count': 1 } }, + probes: 9 * DATE_RANGE_PRESETS.length, + held: 9 * DATE_RANGE_PRESETS.length, + blank: 0, + contradictions: 0, + faults: 0, + notRun: 0, + inert: 0, + narrowing: 0, + hardFailures: 0, + }); + }); +}); + +// ── The controls ──────────────────────────────────────────────────────────── + +const controlSource = { + manifest: { id: 'com.example.dashboard-window-controls', namespace: 'wnc', version: '0.0.1', type: 'app', name: 'Dashboard Window Controls' }, + objects: [ + { + name: 'wnc_case', + label: 'Case', + fields: { + subject: { type: 'text', label: 'Subject', required: true }, + opened_at: { type: 'datetime', label: 'Opened at' }, + logged_at: { type: 'datetime', label: 'Logged at' }, + }, + }, + ], + datasets: [{ name: 'case_metrics', label: 'Cases', object: 'wnc_case', dimensions: [], measures: [{ name: 'case_count', label: 'Cases', aggregate: 'count' }] }], + dashboards: [ + { + name: 'wnc_board', + label: 'Board', + dateRange: { field: 'opened_at', defaultRange: 'last_90_days' }, + widgets: [ + { id: 'cases', type: 'metric', dataset: 'case_metrics', values: ['case_count'] }, + { id: 'logged', type: 'metric', dataset: 'case_metrics', values: ['case_count'], filterBindings: { dateRange: 'logged_at' } }, + ], + }, + ], + hooks: [ + { + // An app hook stamping `logged_at` itself on every insert. + name: 'wnc_stamp_logged', + object: 'wnc_case', + events: ['beforeInsert'], + handler: async (ctx: { input: Record }) => { + ctx.input.logged_at = '2020-01-01T00:00:00.000Z'; + }, + }, + ], +}; + +/** `filter` with every condition on `field` removed, at any depth of `$and`. */ +function without(filter: unknown, field: string): unknown { + if (Array.isArray(filter)) return filter.map((f) => without(f, field)); + if (!filter || typeof filter !== 'object') return filter; + return Object.fromEntries(Object.entries(filter as Record).filter(([k]) => k !== field).map(([k, v]) => [k, without(v, field)])); +} +const mentions = (filter: unknown, field: string): boolean => JSON.stringify(filter ?? {}).includes(`"${field}"`); + +/** + * An analytics service whose bridge to storage does what a broken one would: + * `drop` hands the engine the query with the window's field removed (every row + * passes the window), `zero` answers no row whenever a window is present. + */ +function brokenBridge(mode: 'drop' | 'zero', holder: { engine?: any }): AnalyticsServicePlugin { + return new AnalyticsServicePlugin({ + queryCapabilities: () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false }), + executeAggregate: async (objectName, { groupBy, aggregations, filter, timezone, context }) => { + if (mode === 'zero' && mentions(filter, 'opened_at')) return []; + return holder.engine.aggregate(objectName, { + where: mode === 'drop' ? without(filter, 'opened_at') : filter, + groupBy, + aggregations: aggregations?.map((a) => ({ function: a.method, field: a.field, alias: a.alias, ...(a.filter ? { filter: a.filter } : {}) })), + timezone, + context, + }); + }, + }); +} + +describe.each(['drop', 'zero'] as const)('NEGATIVE CONTROL — an analytics bridge that does not honour the window (%s)', (mode) => { + let stack: VerifyStack; + let report: DashboardWindowReport; + const app = defineStack(controlSource as never) as unknown as typeof controlSource; + const holder: { engine?: any } = {}; + + beforeAll(async () => { + stack = await bootStack(app as never, { analytics: brokenBridge(mode, holder) }); + holder.engine = await stack.kernel.getServiceAsync('data'); + report = await runDashboardWindowProofs(stack, await stack.signIn(), app); + }, BOOT_TIMEOUT + RUN_TIMEOUT); + + afterAll(async () => { + await stack?.stop(); + }); + + it('is found, under every preset', () => { + const r = widgetOf(report, 'cases'); + expect(r.status, formatDashboardWindowReport(report)).toBe('contradicted'); + const m = r.measures[0]!; + // The control leg holds: with no window the bridge counts every fixture row. + expect(m.unwindowed!.answered).toBe(m.days!.length); + if (mode === 'drop') { + expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'contradicted')); + expect(m.presets.every((p) => p.answered === m.days!.length)).toBe(true); + expect(m.presets[0]!.detail).toMatch(/^the window counted \d+ fixture row\(s\), and it selects \d+ \(wider\)$/); + expect(m.inert).toBe(true); + } else { + expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'blank')); + expect(m.inert).toBe(false); + } + expect(report.summary.hardFailures).toBe(1); + expect(formatDashboardWindowReport(report)).toContain('✗ wnc_board/cases (wnc_case.opened_at, datetime) CONTRADICTED'); + }); + + it("ATTRIBUTION CONTROL — a hook rewriting the window's field makes the probe not run, never contradicted", () => { + const r = widgetOf(report, 'logged'); + expect(r.status).toBe('inconclusive'); + expect(r.reason).toMatch(/reads back logged_at = "2020-01-01T00:00:00\.000Z", not \d{4}-\d{2}-\d{2}T12:00:00\.000Z: something rewrote it on write/); + expect(r.measures[0]!.presets.every((p) => p.outcome === 'not-run')).toBe(true); + }); +}); diff --git a/packages/verify/src/dashboard-window.ts b/packages/verify/src/dashboard-window.ts new file mode 100644 index 00000000000..6a3fe141e1a --- /dev/null +++ b/packages/verify/src/dashboard-window.ts @@ -0,0 +1,1029 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// The dashboard date-window family: every dashboard that declares a +// `dateRange` answers, under each preset of its own picker, with the rows that +// window selects. +// +// ## The promise, and where each half of it is read from +// +// A dashboard's `dateRange` is a picker. Each widget it binds is narrowed to +// the picked window on one of the widget's own fields, and the widget's count +// is the count of the rows that window selects. Three published sources: +// +// - WHICH field a widget's window lands on: `DashboardWidgetSchema.filterBindings` +// (`@objectstack/spec/ui`). A string binds the built-in date range to that +// field, `false` opts the widget out, and absent means the filter's own +// field: `dateRange.field ?? 'created_at'`. The field resolves against the +// object behind the widget's dataset (`dataset.object`). Dashboards and +// datasets are read through the spec's own parse (`DashboardSchema`, +// `DatasetSchema`), never as authored. +// - WHAT a preset's window is: `DATE_RANGE_PRESET_MACRO_WINDOWS` +// (`@objectstack/spec/data`), the `{date-macro}` pair each preset of +// `DATE_RANGE_PRESETS` names — its first and last day, inclusive, the last +// open for the three rolling `last_N_days` presets. It is handed to the +// door as written, `{ $gte: start, $lte: end }`, ANDed with the widget's +// own filter (`{ $and: [filter, window] }`, the shipped console's merge): +// ⛔ never lowered to dates here. The server resolves the tokens. +// - WHICH days a window selects, for the ground truth: the same window +// object, resolved by `@objectstack/core`'s `resolveFilterTokens` (the one +// date-macro resolver the analytics executor calls, on the reference +// timezone of the caller's `ExecutionContext`, UTC when unset), compared +// with each fixture row's day by the table's own convention (first and last +// day inclusive). +// +// ## The door, and who walks through it +// +// `POST {basePath}/analytics/dataset/query` with the saved dataset's name and +// the widget's own selection (its dimensions, one count measure, and its +// filter with the window ANDed in as `runtimeFilter`) — the request a +// dashboard widget makes — as the signed-in principal the caller hands in. +// `{current_user_id}` in a widget's filter is that principal. +// +// ## The ground truth, and the control that keeps it honest +// +// Per widget and count measure, fixture rows are written through the seed door +// (system, hooks and validations live) on known days that straddle every +// preset's past edges, each row stamped at UTC noon (a `date` field takes the +// day itself), with every column the widget's own filter, the dataset's filter +// and the measure's filter read held to a value those filters select, and +// owned by the principal. Each row is read back: a row whose stamp or held +// columns do not read back as written (an app hook rewrote them) makes the +// probe not run, never contradicted. Every answer is a DIFFERENCE — the count +// after the fixtures minus the count before, under the same window — so the +// app's own rows cancel out. The control leg is the widget with no window: it +// must count every fixture row, or the fixtures do not meet the widget's own +// selection and no window is judged. +// +// Three verdicts per preset: the widget is not blank where the window holds +// fixture rows; its count equals the rows the window selects; and a narrower +// preset never answers more than a wider one (a window that never narrows is +// inert, a finding). +// +// Run it on a fresh stack of its own: it writes fixtures. + +import { DashboardSchema, DatasetSchema } from '@objectstack/spec/ui'; +import { DATE_RANGE_PRESETS, DATE_RANGE_PRESET_MACRO_WINDOWS, type DateRangePreset } from '@objectstack/spec/data'; +import { resolveFilterTokens } from '@objectstack/core'; +import { declaredCollection } from './artifact-collections.js'; +import { deriveCrudCases, fillRelationalRefs, type CrudCase } from './derive.js'; +import type { VerifyStack } from './harness.js'; + +/** + * The built-in date range's default field, when `dateRange.field` is omitted: + * `DashboardWidgetSchema.filterBindings`' published contract, + * "dateRange: `dateRange.field ?? 'created_at'`". No export carries the + * literal, so it is named once here with that source. + */ +const DATE_RANGE_DEFAULT_FIELD = 'created_at'; +/** The reserved `filterBindings` key of the built-in date range. */ +const DATE_RANGE_BINDING = 'dateRange'; + +/** The field types a window can be compared on. */ +const WINDOWABLE_TYPES = new Set(['date', 'datetime']); + +/** The one principal-scoped placeholder a held column may take. */ +const CURRENT_USER_TOKEN = '{current_user_id}'; + +// ── Types ─────────────────────────────────────────────────────────────────── + +/** Why the family cannot compare a widget, or one of its measures. */ +export type DashboardWindowExclusion = + | 'dashboard-unparsed' + | 'dataset-undeclared' + | 'dataset-unparsed' + | 'object-undeclared' + | 'object-not-derivable' + | 'field-missing' + | 'field-type' + | 'filter-unmodelled' + | 'no-count-measure' + | 'measure-undeclared' + | 'not-a-count' + | 'count-field-unsynthesizable'; + +/** One measure a widget selects, as the family reads it. */ +export interface DashboardWindowMeasureCase { + measure: string; + /** A `count` of a named field counts its non-null values, so the fixtures carry one. */ + countField?: string; + /** The measure's own scoped filter, as parsed. */ + filter?: Record; + excluded?: { kind: DashboardWindowExclusion; reason: string }; +} + +/** One widget of a dashboard that declares a `dateRange`. */ +export interface DashboardWindowCase { + dashboard: string; + /** The widget's `id`, or `#` when it has none. */ + widget: string; + dataset?: string; + object?: string; + /** The field the window lands on. */ + field?: string; + /** `explicit` — `filterBindings.dateRange` names it; `default` — the dashboard's own window field. */ + binding?: 'explicit' | 'default'; + /** The widget's own filter, as parsed. */ + filter?: Record; + /** The dataset's intrinsic filter, as parsed. */ + datasetFilter?: Record; + dimensions: string[]; + measures: DashboardWindowMeasureCase[]; + /** The widget opts out of the window (`filterBindings.dateRange: false`). */ + unbound?: true; + excluded?: { kind: DashboardWindowExclusion; reason: string }; +} + +/** A dashboard the app declares, and whether it is in the population. */ +export interface DashboardWindowDashboard { + name: string; + /** `windowed` — it declares a `dateRange`; `not-windowed` — it does not; `excluded` — it does not parse. */ + status: 'windowed' | 'not-windowed' | 'excluded'; + /** The window field its widgets default to. */ + field?: string; + /** The preset the picker opens on (the parse's default when unauthored). */ + defaultRange?: string; + reason?: string; +} + +/** The window a preset names, as handed to the door and as resolved for the ground truth. */ +export interface DashboardWindowPreset { + preset: DateRangePreset; + /** The `{date-macro}` condition handed to the door, ANDed onto the widget's field. */ + condition: Record; + /** That condition resolved by the platform's resolver at the start of the run. */ + resolved: Record; +} + +/** What the door answered one query. */ +export interface DashboardWindowAnswer { + kind: 'answered' | 'refused' | 'faulted'; + status: number; + /** The measure summed over every returned row. */ + count?: number; + rows?: number; + code?: string; + detail?: string; +} + +/** One preset against one measure. */ +export interface DashboardWindowPresetProbe { + preset: DateRangePreset; + /** Fixture rows the window selects (the ground truth). */ + expected: number; + /** The count the window added once the fixtures were written (after minus before). */ + answered?: number; + /** The count the widget showed after the fixtures, all rows included. */ + shown?: number; + /** + * - `held` — the window counted exactly the fixture rows it selects; + * - `blank` — it showed nothing where it selects fixture rows; + * - `contradicted` — it counted another number of them; + * - `failed` — the door refused or faulted on the windowed query (the unwindowed one answered); + * - `not-run` — the fixtures could not be written, or the day turned mid-probe. + */ + outcome: 'held' | 'blank' | 'contradicted' | 'failed' | 'not-run'; + answer?: DashboardWindowAnswer; + detail?: string; +} + +export interface DashboardWindowMeasureResult { + measure: string; + status: 'probed' | 'excluded'; + reason?: string; + excludedKind?: DashboardWindowExclusion; + /** The fixture days written (UTC calendar days). */ + days?: string[]; + /** The control leg: the widget with no window must count every fixture row. */ + unwindowed?: { expected: number; answered?: number; answer?: DashboardWindowAnswer }; + presets: DashboardWindowPresetProbe[]; + /** Every preset answered what the unwindowed query answered: the window never narrows. */ + inert?: boolean; + /** `narrower ⊂ wider: a > b` — a preset nested inside another answered more. */ + narrowing?: string[]; +} + +/** + * - `proven` — every probed measure held under every preset; + * - `contradicted` — a preset went blank or counted the wrong rows (a hard failure); + * - `failed` — the windowed query was refused or faulted (a hard failure); + * - `inconclusive` — a probe did not run, or the control leg did not hold; + * - `unbound` — the widget opts out of the window; + * - `excluded` — the family cannot compare it (see `excludedKind`, `reason`). + */ +export type DashboardWindowWidgetStatus = 'proven' | 'contradicted' | 'failed' | 'inconclusive' | 'unbound' | 'excluded'; + +export interface DashboardWindowWidgetResult { + dashboard: string; + widget: string; + dataset?: string; + object?: string; + field?: string; + fieldType?: string; + binding?: 'explicit' | 'default'; + status: DashboardWindowWidgetStatus; + excludedKind?: DashboardWindowExclusion; + reason?: string; + measures: DashboardWindowMeasureResult[]; +} + +export interface DashboardWindowReport { + app: string; + /** `POST /analytics/dataset/query` as the signed-in principal. */ + door: 'rest'; + /** The principal's reference timezone, which the server resolves the window on. */ + timezone: string; + presets: DashboardWindowPreset[]; + dashboards: DashboardWindowDashboard[]; + results: DashboardWindowWidgetResult[]; + summary: { + dashboards: { declared: number; windowed: number; notWindowed: number; excluded: number }; + widgets: { + total: number; + unbound: number; + excluded: number; + probed: number; + proven: number; + contradicted: number; + failed: number; + inconclusive: number; + }; + /** Widgets excluded, per reason. */ + excludedBy: Partial>; + measures: { probed: number; excluded: number; excludedBy: Partial> }; + /** Preset × probed measure. */ + probes: number; + held: number; + blank: number; + contradictions: number; + faults: number; + notRun: number; + /** Measures whose window never narrowed. */ + inert: number; + /** Nested preset pairs where the narrower answered more. */ + narrowing: number; + /** `contradicted + failed` widgets — the count a CI gate reads. */ + hardFailures: number; + }; +} + +const clip = (s: unknown, n = 200): string => String(s ?? '').slice(0, n); +const isPlainObject = (v: unknown): v is Record => !!v && typeof v === 'object' && !Array.isArray(v); +const nonEmpty = (f: unknown): f is Record => isPlainObject(f) && Object.keys(f).length > 0; + +// ── The population ────────────────────────────────────────────────────────── + +const firstIssue = (error: any): string => { + const issue = error?.issues?.[0]; + return `${issue?.path?.join('.') || 'root'}: ${clip(issue?.message, 120)}`; +}; + +/** + * Derive one case per widget of every dashboard the app declares with a + * `dateRange`, as the spec's parse of the dashboard and of its datasets states + * it. A widget the family cannot compare is still returned, `unbound` or + * `excluded`, with the reason, so the report counts it. What only a booted + * stack can answer (the bound field's registered type) is judged by the run. + */ +export function deriveDashboardWindowCases(config: any): DashboardWindowCase[] { + return derive(config).cases; +} + +function derive(config: any): { cases: DashboardWindowCase[]; dashboards: DashboardWindowDashboard[] } { + const crud = new Map(deriveCrudCases(config).map((c) => [c.object, c])); + const datasets = new Map(); + for (const ds of declaredCollection(config, 'datasets')) if (ds?.name) datasets.set(ds.name, ds); + const objects = new Set(); + for (const o of declaredCollection(config, 'objects')) if (o?.name) objects.add(o.name); + + const cases: DashboardWindowCase[] = []; + const dashboards: DashboardWindowDashboard[] = []; + for (const raw of declaredCollection(config, 'dashboards')) { + const name = String(raw?.name ?? '?'); + const parsed = DashboardSchema.safeParse(raw); + if (!parsed.success) { + if (raw?.dateRange) { + const reason = `the dashboard does not parse under the spec's DashboardSchema (${firstIssue(parsed.error)})`; + dashboards.push({ name, status: 'excluded', reason }); + cases.push({ dashboard: name, widget: '*', dimensions: [], measures: [], excluded: { kind: 'dashboard-unparsed', reason } }); + } else { + dashboards.push({ name, status: 'not-windowed' }); + } + continue; + } + const dash: any = parsed.data; + if (!dash.dateRange) { + dashboards.push({ name, status: 'not-windowed' }); + continue; + } + const windowField: string = dash.dateRange.field || DATE_RANGE_DEFAULT_FIELD; + dashboards.push({ name, status: 'windowed', field: windowField, defaultRange: dash.dateRange.defaultRange }); + + (dash.widgets as any[]).forEach((w, index) => { + const c: DashboardWindowCase = { + dashboard: name, + widget: typeof w?.id === 'string' && w.id ? w.id : `#${index}`, + dataset: w?.dataset, + dimensions: Array.isArray(w?.dimensions) ? w.dimensions.map(String) : [], + measures: [], + ...(nonEmpty(w?.filter) ? { filter: w.filter } : {}), + }; + cases.push(c); + const binding = w?.filterBindings?.[DATE_RANGE_BINDING]; + if (binding === false) { + c.unbound = true; + return; + } + c.field = typeof binding === 'string' && binding ? binding : windowField; + c.binding = typeof binding === 'string' && binding ? 'explicit' : 'default'; + + const rawDataset = datasets.get(String(w?.dataset)); + if (!rawDataset) { + c.excluded = { kind: 'dataset-undeclared', reason: `the widget binds dataset "${w?.dataset}", which the app does not declare` }; + return; + } + const ds = DatasetSchema.safeParse(rawDataset); + if (!ds.success) { + c.excluded = { kind: 'dataset-unparsed', reason: `dataset "${w.dataset}" does not parse under the spec's DatasetSchema (${firstIssue(ds.error)})` }; + return; + } + const dataset: any = ds.data; + c.object = dataset.object; + if (nonEmpty(dataset.filter)) c.datasetFilter = dataset.filter; + if (!objects.has(dataset.object)) { + c.excluded = { + kind: 'object-undeclared', + reason: `dataset "${w.dataset}" reads ${dataset.object}, which the app does not declare, so no fixture row can be built from its metadata`, + }; + return; + } + const blocked = crud.get(dataset.object)?.blocked; + if (blocked) { + c.excluded = { kind: 'object-not-derivable', reason: `${dataset.object} cannot be synthesized: ${blocked}` }; + return; + } + + const measures = new Map((dataset.measures ?? []).map((m: any) => [m.name, m])); + for (const name of (Array.isArray(w?.values) ? w.values : []).map(String)) { + const m = measures.get(name); + const mc: DashboardWindowMeasureCase = { measure: name }; + c.measures.push(mc); + if (!m) { + mc.excluded = { kind: 'measure-undeclared', reason: `dataset "${w.dataset}" declares no measure "${name}"` }; + continue; + } + if (m.derived) { + mc.excluded = { kind: 'not-a-count', reason: `"${name}" is derived (${m.derived.op} of ${(m.derived.of ?? []).join(', ')}), not a row count` }; + continue; + } + if (m.aggregate !== 'count') { + mc.excluded = { kind: 'not-a-count', reason: `"${name}" aggregates ${m.aggregate ?? 'nothing'}${m.field ? ` of ${m.field}` : ''}, not a row count` }; + continue; + } + if (typeof m.field === 'string' && m.field !== '*') { + if (m.field.includes('.')) { + mc.excluded = { kind: 'count-field-unsynthesizable', reason: `"${name}" counts the relationship path ${m.field}, which a fixture row cannot be held to` }; + continue; + } + mc.countField = m.field; + } + if (nonEmpty(m.filter)) mc.filter = m.filter; + } + if (c.measures.length > 0 && c.measures.every((m) => m.excluded)) { + const counts = c.measures.filter((m) => m.excluded!.kind !== 'not-a-count' && m.excluded!.kind !== 'measure-undeclared'); + c.excluded = counts.length > 0 + ? { kind: counts[0]!.excluded!.kind, reason: counts[0]!.excluded!.reason } + : { kind: 'no-count-measure', reason: `no measure the widget selects is a row count (${c.measures.map((m) => m.excluded!.reason).join('; ')})` }; + } else if (c.measures.length === 0) { + c.excluded = { kind: 'no-count-measure', reason: 'the widget selects no measure' }; + } + }); + } + return { cases, dashboards }; +} + +// ── Held columns: what the widget's own selection asks of a row ───────────── + +/** + * Collect the equality every row must satisfy for `filter` to select it, into + * `held`. Understood: a field compared with a scalar (bare or `$eq`), an `$in` + * list (its first member), `{current_user_id}` (the principal), and `$and` of + * those. Anything else is a selection the ground truth cannot model, and the + * reason is returned — never a guess. + */ +function holdColumns( + filter: unknown, + held: Map, + ctx: { fields: Set; windowField: string; principal: string; object: string }, +): string | undefined { + if (filter == null) return undefined; + if (!isPlainObject(filter)) return `a filter that is not an object (${clip(JSON.stringify(filter), 80)})`; + for (const [key, cond] of Object.entries(filter)) { + if (key === '$and') { + if (!Array.isArray(cond)) return '`$and` that is not a list'; + for (const member of cond) { + const reason = holdColumns(member, held, ctx); + if (reason) return reason; + } + continue; + } + if (key.startsWith('$')) return `the operator ${key}`; + if (key.includes('.')) return `the relationship path ${key}`; + if (key === ctx.windowField) return `a condition on ${key}, the field the window lands on`; + if (!ctx.fields.has(key)) return `${key}, which is not a field of ${ctx.object}`; + let value: unknown = cond; + if (isPlainObject(cond)) { + const ops = Object.keys(cond); + if (ops.length === 1 && ops[0] === '$eq') value = cond.$eq; + else if (ops.length === 1 && ops[0] === '$in' && Array.isArray(cond.$in) && cond.$in.length > 0) value = cond.$in[0]; + else return `the condition ${clip(JSON.stringify({ [key]: cond }), 100)}`; + } + if (typeof value === 'string' && /^\$?\{.*\}$/.test(value)) { + if (value !== CURRENT_USER_TOKEN && value !== `$${CURRENT_USER_TOKEN}`) return `the placeholder ${value} on ${key}`; + value = ctx.principal; + } + if (value === null || !['string', 'number', 'boolean'].includes(typeof value)) { + return `the value ${clip(JSON.stringify(value), 60)} on ${key}`; + } + if (held.has(key) && held.get(key) !== value) { + return `${key} held to ${JSON.stringify(held.get(key))} and to ${JSON.stringify(value)}: no row satisfies both`; + } + held.set(key, value); + } + return undefined; +} + +/** Does a value read back as the one written? (Read coercion answers in the declared type.) */ +function sameValue(written: unknown, read: unknown): boolean { + if (typeof written === 'boolean') return read === written || read === (written ? 1 : 0); + if (typeof written === 'number') return Number(read) === written; + return String(read) === String(written); +} + +// ── Fixtures ──────────────────────────────────────────────────────────────── + +const DAY_MS = 86_400_000; + +/** The calendar day `n` days after (`n < 0`: before) a `YYYY-MM-DD` day. */ +function shiftDay(day: string, n: number): string { + return new Date(Date.parse(`${day}T00:00:00.000Z`) + n * DAY_MS).toISOString().slice(0, 10); +} + +/** A `date` / `datetime` value moved `days` later, in the same spelling. */ +function laterBy(value: string, days: number): string { + const at = Date.parse(value); + if (Number.isNaN(at) || days === 0) return value; + const moved = new Date(at + days * DAY_MS).toISOString(); + return /^\d{4}-\d{2}-\d{2}$/.test(value) ? moved.slice(0, 10) : moved; +} + +/** The `k`-th distinct value of a sample `base`. */ +function vary(base: string | number, k: number): string | number { + if (typeof base === 'number') return base + k; + if (/^\d{4}-\d{2}-\d{2}/.test(base)) return laterBy(base, k); + const at = base.indexOf('@'); + if (at > 0) return `${base.slice(0, at)}-w${k}${base.slice(at)}`; + if (/^https?:\/\//.test(base)) return `${base.replace(/\/$/, '')}/w${k}`; + if (/^\+\d+$/.test(base)) return `${base}${k}`; + return `${base}-w${k}`; +} + +/** The fields of `objectDef` a uniqueness constraint covers: a declared field scope, or a unique index entry. */ +function uniqueFields(objectDef: any): Set { + const out = new Set(); + for (const [name, def] of Object.entries(objectDef?.fields ?? {})) { + if (def?.unique !== undefined && def?.unique !== false) out.add(name); + } + for (const index of Array.isArray(objectDef?.indexes) ? objectDef.indexes : []) { + if (index?.unique) for (const f of Array.isArray(index.fields) ? index.fields : []) out.add(String(f)); + } + return out; +} + +/** + * The smallest record the CRUD derivation can offer (the shape the sibling + * families use): its values for the fields the object REQUIRES, so every + * optional field takes the app's own default; each `date` / `datetime` one day + * after the one declared before it, and each unique-constrained value made + * distinct by `n`. + */ +function fixtureBody(body: Record, objectDef: any, n: number): Record { + const unique = uniqueFields(objectDef); + const out: Record = {}; + let dates = 0; + for (const [name, def] of Object.entries(objectDef?.fields ?? {})) { + if (def?.required !== true || !(name in body)) continue; + let value = body[name]; + if ((def?.type === 'date' || def?.type === 'datetime') && typeof value === 'string') value = laterBy(value, dates++); + if (unique.has(name) && (typeof value === 'string' || typeof value === 'number')) value = vary(value, n); + out[name] = value; + } + return out; +} + +/** The days the fixtures are written on: each preset's first and last day and the day either side, never after today. */ +function fixtureDays(presets: DashboardWindowPreset[], today: string): string[] { + const days = new Set([today]); + for (const p of presets) { + const start = p.resolved.$gte as string; + const end = p.resolved.$lte as string | undefined; + for (const d of [start, shiftDay(start, -1), ...(end ? [end, shiftDay(end, 1)] : [])]) { + if (d <= today) days.add(d); + } + } + return [...days].sort().reverse(); +} + +/** Is `day` inside a resolved window? The table's convention: first and last day inclusive, the rolling end open. */ +const selects = (resolved: Record, day: string): boolean => + day >= String(resolved.$gte) && (resolved.$lte === undefined || day <= String(resolved.$lte)); + +/** Is window `a` nested inside window `b`? */ +const nestedIn = (a: Record, b: Record): boolean => + String(a.$gte) >= String(b.$gte) + && (b.$lte === undefined || (a.$lte !== undefined && String(a.$lte) <= String(b.$lte))); + +// ── The verdict ───────────────────────────────────────────────────────────── + +/** + * Compare one windowed answer against the ground truth. The whole verdict of + * the family is this function: it reads the expectation and the door's two + * answers under the same window, and nothing else. + */ +function judge( + expected: number, + before: DashboardWindowAnswer, + after: DashboardWindowAnswer, +): Pick { + const failed = before.kind !== 'answered' ? before : after.kind !== 'answered' ? after : undefined; + if (failed) { + return { outcome: 'failed', detail: `the windowed query answered ${failed.status}${failed.code ? ` ${failed.code}` : ''}: ${failed.detail ?? ''}` }; + } + const answered = after.count! - before.count!; + const shown = after.count!; + if (expected > 0 && (after.rows === 0 || shown === 0)) { + return { outcome: 'blank', answered, shown, detail: `the widget showed nothing where the window selects ${expected} fixture row(s)` }; + } + if (answered === expected) return { outcome: 'held', answered, shown }; + return { + outcome: 'contradicted', + answered, + shown, + detail: `the window counted ${answered} fixture row(s), and it selects ${expected} (${answered > expected ? 'wider' : 'narrower'})`, + }; +} + +// ── The run ───────────────────────────────────────────────────────────────── + +export interface DashboardWindowProofOptions { + /** The reference instant the ground truth resolves on. Defaults to the start of the run. */ + now?: Date; +} + +/** + * Run the dashboard date-window family against a live stack, as the principal + * `token` signs in: for every widget a `dateRange` binds, every count measure + * it selects is asked through the dataset door with no window and under each + * preset of `DATE_RANGE_PRESETS`, before and after fixture rows are written on + * known days, and each difference is compared with the rows that window + * selects. Returns a structured report; a per-widget failure is collected, + * not thrown. + */ +export async function runDashboardWindowProofs( + stack: VerifyStack, + token: string, + config: any, + options: DashboardWindowProofOptions = {}, +): Promise { + const { cases, dashboards } = derive(config); + const crud = new Map(deriveCrudCases(config).map((c) => [c.object, c])); + const objectDefs = new Map(); + for (const o of declaredCollection(config, 'objects')) if (o?.name) objectDefs.set(o.name, o); + + const caller = await stack.contextFor(token); + const principal = String(caller.userId); + const timezone: string | undefined = (caller as { timezone?: string }).timezone; + const utc = timezone === undefined || timezone === 'UTC'; + const tokenCtx = (now: Date) => ({ now, timezone, userId: principal }); + + const start = options.now ?? new Date(); + const presets: DashboardWindowPreset[] = DATE_RANGE_PRESETS.map((preset) => { + const [from, to] = DATE_RANGE_PRESET_MACRO_WINDOWS[preset]; + const condition: Record = { $gte: from, ...(to ? { $lte: to } : {}) }; + return { preset, condition, resolved: resolveFilterTokens(condition, tokenCtx(start)) as Record }; + }); + const todayAt = (now: Date) => String(resolveFilterTokens('{today}', tokenCtx(now))); + + const registered = (object: string): Record => { + try { + return ((stack.metadata.object(object) as any)?.fields ?? {}) as Record; + } catch { + return {}; + } + }; + + // ── The door ────────────────────────────────────────────────────────────── + async function ask(dataset: string, selection: Record, measure: string): Promise { + try { + const res = await stack.apiAs(token, 'POST', '/analytics/dataset/query', { datasetName: dataset, selection }); + const body = (await res.json().catch(() => undefined)) as { rows?: unknown; code?: unknown; message?: unknown } | undefined; + if (res.status === 200 && Array.isArray(body?.rows)) { + const rows = body!.rows as Array>; + return { kind: 'answered', status: 200, rows: rows.length, count: rows.reduce((s, r) => s + (Number(r?.[measure]) || 0), 0) }; + } + return { + kind: res.status >= 500 ? 'faulted' : 'refused', + status: res.status, + ...(typeof body?.code === 'string' ? { code: body.code } : {}), + detail: clip(body?.message), + }; + } catch (e) { + return { kind: 'faulted', status: 0, detail: clip((e as Error)?.message ?? e) }; + } + } + + // ── Base records for required relations, through the seed door ──────────── + let counter = 0; + const bases = new Map(); + async function baseRecord(object: string): Promise<{ id?: string; reason?: string }> { + const known = bases.get(object); + if (known) return known; + bases.set(object, { reason: `the base ${object} record is being created` }); + const c = crud.get(object); + let made: { id?: string; reason?: string }; + if (!c || c.blocked) { + made = { reason: c?.blocked ?? `object ${object} is not derived` }; + } else { + const ids = new Map(); + let reason: string | undefined; + for (const ref of c.relationalRefs ?? []) { + if (!ref.required) continue; + const dep = await baseRecord(ref.target); + if (!dep.id) { + reason = `required relation ${ref.field} → ${ref.target}: ${dep.reason}`; + break; + } + ids.set(ref.target, dep.id); + } + if (reason) { + made = { reason }; + } else { + const { body } = fillRelationalRefs(c, ids); + try { + const [written] = await stack.seed(object, [fixtureBody(body, objectDefs.get(object), ++counter)]); + made = written?.id != null ? { id: String(written.id) } : { reason: `the seed of ${object} returned no id` }; + } catch (e) { + made = { reason: `the seed of ${object} was refused: ${clip((e as Error)?.message, 160)}` }; + } + } + } + bases.set(object, made); + return made; + } + + // ── Probes ──────────────────────────────────────────────────────────────── + const results: DashboardWindowWidgetResult[] = []; + for (const c of cases) { + const base: DashboardWindowWidgetResult = { + dashboard: c.dashboard, + widget: c.widget, + ...(c.dataset ? { dataset: c.dataset } : {}), + ...(c.object ? { object: c.object } : {}), + ...(c.field ? { field: c.field } : {}), + ...(c.binding ? { binding: c.binding } : {}), + status: 'excluded', + measures: [], + }; + if (c.unbound) { + results.push({ ...base, status: 'unbound', reason: 'the widget opts out of the window (filterBindings.dateRange: false)' }); + continue; + } + const measureResults: DashboardWindowMeasureResult[] = c.measures.map((m) => ({ + measure: m.measure, + status: 'excluded' as const, + ...(m.excluded ? { reason: m.excluded.reason, excludedKind: m.excluded.kind } : {}), + presets: [], + })); + if (c.excluded) { + results.push({ ...base, excludedKind: c.excluded.kind, reason: c.excluded.reason, measures: measureResults }); + continue; + } + + const fields = registered(c.object!); + const fieldType = fields[c.field!]?.type as string | undefined; + if (!fieldType) { + results.push({ + ...base, + excludedKind: 'field-missing', + reason: `the window lands on ${c.field}, which is not a field of ${c.object} as the stack registers it`, + measures: measureResults, + }); + continue; + } + base.fieldType = fieldType; + if (!WINDOWABLE_TYPES.has(fieldType)) { + results.push({ + ...base, + excludedKind: 'field-type', + reason: `the window lands on ${c.object}.${c.field}, a ${fieldType} field: only a date or datetime field is compared`, + measures: measureResults, + }); + continue; + } + + const ownerColumn = 'owner_id' in fields; + const fieldNames = new Set(Object.keys(fields)); + const holdCtx = { fields: fieldNames, windowField: c.field!, principal, object: c.object! }; + const objectHeld = new Map(); + const objectReason = holdColumns(c.datasetFilter, objectHeld, holdCtx) ?? holdColumns(c.filter, objectHeld, holdCtx); + if (objectReason) { + results.push({ + ...base, + excludedKind: 'filter-unmodelled', + reason: `the widget's own selection carries ${objectReason}, which the ground truth cannot model`, + measures: measureResults, + }); + continue; + } + + for (const [i, m] of c.measures.entries()) { + const r = measureResults[i]!; + if (m.excluded) continue; + const held = new Map(objectHeld); + const measureReason = holdColumns(m.filter, held, holdCtx); + if (measureReason) { + Object.assign(r, { reason: `the measure's own filter carries ${measureReason}, which the ground truth cannot model`, excludedKind: 'filter-unmodelled' }); + continue; + } + // A count of a field counts its non-null values: hold one. + if (m.countField && !held.has(m.countField)) { + const cc = crud.get(c.object!)!; + const ref = (cc.relationalRefs ?? []).find((x) => x.field === m.countField); + if (ref) { + const dep = await baseRecord(ref.target); + if (!dep.id) { + Object.assign(r, { reason: `"${m.measure}" counts ${m.countField}, and no ${ref.target} record can be written for it: ${dep.reason}`, excludedKind: 'count-field-unsynthesizable' }); + continue; + } + held.set(m.countField, ref.multiple ? [dep.id] : dep.id); + } else if (cc.body && m.countField in cc.body) { + held.set(m.countField, cc.body[m.countField]); + } else { + Object.assign(r, { reason: `"${m.measure}" counts ${m.countField}, and the CRUD derivation synthesizes no value for it`, excludedKind: 'count-field-unsynthesizable' }); + continue; + } + } + r.status = 'probed'; + delete r.reason; + delete r.excludedKind; + await probeMeasure(c, m, r, held, fieldType, ownerColumn); + } + + const probed = measureResults.filter((m) => m.status === 'probed'); + if (probed.length === 0) { + const first = measureResults.find((m) => m.excludedKind && m.excludedKind !== 'not-a-count' && m.excludedKind !== 'measure-undeclared'); + results.push({ + ...base, + excludedKind: first?.excludedKind ?? 'no-count-measure', + reason: first?.reason ?? 'no measure the widget selects is a row count', + measures: measureResults, + }); + continue; + } + results.push({ ...base, ...widgetStatus(probed), measures: measureResults }); + } + + async function probeMeasure( + c: DashboardWindowCase, + m: DashboardWindowMeasureCase, + r: DashboardWindowMeasureResult, + held: Map, + fieldType: string, + ownerColumn: boolean, + ): Promise { + const selection = (window?: Record): Record => { + const windowed = window ? { [c.field!]: window } : undefined; + const runtimeFilter = c.filter && windowed ? { $and: [c.filter, windowed] } : (windowed ?? c.filter); + return { + ...(c.dimensions.length > 0 ? { dimensions: c.dimensions } : {}), + measures: [m.measure], + ...(runtimeFilter ? { runtimeFilter } : {}), + }; + }; + const notRun = (detail: string) => { + r.presets = presets.map((p) => ({ preset: p.preset, expected: 0, outcome: 'not-run' as const, detail })); + }; + + if (!utc) { + notRun(`the principal's reference timezone is ${timezone}: the fixtures are stamped on UTC calendar days, so the family judges a window resolved on UTC days only`); + return; + } + + // The control leg, before any fixture: the widget's own query must answer. + const before0 = await ask(c.dataset!, selection(), m.measure); + if (before0.kind !== 'answered') { + r.unwindowed = { expected: 0, answer: before0 }; + notRun(`the widget's own query, with no window, answered ${before0.status}${before0.code ? ` ${before0.code}` : ''}: ${before0.detail ?? ''}`); + return; + } + const day0 = todayAt(new Date()); + const before = new Map(); + for (const p of presets) before.set(p.preset, await ask(c.dataset!, selection(p.condition), m.measure)); + + // Fixtures: one row per day, every held column held, owned by the principal. + const cc = crud.get(c.object!)!; + const ids = new Map(); + for (const ref of cc.relationalRefs ?? []) { + if (!ref.required || held.has(ref.field)) continue; + const dep = await baseRecord(ref.target); + if (!dep.id) { + notRun(`required relation ${ref.field} → ${ref.target}: ${dep.reason}`); + return; + } + ids.set(ref.target, dep.id); + } + const refs = (cc.relationalRefs ?? []).filter((x) => !held.has(x.field)); + const { body } = fillRelationalRefs({ ...cc, relationalRefs: refs }, ids); + const days = fixtureDays(presets, day0); + const stamp = (day: string) => (fieldType === 'date' ? day : `${day}T12:00:00.000Z`); + const written: Array<{ day: string; id: string }> = []; + for (const day of days) { + const row: Record = { + ...fixtureBody(body, objectDefs.get(c.object!), ++counter), + ...(ownerColumn ? { owner_id: principal } : {}), + ...Object.fromEntries(held), + [c.field!]: stamp(day), + }; + try { + const [made] = await stack.seed(c.object!, [row]); + if (made?.id == null) throw new Error(`the seed of ${c.object} returned no id`); + written.push({ day, id: String(made.id) }); + } catch (e) { + r.days = days; + notRun(`the fixture row for ${day} was refused by the seed door: ${clip((e as Error)?.message, 160)}`); + return; + } + } + r.days = days; + // Read back: the stamp and every held column must read as written. + for (const { day, id } of written) { + const [got] = await stack.rows(c.object!, { id }); + const stamped = got?.[c.field!]; + const stampHolds = fieldType === 'date' + ? String(stamped ?? '').slice(0, 10) === day + : Date.parse(String(stamped)) === Date.parse(stamp(day)); + const drifted = [...held].find(([k, v]) => !Array.isArray(v) && !sameValue(v, got?.[k])); + if (!got || !stampHolds || drifted) { + notRun( + !got + ? `the fixture row for ${day} does not read back` + : !stampHolds + ? `the fixture row for ${day} reads back ${c.field} = ${clip(JSON.stringify(stamped), 60)}, not ${stamp(day)}: something rewrote it on write` + : `the fixture row for ${day} reads back ${drifted![0]} = ${clip(JSON.stringify(got[drifted![0]]), 60)}, not ${JSON.stringify(drifted![1])}: something rewrote it on write`, + ); + return; + } + } + + const after0 = await ask(c.dataset!, selection(), m.measure); + const after = new Map(); + for (const p of presets) after.set(p.preset, await ask(c.dataset!, selection(p.condition), m.measure)); + if (todayAt(new Date()) !== day0) { + notRun(`the day turned during the probe (${day0} → ${todayAt(new Date())}), so the windows the server resolved moved under the fixtures`); + return; + } + + // The control leg: with no window, the widget counts every fixture row. + const unwindowedAnswered = after0.kind === 'answered' ? after0.count! - before0.count! : undefined; + r.unwindowed = { expected: written.length, ...(unwindowedAnswered !== undefined ? { answered: unwindowedAnswered } : {}), answer: after0 }; + if (unwindowedAnswered !== written.length) { + notRun( + after0.kind !== 'answered' + ? `the widget's own query, with no window, answered ${after0.status} once the fixtures were written: ${after0.detail ?? ''}` + : `with no window the widget counts ${unwindowedAnswered} of the ${written.length} fixture rows, so the fixtures do not meet its own selection and no window can be judged`, + ); + return; + } + + r.presets = presets.map((p) => { + const expected = written.filter((w) => selects(p.resolved, w.day)).length; + const b = before.get(p.preset)!; + const a = after.get(p.preset)!; + return { preset: p.preset, expected, ...judge(expected, b, a), answer: a }; + }); + + // An inert window: every preset answered what no window answered. + r.inert = r.presets.every((p) => p.answered === unwindowedAnswered); + r.narrowing = []; + for (const a of presets) { + for (const b of presets) { + if (a === b || !nestedIn(a.resolved, b.resolved)) continue; + const pa = r.presets.find((x) => x.preset === a.preset)!; + const pb = r.presets.find((x) => x.preset === b.preset)!; + if (pa.answered !== undefined && pb.answered !== undefined && pa.answered > pb.answered) { + r.narrowing.push(`${a.preset} ⊂ ${b.preset}: ${pa.answered} > ${pb.answered}`); + } + } + } + } + + // ── The report ──────────────────────────────────────────────────────────── + const widgetsBy = (s: DashboardWindowWidgetStatus) => results.filter((r) => r.status === s).length; + const tally = (kinds: Array) => { + const out: Partial> = {}; + for (const k of kinds) if (k) out[k] = (out[k] ?? 0) + 1; + return out; + }; + const probedMeasures = results.flatMap((r) => r.measures.filter((m) => m.status === 'probed')); + const excludedMeasures = results.filter((r) => r.status !== 'excluded' && r.status !== 'unbound') + .flatMap((r) => r.measures.filter((m) => m.status === 'excluded')); + const probes = probedMeasures.flatMap((m) => m.presets); + return { + app: config?.manifest?.id ?? config?.manifest?.namespace ?? 'app', + door: 'rest', + timezone: timezone ?? 'UTC', + presets, + dashboards, + results, + summary: { + dashboards: { + declared: dashboards.length, + windowed: dashboards.filter((d) => d.status === 'windowed').length, + notWindowed: dashboards.filter((d) => d.status === 'not-windowed').length, + excluded: dashboards.filter((d) => d.status === 'excluded').length, + }, + widgets: { + total: results.length, + unbound: widgetsBy('unbound'), + excluded: widgetsBy('excluded'), + probed: results.length - widgetsBy('unbound') - widgetsBy('excluded'), + proven: widgetsBy('proven'), + contradicted: widgetsBy('contradicted'), + failed: widgetsBy('failed'), + inconclusive: widgetsBy('inconclusive'), + }, + excludedBy: tally(results.filter((r) => r.status === 'excluded').map((r) => r.excludedKind)), + measures: { probed: probedMeasures.length, excluded: excludedMeasures.length, excludedBy: tally(excludedMeasures.map((m) => m.excludedKind)) }, + probes: probes.length, + held: probes.filter((p) => p.outcome === 'held').length, + blank: probes.filter((p) => p.outcome === 'blank').length, + contradictions: probes.filter((p) => p.outcome === 'contradicted').length, + faults: probes.filter((p) => p.outcome === 'failed').length, + notRun: probes.filter((p) => p.outcome === 'not-run').length, + inert: probedMeasures.filter((m) => m.inert).length, + narrowing: probedMeasures.reduce((s, m) => s + (m.narrowing?.length ?? 0), 0), + hardFailures: widgetsBy('contradicted') + widgetsBy('failed'), + }, + }; +} + +function widgetStatus(measures: DashboardWindowMeasureResult[]): { status: DashboardWindowWidgetStatus; reason?: string } { + const probes = measures.flatMap((m) => m.presets); + const first = (o: DashboardWindowPresetProbe['outcome']) => probes.find((p) => p.outcome === o); + const bad = first('blank') ?? first('contradicted'); + if (bad) return { status: 'contradicted', reason: `${bad.preset}: ${bad.detail ?? ''}` }; + const failed = first('failed'); + if (failed) return { status: 'failed', reason: `${failed.preset}: ${failed.detail ?? ''}` }; + const notRun = first('not-run'); + if (notRun || probes.length === 0) return { status: 'inconclusive', reason: notRun?.detail ?? 'no preset was probed' }; + return { status: 'proven' }; +} + +/** Pretty one-line-per-widget summary for logs. */ +export function formatDashboardWindowReport(report: DashboardWindowReport): string { + const lines: string[] = [`\n=== objectstack verify — dashboard date windows — ${report.app} ===`]; + lines.push( + ` door: POST /analytics/dataset/query as the signed-in principal; ${report.presets.length} preset(s) per count measure, ` + + `each window resolved on ${report.timezone} calendar days`, + ); + for (const d of report.dashboards.filter((x) => x.status === 'excluded')) { + lines.push(` – dashboard ${d.name} excluded: ${d.reason ?? ''}`); + } + for (const r of report.results) { + const at = `${r.dashboard}/${r.widget}${r.field ? ` (${r.object ?? '?'}.${r.field}${r.fieldType ? `, ${r.fieldType}` : ''})` : ''}`; + const probed = r.measures.filter((m) => m.status === 'probed'); + if (r.status === 'proven') { + lines.push(` ✓ ${at} held under ${probed.reduce((s, m) => s + m.presets.length, 0)} window(s) on ${probed.map((m) => m.measure).join(', ')}`); + } else if (r.status === 'contradicted' || r.status === 'failed') { + lines.push(` ✗ ${at} ${r.status === 'contradicted' ? 'CONTRADICTED' : 'FAILED'}:`); + for (const m of probed) { + const bad = m.presets.filter((p) => p.outcome === 'blank' || p.outcome === 'contradicted' || p.outcome === 'failed'); + for (const p of bad.slice(0, 13)) lines.push(` ${m.measure} @ ${p.preset}: ${p.outcome} — ${clip(p.detail, 200)}`); + if (m.inert) lines.push(` ${m.measure}: the window is INERT — every preset answered what no window answered`); + for (const n of (m.narrowing ?? []).slice(0, 6)) lines.push(` ${m.measure}: a narrower preset answered more — ${n}`); + } + } else if (r.status === 'inconclusive') { + lines.push(` ~ ${at} inconclusive: ${clip(r.reason, 220)}`); + } else { + lines.push(` – ${at} ${r.status}${r.excludedKind ? ` (${r.excludedKind})` : ''}: ${clip(r.reason, 220)}`); + } + } + const s = report.summary; + lines.push( + ` ── ${s.widgets.proven} proven, ${s.widgets.contradicted} CONTRADICTED, ${s.widgets.failed} FAILED, ` + + `${s.widgets.inconclusive} inconclusive, ${s.widgets.unbound} unbound, ${s.widgets.excluded} excluded ` + + `(${s.dashboards.windowed} windowed dashboard(s) of ${s.dashboards.declared}; ${s.measures.probed} count measure(s) probed, ` + + `${s.measures.excluded} excluded; ${s.probes} window(s): ${s.held} held, ${s.blank} blank, ${s.contradictions} contradicted, ` + + `${s.faults} faulted, ${s.notRun} not run; ${s.inert} inert, ${s.narrowing} narrowing violation(s))`, + ); + return lines.join('\n'); +} diff --git a/packages/verify/src/index.ts b/packages/verify/src/index.ts index 90ba308c903..1f1a3732d6b 100644 --- a/packages/verify/src/index.ts +++ b/packages/verify/src/index.ts @@ -3,13 +3,14 @@ // @objectstack/verify — public API. // // Boot any ObjectStack app in-process and verify it through the real HTTP -// stack. Six proof families, all app-agnostic (derived from your metadata): +// stack. Seven proof families, all app-agnostic (derived from your metadata): // - data fidelity : runCrudVerification — author → write → read → assert // - authorization : runRlsProofs — "you can't write what you can't read" // - value shape : runReferenceShapeProofs — "a reference refuses a value that is not a record id" // - delete behaviour: runDeleteBehaviorProofs — "a declared deleteBehavior is what a delete does" // - unique scope : runUniqueScopeProofs — "a declared unique collides where its scope says" // - parent write : runParentWriteReachProofs — "a detail write needs edit on its master, and its own verb" +// - dashboard window: runDashboardWindowProofs — "a dashboard's date picker counts the rows its window selects" export { bootStack, bootStackOnce } from './harness.js'; export type { VerifyStack, BootOptions } from './harness.js'; @@ -152,6 +153,31 @@ export type { ParentWriteReachReport, } from './parent-write-reach.js'; +// The dashboard date-window family: every widget a dashboard's `dateRange` +// binds counts, under each preset of the picker, the rows that window selects +// — the window read from the spec (`filterBindings`, the preset macro +// windows), asked of the dataset door as the signed-in principal, and compared +// with fixture rows written on known days. +export { + deriveDashboardWindowCases, + runDashboardWindowProofs, + formatDashboardWindowReport, +} from './dashboard-window.js'; +export type { + DashboardWindowExclusion, + DashboardWindowMeasureCase, + DashboardWindowCase, + DashboardWindowDashboard, + DashboardWindowPreset, + DashboardWindowAnswer, + DashboardWindowPresetProbe, + DashboardWindowMeasureResult, + DashboardWindowWidgetStatus, + DashboardWindowWidgetResult, + DashboardWindowReport, + DashboardWindowProofOptions, +} from './dashboard-window.js'; + // ADR-0060 — reusable conformance-ledger helper (static complement to the // runtime harness): classify every declarable property, fail closed on drift. export { checkLedger } from './conformance.js'; From 89a5fc6681f1c6606442843b5676431ae455b3e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 00:59:33 +0000 Subject: [PATCH 2/6] feat(cli): os verify --dashboard-windows runs the dashboard date-window family (opt-in) A fresh stack of its own, the admin principal, hardFailures counted, the report under --json as dashboardWindows. The family's controls move to an app plugin's aggregate middleware (the engine's published seam), which os verify composes exactly as bootStack does; an omitted column on read back is reported unconfirmed, not rewritten. Claude-Session: https://claude.ai/code/session_01VoSxBQujKLZKPwK2u5ehQ6 Co-authored-by: Claude --- packages/cli/src/commands/verify.ts | 38 +++- .../cli/test/verify-dashboard-windows.test.ts | 176 ++++++++++++++++++ packages/verify/src/dashboard-window.test.ts | 65 ++++--- packages/verify/src/dashboard-window.ts | 8 +- 4 files changed, 252 insertions(+), 35 deletions(-) create mode 100644 packages/cli/test/verify-dashboard-windows.test.ts diff --git a/packages/cli/src/commands/verify.ts b/packages/cli/src/commands/verify.ts index f3f99713a7e..196b60a29ab 100644 --- a/packages/cli/src/commands/verify.ts +++ b/packages/cli/src/commands/verify.ts @@ -24,12 +24,15 @@ import { runParentWriteReachProofs, formatParentWriteReachReport, parentWriteReachSecurity, + runDashboardWindowProofs, + formatDashboardWindowReport, type VerifyReport, type RlsReport, type ReferenceShapeReport, type DeleteBehaviorReport, type UniqueScopeReport, type ParentWriteReachReport, + type DashboardWindowReport, type RlsProbeDescriptor, type RlsPositionPersonaInput, } from '@objectstack/verify'; @@ -108,12 +111,16 @@ export function resolveVerifyMultiTenant(flags: { 'multi-tenant'?: boolean }): b * or create of a detail, written through the engine as the persona, * is admitted exactly when it holds that verb on the detail's object * and can edit the master + * - dashboard date windows (--dashboard-windows): every widget a + * dashboard's `dateRange` binds counts, under each preset of the + * picker, the fixture rows that window selects, asked of the dataset + * door as the signed-in principal * * Exits non-zero on either stage's failures so it drops straight into CI. */ export default class Verify extends Command { static override description = - 'Run the author-time rules os validate runs, then boot the app in-process and verify it through the real HTTP stack (CRUD round-trip fidelity, plus the cross-owner RLS invariant, the reference-value shape proofs, the declared delete-behaviour proofs, the declared unique-scope proofs and the parent-derived write reach proofs on request)'; + 'Run the author-time rules os validate runs, then boot the app in-process and verify it through the real HTTP stack (CRUD round-trip fidelity, plus the cross-owner RLS invariant, the reference-value shape proofs, the declared delete-behaviour proofs, the declared unique-scope proofs, the parent-derived write reach proofs and the dashboard date-window proofs on request)'; static override examples = [ '<%= config.bin %> verify', @@ -123,6 +130,7 @@ export default class Verify extends Command { '<%= config.bin %> verify --delete-behaviors', '<%= config.bin %> verify --unique-scopes --multi-tenant', '<%= config.bin %> verify --parent-write-reach', + '<%= config.bin %> verify --dashboard-windows', ]; static override flags = { @@ -146,6 +154,10 @@ export default class Verify extends Command { description: "Also prove every field declared unique collides as its scope says (true / 'organization': within one organization only; 'global': across the installation), through the REST door and the system door; the cross-organization half needs --multi-tenant and is reported not run without it (ADR-0120 unique scopes)", default: false, }), + 'dashboard-windows': Flags.boolean({ + description: "Also prove, for every dashboard that declares a dateRange, that each widget it binds counts, under each date-range preset of the picker, the rows that window selects: fixture rows written on known days, the widget asked through the dataset door with and without the window (date and datetime fields; count measures)", + default: false, + }), 'parent-write-reach': Flags.boolean({ description: "Also prove, for every controlled_by_parent object, that a persona's edit, delete or create of a detail is admitted exactly when it holds that verb on the detail's object and can edit the master, written through the engine as verifier personas and one persona per declared position (ADR-0055 master-detail write reach)", default: false, @@ -231,6 +243,7 @@ export default class Verify extends Command { 'delete-behaviors': boolean; 'unique-scopes': boolean; 'parent-write-reach': boolean; + 'dashboard-windows': boolean; 'multi-tenant': boolean; json: boolean; }): Promise { @@ -387,6 +400,20 @@ export default class Verify extends Command { } } + // The dashboard date-window proofs run on a fresh stack of their own too: + // they write fixture rows on known days, which another phase should not + // count, and which no other phase's rows should land among. + let dashboardWindows: DashboardWindowReport | undefined; + if (flags['dashboard-windows']) { + const windowStack = await bootStack(config, { multiTenant, hostRoot }); + try { + const adminToken = await windowStack.signIn(); + dashboardWindows = await runDashboardWindowProofs(windowStack, adminToken, config); + } finally { + await windowStack.stop(); + } + } + // Failure contract: a "real" runtime break the app's author must see. // A degraded RLS probe counts: the run reported verdicts it could not have // established, which is worse than no verifier at all. @@ -418,13 +445,17 @@ export default class Verify extends Command { (uniqueScopes?.summary.hardFailures ?? 0) + // A detail write whose answer disagreed with the conjunction, a door // that faulted on one, or a persona that could not be provisioned. - (parentWriteReach?.summary.hardFailures ?? 0); + (parentWriteReach?.summary.hardFailures ?? 0) + + // A bound widget that went blank or counted other rows than its window + // selects, or a windowed query the door refused or faulted. Inconclusive, + // unbound and excluded widgets are reported, not counted. + (dashboardWindows?.summary.hardFailures ?? 0); if (flags.json) { // `emitJson`, not `this.log`: stdout is reserved (see `run()`), so the // report has to leave through the real stream — and awaiting the write // drains a report larger than one pipe buffer before `this.exit` below. - await emitJson({ app: crud.app, config: absolutePath, multiTenant, crud, rls, referenceShapes, deleteBehaviors, uniqueScopes, parentWriteReach, hardFailures }); + await emitJson({ app: crud.app, config: absolutePath, multiTenant, crud, rls, referenceShapes, deleteBehaviors, uniqueScopes, parentWriteReach, dashboardWindows, hardFailures }); } else { this.log(formatReport(crud)); if (rls) this.log(formatRlsReport(rls)); @@ -432,6 +463,7 @@ export default class Verify extends Command { if (deleteBehaviors) this.log(formatDeleteBehaviorReport(deleteBehaviors)); if (uniqueScopes) this.log(formatUniqueScopeReport(uniqueScopes)); if (parentWriteReach) this.log(formatParentWriteReachReport(parentWriteReach)); + if (dashboardWindows) this.log(formatDashboardWindowReport(dashboardWindows)); this.log(''); this.log( hardFailures > 0 diff --git a/packages/cli/test/verify-dashboard-windows.test.ts b/packages/cli/test/verify-dashboard-windows.test.ts new file mode 100644 index 00000000000..5fc0131405c --- /dev/null +++ b/packages/cli/test/verify-dashboard-windows.test.ts @@ -0,0 +1,176 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * PIN — `os verify --dashboard-windows` runs the dashboard date-window family + * and counts what it finds in the exit status. + * + * ## What is pinned + * + * - without the flag the run is what it was: no `dashboardWindows` key in + * the `--json` document, and the exit status reads CRUD only; + * - with the flag, the dashboard's one bound widget counts, under every + * preset of the picker, the fixture rows its window selects: `proven`, + * `hardFailures` stays 0 and the run exits 0; + * - with the flag, on the same metadata plus the app's own plugin that + * removes the window's condition from every aggregate of the object (the + * engine's published middleware seam), the window counts every fixture + * row: the widget is reported `contradicted` and inert, `hardFailures` + * counts it and the run exits 1. + * + * Spawned rather than run in-process: the exit status is the contract a CI + * gate reads, and only a real process has one. + */ + +import { describe, expect, it, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { CLI, TSX, childEnv } from './helpers/serve-process.js'; +import { defineStackSource, defineStackSourceFromLiteral, linkSpec } from './helpers/define-stack-fixture.js'; + +const MANIFEST = { + id: 'com.example.verify-dashboard-windows', + namespace: 'vdw', + version: '1.0.0', + name: 'Verify Dashboard Windows', + type: 'app', + engines: { protocol: '^18' }, +}; + +const OBJECTS = [ + { + name: 'vdw_case', + label: 'Case', + pluralLabel: 'Cases', + fields: { + subject: { type: 'text', label: 'Subject', required: true }, + opened_at: { type: 'datetime', label: 'Opened at' }, + }, + }, +]; + +const DATASETS = [ + { name: 'case_metrics', label: 'Cases', object: 'vdw_case', dimensions: [], measures: [{ name: 'case_count', label: 'Cases', aggregate: 'count' }] }, +]; + +const DASHBOARDS = [ + { + name: 'vdw_board', + label: 'Board', + dateRange: { field: 'opened_at', defaultRange: 'last_30_days' }, + widgets: [{ id: 'cases', type: 'metric', dataset: 'case_metrics', values: ['case_count'] }], + }, +]; + +/** The same app, plus its own plugin removing the window's condition from every aggregate of `vdw_case`. */ +const CONTROL_SOURCE = `{ + manifest: ${JSON.stringify({ ...MANIFEST, id: 'com.example.verify-dashboard-windows-control' })}, + objects: ${JSON.stringify(OBJECTS)}, + datasets: ${JSON.stringify(DATASETS)}, + dashboards: ${JSON.stringify(DASHBOARDS)}, + plugins: [ + { + name: 'com.example.vdw.window-drop', + version: '1.0.0', + init: async () => {}, + start: async (ctx) => { + const without = (f) => Array.isArray(f) ? f.map(without) + : f && typeof f === 'object' ? Object.fromEntries(Object.entries(f).filter(([k]) => k !== 'opened_at').map(([k, v]) => [k, without(v)])) + : f; + ctx.getService('objectql').registerMiddleware(async (op, next) => { + if (op.operation === 'aggregate' && op.ast && op.ast.where) op.ast.where = without(op.ast.where); + await next(); + }, { object: 'vdw_case' }); + }, + }, + ], +}`; + +interface Run { + status: number | null; + stdout: string; + stderr: string; +} + +interface Doc { + hardFailures?: number; + dashboardWindows?: { + door: string; + results: Array<{ + dashboard: string; + widget: string; + status: string; + measures: Array<{ measure: string; inert?: boolean; presets: Array<{ preset: string; outcome: string }> }>; + }>; + summary: { widgets: { proven: number; contradicted: number }; probes: number; held: number; inert: number; hardFailures: number }; + }; +} + +function runCli(dir: string, args: string[]): Run { + // Through tsx, so the child runs this checkout's `src/`. + const r = spawnSync(TSX, [CLI, ...args], { + cwd: dir, + encoding: 'utf8', + env: childEnv({ NO_COLOR: '1' }), + maxBuffer: 64 * 1024 * 1024, + }); + return { status: r.status, stdout: r.stdout ?? '', stderr: r.stderr ?? '' }; +} + +const docOf = (run: Run): Doc => JSON.parse(run.stdout) as Doc; + +describe('os verify --dashboard-windows', () => { + let root: string; + let plain: Run; + let proven: Run; + let control: Run; + + beforeAll(() => { + root = mkdtempSync(join(tmpdir(), 'os-verify-dashboard-windows-')); + const app = join(root, 'app'); + const controlApp = join(root, 'control'); + for (const [dir, source] of [ + [app, defineStackSource({ manifest: MANIFEST, objects: OBJECTS, datasets: DATASETS, dashboards: DASHBOARDS })], + [controlApp, defineStackSourceFromLiteral(CONTROL_SOURCE)], + ] as const) { + mkdirSync(dir); + writeFileSync(join(dir, 'objectstack.config.mjs'), source); + linkSpec(dir); + } + plain = runCli(app, ['verify', '--json']); + proven = runCli(app, ['verify', '--dashboard-windows', '--json']); + control = runCli(controlApp, ['verify', '--dashboard-windows', '--json']); + }, 480_000); + + afterAll(() => { + if (root) rmSync(root, { recursive: true, force: true }); + }); + + it('leaves the run unchanged without the flag', () => { + expect(plain.status, `${plain.stdout}\n${plain.stderr}`).toBe(0); + const doc = docOf(plain); + expect('dashboardWindows' in doc).toBe(false); + expect(doc.hardFailures).toBe(0); + }); + + it('proves the bound widget under every preset of the picker, and exits 0', () => { + expect(proven.status, `${proven.stdout}\n${proven.stderr}`).toBe(0); + const report = docOf(proven).dashboardWindows!; + expect(report.door).toBe('rest'); + expect(report.results).toEqual([expect.objectContaining({ dashboard: 'vdw_board', widget: 'cases', status: 'proven' })]); + expect(report.summary).toMatchObject({ widgets: { proven: 1, contradicted: 0 }, probes: 13, held: 13, inert: 0, hardFailures: 0 }); + expect(docOf(proven).hardFailures).toBe(0); + }); + + it('counts a window that never narrows, and exits 1', () => { + expect(control.status, `${control.stdout}\n${control.stderr}`).toBe(1); + const doc = docOf(control); + const [widget] = doc.dashboardWindows!.results; + expect(widget).toMatchObject({ widget: 'cases', status: 'contradicted' }); + expect(widget!.measures[0]!.inert).toBe(true); + expect(widget!.measures[0]!.presets.every((p) => p.outcome === 'contradicted')).toBe(true); + expect(doc.dashboardWindows!.summary).toMatchObject({ widgets: { contradicted: 1 }, inert: 1, hardFailures: 1 }); + expect(doc.hardFailures).toBe(1); + }); +}); diff --git a/packages/verify/src/dashboard-window.test.ts b/packages/verify/src/dashboard-window.test.ts index 04ec04bc550..3f2b21fa5e1 100644 --- a/packages/verify/src/dashboard-window.test.ts +++ b/packages/verify/src/dashboard-window.test.ts @@ -3,12 +3,12 @@ // The dashboard date-window family — what it reads from the metadata, what it // proves on a real stack, and the controls that keep its green honest: // -// • two NEGATIVE controls, each a deployment whose analytics bridge does not -// honour the window — built with `AnalyticsServicePlugin`'s published -// `executeAggregate` option, no engine change: one drops the window from -// the query that reaches storage (a floor that matches every row), one -// answers nothing whenever a window is present (a window that zeroes the -// widget). Both are found, and the first is named inert; +// • two NEGATIVE controls, each an app whose own plugin keeps the window +// from reaching storage — an aggregate middleware on the engine's +// published seam (`registerMiddleware`), no engine change: one drops the +// window from the query (a floor that matches every row), one makes a +// windowed query match nothing (a window that zeroes the widget). Both are +// found, and the first is named inert; // • the ATTRIBUTION control: an app hook that rewrites the window's field on // write makes the probe not run, never contradicted — the family judges a // window only over rows that read back as written. @@ -17,7 +17,6 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { defineStack } from '@objectstack/spec'; import { DATE_RANGE_PRESETS } from '@objectstack/spec/data'; -import { AnalyticsServicePlugin } from '@objectstack/service-analytics'; import { bootStack, type VerifyStack } from './harness.js'; import { @@ -246,6 +245,7 @@ const controlSource = { subject: { type: 'text', label: 'Subject', required: true }, opened_at: { type: 'datetime', label: 'Opened at' }, logged_at: { type: 'datetime', label: 'Logged at' }, + audited_at: { type: 'datetime', label: 'Audited at', internal: true }, }, }, ], @@ -258,6 +258,7 @@ const controlSource = { widgets: [ { id: 'cases', type: 'metric', dataset: 'case_metrics', values: ['case_count'] }, { id: 'logged', type: 'metric', dataset: 'case_metrics', values: ['case_count'], filterBindings: { dateRange: 'logged_at' } }, + { id: 'audited', type: 'metric', dataset: 'case_metrics', values: ['case_count'], filterBindings: { dateRange: 'audited_at' } }, ], }, ], @@ -283,35 +284,34 @@ function without(filter: unknown, field: string): unknown { const mentions = (filter: unknown, field: string): boolean => JSON.stringify(filter ?? {}).includes(`"${field}"`); /** - * An analytics service whose bridge to storage does what a broken one would: - * `drop` hands the engine the query with the window's field removed (every row - * passes the window), `zero` answers no row whenever a window is present. + * The app's own plugin, as `os serve` and `bootStack` compose it: an aggregate + * middleware on `wnc_case` (the engine's published seam, `registerMiddleware`, + * read by `ctx.operation === 'aggregate'` and `ctx.ast.where`) that does what + * a broken storage path would — `drop` removes the window's condition from the + * query (every row passes the window), `zero` makes a windowed query match no + * row (the window zeroes the widget). */ -function brokenBridge(mode: 'drop' | 'zero', holder: { engine?: any }): AnalyticsServicePlugin { - return new AnalyticsServicePlugin({ - queryCapabilities: () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false }), - executeAggregate: async (objectName, { groupBy, aggregations, filter, timezone, context }) => { - if (mode === 'zero' && mentions(filter, 'opened_at')) return []; - return holder.engine.aggregate(objectName, { - where: mode === 'drop' ? without(filter, 'opened_at') : filter, - groupBy, - aggregations: aggregations?.map((a) => ({ function: a.method, field: a.field, alias: a.alias, ...(a.filter ? { filter: a.filter } : {}) })), - timezone, - context, - }); - }, - }); -} +const windowBreaker = (mode: 'drop' | 'zero') => ({ + name: `com.example.wnc.window-${mode}`, + version: '1.0.0', + init: async () => {}, + start: async (ctx: { getService: (name: string) => any }) => { + ctx.getService('objectql').registerMiddleware(async (op: { operation: string; ast?: { where?: unknown } }, next: () => Promise) => { + if (op.operation === 'aggregate' && op.ast && mentions(op.ast.where, 'opened_at')) { + op.ast.where = mode === 'drop' ? without(op.ast.where, 'opened_at') : { $and: [op.ast.where, { subject: '__no_row__' }] }; + } + await next(); + }, { object: 'wnc_case' }); + }, +}); -describe.each(['drop', 'zero'] as const)('NEGATIVE CONTROL — an analytics bridge that does not honour the window (%s)', (mode) => { +describe.each(['drop', 'zero'] as const)("NEGATIVE CONTROL — an app plugin that keeps the window from storage (%s)", (mode) => { let stack: VerifyStack; let report: DashboardWindowReport; - const app = defineStack(controlSource as never) as unknown as typeof controlSource; - const holder: { engine?: any } = {}; + const app = defineStack({ ...controlSource, plugins: [windowBreaker(mode)] } as never) as unknown as typeof controlSource; beforeAll(async () => { - stack = await bootStack(app as never, { analytics: brokenBridge(mode, holder) }); - holder.engine = await stack.kernel.getServiceAsync('data'); + stack = await bootStack(app as never); report = await runDashboardWindowProofs(stack, await stack.signIn(), app); }, BOOT_TIMEOUT + RUN_TIMEOUT); @@ -343,5 +343,10 @@ describe.each(['drop', 'zero'] as const)('NEGATIVE CONTROL — an analytics brid expect(r.status).toBe('inconclusive'); expect(r.reason).toMatch(/reads back logged_at = "2020-01-01T00:00:00\.000Z", not \d{4}-\d{2}-\d{2}T12:00:00\.000Z: something rewrote it on write/); expect(r.measures[0]!.presets.every((p) => p.outcome === 'not-run')).toBe(true); + // A column the read does not return (`internal: true`) is not called rewritten: it is unconfirmed. + const audited = widgetOf(report, 'audited'); + expect(audited.status).toBe('inconclusive'); + expect(audited.reason).toMatch(/reads back no audited_at \(the read does not return that field\), so what was written there cannot be confirmed$/); + expect(report.summary.widgets).toMatchObject({ contradicted: 1, inconclusive: 2, proven: 0 }); }); }); diff --git a/packages/verify/src/dashboard-window.ts b/packages/verify/src/dashboard-window.ts index 6a3fe141e1a..4ebdda3f0aa 100644 --- a/packages/verify/src/dashboard-window.ts +++ b/packages/verify/src/dashboard-window.ts @@ -872,13 +872,17 @@ export async function runDashboardWindowProofs( ? String(stamped ?? '').slice(0, 10) === day : Date.parse(String(stamped)) === Date.parse(stamp(day)); const drifted = [...held].find(([k, v]) => !Array.isArray(v) && !sameValue(v, got?.[k])); + const misread = (field: string, value: unknown, wrote: unknown): string => + value === undefined + ? `the fixture row for ${day} reads back no ${field} (the read does not return that field), so what was written there cannot be confirmed` + : `the fixture row for ${day} reads back ${field} = ${clip(JSON.stringify(value), 60)}, not ${typeof wrote === 'string' ? wrote : JSON.stringify(wrote)}: something rewrote it on write`; if (!got || !stampHolds || drifted) { notRun( !got ? `the fixture row for ${day} does not read back` : !stampHolds - ? `the fixture row for ${day} reads back ${c.field} = ${clip(JSON.stringify(stamped), 60)}, not ${stamp(day)}: something rewrote it on write` - : `the fixture row for ${day} reads back ${drifted![0]} = ${clip(JSON.stringify(got[drifted![0]]), 60)}, not ${JSON.stringify(drifted![1])}: something rewrote it on write`, + ? misread(c.field!, stamped, stamp(day)) + : misread(drifted![0], got[drifted![0]], drifted![1]), ); return; } From b75f655e32fee81587363f65d2adca61e65d0e84 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 01:25:21 +0000 Subject: [PATCH 3/6] feat(verify): hold a $nin / $ne column to a declared option; the stand-in rows; the docs (#15953 slice 5, WIP) Claude-Session: https://claude.ai/code/session_01VoSxBQujKLZKPwK2u5ehQ6 Co-authored-by: Claude --- content/docs/deployment/cli.mdx | 37 ++++++++++- packages/verify/README.md | 66 +++++++++++++++++++- packages/verify/src/dashboard-window.test.ts | 17 +++-- packages/verify/src/dashboard-window.ts | 26 ++++++-- scripts/check-verify-stand-in-erasure.mjs | 3 + 5 files changed, 132 insertions(+), 17 deletions(-) diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index f83c26a079d..b588b85364f 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -1926,6 +1926,7 @@ os verify --reference-shapes # Also prove reference fields ref os verify --delete-behaviors # Also prove each declared deleteBehavior is what a delete does os verify --unique-scopes --multi-tenant # Also prove each declared unique collides where its scope says os verify --parent-write-reach # Also prove each controlled_by_parent detail write follows its master +os verify --dashboard-windows # Also prove each dashboard date picker counts the rows its window selects ``` 1. **Author-time rules.** The rule registry `os validate` runs, over the stack @@ -2015,6 +2016,36 @@ os verify --parent-write-reach # Also prove each controlled_by_p two levels below its root master (a line under a quote under an account), even where the caller edits the middle master, so an app that declares such a chain fails the run. + With `--dashboard-windows`, another fresh boot proves that every dashboard + declaring a `dateRange` answers under each preset of its picker with the + rows that window selects. A widget's window lands on the field its + `filterBindings.dateRange` names, or on the dashboard's `dateRange.field` + (`created_at` when unauthored), resolved against the object behind the + widget's dataset, and must be a `date` or `datetime` field; a widget with + `filterBindings: { dateRange: false }` is reported `unbound`. For each count + measure of a bound widget, fixture rows are written through the seed door + on every preset's first and last day and the day either side (never after + today), stamped at UTC noon, owned by the caller and holding every column + the widget's own filter, the dataset's filter and the measure's filter + compare, and read back. The widget is then asked through + `POST /analytics/dataset/query` as the signed-in admin with no window and + under each of the 13 presets, before and after the fixtures. A preset is + handed as the spec's `{date-macro}` window for it (`{ $gte: '{7_days_ago}' }` + for `last_7_days`, `{ $gte: '{month_start}', $lte: '{month_end}' }` for + `this_month`), ANDed with the widget's filter, and the server resolves the + tokens; the ground truth resolves the same window with the platform's + resolver, on UTC calendar days unless the caller's context names a + timezone. Each difference must equal the fixture rows the window selects: a + widget that shows nothing there, or counts another number of them, fails + the run, and one whose every preset answers what no window answers is named + inert. The widget with no window must count every fixture row first; a + probe whose fixtures do not meet the widget's own selection, or that cannot + be written or read back as written, is reported inconclusive, never proven. + A held column is an equality, an `$in`, a `$nin` or `$ne` on a field that + declares its options, or `{current_user_id}`; an average, a sum or a derived + measure, and a filter the ground truth cannot model (`$or`, a range, a + relationship path, a condition on the window's own field), are reported + `excluded`. `--multi-tenant` boots org-scoped so tenant-isolation RLS policies apply; a walled tenancy posture (`OS_TENANCY_POSTURE` set to `isolated` or `group`) asks for the same boot. @@ -2046,8 +2077,10 @@ document carries depends on where the run ended: `deleteBehaviors` (with `--delete-behaviors`), `uniqueScopes` (with `--unique-scopes`; its `boot` names the legs the run could prove), `parentWriteReach` (with `--parent-write-reach`; its `personas` name who - wrote and who could not be provisioned) and `hardFailures`, the count - the exit status is read from. + wrote and who could not be provisioned), `dashboardWindows` (with + `--dashboard-windows`; its `presets` name what each preset handed the door + and how it resolved) and `hardFailures`, the count the exit status is read + from. - **Could not run:** `{ "error": "" }`, plus `code` and `httpStatus` when the failure carries them. diff --git a/packages/verify/README.md b/packages/verify/README.md index 5d8705cf861..795d3f85df7 100644 --- a/packages/verify/README.md +++ b/packages/verify/README.md @@ -1,7 +1,7 @@ # @objectstack/verify Boot any ObjectStack app **in-process** and verify it through the **real HTTP -stack** — no mocks, no ports, no sockets. Six app-agnostic proof families, all +stack** — no mocks, no ports, no sockets. Seven app-agnostic proof families, all derived from your own metadata: - **Data fidelity** — author one record per object covering every field type, @@ -36,6 +36,13 @@ derived from your own metadata: persona, each compared with that conjunction — the caller's effective grants on the detail, and its edit of the master MEASURED by an empty update of the master. +- **Dashboard date windows** (opt-in) — *every widget a dashboard's `dateRange` + binds counts, under each preset of the picker, the rows that window selects*: + per bound widget and count measure, fixture rows are written on known days + around every preset's edges, and the widget is asked through the dataset door + with no window and under each of the 13 presets, before and after the + fixtures. Each difference must equal the fixture rows the window selects; a + window that answers nothing is `blank`, and one that never narrows is inert. And, on the same booted stack, an **in-process handle** so an app's own tests can run a hook, a validation rule, a flow, an action, a seed or a read against @@ -83,6 +90,9 @@ objectstack verify --unique-scopes --multi-tenant # also prove every controlled_by_parent detail write follows its master objectstack verify --parent-write-reach + +# also prove every dashboard date picker counts the rows its window selects +objectstack verify --dashboard-windows ``` `--reference-shapes` is opt-in, on a fresh boot of its own: the engine still @@ -138,11 +148,43 @@ two levels below its root master, even where the caller edits the middle master, so on an app that declares such a chain the family reports a contradiction and fails the run. +`--dashboard-windows` is opt-in, on a fresh boot of its own (it writes fixture +rows). The population is every dashboard the spec's parse gives a `dateRange`, +and every widget it binds: the window lands on the field +`filterBindings.dateRange` names, or on `dateRange.field` (`created_at` when +unauthored) where the widget declares no binding, and a widget with +`filterBindings: { dateRange: false }` is `unbound`. The field resolves against +the object behind the widget's dataset, as the stack registers it, and must be +a `date` or `datetime` field. Each preset of `DATE_RANGE_PRESETS` is handed to +`POST /analytics/dataset/query` as its `{date-macro}` window from the spec +(`DATE_RANGE_PRESET_MACRO_WINDOWS`, e.g. `{ $gte: '{7_days_ago}' }`), ANDed +with the widget's own filter as `runtimeFilter`, as the signed-in admin; the +server resolves the tokens, and the ground truth resolves the same window with +`@objectstack/core`'s `resolveFilterTokens` on the caller's reference timezone +(UTC unless the `ExecutionContext` carries one). Fixture rows are written +through the seed door on each preset's first and last day and the day either +side (never after today), each stamped at UTC noon (a `date` field takes the +day), owned by the caller, with every column the widget's filter, the +dataset's filter and the measure's filter compare held to a value they select, +and read back before anything is judged. Every count is a difference (after +the fixtures minus before, under the same window), so the app's own rows +cancel out, and the widget with no window must count every fixture row before +any window is judged. A count measure is probed (a `count` of a named field +gets a non-null value); an average, a sum or a derived measure is counted as +excluded. A held column is an equality, an `$in` (its first member), a +`$nin` or `$ne` on a field that declares its options (the first option it does +not exclude) or `{current_user_id}` (the caller); any other filter the ground +truth cannot model (`$or`, a range, a relationship path, another placeholder, a +condition on the window's own field) is excluded, and so is a dataset over an +object the app does not declare. The shipped console closes the three rolling windows at `{today}` as +well; the fixtures carry no later day, so both select the same rows. + Exit code is **non-zero** on real failures (`create-failed`, `read-failed`, `fidelity-gaps`, `rls-hole`, with `--reference-shapes` a field with `holes` or `failed`, with `--delete-behaviors` or `--unique-scopes` a field -`contradicted` or `failed`, and with `--parent-write-reach` an object -`contradicted` or `failed` or a persona that could not be provisioned) so +`contradicted` or `failed`, with `--parent-write-reach` an object +`contradicted` or `failed` or a persona that could not be provisioned, and +with `--dashboard-windows` a widget `contradicted` or `failed`) so it drops straight into a CI gate. Inconclusive verdicts (`needs-fixture`, `skipped`, `member-visible`) are warnings and exit 0. @@ -445,6 +487,20 @@ is reported not run: that empty answer is also the endpoint's when it could not compute one, so it is not read as "no grant anywhere". `summary.hardFailures` is `contradicted + failed` objects plus personas that could not be provisioned. +**Dashboard date windows** (`runDashboardWindowProofs`), per widget a +dashboard's `dateRange` binds. Probes per count measure: the widget with no +window, then each of the 13 presets, each counted before and after the +fixture rows: + +| status | meaning | +| --- | --- | +| `proven` | under every preset the window counted exactly the fixture rows it selects | +| `contradicted` | under some preset the widget showed nothing where the window selects fixture rows (`blank`), or counted another number of them; a window that answers what no window answers under every preset is also named `inert` **(failure)** | +| `failed` | the door refused or faulted on a windowed query while the widget's own query answered **(failure)** | +| `inconclusive` | a fixture row could not be written or did not read back as written (an app hook rewrote it, or the read omits the column), the widget with no window did not count every fixture row (its own selection does not meet the fixtures), its own query is refused, the day turned mid-probe, or the caller's reference timezone is not UTC | +| `unbound` | the widget opts out of the window (`filterBindings: { dateRange: false }`) | +| `excluded` | the family cannot compare it: no count measure, a filter the ground truth cannot model, a window on a field that is missing or is not `date` / `datetime`, a dataset or dashboard that is undeclared or does not parse, or an object the app does not declare or the CRUD derivation cannot synthesize (`excludedKind` names which) | + `member-visible` everywhere usually means the app is single-tenant; pass `--multi-tenant` (or `{ multiTenant: true }`) to register org-scoping so tenant isolation policies actually apply. @@ -495,6 +551,10 @@ run" must never read like "nothing to find". `SecurityPlugin` to boot with (registers the verifier personas' sets); `runParentWriteReachProofs(stack, config)` → `ParentWriteReachReport` (`personas` names who wrote and who could not be provisioned; `summary.hardFailures` is what a gate reads); `formatParentWriteReachReport(report)`. +- `deriveDashboardWindowCases(config)` → one case per widget of every dashboard with a `dateRange`: its bound field, + dataset, object and count measures, or why it is unbound or not compared; `runDashboardWindowProofs(stack, token, + config, { now? })` → `DashboardWindowReport` (`presets` names what each preset handed the door and how it resolved; + `summary.hardFailures` is what a gate reads); `formatDashboardWindowReport(report)`. `bootStack` options: `admin`, `authSecret`, `security` (a custom `SecurityPlugin` for owner-scoped fixtures), `multiTenant` (also what decides the posture diff --git a/packages/verify/src/dashboard-window.test.ts b/packages/verify/src/dashboard-window.test.ts index 3f2b21fa5e1..0a468a70c49 100644 --- a/packages/verify/src/dashboard-window.test.ts +++ b/packages/verify/src/dashboard-window.test.ts @@ -89,6 +89,8 @@ const source = { { id: 'closed_tickets', type: 'metric', dataset: 'ticket_metrics', values: ['closed_count'] }, { id: 'with_assignee', type: 'metric', dataset: 'ticket_metrics', values: ['assigned_count'] }, { id: 'high_or_low', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { priority: { $in: ['high', 'low'] } } }, + // A declared option the condition does not exclude holds the column. + { id: 'not_closed', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { status: { $nin: ['closed'] } } }, { id: 'mixed', type: 'table', dataset: 'ticket_metrics', dimensions: ['priority'], values: ['ticket_count', 'avg_effort'] }, // Outside the population, each with its reason. { id: 'trend', type: 'line', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: false } }, @@ -96,6 +98,7 @@ const source = { { id: 'rate', type: 'metric', dataset: 'ticket_metrics', values: ['closed_rate'] }, { id: 'recent', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { opened_at: { $gte: '{30_days_ago}' } } }, { id: 'either', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { $or: [{ status: 'open' }, { is_closed: true }] } }, + { id: 'not_x', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { subject: { $ne: 'x' } } }, { id: 'by_subject', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: 'subject' } }, { id: 'ghost', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: 'closed_on' } }, { id: 'users', type: 'metric', dataset: 'user_metrics', values: ['user_count'] }, @@ -112,7 +115,7 @@ const source = { ], }; -const PROVEN = ['open_tickets', 'by_status', 'due_window', 'mine', 'closed_tickets', 'with_assignee', 'high_or_low', 'mixed', 'all_tickets']; +const PROVEN = ['open_tickets', 'by_status', 'due_window', 'mine', 'closed_tickets', 'with_assignee', 'high_or_low', 'not_closed', 'mixed', 'all_tickets']; const widgetOf = (report: DashboardWindowReport, widget: string) => { const r = report.results.find((x) => x.widget === widget); @@ -207,6 +210,8 @@ describe('runDashboardWindowProofs — the fixture app', () => { expect(widgetOf(report, 'recent').reason).toMatch(/a condition on opened_at, the field the window lands on/); expect(excluded('either')).toEqual(['excluded', 'filter-unmodelled']); expect(widgetOf(report, 'either').reason).toMatch(/the operator \$or/); + expect(excluded('not_x')).toEqual(['excluded', 'filter-unmodelled']); + expect(widgetOf(report, 'not_x').reason).toMatch(/on subject, which declares no options to choose from/); expect(excluded('by_subject')).toEqual(['excluded', 'field-type']); expect(excluded('ghost')).toEqual(['excluded', 'field-missing']); expect(excluded('users')).toEqual(['excluded', 'object-undeclared']); @@ -217,11 +222,11 @@ describe('runDashboardWindowProofs — the fixture app', () => { it('counts every dashboard, widget, measure and window', () => { expect(report.summary).toEqual({ dashboards: { declared: 3, windowed: 2, notWindowed: 1, excluded: 0 }, - widgets: { total: 17, unbound: 1, excluded: 7, probed: 9, proven: 9, contradicted: 0, failed: 0, inconclusive: 0 }, - excludedBy: { 'no-count-measure': 2, 'filter-unmodelled': 2, 'field-type': 1, 'field-missing': 1, 'object-undeclared': 1 }, - measures: { probed: 9, excluded: 1, excludedBy: { 'not-a-count': 1 } }, - probes: 9 * DATE_RANGE_PRESETS.length, - held: 9 * DATE_RANGE_PRESETS.length, + widgets: { total: 19, unbound: 1, excluded: 8, probed: 10, proven: 10, contradicted: 0, failed: 0, inconclusive: 0 }, + excludedBy: { 'no-count-measure': 2, 'filter-unmodelled': 3, 'field-type': 1, 'field-missing': 1, 'object-undeclared': 1 }, + measures: { probed: 10, excluded: 1, excludedBy: { 'not-a-count': 1 } }, + probes: 10 * DATE_RANGE_PRESETS.length, + held: 10 * DATE_RANGE_PRESETS.length, blank: 0, contradictions: 0, faults: 0, diff --git a/packages/verify/src/dashboard-window.ts b/packages/verify/src/dashboard-window.ts index 4ebdda3f0aa..3d35d3c15d0 100644 --- a/packages/verify/src/dashboard-window.ts +++ b/packages/verify/src/dashboard-window.ts @@ -403,17 +403,24 @@ function derive(config: any): { cases: DashboardWindowCase[]; dashboards: Dashbo // ── Held columns: what the widget's own selection asks of a row ───────────── +/** A field's declared option values (`select` and its kin), in declaration order. */ +function optionValues(def: any): unknown[] { + return (Array.isArray(def?.options) ? def.options : []).map((o: any) => (isPlainObject(o) ? o.value : o)).filter((v: unknown) => v != null); +} + /** * Collect the equality every row must satisfy for `filter` to select it, into * `held`. Understood: a field compared with a scalar (bare or `$eq`), an `$in` - * list (its first member), `{current_user_id}` (the principal), and `$and` of + * list (its first member), a `$nin` list or a `$ne` on a field that declares + * its options (the first declared option the condition does not exclude, so + * no row is NULL there), `{current_user_id}` (the principal), and `$and` of * those. Anything else is a selection the ground truth cannot model, and the * reason is returned — never a guess. */ function holdColumns( filter: unknown, held: Map, - ctx: { fields: Set; windowField: string; principal: string; object: string }, + ctx: { fields: Record; windowField: string; principal: string; object: string }, ): string | undefined { if (filter == null) return undefined; if (!isPlainObject(filter)) return `a filter that is not an object (${clip(JSON.stringify(filter), 80)})`; @@ -429,13 +436,21 @@ function holdColumns( if (key.startsWith('$')) return `the operator ${key}`; if (key.includes('.')) return `the relationship path ${key}`; if (key === ctx.windowField) return `a condition on ${key}, the field the window lands on`; - if (!ctx.fields.has(key)) return `${key}, which is not a field of ${ctx.object}`; + if (!(key in ctx.fields)) return `${key}, which is not a field of ${ctx.object}`; let value: unknown = cond; if (isPlainObject(cond)) { const ops = Object.keys(cond); + const excluding = ops.length === 1 && ops[0] === '$nin' && Array.isArray(cond.$nin) + ? cond.$nin + : ops.length === 1 && ops[0] === '$ne' && cond.$ne !== null && typeof cond.$ne !== 'object' ? [cond.$ne] : undefined; if (ops.length === 1 && ops[0] === '$eq') value = cond.$eq; else if (ops.length === 1 && ops[0] === '$in' && Array.isArray(cond.$in) && cond.$in.length > 0) value = cond.$in[0]; - else return `the condition ${clip(JSON.stringify({ [key]: cond }), 100)}`; + else if (excluding) { + const options = optionValues(ctx.fields[key]); + if (options.length === 0) return `the condition ${clip(JSON.stringify({ [key]: cond }), 100)} on ${key}, which declares no options to choose from`; + value = options.find((o) => !excluding.some((x: unknown) => String(x) === String(o))); + if (value === undefined) return `the condition ${clip(JSON.stringify({ [key]: cond }), 100)}, which excludes every option ${key} declares`; + } else return `the condition ${clip(JSON.stringify({ [key]: cond }), 100)}`; } if (typeof value === 'string' && /^\$?\{.*\}$/.test(value)) { if (value !== CURRENT_USER_TOKEN && value !== `$${CURRENT_USER_TOKEN}`) return `the placeholder ${value} on ${key}`; @@ -731,8 +746,7 @@ export async function runDashboardWindowProofs( } const ownerColumn = 'owner_id' in fields; - const fieldNames = new Set(Object.keys(fields)); - const holdCtx = { fields: fieldNames, windowField: c.field!, principal, object: c.object! }; + const holdCtx = { fields, windowField: c.field!, principal, object: c.object! }; const objectHeld = new Map(); const objectReason = holdColumns(c.datasetFilter, objectHeld, holdCtx) ?? holdColumns(c.filter, objectHeld, holdCtx); if (objectReason) { diff --git a/scripts/check-verify-stand-in-erasure.mjs b/scripts/check-verify-stand-in-erasure.mjs index 8059a78e85b..979df50213c 100644 --- a/scripts/check-verify-stand-in-erasure.mjs +++ b/scripts/check-verify-stand-in-erasure.mjs @@ -233,6 +233,9 @@ const NOT_A_STAND_IN = { runParentWriteReachProofs: 'same `VerifyStack` handle as runCrudVerification.', formatParentWriteReachReport: 'formats a `ParentWriteReachReport` this package produced; presentation, not conformance.', + runDashboardWindowProofs: 'same `VerifyStack` handle as runCrudVerification.', + formatDashboardWindowReport: + 'formats a `DashboardWindowReport` this package produced; presentation, not conformance.', fillRelationalRefs: 'takes one `CrudCase` derived by `deriveCrudCases` — a data record this package built, with no ' + 'external implementer.', From 769f3fba6c9d12187debd40b4129f62715fc1206 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 01:35:23 +0000 Subject: [PATCH 4/6] test(cli): the dashboard-windows fixture declares its sharingModel (#15953 slice 5, WIP) Claude-Session: https://claude.ai/code/session_01VoSxBQujKLZKPwK2u5ehQ6 Co-authored-by: Claude --- packages/cli/test/verify-dashboard-windows.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/test/verify-dashboard-windows.test.ts b/packages/cli/test/verify-dashboard-windows.test.ts index 5fc0131405c..ae4a58710c4 100644 --- a/packages/cli/test/verify-dashboard-windows.test.ts +++ b/packages/cli/test/verify-dashboard-windows.test.ts @@ -43,6 +43,7 @@ const OBJECTS = [ name: 'vdw_case', label: 'Case', pluralLabel: 'Cases', + sharingModel: 'private', fields: { subject: { type: 'text', label: 'Subject', required: true }, opened_at: { type: 'datetime', label: 'Opened at' }, From 6150d2a1ba17a631442e081dc26c3ecd44ef895b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 01:44:32 +0000 Subject: [PATCH 5/6] feat(verify): a refused fixture row is retried with every synthesized field, and the report names the shape (#15953 slice 5, WIP) Claude-Session: https://claude.ai/code/session_01VoSxBQujKLZKPwK2u5ehQ6 Co-authored-by: Claude --- content/docs/deployment/cli.mdx | 3 +- packages/verify/README.md | 4 +- packages/verify/src/dashboard-window.test.ts | 16 ++++++++ packages/verify/src/dashboard-window.ts | 43 ++++++++++++++------ 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index b588b85364f..eb033d08ad9 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -2027,7 +2027,8 @@ os verify --dashboard-windows # Also prove each dashboard date on every preset's first and last day and the day either side (never after today), stamped at UTC noon, owned by the caller and holding every column the widget's own filter, the dataset's filter and the measure's filter - compare, and read back. The widget is then asked through + compare (the smallest record first, every synthesized field if the seed + door refuses it), and read back. The widget is then asked through `POST /analytics/dataset/query` as the signed-in admin with no window and under each of the 13 presets, before and after the fixtures. A preset is handed as the spec's `{date-macro}` window for it (`{ $gte: '{7_days_ago}' }` diff --git a/packages/verify/README.md b/packages/verify/README.md index 795d3f85df7..14ad09ec108 100644 --- a/packages/verify/README.md +++ b/packages/verify/README.md @@ -166,7 +166,9 @@ through the seed door on each preset's first and last day and the day either side (never after today), each stamped at UTC noon (a `date` field takes the day), owned by the caller, with every column the widget's filter, the dataset's filter and the measure's filter compare held to a value they select, -and read back before anything is judged. Every count is a difference (after +and read back before anything is judged. A row is the smallest record the CRUD +derivation offers (its required fields); if the seed door refuses that (an app +rule that requires a field only in some state), every field it synthesizes. Every count is a difference (after the fixtures minus before, under the same window), so the app's own rows cancel out, and the widget with no window must count every fixture row before any window is judged. A count measure is probed (a `count` of a named field diff --git a/packages/verify/src/dashboard-window.test.ts b/packages/verify/src/dashboard-window.test.ts index 0a468a70c49..024724c1dd5 100644 --- a/packages/verify/src/dashboard-window.test.ts +++ b/packages/verify/src/dashboard-window.test.ts @@ -47,6 +47,19 @@ const ticket = { const source = { manifest: { id: 'com.example.dashboard-window', namespace: 'wnd', version: '0.0.1', type: 'app', name: 'Dashboard Window Fixture' }, + hooks: [ + { + // An app rule requiring a field only in some state: a closed ticket needs its effort. + name: 'wnd_closed_needs_effort', + object: 'wnd_ticket', + events: ['beforeInsert'], + handler: async (ctx: { input: Record }) => { + if (ctx.input.is_closed === true && ctx.input.effort == null) { + throw Object.assign(new Error('Effort is required on a closed ticket'), { code: 'VALIDATION_FAILED', status: 400 }); + } + }, + }, + ], objects: [ { name: 'wnd_agent', label: 'Agent', fields: { name: { type: 'text', label: 'Name', required: true } } }, ticket, @@ -197,6 +210,9 @@ describe('runDashboardWindowProofs — the fixture app', () => { } } expect(widgetOf(report, 'open_tickets').fieldType).toBe('datetime'); + // The closed tickets meet the app's own rule only with every synthesized field. + expect(widgetOf(report, 'closed_tickets').measures[0]!.fixture).toBe('every-field'); + expect(widgetOf(report, 'open_tickets').measures[0]!.fixture).toBe('smallest'); expect(widgetOf(report, 'due_window').fieldType).toBe('date'); expect(widgetOf(report, 'all_tickets').fieldType).toBe('datetime'); }); diff --git a/packages/verify/src/dashboard-window.ts b/packages/verify/src/dashboard-window.ts index 3d35d3c15d0..36251c048f3 100644 --- a/packages/verify/src/dashboard-window.ts +++ b/packages/verify/src/dashboard-window.ts @@ -193,6 +193,8 @@ export interface DashboardWindowMeasureResult { excludedKind?: DashboardWindowExclusion; /** The fixture days written (UTC calendar days). */ days?: string[]; + /** `smallest` — the required fields only; `every-field` — the seed door refused that, so every field the CRUD derivation synthesizes. */ + fixture?: 'smallest' | 'every-field'; /** The control leg: the widget with no window must count every fixture row. */ unwindowed?: { expected: number; answered?: number; answer?: DashboardWindowAnswer }; presets: DashboardWindowPresetProbe[]; @@ -515,18 +517,19 @@ function uniqueFields(objectDef: any): Set { } /** - * The smallest record the CRUD derivation can offer (the shape the sibling - * families use): its values for the fields the object REQUIRES, so every - * optional field takes the app's own default; each `date` / `datetime` one day - * after the one declared before it, and each unique-constrained value made - * distinct by `n`. + * A record the CRUD derivation can offer: by default the smallest (the shape + * the sibling families use) — its values for the fields the object REQUIRES, + * so every optional field takes the app's own default; with `full`, every + * field it synthesizes (what a rule that requires a field only in some state + * asks for). Each `date` / `datetime` is one day after the one declared before + * it, and each unique-constrained value is made distinct by `n`. */ -function fixtureBody(body: Record, objectDef: any, n: number): Record { +function fixtureBody(body: Record, objectDef: any, n: number, full = false): Record { const unique = uniqueFields(objectDef); const out: Record = {}; let dates = 0; for (const [name, def] of Object.entries(objectDef?.fields ?? {})) { - if (def?.required !== true || !(name in body)) continue; + if ((!full && def?.required !== true) || !(name in body)) continue; let value = body[name]; if ((def?.type === 'date' || def?.type === 'datetime') && typeof value === 'string') value = laterBy(value, dates++); if (unique.has(name) && (typeof value === 'string' || typeof value === 'number')) value = vary(value, n); @@ -860,17 +863,32 @@ export async function runDashboardWindowProofs( const days = fixtureDays(presets, day0); const stamp = (day: string) => (fieldType === 'date' ? day : `${day}T12:00:00.000Z`); const written: Array<{ day: string; id: string }> = []; + // The smallest record first; if the seed door refuses it (an app rule that + // requires a field in the state the held columns put the row in), every + // field the CRUD derivation synthesizes, and that shape for the rest. + let full = false; for (const day of days) { - const row: Record = { - ...fixtureBody(body, objectDefs.get(c.object!), ++counter), + const row = (wide: boolean): Record => ({ + ...fixtureBody(body, objectDefs.get(c.object!), ++counter, wide), ...(ownerColumn ? { owner_id: principal } : {}), ...Object.fromEntries(held), [c.field!]: stamp(day), + }); + const seedOne = async (wide: boolean): Promise => { + const [made] = await stack.seed(c.object!, [row(wide)]); + if (made?.id == null) throw new Error(`the seed of ${c.object} returned no id`); + return String(made.id); }; try { - const [made] = await stack.seed(c.object!, [row]); - if (made?.id == null) throw new Error(`the seed of ${c.object} returned no id`); - written.push({ day, id: String(made.id) }); + let id: string; + try { + id = await seedOne(full); + } catch (first) { + if (full || written.length > 0) throw first; + full = true; + id = await seedOne(true); + } + written.push({ day, id }); } catch (e) { r.days = days; notRun(`the fixture row for ${day} was refused by the seed door: ${clip((e as Error)?.message, 160)}`); @@ -878,6 +896,7 @@ export async function runDashboardWindowProofs( } } r.days = days; + r.fixture = full ? 'every-field' : 'smallest'; // Read back: the stamp and every held column must read as written. for (const { day, id } of written) { const [got] = await stack.rows(c.object!, { id }); From 78bfcefa4e4ae3ad9b377cae73a27c2a2f477bfd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Oct 2026 02:13:40 +0000 Subject: [PATCH 6/6] =?UTF-8?q?chore(changeset):=20@objectstack/verify=20m?= =?UTF-8?q?inor,=20@objectstack/cli=20minor=20=E2=80=94=20the=20dashboard?= =?UTF-8?q?=20date-window=20family=20(#15953=20slice=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01VoSxBQujKLZKPwK2u5ehQ6 Co-authored-by: Claude --- .../15953-verify-dashboard-window-family.md | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .changeset/15953-verify-dashboard-window-family.md diff --git a/.changeset/15953-verify-dashboard-window-family.md b/.changeset/15953-verify-dashboard-window-family.md new file mode 100644 index 00000000000..144d744b534 --- /dev/null +++ b/.changeset/15953-verify-dashboard-window-family.md @@ -0,0 +1,38 @@ +--- +'@objectstack/verify': minor +'@objectstack/cli': minor +--- + +feat(verify): the dashboard date-window family — every widget a dashboard's `dateRange` binds counts, under each preset of the picker, the rows that window selects, runnable as `os verify --dashboard-windows` (#15953) + +Clause-②: yes + +A seventh derived proof family beside `runCrudVerification`, `runRlsProofs`, `runReferenceShapeProofs`, `runDeleteBehaviorProofs`, `runUniqueScopeProofs` and `runParentWriteReachProofs`. It needs no hand-written test: from the app's own dashboards and datasets, read through the spec's own parse (`DashboardSchema`, `DatasetSchema`), it takes every dashboard that declares a `dateRange` and every widget that window binds, and asks the dataset door what each one counts. + +Where each half of the promise is read from: + +- the field a widget's window lands on is `DashboardWidgetSchema.filterBindings`' contract: `filterBindings.dateRange` names it, `false` opts the widget out (reported `unbound`), and absent means the dashboard's `dateRange.field` (`created_at` when unauthored). It resolves against the object behind the widget's dataset, as the stack registers it, and must be a `date` or `datetime` field; +- a preset's window is `DATE_RANGE_PRESET_MACRO_WINDOWS`, the `{date-macro}` pair each of the 13 `DATE_RANGE_PRESETS` names. It is handed to `POST /analytics/dataset/query` as written (`{ $gte: '{7_days_ago}' }`, `{ $gte: '{month_start}', $lte: '{month_end}' }`), ANDed with the widget's own filter as `runtimeFilter`, and the server resolves the tokens; +- the days a window selects, for the ground truth, are that same window resolved by `@objectstack/core`'s `resolveFilterTokens` on the caller's reference timezone (UTC unless the `ExecutionContext` carries one), first and last day inclusive. + +Per bound widget and count measure, fixture rows are written through the seed door on every preset's first and last day and the day either side (never after today), each stamped at UTC noon (a `date` field takes the day itself), owned by the caller, with every column the widget's filter, the dataset's filter and the measure's filter compare held to a value they select, and read back before anything is judged. The widget is asked as the signed-in caller with no window and under every preset, before and after the fixtures, so each count is a difference and the app's own rows cancel out. With no window it must count every fixture row before any window is judged. Under each preset it must count exactly the fixture rows that window selects: a widget that shows nothing there is `blank`, one that counts another number is contradicted, and one whose every preset answers what no window answers is named inert. + +A widget is `proven`, `contradicted` (a preset went blank or counted other rows), `failed` (the door refused or faulted on a windowed query while the widget's own query answered), `inconclusive` (a fixture could not be written or did not read back as written, the widget with no window did not count every fixture row, its own query is refused, the day turned mid-probe, or the caller's timezone is not UTC), `unbound` or `excluded` (`excludedKind` names why: no count measure, a filter the ground truth cannot model, a window on a missing or non-temporal field, an undeclared or unparsed dataset or dashboard, an object the app does not declare or the CRUD derivation cannot synthesize). `contradicted + failed` widgets are the report's `hardFailures`. An average, a sum or a derived measure is counted as excluded; a held column is an equality, an `$in`, a `$nin` or `$ne` on a field that declares its options, or `{current_user_id}`. + +**`@objectstack/verify` — new exports.** + +- `runDashboardWindowProofs(stack, token, config, { now? })` → `DashboardWindowReport`. Run it on a fresh stack of its own, as a principal that reads the dashboards' objects: it writes fixture rows. +- `deriveDashboardWindowCases(config)`, `formatDashboardWindowReport(report)`. +- Types: `DashboardWindowExclusion`, `DashboardWindowMeasureCase`, `DashboardWindowCase`, `DashboardWindowDashboard`, `DashboardWindowPreset`, `DashboardWindowAnswer`, `DashboardWindowPresetProbe`, `DashboardWindowMeasureResult`, `DashboardWindowWidgetStatus`, `DashboardWindowWidgetResult`, `DashboardWindowReport`, `DashboardWindowProofOptions`. + +```ts +import { bootStack, runDashboardWindowProofs, formatDashboardWindowReport } from '@objectstack/verify'; + +const stack = await bootStack(myApp); +const report = await runDashboardWindowProofs(stack, await stack.signIn(), myApp); +console.log(formatDashboardWindowReport(report)); // report.presets names what each preset handed the door +expect(report.summary.hardFailures).toBe(0); +await stack.stop(); +``` + +**`@objectstack/cli` — `os verify --dashboard-windows`.** Opt-in, like `--unique-scopes` and `--parent-write-reach`, on a fresh boot of its own, as the signed-in admin. A widget that went blank or counted other rows than its window selects, or a windowed query the door refused or faulted, counts in `hardFailures` and fails the run. The `--json` document carries the report as `dashboardWindows` when the flag is given; without the flag the run and its document are unchanged.