From fbdf9fd70d693d0da6bbe31f1d06fd672be1cf20 Mon Sep 17 00:00:00 2001 From: fretchen Date: Fri, 6 Nov 2026 06:28:53 +0100 Subject: [PATCH 1/6] Create bunny-edge-script-observability-kiss-implementation-plan.md --- ...-observability-kiss-implementation-plan.md | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 observability/bunny-edge-script-observability-kiss-implementation-plan.md diff --git a/observability/bunny-edge-script-observability-kiss-implementation-plan.md b/observability/bunny-edge-script-observability-kiss-implementation-plan.md new file mode 100644 index 000000000..2e642acab --- /dev/null +++ b/observability/bunny-edge-script-observability-kiss-implementation-plan.md @@ -0,0 +1,75 @@ +# Observability for `bunny_js` — KISS Implementation Plan + +**Architecture:** script logs → Better Stack (ingest + query + alert) → email. +Everything runs on the Better Stack free plan (3 GB logs / month, 10 monitors + heartbeats, email alerts). +No phone push, no webhooks, no self-hosted components. + +--- + +## 1. Better Stack setup (one-time, ~10 min) + +- [ ] Create account → create a **Source** (e.g. `bunny-web-agent`). Note the **ingesting host URL** and the **source token**. +- [ ] Create a **Log alert**: saved query `level = error` → notify via **email**. This one rule covers every `logger.error` call site. +- [ ] Create an **uptime monitor**: `GET https:///search` with no payment headers, **expected status code 402** (the unpaid challenge = proof that routing, script init, and the 402 machinery all work). Interval 5 min. This is the outside-in check that catches "the whole pipeline is dead" — the failure mode log-based alerting structurally cannot see. + +## 2. Code changes (~15 lines) + +**`logger.ts`** — keep the pino-shaped API and the console shim (dashboard debugging stays free); add a buffer and a flush: + +```ts +const lines: object[] = []; + +// in info/warn/error: push { level, message, ...fields } into lines +// (plus the existing console output) + +export function flushLogs(): void { + const batch = lines.splice(0); + if (!batch.length) return; + fetch(BETTERSTACK_INGEST_HOST, { + method: "POST", + headers: { + Authorization: `Bearer ${Deno.env.get("BETTERSTACK_SOURCE_TOKEN")}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(batch), // array of events + }).catch(() => {}); // best-effort: never blocks, no retry +} +``` + +**`routes.ts`** — flush per request, after the handler, before returning the response: + +```ts +const res = await handler(req); +flushLogs(); +return res; +``` + +## 3. Environment (per the repo's own secret criterion: authenticates ⇒ secret) + +| Variable | bunny kind | Why | +| --- | --- | --- | +| `BETTERSTACK_SOURCE_TOKEN` | `--secret` | authenticates the ingest | +| `BETTERSTACK_INGEST_HOST` | plain | public host, inert without the token | + +```bash +bunny scripts env set BETTERSTACK_SOURCE_TOKEN --secret +bunny scripts env set BETTERSTACK_INGEST_HOST +``` + +## 4. Verification + +- [ ] Deploy with a temporary test route (or dev server) that logs `level: "error"` once → line appears in Better Stack, alert email arrives. +- [ ] `GET /search` without payment → monitor reports up (expecting 402). +- [ ] Remove the test trigger; confirm the query `level = error` is empty on a normal traffic day. + +## 5. Convention changes (vs the Scaleway setup) + +- `logger.error` semantics unchanged: "ours, and should page" — now it pages by email via the single `level = error` alert. +- The `alert_coverage` test simplifies: instead of "every `logger.error` has a matching Loki rule", assert **every `logger.error` writes `{ level: "error" }`** (the one query covers all call sites). +- The pino-shaped console shim stays as-is; Better Stack replaces Loki + alert rules + the syslog forwarding ideas entirely. + +## 6. Accepted trade-offs (deliberate) + +- **Fire-and-forget:** occasional lost log lines on isolate eviction — no retry, no queue. The uptime monitor covers the systemic case; a lost line from a €0.01 query is not worth infrastructure. +- **Minutes, not seconds:** alert latency = query evaluation + email delivery. Inherent to email-grade alerting. +- **Single-platform risk:** if Better Stack itself is down, alerts are silent. Accepted at personal scale; the escape hatch is a direct webhook channel (ntfy/Telegram) if this ever stops being acceptable. \ No newline at end of file From a25915bf12bf11c6d4021b2a3491bd2dd8e433f1 Mon Sep 17 00:00:00 2001 From: fretchen Date: Fri, 6 Nov 2026 14:32:14 +0100 Subject: [PATCH 2/6] Update bunny-edge-script-observability-kiss-implementation-plan.md --- ...-observability-kiss-implementation-plan.md | 99 +++++++++++++------ 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/observability/bunny-edge-script-observability-kiss-implementation-plan.md b/observability/bunny-edge-script-observability-kiss-implementation-plan.md index 2e642acab..7523a5be0 100644 --- a/observability/bunny-edge-script-observability-kiss-implementation-plan.md +++ b/observability/bunny-edge-script-observability-kiss-implementation-plan.md @@ -8,42 +8,75 @@ No phone push, no webhooks, no self-hosted components. ## 1. Better Stack setup (one-time, ~10 min) -- [ ] Create account → create a **Source** (e.g. `bunny-web-agent`). Note the **ingesting host URL** and the **source token**. -- [ ] Create a **Log alert**: saved query `level = error` → notify via **email**. This one rule covers every `logger.error` call site. -- [ ] Create an **uptime monitor**: `GET https:///search` with no payment headers, **expected status code 402** (the unpaid challenge = proof that routing, script init, and the 402 machinery all work). Interval 5 min. This is the outside-in check that catches "the whole pipeline is dead" — the failure mode log-based alerting structurally cannot see. +- [ ] Create account → create a **Source** (e.g. `bunny-web-agent`). Note the **ingesting host URL** and the **source token**. The ingest endpoint accepts a single event or a JSON **array** of events with `Authorization: Bearer `. +- [ ] Create a **Log alert** (Log SQL): saved query `level = error` → notify via **email**. This one rule covers every `logger.error` call site. +- [ ] Create **three uptime monitors** — one per script, since each twin is its own deployment: + - **searchapi:** `GET https:///search` with no payment headers, **expected status code 402**. The unpaid challenge is proof that routing, script init, and the shared 402 machinery all work — the outside-in check that catches "the whole pipeline is dead", the failure mode log-based alerting structurally cannot see. + - **llm and mcp:** `GET https:///openapi.json`, **expected status code 200**. This proves that each script's isolate boots and its routing works. They deliberately do not probe the 402 machinery: that code (`x402_server.ts`) is shared with searchapi, whose monitor already approves it, and a POST-without-payment 402 probe is just extra fragility for no additional coverage. Interval 5 min each. Three of the free plan's ten monitors. -## 2. Code changes (~15 lines) +## 2. Code changes (~15 lines, plus one unit test) -**`logger.ts`** — keep the pino-shaped API and the console shim (dashboard debugging stays free); add a buffer and a flush: +**`logger.ts`** — keep the pino-shaped API and the console shim (dashboard debugging stays free); add a buffer and an awaited flush: ```ts const lines: object[] = []; // in info/warn/error: push { level, message, ...fields } into lines -// (plus the existing console output) +// (plus the existing console output); emit() already builds the line — +// reuse it and map fields for Better Stack: msg -> message, time -> timestamp -export function flushLogs(): void { +export async function flushLogs(): Promise { const batch = lines.splice(0); - if (!batch.length) return; - fetch(BETTERSTACK_INGEST_HOST, { - method: "POST", - headers: { - Authorization: `Bearer ${Deno.env.get("BETTERSTACK_SOURCE_TOKEN")}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(batch), // array of events - }).catch(() => {}); // best-effort: never blocks, no retry + if (!batch.length) return; // common path: zero latency, no subrequest + try { + const res = await fetch(process.env.BETTERSTACK_INGEST_HOST ?? "", { + method: "POST", + headers: { + Authorization: `Bearer ${process.env.BETTERSTACK_SOURCE_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(batch), // array of events + }); + if (!res.ok) { + console.error(JSON.stringify({ level: "error", msg: "log flush failed", status: res.status })); + } + } catch (err) { + // swallowed: an ingest outage must never fail the user's request — but it is + // console.error'd so it stays visible in the bunny dashboard (the free tier of truth) + console.error(JSON.stringify({ level: "error", msg: "log flush failed", err: String(err) })); + } } ``` -**`routes.ts`** — flush per request, after the handler, before returning the response: +Design points, deliberately: + +- **`await` on the non-empty path.** The buffer is almost only non-empty on errors, so the await costs ~nothing on the common path and nothing at all when it is empty; fire-and-forget on the edge is closer to fire-and-always-lost than "occasional" — there is no `waitUntil` in Bunny's SDK, and the isolate's guaranteed lifetime ends the moment the handler returns. +- **`process.env`, not `Deno.env.get`** — the runtime shims pin `process` and every entry imports them first; bare globals are not guaranteed (bunny Deno 1.x). +- **Field mapping, not pass-through** (`msg → message`, `time → timestamp`): Better Stack renders the message from `message` and parses `timestamp` natively; relying on undocumented auto-detection of `msg`/`time` would make the query and the event rendering fragile. +- Flush failure is visible (console.error → bunny dashboard) but never blocks or fails the response. + +**Entry files (`searchapi.ts`, `llm.ts`, `mcp.ts`) — not the route files.** There is no single `routes.ts`: three handler files with ten-plus early `return` paths each (OPTIONS, discovery, 402, 404). Editing returns per file is unmaintainable and misses paths. Instead, one decorator, applied once per entry: ```ts -const res = await handler(req); -flushLogs(); -return res; +// logger.ts (or a small flush.ts) +export function withLogFlush(handler: (req: Request) => Promise) { + return async (req: Request): Promise => { + try { + return await handler(req); + } finally { + await flushLogs(); // covers early returns AND handler throws + } + }; +} + +// each entry file +BunnySDK.net.http.serve(withLogFlush(handleRequest)); ``` +Subrequest budget: +1 per request at most (only when the buffer is non-empty) — well within the 50/request cap. + +**New unit test (`bunny_js/test/`)** — the buffer/flush mechanics, not a tautology. The old `alert_coverage` idea ("assert every `logger.error` writes `level: "error"`") is true by construction of the single `emit` function and can never catch anything, so there is no `alert_coverage` test for bunny_js. Instead, one small test asserts: buffer fills from `logger.error`; a single non-empty flush posts one batched request (mock fetch); an empty buffer does no fetch at all; a failing fetch does not throw out of the decorator. + ## 3. Environment (per the repo's own secret criterion: authenticates ⇒ secret) | Variable | bunny kind | Why | @@ -51,25 +84,35 @@ return res; | `BETTERSTACK_SOURCE_TOKEN` | `--secret` | authenticates the ingest | | `BETTERSTACK_INGEST_HOST` | plain | public host, inert without the token | +**Per script — all three.** `bunny scripts env set` acts on the directory-linked script (searchapi) only; llm and mcp need their values set against their own script ids explicitly. The GitHub Action ships code, not env — a script without its env vars silently logs only to the dashboard. + ```bash +# searchapi (linked): bunny scripts env set BETTERSTACK_SOURCE_TOKEN --secret -bunny scripts env set BETTERSTACK_INGEST_HOST +bunny scripts env set BETTERSTACK_INGEST_HOST +# llm + mcp: same two commands against / ``` ## 4. Verification -- [ ] Deploy with a temporary test route (or dev server) that logs `level: "error"` once → line appears in Better Stack, alert email arrives. -- [ ] `GET /search` without payment → monitor reports up (expecting 402). +- [ ] Deploy with a temporary test route (or the dev server) that logs `level: "error"` once → line appears in Better Stack with the message rendered (confirms the `msg → message` mapping) and the alert email arrives. Per the repo doctrine: a rule that has never fired is not known to work. +- [ ] `GET /search` without payment → searchapi monitor reports up (expecting 402); `GET /openapi.json` on llm and mcp → their monitors report up (expecting 200). +- [ ] Confirm a normal request produces the flush only when log lines exist (dev server) — no fetch per request on the happy path. - [ ] Remove the test trigger; confirm the query `level = error` is empty on a normal traffic day. +- [ ] After one week, check the Better Stack volume dashboard: if the buffer's info lines ever threaten the 3 GB budget, the lever is to buffer warn+error only (the alert needs errors anyway) — info stays console-only. Decide on data, not now. ## 5. Convention changes (vs the Scaleway setup) -- `logger.error` semantics unchanged: "ours, and should page" — now it pages by email via the single `level = error` alert. -- The `alert_coverage` test simplifies: instead of "every `logger.error` has a matching Loki rule", assert **every `logger.error` writes `{ level: "error" }`** (the one query covers all call sites). -- The pino-shaped console shim stays as-is; Better Stack replaces Loki + alert rules + the syslog forwarding ideas entirely. +- `logger.error` semantics unchanged: "ours, and should page" — now it pages by email via the single `level = error` alert. A caller's fault is `logger.warn`, as before. +- The pino-shaped console shim stays as-is; Better Stack replaces Loki + alert rules + the syslog forwarding ideas entirely. No `alert_coverage` test for bunny_js (see §2). +- **Three docs become wrong the moment this ships — update them in the same change:** + - `bunny_js/README.md`: the bullet "bunny logs go to the dashboard … the `logger.error` alert convention does not apply" → point to this setup. + - `bunny_js/src/logger.ts` header comment: same claim, same fix. + - `observability/README.md`: "(The llm/search/mcp sellers moved to bunny_js — Edge Script logging has no Loki rules here)" → note they alert via Better Stack instead (this plan), Loki stays Scaleway-only. ## 6. Accepted trade-offs (deliberate) -- **Fire-and-forget:** occasional lost log lines on isolate eviction — no retry, no queue. The uptime monitor covers the systemic case; a lost line from a €0.01 query is not worth infrastructure. +- **No retry, no queue:** a genuinely lost line happens only if the isolate dies between error and flush — the uptime monitor covers the systemic case; a retry queue for a €0.01 query is not worth infrastructure. The awaited non-empty flush removes the much more mundane loss mode (fetch cancelled at response time). - **Minutes, not seconds:** alert latency = query evaluation + email delivery. Inherent to email-grade alerting. -- **Single-platform risk:** if Better Stack itself is down, alerts are silent. Accepted at personal scale; the escape hatch is a direct webhook channel (ntfy/Telegram) if this ever stops being acceptable. \ No newline at end of file +- **Single-platform risk:** if Better Stack itself is down, alerts are silent — but flush failures surface in the bunny dashboard. Accepted at personal scale; the escape hatch is a direct webhook channel (ntfy/Telegram) if this ever stops being acceptable. +- **llm/mcp monitors check liveness, not payments:** their 402 machinery is covered transitively via the shared `x402_server.ts` through the searchapi monitor. If the three scripts ever diverge in their payment paths, revisit (a POST 402-monitor each). From d3c59b60a3b24c7646bd3a75542eee09edfc81b5 Mon Sep 17 00:00:00 2001 From: fretchen Date: Fri, 6 Nov 2026 16:12:49 +0100 Subject: [PATCH 3/6] first tries --- bunny_js/README.md | 16 +- bunny_js/src/llm.ts | 3 +- bunny_js/src/logger.ts | 91 ++++++++- bunny_js/src/mcp.ts | 3 +- bunny_js/src/searchapi.ts | 3 +- bunny_js/test/logger.test.ts | 178 ++++++++++++++++++ observability/README.md | 12 +- ...-observability-kiss-implementation-plan.md | 124 +++++++----- 8 files changed, 373 insertions(+), 57 deletions(-) create mode 100644 bunny_js/test/logger.test.ts diff --git a/bunny_js/README.md b/bunny_js/README.md index c31dacc1f..37f2492fa 100644 --- a/bunny_js/README.md +++ b/bunny_js/README.md @@ -19,8 +19,12 @@ Only `scw_js`'s `llmx402cron` remains, claiming the channels these twins charge. the shared seller stack (`x402_server.ts`, channel storage/sync), the services (`search_service.ts`, `web_fetch_service.ts` with the SSRF defence), schemas, discovery, favicon, `openapi.search.json`. Two deliberate deviations: - - `logger.ts` is a pino-shaped console shim — pino is a Node-runtime library; bunny logs go - to the dashboard, and the observability `logger.error` alert convention does not apply. + - `logger.ts` is a pino-shaped console shim — pino is a Node-runtime library; the lines go to + the bunny dashboard AND, since the Cockpit integration, to Scaleway's custom Loki data + source: `withLogFlush()` wraps each entry handler, so buffered lines are pushed per + request to Cockpit, where one datasource-managed alert rule on `level="error"` pages by + email (the `logger.error` convention applies again — see + [`observability/bunny-edge-script-observability-kiss-implementation-plan.md`](../observability/bunny-edge-script-observability-kiss-implementation-plan.md)). - `utils.ts` ports only the platform-neutral half (CORS/parseJsonBody); the ScwResponse envelope helpers are not ported — there is no Scaleway envelope here. - `src/runtime-shims.ts` — pins the Node `Buffer`/`process` globals the ported code assumes, @@ -66,6 +70,8 @@ it is inert without another credential, plain — and plain stays inspectable wh | `FACILITATOR_URL` | plain (code default ok) | `https://facilitator.fretchen.eu` | | `LLM_WITHDRAW_DELAY_SECONDS` | plain (code default ok) | 86400 — must match production | | `SEARCH_SERVICE_URL` | plain | advertised `resource.url` base; the bunny trial URL while trialing, production default otherwise | +| `SCW_COCKPIT_PUSH_TOKEN` | secret (`--secret`) | Cockpit push-only token — authenticates the Loki log flush to the observability data source (see the observability README/implementation plan) | +| `COCKPIT_LOKI_URL` | plain | push URL of the custom Cockpit logs data source — public and inert without the token | ## Known limits / decisions @@ -158,8 +164,10 @@ and the `.bunny/script.json` link comes out of the create step. One-time sequenc other twins take their id from the create output (or `bunny scripts list`) instead. 2. Upload the env from `.env` with `bunny scripts env set`, marking the sensitive keys with `--secret` (RECEIVER_AUTHORIZER_PRIVATE_KEY, BRAVE_API_KEY, MISTRAL_API_KEY, - SCW_ACCESS_KEY, SCW_SECRET_KEY, RPC_URL_*). Skip `TEST_WALLET_PRIVATE_KEY` — buyer-side - only, the server must never see it. Verify with `bunny scripts env list`. Per script: + SCW_ACCESS_KEY, SCW_SECRET_KEY, RPC_URL_*, SCW_COCKPIT_PUSH_TOKEN). Skip + `TEST_WALLET_PRIVATE_KEY` — buyer-side only, the server must never see it. Verify with + `bunny scripts env list`. Per script — all three also get the Cockpit observability pair + (`SCW_COCKPIT_PUSH_TOKEN` as `--secret`, `COCKPIT_LOKI_URL` plain): - **searchapi** — the channel stack + `BRAVE_API_KEY` (web_search/fetch) + `SCW_S3_*`. - **llm** — the channel stack + `MISTRAL_API_KEY` (chat_completion) + `SCW_S3_*`. - **mcp** — the union: both `BRAVE_API_KEY` and `MISTRAL_API_KEY` (its bundle embeds diff --git a/bunny_js/src/llm.ts b/bunny_js/src/llm.ts index 30b076188..7dbcd41c8 100644 --- a/bunny_js/src/llm.ts +++ b/bunny_js/src/llm.ts @@ -7,6 +7,7 @@ */ import "./runtime-shims.js"; import * as BunnySDK from "@bunny.net/edgescript-sdk"; +import { withLogFlush } from "./logger.js"; import { handleLLMRequest } from "./llm_routes.js"; -BunnySDK.net.http.serve(handleLLMRequest); +BunnySDK.net.http.serve(withLogFlush("bunny-llm-agent", handleLLMRequest)); diff --git a/bunny_js/src/logger.ts b/bunny_js/src/logger.ts index 8472bec11..b0a6b58f1 100644 --- a/bunny_js/src/logger.ts +++ b/bunny_js/src/logger.ts @@ -4,11 +4,31 @@ * Deno isolate bundle. The call surface is kept pino-shaped — `logger.info(obj, msg)`, * `logger.error({ err }, msg)` — because the ported files use that convention verbatim. * - * The `logger.error` convention (observability alert rules key on it) does not carry over - * to bunny: the edge script's logs live in the bunny dashboard, not Loki, and shipping - * structured logs out is a documented follow-up if this twin graduates. + * On top of the console output (the bunny dashboard's own log view), every line is buffered + * and flushed to Scaleway Cockpit's custom Loki data source at the end of the request + * (observability/bunny-edge-script-observability-kiss-implementation-plan.md), where a + * single datasource-managed alert rule on `level="error"` pages by email — the same + * `logger.error` = "ours, and should be alertable" convention the Scaleway functions have. */ +type BufferedLine = { level: string; event: object }; + +const lines: BufferedLine[] = []; + +/** Per-script `service` label (searchapi/llm/mcp differ); set by withLogFlush at entry setup. */ +let serviceLabel = "bunny-js"; + +/** + * Loki rejects duplicate timestamps within one push (`duplicate timestamp for series`) by + * dropping the whole batch, so a monotonic per-isolate counter keeps same-millisecond lines + * strictly increasing. String concat for the nanosecond part: `Date.now() * 1e6` as a number + * exceeds Number.MAX_SAFE_INTEGER. + */ +let nsCounter = 0; +function nsTimestamp(): string { + return `${Date.now()}${String(++nsCounter).padStart(6, "0")}`; +} + function emit(level: string, arg: unknown, maybeMsg?: string): void { const [obj, msg] = typeof arg === "string" ? [{}, arg] : [(arg as object) ?? {}, maybeMsg ?? ""]; // pino prints `level then message then context`; JSON-lines keeps the same fields. @@ -20,6 +40,71 @@ function emit(level: string, arg: unknown, maybeMsg?: string): void { } else { console.log(line); } + // The same fields go to the Cockpit flush, under the message key the alert/query side + // expects; everything beyond `message` rides in the line payload (queryable via `| json`). + lines.push({ level, event: { message: msg, ...obj } }); +} + +/** + * Pushes the buffered lines to Cockpit as one Loki batch. Best-effort by design — no retry, + * no queue — but AWAITED when the buffer is non-empty: the edge runtime has no waitUntil, so + * an un-awaited fetch racing the handler's return would be lost most of the time. A failed + * push is console.error'd, so an ingest outage stays visible in the bunny dashboard instead + * of silently eating every alert. Auth is Scaleway's documented `X-Token` header; unpushed + * when URL/token are unset (local dev, tests). + */ +export async function flushLogs(): Promise { + const batch = lines.splice(0); + const url = process.env.COCKPIT_LOKI_URL; + const token = process.env.SCW_COCKPIT_PUSH_TOKEN; + if (batch.length === 0 || !url || !token) { + return; + } + const byLevel = new Map(); + for (const { level, event } of batch) { + const vals = byLevel.get(level) ?? []; + vals.push([nsTimestamp(), JSON.stringify(event)]); + byLevel.set(level, vals); + } + try { + const res = await fetch(url, { + method: "POST", + // Labels stay {service, level} only — high-cardinality labels are the classic Loki trap. + headers: { "X-Token": token, "Content-Type": "application/json" }, + body: JSON.stringify({ + streams: [...byLevel].map(([level, values]) => ({ + stream: { service: serviceLabel, level }, + values, + })), + }), + }); + if (!res.ok) { + console.error( + JSON.stringify({ level: "error", msg: "log flush failed", status: res.status }), + ); + } + } catch (err) { + console.error(JSON.stringify({ level: "error", msg: "log flush failed", err: String(err) })); + } +} + +/** + * Wraps a script entry handler so the buffered lines reach Cockpit however the request ended — + * early returns (OPTIONS, discovery, the 404, the unpaid 402) and throws alike, which is why + * this wraps the entry's top-level handler instead of touching any route file's returns. + */ +export function withLogFlush( + service: string, + handler: (request: Request) => Promise, +): (request: Request) => Promise { + serviceLabel = service; + return async (request: Request): Promise => { + try { + return await handler(request); + } finally { + await flushLogs(); + } + }; } export const logger = { diff --git a/bunny_js/src/mcp.ts b/bunny_js/src/mcp.ts index d7a159987..e240a262c 100644 --- a/bunny_js/src/mcp.ts +++ b/bunny_js/src/mcp.ts @@ -7,6 +7,7 @@ */ import "./runtime-shims.js"; import * as BunnySDK from "@bunny.net/edgescript-sdk"; +import { withLogFlush } from "./logger.js"; import { handleMCPRequest } from "./mcp_routes.js"; -BunnySDK.net.http.serve(handleMCPRequest); +BunnySDK.net.http.serve(withLogFlush("bunny-mcp-agent", handleMCPRequest)); diff --git a/bunny_js/src/searchapi.ts b/bunny_js/src/searchapi.ts index c4c353be8..1d6fb8d96 100644 --- a/bunny_js/src/searchapi.ts +++ b/bunny_js/src/searchapi.ts @@ -9,6 +9,7 @@ */ import "./runtime-shims.js"; import * as BunnySDK from "@bunny.net/edgescript-sdk"; +import { withLogFlush } from "./logger.js"; import { handleRequest } from "./searchapi_routes.js"; -BunnySDK.net.http.serve(handleRequest); +BunnySDK.net.http.serve(withLogFlush("bunny-web-agent", handleRequest)); diff --git a/bunny_js/test/logger.test.ts b/bunny_js/test/logger.test.ts new file mode 100644 index 000000000..44cd40877 --- /dev/null +++ b/bunny_js/test/logger.test.ts @@ -0,0 +1,178 @@ +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { flushLogs, logger, withLogFlush } from "../src/logger.js"; + +// The flush mechanics only — no network. fetch is stubbed per test; env vars are set/cleared +// so a test can exercise both ends deliberately (hermetic even on a machine whose shell +// happens to export COCKPIT_* variables). +const realFetch = globalThis.fetch; +const mockFetch = vi.fn(); + +const TEST_URL = "https://logs.example.invalid/loki/api/v1/push"; + +beforeEach(() => { + mockFetch.mockReset(); + mockFetch.mockResolvedValue({ ok: true, status: 204 }); + globalThis.fetch = mockFetch as unknown as typeof fetch; + process.env.COCKPIT_LOKI_URL = TEST_URL; + process.env.SCW_COCKPIT_PUSH_TOKEN = "test-token"; +}); + +afterEach(async () => { + globalThis.fetch = realFetch; + delete process.env.COCKPIT_LOKI_URL; + delete process.env.SCW_COCKPIT_PUSH_TOKEN; + // Drain whatever a test left behind, with the real fetch restored and no config: a no-op. + await flushLogs(); +}); + +function singlePushBody() { + expect(mockFetch).toHaveBeenCalledTimes(1); + const [calledUrl, init] = mockFetch.mock.calls[0] as [string, RequestInit]; + expect(calledUrl).toBe(TEST_URL); + return { + init, + body: JSON.parse(init.body as string), + }; +} + +describe("flushLogs", () => { + test("batches one Loki push per flush, X-Token-authenticated, with the fields the alert side keys on", async () => { + logger.error({ err: "boom" }, "Request handler error"); + logger.error({ err: "boom2" }, "Paid request handler error"); + await flushLogs(); + + const { init, body } = singlePushBody(); + const headers = init.headers as Record; + expect(headers["X-Token"]).toBe("test-token"); + expect(headers["Content-Type"]).toBe("application/json"); + + // One stream for the two error lines; the alert rule keys on exactly these labels. + expect(body.streams).toHaveLength(1); + const stream = body.streams[0]; + expect(stream.stream).toEqual({ service: "bunny-js", level: "error" }); + expect(stream.values).toHaveLength(2); + expect(JSON.parse(stream.values[0][1])).toEqual({ + message: "Request handler error", + err: "boom", + }); + expect(JSON.parse(stream.values[1][1])).toEqual({ + message: "Paid request handler error", + err: "boom2", + }); + + // pino-shaped string call: the string IS the message. + mockFetch.mockClear(); + logger.warn("brave timeout"); + await flushLogs(); + const second = singlePushBody().body.streams[0]; + expect(second.stream.level).toBe("warn"); + expect(JSON.parse(second.values[0][1])).toEqual({ message: "brave timeout" }); + }); + + test("same-millisecond lines get strictly increasing nanosecond timestamps as strings", async () => { + logger.error({ attempt: 1 }, "same ms"); + logger.error({ attempt: 2 }, "same ms"); + logger.error({ attempt: 3 }, "same ms"); + await flushLogs(); + + const { body } = singlePushBody(); + const [ts1, ts2, ts3] = body.streams[0].values.map(([ts]: [string]) => ts); + expect(ts1).toMatch(/^\d{19}$/); // nanoseconds, not the ms Date.now() would give + expect(BigInt(ts2)).toBeGreaterThan(BigInt(ts1)); + expect(BigInt(ts3)).toBeGreaterThan(BigInt(ts2)); + }); + + test("levels split into separate streams; labels stay {service, level} only", async () => { + logger.info("quiet"); + logger.warn("caller's fault"); + logger.error("ours"); + await flushLogs(); + + const { body } = singlePushBody(); + expect(body.streams).toHaveLength(3); + const byLevel = new Map( + body.streams.map((s: { stream: { level: string } }) => [s.stream.level, s.stream] as const), + ); + expect([...byLevel.keys()].sort()).toEqual(["error", "info", "warn"]); + for (const labels of byLevel.values()) { + expect(Object.keys(labels).sort()).toEqual(["level", "service"]); + } + }); + + test("empty buffer sends no fetch at all (the happy-path request stays free)", async () => { + logger.error("seen"); + await flushLogs(); + expect(mockFetch).toHaveBeenCalledTimes(1); + mockFetch.mockClear(); + + await flushLogs(); + expect(mockFetch).not.toHaveBeenCalled(); + }); + + test("a rejected fetch never throws out of flushLogs, and surfaces in the console", async () => { + mockFetch.mockRejectedValue(new Error("network gone")); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + logger.error({ err: "x" }, "Request handler error"); + try { + await expect(flushLogs()).resolves.toBeUndefined(); + // mockRestore would reset the spy's history — assert while the spy still records. + expect( + consoleError.mock.calls.filter((c) => /log flush failed/.test(String(c[0]))), + ).toHaveLength(1); + } finally { + consoleError.mockRestore(); + } + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + + test("a non-2xx push is reported, and an ingest outage never throws", async () => { + mockFetch.mockResolvedValue({ ok: false, status: 401 }); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + logger.error({ err: "x" }, "Request handler error"); + let flushFailureLine: unknown; + try { + await expect(flushLogs()).resolves.toBeUndefined(); + const failure = consoleError.mock.calls + .map((c) => c[0]) + .find((l) => /log flush failed/.test(String(l))); + expect(failure).toBeDefined(); + flushFailureLine = failure; + } finally { + consoleError.mockRestore(); + } + expect(JSON.parse(flushFailureLine as string)).toMatchObject({ + msg: "log flush failed", + status: 401, + }); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); +}); + +describe("withLogFlush", () => { + test("flushes when the handler returns early — the wrapper, not per-return call sites", async () => { + const handler = vi.fn(async () => { + logger.error({ path: "/nope" }, "Route not found"); + return new Response("404", { status: 404 }); + }); + const wrapped = withLogFlush("bunny-web-agent", handler); + const res = await wrapped(new Request("https://x.example/nope")); + expect(res.status).toBe(404); + expect(mockFetch).toHaveBeenCalledTimes(1); + + // The per-script service label from the wrapper reaches the stream labels. + const { body } = singlePushBody(); + expect(body.streams[0].stream).toEqual(expect.objectContaining({ service: "bunny-web-agent" })); + }); + + test("flushes even when the handler throws — and does not swallow the error", async () => { + const handler = vi.fn(async () => { + logger.error({ err: "uncaught" }, "Request handler error"); + throw new Error("handler blew up"); + }); + const wrapped = withLogFlush("bunny-mcp-agent", handler); + await expect(wrapped(new Request("https://x.example/"))).rejects.toThrow("handler blew up"); + expect(mockFetch).toHaveBeenCalledTimes(1); + const { body } = singlePushBody(); + expect(body.streams[0].stream).toEqual(expect.objectContaining({ service: "bunny-mcp-agent" })); + }); +}); diff --git a/observability/README.md b/observability/README.md index 0ee4a498d..bbaca976c 100644 --- a/observability/README.md +++ b/observability/README.md @@ -4,7 +4,13 @@ Project-wide log access and alerting for every Scaleway function in this repo: ` `walletreportcron`, `llmx402cron`, `genimgx402token`, `growthapi`, the comment service and analytics. It belongs to no single service. Cockpit is scoped to the Scaleway _project_, so one token and one set of rules cover all of them. (The llm/search/mcp sellers moved -to bunny_js — Edge Script logging has no Loki rules here.) +to bunny_js — they push to a **custom logs data source** in the same Cockpit: the scripts buffer +their `logger` lines and flush them per request to the Loki push API with a push-only Cockpit +token; a single **datasource-managed** alert rule on `{level="error"}` pages by email. That rule +is created by hand in the Cockpit Grafana and is deliberately outside `alerts/*.yaml` and its +budget test — see [`bunny-edge-script-observability-kiss-implementation-plan.md`](./bunny-edge-script-observability-kiss-implementation-plan.md). +`scripts/logs.ts` still queries only the default (Scaleway) source; read the bunny data source in +Grafana → Explore.) | Path | What | | ------------------- | -------------------------------------------------------------------------------------------------------- | @@ -48,7 +54,9 @@ npx tsx scripts/alerts.ts --delete payments (`ServicesNeedAttention`) and `payments.yaml` two (`PaymentCronNeedsAttention`, `FacilitatorNeedsAttention`). A new failure mode is a new alternative in the matching rule's line filter, never a new rule; `test/alerts_budget.test.ts` fails if the count grows. The comment at the -top of each file explains the budget and what each branch of the filter means. +top of each file explains the budget and what each branch of the filter means. One rule exists +outside this mechanism: the hand-created, datasource-managed bunny rule (see the intro) — a +deliberate fourth billable rule this test cannot see, documented in the implementation plan. **Push only what is committed.** `--push` refuses a file with uncommitted changes. On 2026-10-04 the three-rule consolidation was live from a file that had never been committed, git still held the diff --git a/observability/bunny-edge-script-observability-kiss-implementation-plan.md b/observability/bunny-edge-script-observability-kiss-implementation-plan.md index 7523a5be0..485fa08b4 100644 --- a/observability/bunny-edge-script-observability-kiss-implementation-plan.md +++ b/observability/bunny-edge-script-observability-kiss-implementation-plan.md @@ -1,59 +1,89 @@ # Observability for `bunny_js` — KISS Implementation Plan -**Architecture:** script logs → Better Stack (ingest + query + alert) → email. -Everything runs on the Better Stack free plan (3 GB logs / month, 10 monitors + heartbeats, email alerts). -No phone push, no webhooks, no self-hosted components. +**Architecture:** script logs → Scaleway Cockpit (custom Loki data source, `fr-par`) → datasource-managed alert rule → Scaleway Alert manager → email. +Logs from the remaining Scaleway functions (cron, growthapi, facilitator later) land in the same Cockpit **natively and free** — one Grafana for the whole stack. +No phone push, no webhooks, no self-hosted components, no third-party log vendor. **No uptime/heartbeat layer** — there is none today either (the repo's only "health check" is growth-agent's own `/health` route; nothing ever probes the twins from outside), and log-based alerting alone is the accepted scope. The known blind spot this leaves — "the whole pipeline is dead" produces *no* error lines, so no alert — is accepted for now (see trade-offs). + +**Why Cockpit over Better Stack:** EU jurisdiction (French company, no CLOUD Act ambiguity); unified view with the Scaleway half of the stack; retention 7 days free vs. Better Stack's 3; ingestion is metered but rounds to ~€0 at this volume (custom-data ingestion is billed per GB; pushing your own data is not free-tier, just negligible here). --- -## 1. Better Stack setup (one-time, ~10 min) +## 1. Cockpit setup (one-time, ~15 min) + +- [x] Console → **Cockpit → Data sources** → create a custom **logs data source** (`bunny-logs`) in region `fr-par`. Push URL: `https://db123d37-9094-4e15-badc-706071190ed7.logs.cockpit.fr-par.scw.cloud/loki/api/v1/push`. +- [x] Create a **Cockpit token with Push Logs permission** (`push_logs` scope, same region as the data source). Called it `bunny_push_logs`; env var `SCW_COCKPIT_PUSH_TOKEN` (NOT `COCKPIT_LOGS_TOKEN` — `observability/.env` already has `SCW_COCKPIT_LOGS_TOKEN`, the read-only query token; the names must not collide). Lives in `bunny_js/.env`. +- [ ] **Alerts tab → enable the alert manager in the same region** as the data source (region must match, otherwise alerts are not delivered). +- [ ] **Contacts → Add email**; keep "Resolved notifications" ticked. Scaleway sends the mail itself — **no TEM needed** for alerting. +- [ ] In the Cockpit Grafana: **Alerting → New alert rule** — must be **datasource-managed** (Grafana-managed alerting is *not supported* in Cockpit). Query on the Loki data source: + ```logql + sum(count_over_time({service=~"bunny-web-agent|bunny-llm-agent|bunny-mcp-agent", level="error"}[5m])) > 0 + ``` + One rule covers every `logger.error` call site in all three twins. (Per-script `service` labels — see §3 — so a query can distinguish the twins; the rule itself stays single.) +- [ ] **Rule budget note:** the repo deliberately caps ruler rules at three (Scaleway bills each active rule ~€0.015/day, `observability/test/alerts_budget.test.ts` enforces it for `observability/alerts/*.yaml`). This rule is a fourth billed one, created by hand in Grafana where that test cannot see it — the exact cost should be verified in the Cockpit FAQ/pricing before relying on it. The rule's LogQL lives in this committed file (successor: the observability README), honoring the "push only what is committed" doctrine as far as a Grafana-managed rule allows. + +**Auth, verified 2026-11-06 by curl probe against the live endpoint:** `X-Token: ` → **204**, and (contrary to the old assumption in this plan) `Authorization: Bearer ` → **204** as well; no auth → 401. Both work — the implementation uses Scaleway's documented `X-Token`. A probe line sits in Cockpit as `service="bunny-web-agent", level="info"` (harmless). -- [ ] Create account → create a **Source** (e.g. `bunny-web-agent`). Note the **ingesting host URL** and the **source token**. The ingest endpoint accepts a single event or a JSON **array** of events with `Authorization: Bearer `. -- [ ] Create a **Log alert** (Log SQL): saved query `level = error` → notify via **email**. This one rule covers every `logger.error` call site. -- [ ] Create **three uptime monitors** — one per script, since each twin is its own deployment: - - **searchapi:** `GET https:///search` with no payment headers, **expected status code 402**. The unpaid challenge is proof that routing, script init, and the shared 402 machinery all work — the outside-in check that catches "the whole pipeline is dead", the failure mode log-based alerting structurally cannot see. - - **llm and mcp:** `GET https:///openapi.json`, **expected status code 200**. This proves that each script's isolate boots and its routing works. They deliberately do not probe the 402 machinery: that code (`x402_server.ts`) is shared with searchapi, whose monitor already approves it, and a POST-without-payment 402 probe is just extra fragility for no additional coverage. Interval 5 min each. Three of the free plan's ten monitors. +## 2. Code changes (~25 lines, plus one unit test) — IMPLEMENTED -## 2. Code changes (~15 lines, plus one unit test) +Implemented in `bunny_js/src/logger.ts` (buffer, awaited `flushLogs`, monotonic ns counter, `withLogFlush`), wired into all three entries (`searchapi.ts` / `llm.ts` / `mcp.ts` with `bunny-web-agent` / `bunny-llm-agent` / `bunny-mcp-agent`), covered by `bunny_js/test/logger.test.ts` (8 tests). `npm run check` green (tsc, eslint, prettier, 266 tests). -**`logger.ts`** — keep the pino-shaped API and the console shim (dashboard debugging stays free); add a buffer and an awaited flush: +**`logger.ts`** — keep the pino-shaped API and the console shim (dashboard debugging stays free); add a buffer and an awaited, Loki-shaped flush: ```ts -const lines: object[] = []; +const lines: { level: string; event: object }[] = []; +let nsCounter = 0; // monotonic: two errors in the same ms must not collide -// in info/warn/error: push { level, message, ...fields } into lines -// (plus the existing console output); emit() already builds the line — -// reuse it and map fields for Better Stack: msg -> message, time -> timestamp +// in info/warn/error: push { level, event: { message, ...fields } } into lines +// (plus the existing console output) export async function flushLogs(): Promise { const batch = lines.splice(0); if (!batch.length) return; // common path: zero latency, no subrequest + // one stream per level; labels stay low-cardinality + const byLevel = new Map(); + for (const { level, event } of batch) { + const vals = byLevel.get(level) ?? []; + // Loki wants nanosecond STRINGS: string-concat, never Date.now() * 1e6 + // (that float exceeds Number.MAX_SAFE_INTEGER). The counter guards Loki's + // "duplicate timestamp for series" rejection for same-millisecond lines + // in one batch — a collision loses the WHOLE push, not one line. + vals.push([`${Date.now()}${String(++nsCounter).padStart(6, "0")}`, JSON.stringify(event)]); + byLevel.set(level, vals); + } try { - const res = await fetch(process.env.BETTERSTACK_INGEST_HOST ?? "", { + // Cockpit does NOT use Authorization: Bearer — the token goes in X-Token + // (or X-Scope-OrgID for Promtail-style agents). Bearer = 401. + const res = await fetch(process.env.COCKPIT_LOKI_URL ?? "", { method: "POST", headers: { - Authorization: `Bearer ${process.env.BETTERSTACK_SOURCE_TOKEN}`, + "X-Token": process.env.SCW_COCKPIT_PUSH_TOKEN ?? "", "Content-Type": "application/json", }, - body: JSON.stringify(batch), // array of events + body: JSON.stringify({ + streams: [...byLevel].map(([level, values]) => ({ + stream: { service: SERVICE_LABEL, level }, + values, + })), + }), }); if (!res.ok) { console.error(JSON.stringify({ level: "error", msg: "log flush failed", status: res.status })); } } catch (err) { // swallowed: an ingest outage must never fail the user's request — but it is - // console.error'd so it stays visible in the bunny dashboard (the free tier of truth) + // console.error'd so it stays visible in the bunny dashboard console.error(JSON.stringify({ level: "error", msg: "log flush failed", err: String(err) })); } } ``` -Design points, deliberately: +Key points, deliberately: -- **`await` on the non-empty path.** The buffer is almost only non-empty on errors, so the await costs ~nothing on the common path and nothing at all when it is empty; fire-and-forget on the edge is closer to fire-and-always-lost than "occasional" — there is no `waitUntil` in Bunny's SDK, and the isolate's guaranteed lifetime ends the moment the handler returns. +- **`await` on the non-empty path.** There is no `waitUntil` in Bunny's edge SDK — the isolate's guaranteed lifetime ends when the handler returns, so a fire-and-forget fetch is closer to fire-and-always-lost than "occasional". The buffer is non-empty almost only on errors, so the await costs ~nothing on the common path and zero when the buffer is empty. +- **`X-Token`, not `Authorization: Bearer`** — Scaleway's Cockpit endpoints take the token in the `X-Token` header (their own Grafana integration does the same); a Bearer header gets a 401 and the experiment looks broken for the wrong reason. - **`process.env`, not `Deno.env.get`** — the runtime shims pin `process` and every entry imports them first; bare globals are not guaranteed (bunny Deno 1.x). -- **Field mapping, not pass-through** (`msg → message`, `time → timestamp`): Better Stack renders the message from `message` and parses `timestamp` natively; relying on undocumented auto-detection of `msg`/`time` would make the query and the event rendering fragile. -- Flush failure is visible (console.error → bunny dashboard) but never blocks or fails the response. +- **Monotonic nanosecond suffix** — two same-millisecond lines in one batch would trigger Loki's `duplicate timestamp for series` rejection, which drops the whole push. +- **Per-script `service` label** (via the `SERVICE_LABEL` constant set from the entry: `bunny-web-agent` / `bunny-llm-agent` / `bunny-mcp-agent`) — low-cardinality, distinguishes the twins in queries; `level` is the only other label. **Entry files (`searchapi.ts`, `llm.ts`, `mcp.ts`) — not the route files.** There is no single `routes.ts`: three handler files with ten-plus early `return` paths each (OPTIONS, discovery, 402, 404). Editing returns per file is unmaintainable and misses paths. Instead, one decorator, applied once per entry: @@ -75,44 +105,48 @@ BunnySDK.net.http.serve(withLogFlush(handleRequest)); Subrequest budget: +1 per request at most (only when the buffer is non-empty) — well within the 50/request cap. -**New unit test (`bunny_js/test/`)** — the buffer/flush mechanics, not a tautology. The old `alert_coverage` idea ("assert every `logger.error` writes `level: "error"`") is true by construction of the single `emit` function and can never catch anything, so there is no `alert_coverage` test for bunny_js. Instead, one small test asserts: buffer fills from `logger.error`; a single non-empty flush posts one batched request (mock fetch); an empty buffer does no fetch at all; a failing fetch does not throw out of the decorator. +**New unit test (`bunny_js/test/`)** — the buffer/flush mechanics, not a tautology. The old `alert_coverage` idea ("assert every `logger.error` writes a `level: "error"` stream") is true by construction of the single `emit` function and can never catch anything, so there is no `alert_coverage` test for bunny_js. Instead, one small test asserts: buffer fills from `logger.error`; a single non-empty flush posts one batched `{streams}` payload with `X-Token` auth (mock fetch); an empty buffer does no fetch at all; same-millisecond lines get strictly increasing timestamps (no duplicate); a failing fetch does not throw out of the decorator. + +## 3. Environment (per the repo's own secret criterion: authenticates ⇒ secret) — SET ON ALL THREE SCRIPTS -## 3. Environment (per the repo's own secret criterion: authenticates ⇒ secret) +Both values are in `bunny_js/.env` for local dev (token linespacing fixed: `KEY = value` was normalized to `KEY=value` — Deno `--env-file` parsing), and were set live via the bunny CLI (2026-11-06): searchapi (linked), `fretchen-llm` (--id), `fretchen-mcp` (--id). The env values are inert until the next deploy ships the flush code. -| Variable | bunny kind | Why | -| --- | --- | --- | -| `BETTERSTACK_SOURCE_TOKEN` | `--secret` | authenticates the ingest | -| `BETTERSTACK_INGEST_HOST` | plain | public host, inert without the token | +| Variable | bunny kind | Why | +| -------------------------- | ------------ | ---------------------------------------------------------- | +| `SCW_COCKPIT_PUSH_TOKEN` | `--secret` | authenticates the Loki push (`push_logs` scope) | +| `COCKPIT_LOKI_URL` | plain | public host, inert without the token | **Per script — all three.** `bunny scripts env set` acts on the directory-linked script (searchapi) only; llm and mcp need their values set against their own script ids explicitly. The GitHub Action ships code, not env — a script without its env vars silently logs only to the dashboard. ```bash # searchapi (linked): -bunny scripts env set BETTERSTACK_SOURCE_TOKEN --secret -bunny scripts env set BETTERSTACK_INGEST_HOST +bunny scripts env set SCW_COCKPIT_PUSH_TOKEN --secret +bunny scripts env set COCKPIT_LOKI_URL .logs.cockpit.fr-par.scw.cloud/loki/api/v1/push> # llm + mcp: same two commands against / ``` ## 4. Verification -- [ ] Deploy with a temporary test route (or the dev server) that logs `level: "error"` once → line appears in Better Stack with the message rendered (confirms the `msg → message` mapping) and the alert email arrives. Per the repo doctrine: a rule that has never fired is not known to work. -- [ ] `GET /search` without payment → searchapi monitor reports up (expecting 402); `GET /openapi.json` on llm and mcp → their monitors report up (expecting 200). -- [ ] Confirm a normal request produces the flush only when log lines exist (dev server) — no fetch per request on the happy path. -- [ ] Remove the test trigger; confirm the query `level = error` is empty on a normal traffic day. -- [ ] After one week, check the Better Stack volume dashboard: if the buffer's info lines ever threaten the 3 GB budget, the lever is to buffer warn+error only (the alert needs errors anyway) — info stays console-only. Decide on data, not now. +- [x] curl probe against the live push URL: `X-Token` → 204 (`Authorization: Bearer` also 204 — both accepted; no auth → 401). A probe line sits in the data source as `level="info"`. +- [x] Unit-tested (`test/logger.test.ts`): one batched `{streams}` push with `X-Token` auth; one stream per level with labels `{service, level}` only; same-millisecond lines get strictly increasing 19-digit ns timestamps; empty buffer sends no fetch; rejected fetch and non-2xx pushes never throw and surface via `console.error`; `withLogFlush` flushes on early returns AND handler throws. +- [ ] Deploy with a temporary test route that logs `level: "error"` once → line appears in Cockpit Grafana → Explore (`{service=~"bunny-web.*"}`), alert email arrives. Per the repo doctrine: a rule that has never fired is not known to work. +- [ ] Remove the test trigger; confirm `{level="error"}` is empty on a normal traffic day. +- [ ] Glance at Cockpit usage in the console after a week: ingestion should read ~0 GB (custom-data ingestion is billed per GB, but rounds to ~€0 here); while there, confirm the fourth alert rule's actual billing. ## 5. Convention changes (vs the Scaleway setup) -- `logger.error` semantics unchanged: "ours, and should page" — now it pages by email via the single `level = error` alert. A caller's fault is `logger.warn`, as before. -- The pino-shaped console shim stays as-is; Better Stack replaces Loki + alert rules + the syslog forwarding ideas entirely. No `alert_coverage` test for bunny_js (see §2). -- **Three docs become wrong the moment this ships — update them in the same change:** - - `bunny_js/README.md`: the bullet "bunny logs go to the dashboard … the `logger.error` alert convention does not apply" → point to this setup. +- `logger.error` semantics unchanged: "ours, and should page" — now it pages by email via the single `level = error` alert rule. A caller's fault is `logger.warn`, as before. +- The pino-shaped console shim stays as-is; Cockpit replaces the Better Stack plan (and the old per-call-site Loki-rule idea) entirely. No `alert_coverage` test for bunny_js (see §2). +- **Three docs updated in the same change (done):** + - `bunny_js/README.md`: logger deviation bullet now describes the flush + alerted `logger.error` convention; env table gained the two Cockpit rows; the first-deploy env list includes the pair. - `bunny_js/src/logger.ts` header comment: same claim, same fix. - - `observability/README.md`: "(The llm/search/mcp sellers moved to bunny_js — Edge Script logging has no Loki rules here)" → note they alert via Better Stack instead (this plan), Loki stays Scaleway-only. + - `observability/README.md`: intro now describes the custom `bunny-logs` data source and the hand-created datasource-managed fourth rule (and the alerting section names it as deliberately outside the `alerts_budget` test). `scripts/logs.ts` still queries only the default source — the bunny data source is read via Grafana → Explore for now. ## 6. Accepted trade-offs (deliberate) -- **No retry, no queue:** a genuinely lost line happens only if the isolate dies between error and flush — the uptime monitor covers the systemic case; a retry queue for a €0.01 query is not worth infrastructure. The awaited non-empty flush removes the much more mundane loss mode (fetch cancelled at response time). -- **Minutes, not seconds:** alert latency = query evaluation + email delivery. Inherent to email-grade alerting. -- **Single-platform risk:** if Better Stack itself is down, alerts are silent — but flush failures surface in the bunny dashboard. Accepted at personal scale; the escape hatch is a direct webhook channel (ntfy/Telegram) if this ever stops being acceptable. -- **llm/mcp monitors check liveness, not payments:** their 402 machinery is covered transitively via the shared `x402_server.ts` through the searchapi monitor. If the three scripts ever diverge in their payment paths, revisit (a POST 402-monitor each). +- **No outside-in liveness check** — deliberately out of scope now. A dead pipeline emits no error lines, so no alert fires; discovery is via users (or the bunny dashboard showing silence). The earlier draft considered a tiny `*/5` Scaleway cron probing `GET /search` → 402 and pushing an error line on failure; dropped because no such layer has ever existed for the twins and it is outside the KISS line. If silent-death ever bites, that cron sketch is the escape hatch — but note it must be a NEW function: attaching a 5-minute trigger to `llmx402cron` would run its channel-claim logic 144× more often. +- **No retry, no queue:** a genuinely lost line happens only if the isolate dies between error and flush. The awaited non-empty flush removes the much more mundane loss mode (fetch cancelled at response time). +- **Minutes, not seconds:** alert latency = rule evaluation (5 min window) + email delivery. Inherent to email-grade alerting. +- **Higher platform concentration:** Scaleway now hosts compute (cron), logs, alerting, and the email delivery — if Scaleway itself is down, alerts are silent. Accepted at personal scale; escape hatches are (a) TEM as a direct secondary channel for rare money-path events (`mint_failed`), or (b) an alert-manager webhook (ntfy/Telegram) if this ever stops being acceptable. +- **Metered, not free-tier:** ingestion is billed per GB (rounds to ~€0 here), custom-log retention is free for 7 days, extendable 1 day–5 years at cost; the fourth alert rule adds ~€0.015/day outside the `alerts_budget.test.ts` budget (see §1). +- **Loki quirks:** nanosecond string timestamps + duplicate-rejection (see code comment), `X-Token` auth, datasource-managed alert rules only — all one-time gotchas, not ongoing work. From a8086d76c6e2bcd228990822e96a564a5f7046a8 Mon Sep 17 00:00:00 2001 From: fretchen Date: Fri, 6 Nov 2026 20:02:16 +0100 Subject: [PATCH 4/6] bunny_js observability: flush logs to Cockpit + one alert rule The bunny twins lost all log-based alerting at the scw_js -> bunny_js move; this restores it on Scaleway Cockpit (EU, same Grafana as the rest). - logger.ts: buffer + awaited flushLogs per request (no waitUntil on the edge; empty buffer = no fetch), {service, level} labels only, monotonic ns counter against Loki's duplicate-timestamp batch rejection, X-Token auth, failures console.error'd to the bunny dashboard. withLogFlush wraps the three entries' handlers (early returns AND throws). - alerts/bunny.yaml: BunnySellersNeedAttention on level="error", pushed to the custom bunny-logs data source's ruler (alerts.ts --url override); budget test pins the count at four, raised deliberately. - env set on all three bunny scripts (SCW_COCKPIT_PUSH_TOKEN secret, COCKPIT_LOKI_URL plain); the deploy Action ships code only, unchanged. Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- .github/workflows/deploy-bunny-js.yml | 3 +- observability/README.md | 27 +++++------ observability/alerts/bunny.yaml | 45 +++++++++++++++++++ observability/alerts/services.yaml | 5 ++- ...-observability-kiss-implementation-plan.md | 31 ++++++++----- observability/scripts/alerts.ts | 8 +++- observability/test/alerts_budget.test.ts | 4 ++ 7 files changed, 94 insertions(+), 29 deletions(-) create mode 100644 observability/alerts/bunny.yaml diff --git a/.github/workflows/deploy-bunny-js.yml b/.github/workflows/deploy-bunny-js.yml index 132bc3e49..a416eb899 100644 --- a/.github/workflows/deploy-bunny-js.yml +++ b/.github/workflows/deploy-bunny-js.yml @@ -8,7 +8,8 @@ name: Deploy Bunny JS twins # # This workflow ships CODE only. Each script's environment (BRAVE_API_KEY, # MISTRAL_API_KEY, the receiver authorizer key, the S3 channel-store credentials, RPC -# URLs, SEARCH_SERVICE_URL) is configured on the script itself — bunny dashboard or +# URLs, SEARCH_SERVICE_URL, the Cockpit observability pair SCW_COCKPIT_PUSH_TOKEN + +# COCKPIT_LOKI_URL) is configured on the script itself — bunny dashboard or # `bunny scripts env set` — and is deliberately not part of CI: a repo leak must not be # a secrets leak. # diff --git a/observability/README.md b/observability/README.md index bbaca976c..a87a18a33 100644 --- a/observability/README.md +++ b/observability/README.md @@ -4,17 +4,16 @@ Project-wide log access and alerting for every Scaleway function in this repo: ` `walletreportcron`, `llmx402cron`, `genimgx402token`, `growthapi`, the comment service and analytics. It belongs to no single service. Cockpit is scoped to the Scaleway _project_, so one token and one set of rules cover all of them. (The llm/search/mcp sellers moved -to bunny_js — they push to a **custom logs data source** in the same Cockpit: the scripts buffer -their `logger` lines and flush them per request to the Loki push API with a push-only Cockpit -token; a single **datasource-managed** alert rule on `{level="error"}` pages by email. That rule -is created by hand in the Cockpit Grafana and is deliberately outside `alerts/*.yaml` and its -budget test — see [`bunny-edge-script-observability-kiss-implementation-plan.md`](./bunny-edge-script-observability-kiss-implementation-plan.md). -`scripts/logs.ts` still queries only the default (Scaleway) source; read the bunny data source in -Grafana → Explore.) +to bunny_js — they flush per request to a **custom logs data source** (`bunny-logs`) in the same +Cockpit via the Loki push API with a push-only Cockpit token, and their one rule +(`BunnySellersNeedAttention`, on `level="error"`) lives in `alerts/bunny.yaml`, pushed to that +data source's own ruler with `--url` — a ruler rule only sees its own source, so the +default-source rules cannot cover them. Read the bunny data source in Grafana → Explore; +`scripts/logs.ts` still queries only the default (Scaleway) source.) | Path | What | | ------------------- | -------------------------------------------------------------------------------------------------------- | -| `alerts/*.yaml` | Loki ruler rules. `payments.yaml`: facilitator + claim/refund cron; `services.yaml`: the scw_js services | +| `alerts/*.yaml` | Loki ruler rules. `payments.yaml`: facilitator + claim/refund cron; `services.yaml`: the scw_js services. `bunny.yaml`: the bunny twins, on the custom data source's own ruler | | `scripts/logs.ts` | Read any function's logs from the terminal | | `scripts/alerts.ts` | List / push / delete rule groups (namespace `fretchen`) | | `alertCoverage.ts` | Parser behind `scw_js/` and `x402_facilitator/`'s `test/alert_coverage.test.ts` | @@ -46,17 +45,19 @@ watch for it directly. npx tsx scripts/alerts.ts # list the rule groups currently on the ruler npx tsx scripts/alerts.ts --push # push alerts/payments.yaml npx tsx scripts/alerts.ts --push alerts/services.yaml +npx tsx scripts/alerts.ts --push alerts/bunny.yaml --url https://db123d37-9094-4e15-badc-706071190ed7.logs.cockpit.fr-par.scw.cloud npx tsx scripts/alerts.ts --delete payments ``` -**Three rules in total, on purpose: Scaleway bills each active alert rule** (€0.015/day, checked +**Four rules in total, on purpose: Scaleway bills each active alert rule** (€0.015/day, checked 2026-10-04), while the phrases a rule matches are free. `services.yaml` holds one rule (`ServicesNeedAttention`) and `payments.yaml` two (`PaymentCronNeedsAttention`, -`FacilitatorNeedsAttention`). A new failure mode is a new alternative in the matching rule's line +`FacilitatorNeedsAttention`) on the default logs source; `bunny.yaml` holds the fourth +(`BunnySellersNeedAttention`) on the custom bunny-logs data source's own ruler — the count was +raised from three deliberately, because no phrase in a default-source rule can see that source's +logs. A new failure mode is a new alternative in the matching rule's line filter, never a new rule; `test/alerts_budget.test.ts` fails if the count grows. The comment at the -top of each file explains the budget and what each branch of the filter means. One rule exists -outside this mechanism: the hand-created, datasource-managed bunny rule (see the intro) — a -deliberate fourth billable rule this test cannot see, documented in the implementation plan. +top of each file explains the budget and what each branch of the filter means. **Push only what is committed.** `--push` refuses a file with uncommitted changes. On 2026-10-04 the three-rule consolidation was live from a file that had never been committed, git still held the diff --git a/observability/alerts/bunny.yaml b/observability/alerts/bunny.yaml new file mode 100644 index 000000000..f3c3c1bef --- /dev/null +++ b/observability/alerts/bunny.yaml @@ -0,0 +1,45 @@ +# Loki alerting rule for the bunny_js Edge Script twins (searchapi / llm / mcp) — the x402 +# sellers that left scw_js. Same namespace and doctrine as the other files, but pushed to a +# DIFFERENT ruler: the twins flush their logs to the custom `bunny-logs` data source, not the +# default one the Scaleway functions write to, and a ruler rule only ever sees its own data +# source's logs. The push therefore names that source's ruler URL explicitly: +# +# npx tsx scripts/alerts.ts --push alerts/bunny.yaml \ +# --url https://db123d37-9094-4e15-badc-706071190ed7.logs.cockpit.fr-par.scw.cloud +# +# ── ONE RULE, FOURTH OF FOUR (cost) ───────────────────────────────────────────────────── +# Same budget doctrine as services.yaml (every active rule is billed ~€0.015/day, checked +# 2026-10-04). The count was raised from three to four DELIBERATELY: phrases in a rule are +# free, but no phrase can make a rule on the DEFAULT source see this source's logs — the +# twins' phrases were removed from services.yaml when they moved for exactly that reason. +# test/alerts_budget.test.ts pins bunny.yaml at 1 so a fifth rule fails CI, not the bill. +# +# ── Why level="error" and no phrase list ──────────────────────────────────────────────── +# The twins push via the Loki push API with `level` as a REAL stream label, so one label match +# covers every `logger.error` call site (the "ours, and should be alertable" convention) — unlike +# the Scaleway functions, whose lines arrive JSON-encoded inside an envelope, forcing the bare- +# phrase style of services.yaml and its `msg=` trap. WHICH case fired is in the line payload: +# read it with `| json` in Explore. (`service` distinguishes the three twins.) +# +# ── Window ────────────────────────────────────────────────────────────────────────────── +# [1h], like every rule here: the ruler's maximum, and the doctrine that a "resolved" +# notification means "no new occurrence in the last hour", not "fixed". +name: bunny +interval: 1m +rules: + - alert: BunnySellersNeedAttention + expr: | + sum(count_over_time({service=~"bunny-web-agent|bunny-llm-agent|bunny-mcp-agent", level="error"}[1h])) > 0 + for: 0m + labels: + severity: critical + area: services + annotations: + summary: "A bunny Edge Script seller needs attention (searchapi / llm / mcp x402 sellers)" + description: >- + A logger.error line arrived from one of the bunny twins — by convention that is "ours, and + should be alertable" (a caller's fault is logger.warn and does not fire this). Open Explore + on the bunny-logs data source with + {service=~"bunny-web-agent|bunny-llm-agent|bunny-mcp-agent", level="error"} | json and read + the message. A "log flush failed" line means the ingest itself was down; it is + console.error'd to the bunny dashboard too. diff --git a/observability/alerts/services.yaml b/observability/alerts/services.yaml index b540ff268..d5af244c3 100644 --- a/observability/alerts/services.yaml +++ b/observability/alerts/services.yaml @@ -15,8 +15,9 @@ # description and the log line say which phrase fired. # # Covers genimgx402token and growthapi. The claim/refund cron is NOT here; `payments.yaml` owns -# llmx402cron and the facilitator. The llm/search/mcp sellers moved to bunny_js (Edge Script -# logging, no Loki rules) — they no longer appear in any selector here. +# llmx402cron and the facilitator. The llm/search/mcp sellers moved to bunny_js — they no longer +# appear in any selector here, because their logs now live in the custom bunny-logs data source, +# which this ruler cannot see (they are covered by `bunny.yaml`, pushed with --url). # # ── Why now ──────────────────────────────────────────────────────────────────────────────────── # Because /imagegen is a paid, public route: a BFL outage breaks something a stranger has paid diff --git a/observability/bunny-edge-script-observability-kiss-implementation-plan.md b/observability/bunny-edge-script-observability-kiss-implementation-plan.md index 485fa08b4..43f2ece74 100644 --- a/observability/bunny-edge-script-observability-kiss-implementation-plan.md +++ b/observability/bunny-edge-script-observability-kiss-implementation-plan.md @@ -12,16 +12,20 @@ No phone push, no webhooks, no self-hosted components, no third-party log vendor - [x] Console → **Cockpit → Data sources** → create a custom **logs data source** (`bunny-logs`) in region `fr-par`. Push URL: `https://db123d37-9094-4e15-badc-706071190ed7.logs.cockpit.fr-par.scw.cloud/loki/api/v1/push`. - [x] Create a **Cockpit token with Push Logs permission** (`push_logs` scope, same region as the data source). Called it `bunny_push_logs`; env var `SCW_COCKPIT_PUSH_TOKEN` (NOT `COCKPIT_LOGS_TOKEN` — `observability/.env` already has `SCW_COCKPIT_LOGS_TOKEN`, the read-only query token; the names must not collide). Lives in `bunny_js/.env`. -- [ ] **Alerts tab → enable the alert manager in the same region** as the data source (region must match, otherwise alerts are not delivered). -- [ ] **Contacts → Add email**; keep "Resolved notifications" ticked. Scaleway sends the mail itself — **no TEM needed** for alerting. -- [ ] In the Cockpit Grafana: **Alerting → New alert rule** — must be **datasource-managed** (Grafana-managed alerting is *not supported* in Cockpit). Query on the Loki data source: +- [x] **Alert manager enabled + email contact existing** — already active for the Scaleway functions' ruler rules (they page through it), so the bunny rule reuses it. No new contact needed. +- [x] **Alert rule live — pushed via `scripts/alerts.ts` to the custom source's ruler, not hand-created in Grafana.** The rules token (`full_access_logs_rules`) reaches the custom data source's ruler too (probed 2026-11-06: GET → 404 "no rule groups found" = empty; the push-scoped token correctly gets 403). The rule is committed as `observability/alerts/bunny.yaml` (`BunnySellersNeedAttention`), `alerts.ts` gained a `--url` override, and `test/alerts_budget.test.ts` pins the count at **four** — better doctrine fidelity than the original Grafana-by-hand idea: committed file, budget test, no dashboard fiddling. Pushed with: + ```bash + npx tsx scripts/alerts.ts --push alerts/bunny.yaml --url https://db123d37-9094-4e15-badc-706071190ed7.logs.cockpit.fr-par.scw.cloud + ``` + Rule query (window `[1h]` per the repo-wide ruler convention, which the budget test enforces): ```logql - sum(count_over_time({service=~"bunny-web-agent|bunny-llm-agent|bunny-mcp-agent", level="error"}[5m])) > 0 + sum(count_over_time({service=~"bunny-web-agent|bunny-llm-agent|bunny-mcp-agent", level="error"}[1h])) > 0 ``` - One rule covers every `logger.error` call site in all three twins. (Per-script `service` labels — see §3 — so a query can distinguish the twins; the rule itself stays single.) -- [ ] **Rule budget note:** the repo deliberately caps ruler rules at three (Scaleway bills each active rule ~€0.015/day, `observability/test/alerts_budget.test.ts` enforces it for `observability/alerts/*.yaml`). This rule is a fourth billed one, created by hand in Grafana where that test cannot see it — the exact cost should be verified in the Cockpit FAQ/pricing before relying on it. The rule's LogQL lives in this committed file (successor: the observability README), honoring the "push only what is committed" doctrine as far as a Grafana-managed rule allows. + One rule covers every `logger.error` call site in all three twins. (Per-script `service` labels — see §2 — so a query can distinguish the twins; the rule itself stays single.) +- [x] **Why a NEW rule (the twins were covered in scw_js):** a ruler rule evaluates only the data source it lives in. The twins' phrases were removed from `alerts/services.yaml` at the move (its header says so), because their logs left the DEFAULT logs source — and they now return in the CUSTOM `bunny-logs` source, which `ServicesNeedAttention` cannot query, phrases or no phrases. Scaleway accepts external pushes only into custom sources, and the twins' own `level` arrives as a real Loki label (unlike scw's JSON-encoded envelope), so one `{level="error"}` match replaces the whole phrase list. +- [x] **Rule budget:** the repo deliberately caps ruler rules (Scaleway bills each active rule ~€0.015/day). The bunny rule is the deliberate fourth — post hoc verified in the Cockpit billing dashboards at the §4 weekly check — and `test/alerts_budget.test.ts` now pins `bunny.yaml: 1`, so a fifth rule fails CI instead of the bill. -**Auth, verified 2026-11-06 by curl probe against the live endpoint:** `X-Token: ` → **204**, and (contrary to the old assumption in this plan) `Authorization: Bearer ` → **204** as well; no auth → 401. Both work — the implementation uses Scaleway's documented `X-Token`. A probe line sits in Cockpit as `service="bunny-web-agent", level="info"` (harmless). +**Auth, verified 2026-11-06 by curl probe against the live endpoint:** `X-Token: ` → **204**, and (contrary to the old assumption in this plan) `Authorization: Bearer ` → **204** as well; no auth → 401. Both work — the implementation uses Scaleway's documented `X-Token`. Bonus finding: the observability **read-only logs token** (`SCW_COCKPIT_LOGS_TOKEN`) queries the custom source's `query_range` fine (the rules token 403s on queries — its scope is exactly rules, as documented), so extending `scripts/logs.ts` to the bunny source later is trivial. A probe info line from setup sits in the source (harmless). ## 2. Code changes (~25 lines, plus one unit test) — IMPLEMENTED @@ -111,6 +115,8 @@ Subrequest budget: +1 per request at most (only when the buffer is non-empty) Both values are in `bunny_js/.env` for local dev (token linespacing fixed: `KEY = value` was normalized to `KEY=value` — Deno `--env-file` parsing), and were set live via the bunny CLI (2026-11-06): searchapi (linked), `fretchen-llm` (--id), `fretchen-mcp` (--id). The env values are inert until the next deploy ships the flush code. +**Nothing goes into the GitHub deploy Action.** `deploy-bunny-js.yml` ships code only — each script's env is deliberately bunny-side ("a repo leak must not be a secrets leak"), so the deploy (Action or manual CLI) picks the pair up automatically; carrying the token as a repo/CI secret would weaken that design. + | Variable | bunny kind | Why | | -------------------------- | ------------ | ---------------------------------------------------------- | | `SCW_COCKPIT_PUSH_TOKEN` | `--secret` | authenticates the Loki push (`push_logs` scope) | @@ -129,8 +135,9 @@ bunny scripts env set COCKPIT_LOKI_URL .logs.cockpit.fr-par.scw.clo - [x] curl probe against the live push URL: `X-Token` → 204 (`Authorization: Bearer` also 204 — both accepted; no auth → 401). A probe line sits in the data source as `level="info"`. - [x] Unit-tested (`test/logger.test.ts`): one batched `{streams}` push with `X-Token` auth; one stream per level with labels `{service, level}` only; same-millisecond lines get strictly increasing 19-digit ns timestamps; empty buffer sends no fetch; rejected fetch and non-2xx pushes never throw and surface via `console.error`; `withLogFlush` flushes on early returns AND handler throws. -- [ ] Deploy with a temporary test route that logs `level: "error"` once → line appears in Cockpit Grafana → Explore (`{service=~"bunny-web.*"}`), alert email arrives. Per the repo doctrine: a rule that has never fired is not known to work. -- [ ] Remove the test trigger; confirm `{level="error"}` is empty on a normal traffic day. +- [x] **Rule fired end-to-end at setup (2026-11-06):** one `level="error"` probe line was pushed through the exact ingest path the scripts will use (X-Token → 204), confirmed stored via `query_range` `{service="bunny-web-agent", level="error"}`, the mounted rule matched it (1m interval, `[1h]`, `for: 0m`), and **the alert-manager email arrived** — the full chain is proven, per the doctrine that a rule that has never fired is not known to work. Expect one "resolved" email roughly an hour later when the probe ages out of the window; the probe lines themselves age out with the free 7-day retention. No test trigger to remove — there was no temporary route, only the probe line. +- [ ] **Ship the flush code: merge `observ` → `main` → `deploy-bunny-js.yml` publishes all three twins (it triggers on push to `main` touching `bunny_js/**`; PRs only run CI).** Env is already set on the scripts, the rule is live — no config changes ride along with the deploy. The manual three-deploy in the README (`npm run deploy` + llm/mcp by script id) is the out-of-band laptop path and NOT needed here: a laptop deploy publishes the same new code minutes earlier, but the next merge to main would republish anyway (last deployment wins), and deploying unmerged state from a branch is the exact divergence that doctrine avoids. After the Action's deploys, the first real `logger.error` from a paid request flows the probe's path from inside the isolate. +- [ ] Confirm `{level="error"}` is empty on a normal traffic day (the probe ages out within the hour). - [ ] Glance at Cockpit usage in the console after a week: ingestion should read ~0 GB (custom-data ingestion is billed per GB, but rounds to ~€0 here); while there, confirm the fourth alert rule's actual billing. ## 5. Convention changes (vs the Scaleway setup) @@ -140,13 +147,13 @@ bunny scripts env set COCKPIT_LOKI_URL .logs.cockpit.fr-par.scw.clo - **Three docs updated in the same change (done):** - `bunny_js/README.md`: logger deviation bullet now describes the flush + alerted `logger.error` convention; env table gained the two Cockpit rows; the first-deploy env list includes the pair. - `bunny_js/src/logger.ts` header comment: same claim, same fix. - - `observability/README.md`: intro now describes the custom `bunny-logs` data source and the hand-created datasource-managed fourth rule (and the alerting section names it as deliberately outside the `alerts_budget` test). `scripts/logs.ts` still queries only the default source — the bunny data source is read via Grafana → Explore for now. + - `observability/README.md`: intro now describes the custom `bunny-logs` data source and the found-a-place-for-everything-new rule (`alerts/bunny.yaml`, pushed to its own ruler via `--url`, pinned in the budget test at four). `scripts/logs.ts` still queries only the default source — the bunny data source is read via Grafana → Explore for now (its read token verifies against the custom source, so extending the CLI later is trivial). ## 6. Accepted trade-offs (deliberate) - **No outside-in liveness check** — deliberately out of scope now. A dead pipeline emits no error lines, so no alert fires; discovery is via users (or the bunny dashboard showing silence). The earlier draft considered a tiny `*/5` Scaleway cron probing `GET /search` → 402 and pushing an error line on failure; dropped because no such layer has ever existed for the twins and it is outside the KISS line. If silent-death ever bites, that cron sketch is the escape hatch — but note it must be a NEW function: attaching a 5-minute trigger to `llmx402cron` would run its channel-claim logic 144× more often. - **No retry, no queue:** a genuinely lost line happens only if the isolate dies between error and flush. The awaited non-empty flush removes the much more mundane loss mode (fetch cancelled at response time). -- **Minutes, not seconds:** alert latency = rule evaluation (5 min window) + email delivery. Inherent to email-grade alerting. +- **Minutes, not seconds:** alert latency = rule evaluation (1m interval) + email delivery. Inherent to email-grade alerting. - **Higher platform concentration:** Scaleway now hosts compute (cron), logs, alerting, and the email delivery — if Scaleway itself is down, alerts are silent. Accepted at personal scale; escape hatches are (a) TEM as a direct secondary channel for rare money-path events (`mint_failed`), or (b) an alert-manager webhook (ntfy/Telegram) if this ever stops being acceptable. -- **Metered, not free-tier:** ingestion is billed per GB (rounds to ~€0 here), custom-log retention is free for 7 days, extendable 1 day–5 years at cost; the fourth alert rule adds ~€0.015/day outside the `alerts_budget.test.ts` budget (see §1). +- **Metered, not free-tier:** ingestion is billed per GB (rounds to ~€0 here), custom-log retention is free for 7 days, extendable 1 day–5 years at cost; the fourth alert rule adds ~€0.015/day, pinned inside the `alerts_budget.test.ts` budget (see §1). - **Loki quirks:** nanosecond string timestamps + duplicate-rejection (see code comment), `X-Token` auth, datasource-managed alert rules only — all one-time gotchas, not ongoing work. diff --git a/observability/scripts/alerts.ts b/observability/scripts/alerts.ts index 108fa45c3..d2b7877df 100644 --- a/observability/scripts/alerts.ts +++ b/observability/scripts/alerts.ts @@ -16,10 +16,16 @@ * npx tsx scripts/alerts.ts # list the rule groups on the ruler * npx tsx scripts/alerts.ts --push # upload alerts/payments.yaml * npx tsx scripts/alerts.ts --push alerts/services.yaml + * npx tsx scripts/alerts.ts --push alerts/bunny.yaml --url https://.logs.cockpit.fr-par.scw.cloud * npx tsx scripts/alerts.ts --delete payments * * `--push` refuses a file with uncommitted changes (see assertCommitted): commit, then push. * + * `--url ` targets another data source's ruler. The default is the Scaleway functions' + * logs source; alerts/bunny.yaml lives on the custom bunny-logs data source, which that ruler + * cannot see — a ruler rule only ever evaluates its own data source's logs, so it must be + * pushed at its own URL (same token, same namespace). + * * Push is never wired into deploy. Alerting config that changes as a side effect of shipping code * is its own kind of surprise — and a rule silently removed by a deploy is indistinguishable from * a system that is simply quiet. @@ -30,7 +36,7 @@ import dotenv from "dotenv"; dotenv.config(); -const URL_BASE = process.env.SCW_COCKPIT_LOGS_URL; +const URL_BASE = flag("url") ?? process.env.SCW_COCKPIT_LOGS_URL; const TOKEN = process.env.SCW_COCKPIT_RULES_TOKEN; if (!URL_BASE || !TOKEN) { diff --git a/observability/test/alerts_budget.test.ts b/observability/test/alerts_budget.test.ts index bdf550081..cbb17a55d 100644 --- a/observability/test/alerts_budget.test.ts +++ b/observability/test/alerts_budget.test.ts @@ -16,6 +16,10 @@ import { fileURLToPath } from "node:url"; const RULE_BUDGET: Record = { "services.yaml": 1, // ServicesNeedAttention "payments.yaml": 2, // PaymentCronNeedsAttention, FacilitatorNeedsAttention + // Fourth of four, raised from three DELIBERATELY (2026-11-06): the bunny twins' logs live in + // the custom bunny-logs data source, which no phrase in the rules above can see — so their + // coverage is necessarily its own (pushed) rule, not another line filter. + "bunny.yaml": 1, // BunnySellersNeedAttention }; const read = (file: string) => From 6296c884cb018883641f1344ff5b0cb237c03902 Mon Sep 17 00:00:00 2001 From: fretchen Date: Sat, 7 Nov 2026 12:39:56 +0100 Subject: [PATCH 5/6] prettier --- bunny_js/README.md | 4 ++-- observability/README.md | 10 +++++----- ...cript-observability-kiss-implementation-plan.md | 14 ++++++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/bunny_js/README.md b/bunny_js/README.md index 37f2492fa..89ad7c51d 100644 --- a/bunny_js/README.md +++ b/bunny_js/README.md @@ -70,8 +70,8 @@ it is inert without another credential, plain — and plain stays inspectable wh | `FACILITATOR_URL` | plain (code default ok) | `https://facilitator.fretchen.eu` | | `LLM_WITHDRAW_DELAY_SECONDS` | plain (code default ok) | 86400 — must match production | | `SEARCH_SERVICE_URL` | plain | advertised `resource.url` base; the bunny trial URL while trialing, production default otherwise | -| `SCW_COCKPIT_PUSH_TOKEN` | secret (`--secret`) | Cockpit push-only token — authenticates the Loki log flush to the observability data source (see the observability README/implementation plan) | -| `COCKPIT_LOKI_URL` | plain | push URL of the custom Cockpit logs data source — public and inert without the token | +| `SCW_COCKPIT_PUSH_TOKEN` | secret (`--secret`) | Cockpit push-only token — authenticates the Loki log flush to the observability data source (see the observability README/implementation plan) | +| `COCKPIT_LOKI_URL` | plain | push URL of the custom Cockpit logs data source — public and inert without the token | ## Known limits / decisions diff --git a/observability/README.md b/observability/README.md index a87a18a33..388336a20 100644 --- a/observability/README.md +++ b/observability/README.md @@ -11,12 +11,12 @@ data source's own ruler with `--url` — a ruler rule only sees its own source, default-source rules cannot cover them. Read the bunny data source in Grafana → Explore; `scripts/logs.ts` still queries only the default (Scaleway) source.) -| Path | What | -| ------------------- | -------------------------------------------------------------------------------------------------------- | +| Path | What | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | `alerts/*.yaml` | Loki ruler rules. `payments.yaml`: facilitator + claim/refund cron; `services.yaml`: the scw_js services. `bunny.yaml`: the bunny twins, on the custom data source's own ruler | -| `scripts/logs.ts` | Read any function's logs from the terminal | -| `scripts/alerts.ts` | List / push / delete rule groups (namespace `fretchen`) | -| `alertCoverage.ts` | Parser behind `scw_js/` and `x402_facilitator/`'s `test/alert_coverage.test.ts` | +| `scripts/logs.ts` | Read any function's logs from the terminal | +| `scripts/alerts.ts` | List / push / delete rule groups (namespace `fretchen`) | +| `alertCoverage.ts` | Parser behind `scw_js/` and `x402_facilitator/`'s `test/alert_coverage.test.ts` | Setup: `npm install`, then create `.env` as described under [Credentials](#credentials). diff --git a/observability/bunny-edge-script-observability-kiss-implementation-plan.md b/observability/bunny-edge-script-observability-kiss-implementation-plan.md index 43f2ece74..ec9507af3 100644 --- a/observability/bunny-edge-script-observability-kiss-implementation-plan.md +++ b/observability/bunny-edge-script-observability-kiss-implementation-plan.md @@ -2,7 +2,7 @@ **Architecture:** script logs → Scaleway Cockpit (custom Loki data source, `fr-par`) → datasource-managed alert rule → Scaleway Alert manager → email. Logs from the remaining Scaleway functions (cron, growthapi, facilitator later) land in the same Cockpit **natively and free** — one Grafana for the whole stack. -No phone push, no webhooks, no self-hosted components, no third-party log vendor. **No uptime/heartbeat layer** — there is none today either (the repo's only "health check" is growth-agent's own `/health` route; nothing ever probes the twins from outside), and log-based alerting alone is the accepted scope. The known blind spot this leaves — "the whole pipeline is dead" produces *no* error lines, so no alert — is accepted for now (see trade-offs). +No phone push, no webhooks, no self-hosted components, no third-party log vendor. **No uptime/heartbeat layer** — there is none today either (the repo's only "health check" is growth-agent's own `/health` route; nothing ever probes the twins from outside), and log-based alerting alone is the accepted scope. The known blind spot this leaves — "the whole pipeline is dead" produces _no_ error lines, so no alert — is accepted for now (see trade-offs). **Why Cockpit over Better Stack:** EU jurisdiction (French company, no CLOUD Act ambiguity); unified view with the Scaleway half of the stack; retention 7 days free vs. Better Stack's 3; ingestion is metered but rounds to ~€0 at this volume (custom-data ingestion is billed per GB; pushing your own data is not free-tier, just negligible here). @@ -71,7 +71,9 @@ export async function flushLogs(): Promise { }), }); if (!res.ok) { - console.error(JSON.stringify({ level: "error", msg: "log flush failed", status: res.status })); + console.error( + JSON.stringify({ level: "error", msg: "log flush failed", status: res.status }), + ); } } catch (err) { // swallowed: an ingest outage must never fail the user's request — but it is @@ -117,10 +119,10 @@ Both values are in `bunny_js/.env` for local dev (token linespacing fixed: `KEY **Nothing goes into the GitHub deploy Action.** `deploy-bunny-js.yml` ships code only — each script's env is deliberately bunny-side ("a repo leak must not be a secrets leak"), so the deploy (Action or manual CLI) picks the pair up automatically; carrying the token as a repo/CI secret would weaken that design. -| Variable | bunny kind | Why | -| -------------------------- | ------------ | ---------------------------------------------------------- | -| `SCW_COCKPIT_PUSH_TOKEN` | `--secret` | authenticates the Loki push (`push_logs` scope) | -| `COCKPIT_LOKI_URL` | plain | public host, inert without the token | +| Variable | bunny kind | Why | +| ------------------------ | ---------- | ----------------------------------------------- | +| `SCW_COCKPIT_PUSH_TOKEN` | `--secret` | authenticates the Loki push (`push_logs` scope) | +| `COCKPIT_LOKI_URL` | plain | public host, inert without the token | **Per script — all three.** `bunny scripts env set` acts on the directory-linked script (searchapi) only; llm and mcp need their values set against their own script ids explicitly. The GitHub Action ships code, not env — a script without its env vars silently logs only to the dashboard. From 8f9c514f1574c0ca0f05bed439bbdf2ac648f550 Mon Sep 17 00:00:00 2001 From: fretchen Date: Sat, 7 Nov 2026 12:46:40 +0100 Subject: [PATCH 6/6] bunny_js observability: review fixes (flush bound, debug console-only, ruler guard) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the pre-merge code + security review of the Cockpit flush: - P1: the awaited flush fetch now carries AbortSignal.timeout(1500) — without a bound, a slow Cockpit stalls every logged request (unpaid 402s included). Regression test asserts an AbortSignal on the fetch init and that an abort surfaces via console.error instead of throwing. - Security: debug lines are no longer buffered — the llm twins log { prompt } at debug and scw's pino gate (LOG_LEVEL ?? "info") suppressed them pre-move, so the flush would have been a NEW egress of user content, not parity. Regression test: debug alone sends no fetch; mixed payloads exclude it. - Op-safety: alerts.ts refuses --push alerts/bunny.yaml without --url — the group would mount on the default-source ruler, bill daily, match nothing. Verified live: refusal fires, ruler untouched. - Plan §2 rewritten to match the shipped logger.ts (Bearer claim corrected to the probe result, sketch marked as sketch). Generated by Mistral Vibe. Co-Authored-By: Mistral Vibe --- bunny_js/src/logger.ts | 15 +++- bunny_js/test/logger.test.ts | 48 +++++++++++++ ...-observability-kiss-implementation-plan.md | 68 ++++++------------- observability/scripts/alerts.ts | 23 +++++++ 4 files changed, 103 insertions(+), 51 deletions(-) diff --git a/bunny_js/src/logger.ts b/bunny_js/src/logger.ts index b0a6b58f1..928caa58e 100644 --- a/bunny_js/src/logger.ts +++ b/bunny_js/src/logger.ts @@ -40,9 +40,13 @@ function emit(level: string, arg: unknown, maybeMsg?: string): void { } else { console.log(line); } - // The same fields go to the Cockpit flush, under the message key the alert/query side - // expects; everything beyond `message` rides in the line payload (queryable via `| json`). - lines.push({ level, event: { message: msg, ...obj } }); + // debug stays console-only — never buffered: the llm twins log { prompt } at + // debug, and user content must not egress to Cockpit. On scw the pino gate + // (LOG_LEVEL ?? "info") suppressed these lines; buffering them here would be + // a NEW egress, not parity with the pre-move setup. + if (level !== "debug") { + lines.push({ level, event: { message: msg, ...obj } }); + } } /** @@ -69,6 +73,11 @@ export async function flushLogs(): Promise { try { const res = await fetch(url, { method: "POST", + // A slow/brownout Cockpit must not stall every logged request: the flush is + // awaited precisely because the runtime has no waitUntil, and this timeout + // is what keeps that await from becoming a latency coupling to ingest + // health. An abort lands in the catch below like any other failure. + signal: AbortSignal.timeout(1500), // Labels stay {service, level} only — high-cardinality labels are the classic Loki trap. headers: { "X-Token": token, "Content-Type": "application/json" }, body: JSON.stringify({ diff --git a/bunny_js/test/logger.test.ts b/bunny_js/test/logger.test.ts index 44cd40877..9ce2849a4 100644 --- a/bunny_js/test/logger.test.ts +++ b/bunny_js/test/logger.test.ts @@ -99,6 +99,54 @@ describe("flushLogs", () => { } }); + test("debug lines are console-only — they never reach the Cockpit buffer", async () => { + // The llm twins log { prompt } at debug; scw's pino gate (LOG_LEVEL ?? "info") + // suppressed those lines pre-move, so shipping them would be a new egress of + // user content, not parity. Regression guard: debug alone must not even fetch. + logger.debug({ prompt: "user's private chat message" }, "Generating answer for prompt"); + await flushLogs(); + expect(mockFetch).not.toHaveBeenCalled(); + + // And mixed with a buffered level, debug stays out of the payload. + logger.debug({ prompt: "another one" }, "Generating answer for prompt"); + logger.info("settlement bookkeeping"); + await flushLogs(); + const { body } = singlePushBody(); + expect(body.streams).toHaveLength(1); + expect(body.streams[0].stream.level).toBe("info"); + expect(body.streams[0].values).toHaveLength(1); + expect(JSON.stringify(body.streams[0].values)).not.toContain("prompt"); + }); + + test("the flush fetch is bounded by an abort signal (timeout regression guard)", async () => { + // The flush is awaited (no waitUntil at the edge), so without a timeout a + // brownout Cockpit stalls every logged request. The fetch init must carry + // an AbortSignal — AbortSignal.timeout(1500) in the implementation. + logger.error({ err: "x" }, "Request handler error"); + await flushLogs(); + const [, init] = mockFetch.mock.calls[0] as [string, RequestInit]; + expect(init.signal).toBeInstanceOf(AbortSignal); + }); + + test("a timed-out (aborted) fetch never throws out of flushLogs, and surfaces in the console", async () => { + mockFetch.mockRejectedValue(new Error("This operation was aborted")); + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + logger.error({ err: "x" }, "Request handler error"); + let failureLine: unknown; + try { + await expect(flushLogs()).resolves.toBeUndefined(); + const failure = consoleError.mock.calls + .map((c) => c[0]) + .find((l) => /log flush failed/.test(String(l))); + expect(failure).toBeDefined(); + failureLine = failure; + } finally { + consoleError.mockRestore(); + } + expect(JSON.parse(failureLine as string)).toMatchObject({ msg: "log flush failed" }); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + test("empty buffer sends no fetch at all (the happy-path request stays free)", async () => { logger.error("seen"); await flushLogs(); diff --git a/observability/bunny-edge-script-observability-kiss-implementation-plan.md b/observability/bunny-edge-script-observability-kiss-implementation-plan.md index ec9507af3..dd5276162 100644 --- a/observability/bunny-edge-script-observability-kiss-implementation-plan.md +++ b/observability/bunny-edge-script-observability-kiss-implementation-plan.md @@ -27,70 +27,41 @@ No phone push, no webhooks, no self-hosted components, no third-party log vendor **Auth, verified 2026-11-06 by curl probe against the live endpoint:** `X-Token: ` → **204**, and (contrary to the old assumption in this plan) `Authorization: Bearer ` → **204** as well; no auth → 401. Both work — the implementation uses Scaleway's documented `X-Token`. Bonus finding: the observability **read-only logs token** (`SCW_COCKPIT_LOGS_TOKEN`) queries the custom source's `query_range` fine (the rules token 403s on queries — its scope is exactly rules, as documented), so extending `scripts/logs.ts` to the bunny source later is trivial. A probe info line from setup sits in the source (harmless). -## 2. Code changes (~25 lines, plus one unit test) — IMPLEMENTED +## 2. Code changes — IMPLEMENTED (post-review fixes included) -Implemented in `bunny_js/src/logger.ts` (buffer, awaited `flushLogs`, monotonic ns counter, `withLogFlush`), wired into all three entries (`searchapi.ts` / `llm.ts` / `mcp.ts` with `bunny-web-agent` / `bunny-llm-agent` / `bunny-mcp-agent`), covered by `bunny_js/test/logger.test.ts` (8 tests). `npm run check` green (tsc, eslint, prettier, 266 tests). +Implemented in `bunny_js/src/logger.ts` (buffer, awaited `flushLogs`, monotonic ns counter, `withLogFlush`), wired into all three entries (`searchapi.ts` / `llm.ts` / `mcp.ts` with `bunny-web-agent` / `bunny-llm-agent` / `bunny-mcp-agent`), covered by `bunny_js/test/logger.test.ts` (11 tests, including the review's regression guards). `npm run check` green. The authoritative code is the shipped `logger.ts` — the earlier sketch below is kept only for the design rationale; three of its details were corrected in review: -**`logger.ts`** — keep the pino-shaped API and the console shim (dashboard debugging stays free); add a buffer and an awaited, Loki-shaped flush: +- **Auth:** both `X-Token` and `Authorization: Bearer` are accepted by Cockpit (probed: 204/204; only no-auth gets 401) — the implementation uses the documented `X-Token`. +- **The flush is AWAITED when non-empty** (no `waitUntil` in Bunny's SDK), and the fetch carries **`signal: AbortSignal.timeout(1500)`** — without a bound, the await couples every logged request's latency to ingest health. +- **`debug` is console-only, never buffered** — the llm twins log `{ prompt }` at debug, scw's pino gate (`LOG_LEVEL ?? "info"`) suppressed those lines pre-move, and shipping them would be a new egress of user content, not parity. ```ts +// sketch — see src/logger.ts for the shipped version const lines: { level: string; event: object }[] = []; -let nsCounter = 0; // monotonic: two errors in the same ms must not collide +let nsCounter = 0; // two errors in the same ms must not collide -// in info/warn/error: push { level, event: { message, ...fields } } into lines -// (plus the existing console output) +// info/warn/error buffer { level, event: { message, ...fields } }; debug does NOT +// (user content stays dashboard-only). Console output keeps its old shape. export async function flushLogs(): Promise { const batch = lines.splice(0); - if (!batch.length) return; // common path: zero latency, no subrequest - // one stream per level; labels stay low-cardinality - const byLevel = new Map(); - for (const { level, event } of batch) { - const vals = byLevel.get(level) ?? []; - // Loki wants nanosecond STRINGS: string-concat, never Date.now() * 1e6 - // (that float exceeds Number.MAX_SAFE_INTEGER). The counter guards Loki's - // "duplicate timestamp for series" rejection for same-millisecond lines - // in one batch — a collision loses the WHOLE push, not one line. - vals.push([`${Date.now()}${String(++nsCounter).padStart(6, "0")}`, JSON.stringify(event)]); - byLevel.set(level, vals); - } + if (!batch.length || !url || !token) return; // happy path: zero latency, no subrequest + // group one Loki stream per level; ns suffix keeps same-ms lines strictly + // increasing (a duplicate timestamp loses the WHOLE push) try { - // Cockpit does NOT use Authorization: Bearer — the token goes in X-Token - // (or X-Scope-OrgID for Promtail-style agents). Bearer = 401. - const res = await fetch(process.env.COCKPIT_LOKI_URL ?? "", { + const res = await fetch(url, { method: "POST", - headers: { - "X-Token": process.env.SCW_COCKPIT_PUSH_TOKEN ?? "", - "Content-Type": "application/json", - }, - body: JSON.stringify({ - streams: [...byLevel].map(([level, values]) => ({ - stream: { service: SERVICE_LABEL, level }, - values, - })), - }), + signal: AbortSignal.timeout(1500), // awaited => must be bounded + headers: { "X-Token": token, "Content-Type": "application/json" }, + body: JSON.stringify({ streams: [...byLevel].map(...) }), }); - if (!res.ok) { - console.error( - JSON.stringify({ level: "error", msg: "log flush failed", status: res.status }), - ); - } + if (!res.ok) console.error(...); // visible in the bunny dashboard } catch (err) { - // swallowed: an ingest outage must never fail the user's request — but it is - // console.error'd so it stays visible in the bunny dashboard - console.error(JSON.stringify({ level: "error", msg: "log flush failed", err: String(err) })); + console.error(...); // never breaks the response } } ``` -Key points, deliberately: - -- **`await` on the non-empty path.** There is no `waitUntil` in Bunny's edge SDK — the isolate's guaranteed lifetime ends when the handler returns, so a fire-and-forget fetch is closer to fire-and-always-lost than "occasional". The buffer is non-empty almost only on errors, so the await costs ~nothing on the common path and zero when the buffer is empty. -- **`X-Token`, not `Authorization: Bearer`** — Scaleway's Cockpit endpoints take the token in the `X-Token` header (their own Grafana integration does the same); a Bearer header gets a 401 and the experiment looks broken for the wrong reason. -- **`process.env`, not `Deno.env.get`** — the runtime shims pin `process` and every entry imports them first; bare globals are not guaranteed (bunny Deno 1.x). -- **Monotonic nanosecond suffix** — two same-millisecond lines in one batch would trigger Loki's `duplicate timestamp for series` rejection, which drops the whole push. -- **Per-script `service` label** (via the `SERVICE_LABEL` constant set from the entry: `bunny-web-agent` / `bunny-llm-agent` / `bunny-mcp-agent`) — low-cardinality, distinguishes the twins in queries; `level` is the only other label. - **Entry files (`searchapi.ts`, `llm.ts`, `mcp.ts`) — not the route files.** There is no single `routes.ts`: three handler files with ten-plus early `return` paths each (OPTIONS, discovery, 402, 404). Editing returns per file is unmaintainable and misses paths. Instead, one decorator, applied once per entry: ```ts @@ -136,7 +107,8 @@ bunny scripts env set COCKPIT_LOKI_URL .logs.cockpit.fr-par.scw.clo ## 4. Verification - [x] curl probe against the live push URL: `X-Token` → 204 (`Authorization: Bearer` also 204 — both accepted; no auth → 401). A probe line sits in the data source as `level="info"`. -- [x] Unit-tested (`test/logger.test.ts`): one batched `{streams}` push with `X-Token` auth; one stream per level with labels `{service, level}` only; same-millisecond lines get strictly increasing 19-digit ns timestamps; empty buffer sends no fetch; rejected fetch and non-2xx pushes never throw and surface via `console.error`; `withLogFlush` flushes on early returns AND handler throws. +- [x] Unit-tested (`test/logger.test.ts`, 11 tests): one batched `{streams}` push with `X-Token` auth; one stream per level with labels `{service, level}` only; same-millisecond lines get strictly increasing 19-digit ns timestamps; empty buffer sends no fetch; **debug never reaches the buffer (privacy regression guard)**; **the fetch init carries an AbortSignal (timeout regression guard)**; rejected, non-2xx and aborted pushes never throw and surface via `console.error`; `withLogFlush` flushes on early returns AND handler throws. +- [x] **Review fixes (2026-11-07)**: `AbortSignal.timeout(1500)` bounds the awaited flush; `debug` lines stopped buffering (prompt egress closed); `alerts.ts` refuses `--push alerts/bunny.yaml` without `--url` (silent wrong-ruler push closed). Remaining post-merge findings-flow actions: THREAT_MODEL token row + `bunny_js/SECURITY.md` line for the push-only Cockpit token. - [x] **Rule fired end-to-end at setup (2026-11-06):** one `level="error"` probe line was pushed through the exact ingest path the scripts will use (X-Token → 204), confirmed stored via `query_range` `{service="bunny-web-agent", level="error"}`, the mounted rule matched it (1m interval, `[1h]`, `for: 0m`), and **the alert-manager email arrived** — the full chain is proven, per the doctrine that a rule that has never fired is not known to work. Expect one "resolved" email roughly an hour later when the probe ages out of the window; the probe lines themselves age out with the free 7-day retention. No test trigger to remove — there was no temporary route, only the probe line. - [ ] **Ship the flush code: merge `observ` → `main` → `deploy-bunny-js.yml` publishes all three twins (it triggers on push to `main` touching `bunny_js/**`; PRs only run CI).** Env is already set on the scripts, the rule is live — no config changes ride along with the deploy. The manual three-deploy in the README (`npm run deploy` + llm/mcp by script id) is the out-of-band laptop path and NOT needed here: a laptop deploy publishes the same new code minutes earlier, but the next merge to main would republish anyway (last deployment wins), and deploying unmerged state from a branch is the exact divergence that doctrine avoids. After the Action's deploys, the first real `logger.error` from a paid request flows the probe's path from inside the isolate. - [ ] Confirm `{level="error"}` is empty on a normal traffic day (the probe ages out within the hour). diff --git a/observability/scripts/alerts.ts b/observability/scripts/alerts.ts index d2b7877df..c7271dd4d 100644 --- a/observability/scripts/alerts.ts +++ b/observability/scripts/alerts.ts @@ -95,8 +95,31 @@ function assertCommitted(file: string): void { } } +/** + * Rules whose group must NOT land on the default ruler — they live on a custom data source's own + * ruler (a ruler rule only ever sees its own data source's logs). Pushing one with the default + * URL succeeds silently, bills ~€0.015/day and matches nothing, so this refuses instead. + * Value: the custom source's API base, for the error message. + */ +const CUSTOM_RULER_FILES: Record = { + "alerts/bunny.yaml": "https://db123d37-9094-4e15-badc-706071190ed7.logs.cockpit.fr-par.scw.cloud", +}; + +function assertRulerUrl(file: string): void { + const customBase = CUSTOM_RULER_FILES[file]; + if (customBase && !flag("url")) { + console.error( + `refusing to push ${file} to the default ruler: its rules evaluate the custom bunny-logs ` + + `data source, which that ruler cannot see — the group would be billed daily and match ` + + `nothing. Add:\n --url ${customBase}`, + ); + process.exit(1); + } +} + async function push(file: string): Promise { assertCommitted(file); + assertRulerUrl(file); const yaml = readFileSync(file, "utf8"); await ruler(`/${NAMESPACE}`, { method: "POST",