Observability

.github/workflows/deploy-bunny-js.yml

@@ -8,7 +8,8 @@ name: Deploy Bunny JS twins
 #
 # This workflow ships CODE only. Each script's environment (BRAVE_API_KEY,
 # MISTRAL_API_KEY, the receiver authorizer key, the S3 channel-store credentials, RPC
-# URLs, SEARCH_SERVICE_URL) is configured on the script itself — bunny dashboard or
+# URLs, SEARCH_SERVICE_URL, the Cockpit observability pair SCW_COCKPIT_PUSH_TOKEN +
+# COCKPIT_LOKI_URL) is configured on the script itself — bunny dashboard or
 # `bunny scripts env set` — and is deliberately not part of CI: a repo leak must not be
 # a secrets leak.
 #

bunny_js/README.md

@@ -19,8 +19,12 @@ Only `scw_js`'s `llmx402cron` remains, claiming the channels these twins charge.
   the shared seller stack (`x402_server.ts`, channel storage/sync), the services
   (`search_service.ts`, `web_fetch_service.ts` with the SSRF defence), schemas, discovery,
   favicon, `openapi.search.json`. Two deliberate deviations:
-  - `logger.ts` is a pino-shaped console shim — pino is a Node-runtime library; bunny logs go
-    to the dashboard, and the observability `logger.error` alert convention does not apply.
+  - `logger.ts` is a pino-shaped console shim — pino is a Node-runtime library; the lines go to
+    the bunny dashboard AND, since the Cockpit integration, to Scaleway's custom Loki data
+    source: `withLogFlush()` wraps each entry handler, so buffered lines are pushed per
+    request to Cockpit, where one datasource-managed alert rule on `level="error"` pages by
+    email (the `logger.error` convention applies again — see
+    [`observability/bunny-edge-script-observability-kiss-implementation-plan.md`](../observability/bunny-edge-script-observability-kiss-implementation-plan.md)).
   - `utils.ts` ports only the platform-neutral half (CORS/parseJsonBody); the ScwResponse
     envelope helpers are not ported — there is no Scaleway envelope here.
 - `src/runtime-shims.ts` — pins the Node `Buffer`/`process` globals the ported code assumes,
@@ -66,6 +70,8 @@ it is inert without another credential, plain — and plain stays inspectable wh
 | `FACILITATOR_URL`                          | plain (code default ok) | `https://facilitator.fretchen.eu`                                                                                                                                                                               |
 | `LLM_WITHDRAW_DELAY_SECONDS`               | plain (code default ok) | 86400 — must match production                                                                                                                                                                                   |
 | `SEARCH_SERVICE_URL`                       | plain                   | advertised `resource.url` base; the bunny trial URL while trialing, production default otherwise                                                                                                                |
+| `SCW_COCKPIT_PUSH_TOKEN`                   | secret (`--secret`)     | Cockpit push-only token — authenticates the Loki log flush to the observability data source (see the observability README/implementation plan)                                                                  |
+| `COCKPIT_LOKI_URL`                         | plain                   | push URL of the custom Cockpit logs data source — public and inert without the token                                                                                                                            |
 
 ## Known limits / decisions
 
@@ -158,8 +164,10 @@ and the `.bunny/script.json` link comes out of the create step. One-time sequenc
    other twins take their id from the create output (or `bunny scripts list`) instead.
 2. Upload the env from `.env` with `bunny scripts env set`, marking the sensitive keys with
    `--secret` (RECEIVER_AUTHORIZER_PRIVATE_KEY, BRAVE_API_KEY, MISTRAL_API_KEY,
-   SCW_ACCESS_KEY, SCW_SECRET_KEY, RPC_URL_*). Skip `TEST_WALLET_PRIVATE_KEY` — buyer-side
-   only, the server must never see it. Verify with `bunny scripts env list`. Per script:
+   SCW_ACCESS_KEY, SCW_SECRET_KEY, RPC_URL_*, SCW_COCKPIT_PUSH_TOKEN). Skip
+   `TEST_WALLET_PRIVATE_KEY` — buyer-side only, the server must never see it. Verify with
+   `bunny scripts env list`. Per script — all three also get the Cockpit observability pair
+   (`SCW_COCKPIT_PUSH_TOKEN` as `--secret`, `COCKPIT_LOKI_URL` plain):
    - **searchapi** — the channel stack + `BRAVE_API_KEY` (web_search/fetch) + `SCW_S3_*`.
    - **llm** — the channel stack + `MISTRAL_API_KEY` (chat_completion) + `SCW_S3_*`.
    - **mcp** — the union: both `BRAVE_API_KEY` and `MISTRAL_API_KEY` (its bundle embeds

bunny_js/src/llm.ts

@@ -7,6 +7,7 @@
  */
 import "./runtime-shims.js";
 import * as BunnySDK from "@bunny.net/edgescript-sdk";
+import { withLogFlush } from "./logger.js";
 import { handleLLMRequest } from "./llm_routes.js";
 
-BunnySDK.net.http.serve(handleLLMRequest);
+BunnySDK.net.http.serve(withLogFlush("bunny-llm-agent", handleLLMRequest));

bunny_js/src/logger.ts

@@ -4,11 +4,31 @@
  * Deno isolate bundle. The call surface is kept pino-shaped — `logger.info(obj, msg)`,
  * `logger.error({ err }, msg)` — because the ported files use that convention verbatim.
  *
- * The `logger.error` convention (observability alert rules key on it) does not carry over
- * to bunny: the edge script's logs live in the bunny dashboard, not Loki, and shipping
- * structured logs out is a documented follow-up if this twin graduates.
+ * On top of the console output (the bunny dashboard's own log view), every line is buffered
+ * and flushed to Scaleway Cockpit's custom Loki data source at the end of the request
+ * (observability/bunny-edge-script-observability-kiss-implementation-plan.md), where a
+ * single datasource-managed alert rule on `level="error"` pages by email — the same
+ * `logger.error` = "ours, and should be alertable" convention the Scaleway functions have.
  */
 
+type BufferedLine = { level: string; event: object };
+
+const lines: BufferedLine[] = [];
+
+/** Per-script `service` label (searchapi/llm/mcp differ); set by withLogFlush at entry setup. */
+let serviceLabel = "bunny-js";
+
+/**
+ * Loki rejects duplicate timestamps within one push (`duplicate timestamp for series`) by
+ * dropping the whole batch, so a monotonic per-isolate counter keeps same-millisecond lines
+ * strictly increasing. String concat for the nanosecond part: `Date.now() * 1e6` as a number
+ * exceeds Number.MAX_SAFE_INTEGER.
+ */
+let nsCounter = 0;
+function nsTimestamp(): string {
+  return `${Date.now()}${String(++nsCounter).padStart(6, "0")}`;
+}
+
 function emit(level: string, arg: unknown, maybeMsg?: string): void {
   const [obj, msg] = typeof arg === "string" ? [{}, arg] : [(arg as object) ?? {}, maybeMsg ?? ""];
   // pino prints `level then message then context`; JSON-lines keeps the same fields.
@@ -20,6 +40,80 @@ function emit(level: string, arg: unknown, maybeMsg?: string): void {
   } else {
     console.log(line);
   }
+  // debug stays console-only — never buffered: the llm twins log { prompt } at
+  // debug, and user content must not egress to Cockpit. On scw the pino gate
+  // (LOG_LEVEL ?? "info") suppressed these lines; buffering them here would be
+  // a NEW egress, not parity with the pre-move setup.
+  if (level !== "debug") {
+    lines.push({ level, event: { message: msg, ...obj } });
+  }
+}
+
+/**
+ * Pushes the buffered lines to Cockpit as one Loki batch. Best-effort by design — no retry,
+ * no queue — but AWAITED when the buffer is non-empty: the edge runtime has no waitUntil, so
+ * an un-awaited fetch racing the handler's return would be lost most of the time. A failed
+ * push is console.error'd, so an ingest outage stays visible in the bunny dashboard instead
+ * of silently eating every alert. Auth is Scaleway's documented `X-Token` header; unpushed
+ * when URL/token are unset (local dev, tests).
+ */
+export async function flushLogs(): Promise<void> {
+  const batch = lines.splice(0);
+  const url = process.env.COCKPIT_LOKI_URL;
+  const token = process.env.SCW_COCKPIT_PUSH_TOKEN;
+  if (batch.length === 0 || !url || !token) {
+    return;
+  }
+  const byLevel = new Map<string, [string, string][]>();
+  for (const { level, event } of batch) {
+    const vals = byLevel.get(level) ?? [];
+    vals.push([nsTimestamp(), JSON.stringify(event)]);
+    byLevel.set(level, vals);
+  }
+  try {
+    const res = await fetch(url, {
+      method: "POST",
+      // A slow/brownout Cockpit must not stall every logged request: the flush is
+      // awaited precisely because the runtime has no waitUntil, and this timeout
+      // is what keeps that await from becoming a latency coupling to ingest
+      // health. An abort lands in the catch below like any other failure.
+      signal: AbortSignal.timeout(1500),
+      // Labels stay {service, level} only — high-cardinality labels are the classic Loki trap.
+      headers: { "X-Token": token, "Content-Type": "application/json" },
+      body: JSON.stringify({
+        streams: [...byLevel].map(([level, values]) => ({
+          stream: { service: serviceLabel, level },
+          values,
+        })),
+      }),
+    });
+    if (!res.ok) {
+      console.error(
+        JSON.stringify({ level: "error", msg: "log flush failed", status: res.status }),
+      );
+    }
+  } catch (err) {
+    console.error(JSON.stringify({ level: "error", msg: "log flush failed", err: String(err) }));
+  }
+}
+
+/**
+ * Wraps a script entry handler so the buffered lines reach Cockpit however the request ended —
+ * early returns (OPTIONS, discovery, the 404, the unpaid 402) and throws alike, which is why
+ * this wraps the entry's top-level handler instead of touching any route file's returns.
+ */
+export function withLogFlush(
+  service: string,
+  handler: (request: Request) => Promise<Response>,
+): (request: Request) => Promise<Response> {
+  serviceLabel = service;
+  return async (request: Request): Promise<Response> => {
+    try {
+      return await handler(request);
+    } finally {
+      await flushLogs();
+    }
+  };
 }
 
 export const logger = {

bunny_js/src/mcp.ts

@@ -7,6 +7,7 @@
  */
 import "./runtime-shims.js";
 import * as BunnySDK from "@bunny.net/edgescript-sdk";
+import { withLogFlush } from "./logger.js";
 import { handleMCPRequest } from "./mcp_routes.js";
 
-BunnySDK.net.http.serve(handleMCPRequest);
+BunnySDK.net.http.serve(withLogFlush("bunny-mcp-agent", handleMCPRequest));

bunny_js/src/searchapi.ts

@@ -9,6 +9,7 @@
  */
 import "./runtime-shims.js";
 import * as BunnySDK from "@bunny.net/edgescript-sdk";
+import { withLogFlush } from "./logger.js";
 import { handleRequest } from "./searchapi_routes.js";
 
-BunnySDK.net.http.serve(handleRequest);
+BunnySDK.net.http.serve(withLogFlush("bunny-web-agent", handleRequest));