diff --git a/.agentworkforce/trajectories/completed/2026-10/traj_rckjtzs1kvmy/summary.md b/.agentworkforce/trajectories/completed/2026-10/traj_rckjtzs1kvmy/summary.md new file mode 100644 index 000000000..66c2180d8 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-10/traj_rckjtzs1kvmy/summary.md @@ -0,0 +1,41 @@ +# Trajectory: Ship @relaycast/engine as standalone bun-compiled binaries + +> **Status:** ✅ Completed +> **Confidence:** 85% +> **Started:** October 26, 2026 at 01:19 AM +> **Completed:** October 26, 2026 at 01:19 AM + +--- + +## Summary + +Standalone relaycast-engine binaries for 7 targets via bun build --compile with a bun:sqlite driver seam; full engine suite passes on bun:sqlite; CI builds, smoke-tests natively, and attaches assets to releases + +**Approach:** Standard approach + +--- + +## Key Decisions + +### Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection +- **Chose:** Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection +- **Reasoning:** Node path keeps static better-sqlite3 imports with zero behaviour change and no top-level await; the binary never bundles the native addon (build fails if it would); vitest under Bun aliases the same module so the full suite runs on bun:sqlite + +### Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it +- **Chose:** Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it +- **Reasoning:** bun:sqlite swallows errors from non-final statements in exec, silently drops prepare tails, returns null from get, and counts FK/trigger rows in changes; the facade normalizes all of these so migrations and tests behave identically + +### Use a lookup-pinned node:https transport for outbound webhooks under Bun +- **Chose:** Use a lookup-pinned node:https transport for outbound webhooks under Bun +- **Reasoning:** Bun's undici shim ignores dispatcher, which silently disabled connect-time DNS pinning (SSRF guard); Bun honours lookup on node:https, verified by positive control and a mutation-checked regression test + +--- + +## Chapters + +### 1. Work +*Agent: default* + +- Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection: Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection +- Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it: Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it +- Use a lookup-pinned node:https transport for outbound webhooks under Bun: Use a lookup-pinned node:https transport for outbound webhooks under Bun diff --git a/.agentworkforce/trajectories/completed/2026-10/traj_rckjtzs1kvmy/trajectory.json b/.agentworkforce/trajectories/completed/2026-10/traj_rckjtzs1kvmy/trajectory.json new file mode 100644 index 000000000..522b3f5e0 --- /dev/null +++ b/.agentworkforce/trajectories/completed/2026-10/traj_rckjtzs1kvmy/trajectory.json @@ -0,0 +1,77 @@ +{ + "id": "traj_rckjtzs1kvmy", + "version": 1, + "task": { + "title": "Ship @relaycast/engine as standalone bun-compiled binaries" + }, + "status": "completed", + "startedAt": "2026-10-26T08:19:03.527Z", + "completedAt": "2026-10-26T08:19:04.512Z", + "agents": [ + { + "name": "default", + "role": "lead", + "joinedAt": "2026-10-26T08:19:03.788Z" + } + ], + "chapters": [ + { + "id": "chap_b98bhicq6w3x", + "title": "Work", + "agentName": "default", + "startedAt": "2026-10-26T08:19:03.788Z", + "endedAt": "2026-10-26T08:19:04.512Z", + "events": [ + { + "ts": 1793002743789, + "type": "decision", + "content": "Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection: Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection", + "raw": { + "question": "Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection", + "chosen": "Build-entry module swap (sqlite-driver, migration-source, version) via a Bun.build plugin instead of runtime driver detection", + "alternatives": [], + "reasoning": "Node path keeps static better-sqlite3 imports with zero behaviour change and no top-level await; the binary never bundles the native addon (build fails if it would); vitest under Bun aliases the same module so the full suite runs on bun:sqlite" + }, + "significance": "high" + }, + { + "ts": 1793002744042, + "type": "decision", + "content": "Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it: Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it", + "raw": { + "question": "Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it", + "chosen": "Wrap bun:sqlite in a better-sqlite3-compatible facade and route drizzle through it", + "alternatives": [], + "reasoning": "bun:sqlite swallows errors from non-final statements in exec, silently drops prepare tails, returns null from get, and counts FK/trigger rows in changes; the facade normalizes all of these so migrations and tests behave identically" + }, + "significance": "high" + }, + { + "ts": 1793002744302, + "type": "decision", + "content": "Use a lookup-pinned node:https transport for outbound webhooks under Bun: Use a lookup-pinned node:https transport for outbound webhooks under Bun", + "raw": { + "question": "Use a lookup-pinned node:https transport for outbound webhooks under Bun", + "chosen": "Use a lookup-pinned node:https transport for outbound webhooks under Bun", + "alternatives": [], + "reasoning": "Bun's undici shim ignores dispatcher, which silently disabled connect-time DNS pinning (SSRF guard); Bun honours lookup on node:https, verified by positive control and a mutation-checked regression test" + }, + "significance": "high" + } + ] + } + ], + "retrospective": { + "summary": "Standalone relaycast-engine binaries for 7 targets via bun build --compile with a bun:sqlite driver seam; full engine suite passes on bun:sqlite; CI builds, smoke-tests natively, and attaches assets to releases", + "approach": "Standard approach", + "confidence": 0.85 + }, + "commits": [], + "filesChanged": [], + "projectId": "AgentWorkforce/relaycast", + "tags": [], + "_trace": { + "startRef": "752d660a9ba6cdb7d1fe692a9fd35621f45f1590", + "endRef": "752d660a9ba6cdb7d1fe692a9fd35621f45f1590" + } +} \ No newline at end of file diff --git a/.github/workflows/engine-binaries.yml b/.github/workflows/engine-binaries.yml new file mode 100644 index 000000000..3166d2783 --- /dev/null +++ b/.github/workflows/engine-binaries.yml @@ -0,0 +1,199 @@ +name: Engine binaries + +# Standalone `relaycast-engine` executables (bun build --compile + bun:sqlite): +# no Node, npm, compiler, or native addon needed to run them. +# +# - pull_request / workflow_dispatch: build every target, run the engine suite +# on bun:sqlite, and smoke-test each binary on a runner that can execute it. +# Nothing is uploaded to a release. +# - workflow_call (from publish-npm.yml, after Create Release): the same, from +# the release tag, then attach the binaries and their .sha256 files to that +# release as relaycast-engine---[.exe]. + +on: + pull_request: + branches: [main] + paths: + - "packages/engine/**" + - "packages/types/**" + - "packages/a2a/**" + - "package-lock.json" + - ".github/workflows/engine-binaries.yml" + workflow_dispatch: + workflow_call: + inputs: + release_tag: + description: "Existing release tag (v) to build from and attach binaries to" + required: true + type: string + +concurrency: + group: engine-binaries-${{ inputs.release_tag || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + BUN_VERSION: "1.4.2" + +jobs: + build: + name: Build + runs-on: ubuntu-latest + timeout-minutes: 20 + outputs: + version: ${{ steps.version.outputs.version }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.release_tag || github.sha }} + persist-credentials: false + - uses: ./.github/actions/setup + - uses: oven-sh/setup-bun@v2 + with: + bun-version: ${{ env.BUN_VERSION }} + - name: Build workspace dependencies + run: npx turbo build --filter=@relaycast/engine^... + - name: Resolve version + id: version + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + version="$(node -p "require('./packages/engine/package.json').version")" + if [ -n "$RELEASE_TAG" ] && [ "v${version}" != "$RELEASE_TAG" ]; then + echo "::error::engine version ${version} does not match release tag ${RELEASE_TAG}" + exit 1 + fi + echo "version=${version}" >> "$GITHUB_OUTPUT" + - name: Build all targets + working-directory: packages/engine + run: bun standalone/build.ts --outdir dist-standalone + - name: Verify checksums + working-directory: packages/engine/dist-standalone + run: | + sha256sum -c ./*.sha256 + ls -l + - uses: actions/upload-artifact@v4 + with: + name: engine-binaries + path: packages/engine/dist-standalone/ + if-no-files-found: error + retention-days: 7 + + test-bun: + name: Engine suite on bun:sqlite + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.release_tag || github.sha }} + persist-credentials: false + - uses: ./.github/actions/setup + - uses: oven-sh/setup-bun@v2 + with: + bun-version: ${{ env.BUN_VERSION }} + - name: Build workspace dependencies + run: npx turbo build --filter=@relaycast/engine^... + - name: Typecheck standalone sources + working-directory: packages/engine + run: npm run typecheck:standalone + - name: bun:sqlite facade tests + working-directory: packages/engine + run: bun test standalone + - name: Engine suite under Bun + working-directory: packages/engine + env: + DO_NOT_TRACK: "1" + run: npm run test:bun + + smoke: + name: Smoke (${{ matrix.target }}) + needs: build + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - { target: linux-x64, runner: ubuntu-latest } + - { target: linux-arm64, runner: ubuntu-24.04-arm } + - { target: linux-musl-x64, runner: ubuntu-latest, alpine: true } + - { target: linux-musl-arm64, runner: ubuntu-24.04-arm, alpine: true } + - { target: darwin-x64, runner: macos-15-intel } + - { target: darwin-arm64, runner: macos-latest } + - { target: windows-x64, runner: windows-latest, exe: ".exe" } + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.release_tag || github.sha }} + persist-credentials: false + sparse-checkout: packages/engine/standalone + - uses: actions/download-artifact@v4 + with: + name: engine-binaries + path: dist-standalone + - name: Select binary + id: binary + shell: bash + env: + VERSION: ${{ needs.build.outputs.version }} + TARGET: ${{ matrix.target }} + EXE: ${{ matrix.exe }} + run: | + path="dist-standalone/relaycast-engine-${VERSION}-${TARGET}${EXE}" + test -f "$path" + chmod +x "$path" + echo "path=$path" >> "$GITHUB_OUTPUT" + - uses: actions/setup-node@v4 + if: ${{ !matrix.alpine }} + with: + node-version: "22.14.0" + - name: Smoke + if: ${{ !matrix.alpine }} + shell: bash + run: node packages/engine/standalone/smoke.mjs "${{ steps.binary.outputs.path }}" --expect-version "${{ needs.build.outputs.version }}" + - name: Smoke (Alpine) + if: ${{ matrix.alpine }} + # Node is only the test driver; the binary itself needs just musl and + # libstdc++ (Bun's musl builds link it dynamically). + run: | + docker run --rm -v "$PWD:/w" -w /w alpine:3.21 sh -c ' + apk add --no-cache libstdc++ nodejs >/dev/null && + node packages/engine/standalone/smoke.mjs "${{ steps.binary.outputs.path }}" --expect-version "${{ needs.build.outputs.version }}" + ' + + release: + name: Attach to release + needs: [build, test-bun, smoke] + if: ${{ inputs.release_tag }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + name: engine-binaries + path: dist-standalone + - name: Upload release assets + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RELEASE_TAG: ${{ inputs.release_tag }} + VERSION: ${{ needs.build.outputs.version }} + run: | + set -euo pipefail + cd dist-standalone + sha256sum -c ./*.sha256 + shopt -s nullglob + assets=(relaycast-engine-"${VERSION}"-*) + if [ "${#assets[@]}" -ne 14 ]; then + echo "::error::expected 7 binaries plus 7 checksums, found ${#assets[@]}: ${assets[*]}" + exit 1 + fi + gh release upload "$RELEASE_TAG" "${assets[@]}" --clobber --repo "$GITHUB_REPOSITORY" + { + echo "### Standalone engine binaries attached to ${RELEASE_TAG}" + for asset in "${assets[@]}"; do echo "- \`${asset}\`"; done + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/publish-npm.yml b/.github/workflows/publish-npm.yml index 26dc5efc5..ee1717727 100644 --- a/.github/workflows/publish-npm.yml +++ b/.github/workflows/publish-npm.yml @@ -846,10 +846,22 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Standalone relaycast-engine binaries (no Node/npm/native addon), built from + # the release tag and attached to the GitHub release created above. Runs only + # after Create Release succeeded, so dry runs and failed publishes skip it. + engine-binaries: + name: Engine binaries + needs: [build, create-release] + permissions: + contents: write + uses: ./.github/workflows/engine-binaries.yml + with: + release_tag: v${{ needs.build.outputs.new_version }} + # Summary summary: name: Summary - needs: [build, publish-packages, verify-publish, create-release] + needs: [build, publish-packages, verify-publish, create-release, engine-binaries] runs-on: ubuntu-latest if: always() @@ -873,4 +885,5 @@ jobs: echo "| Publish NPM Packages | ${{ needs.publish-packages.result == 'success' && '✅' || (needs.publish-packages.result == 'skipped' && '⏭️' || '❌') }} ${{ needs.publish-packages.result }} |" echo "| Verify Publish | ${{ needs.verify-publish.result == 'success' && '✅' || (needs.verify-publish.result == 'skipped' && '⏭️' || '❌') }} ${{ needs.verify-publish.result }} |" echo "| Create Release | ${{ needs.create-release.result == 'success' && '✅' || (needs.create-release.result == 'skipped' && '⏭️' || '❌') }} ${{ needs.create-release.result }} |" + echo "| Engine Binaries | ${{ needs.engine-binaries.result == 'success' && '✅' || (needs.engine-binaries.result == 'skipped' && '⏭️' || '❌') }} ${{ needs.engine-binaries.result }} |" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 03a8cd6c7..9f47a310b 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ relaycast-files/ # OpenNext Cloudflare build output .open-next/ + +# Standalone engine binaries (packages/engine/standalone/build.ts) +dist-standalone/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e358cf7f..8b579fc5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ Packages without a separate changelog are covered by the cross-package notes bel - New `trial` plan for hosted anonymous workspaces: a one-time allowance of 100 messages (channel posts, DMs, group DMs, thread replies) that never resets. `free`, paid plans and self-hosted `selfhost` are unchanged. - `plan_limit_exceeded` errors now carry `metric`, `plan`, `limit`, `usage`, and, when the deployment has an upgrade path, `upgrade_url` and `upgrade_action` (`sign_in` or `upgrade`). +- Standalone `relaycast-engine` executables for Linux (glibc and musl, x64 and arm64), macOS (x64 and arm64), and Windows x64 are attached to each GitHub release as `relaycast-engine---[.exe]` with `.sha256` files; they run the self-host server with no Node.js, npm, compiler, or native addon. +- `relaycast-engine --version` prints the engine version. ### Fixed diff --git a/README.md b/README.md index 2e90773a1..9a1038cd5 100644 --- a/README.md +++ b/README.md @@ -1274,6 +1274,10 @@ It listens on `http://localhost:8787` and stores state in a local SQLite file (o `--db ` or `$RELAYCAST_DB_PATH`). To run it as a container, build a small image around the `relaycast-engine` bin and expose port 8787 — any Docker/OCI host works. +No Node.js? Each GitHub release also attaches standalone `relaycast-engine` executables for Linux +(glibc and musl), macOS, and Windows — see +[Standalone binary](./docs/self-hosting.md#standalone-binary-no-nodejs). + Point any SDK at it with `baseUrl`: ```ts diff --git a/docs/self-hosting.md b/docs/self-hosting.md index c48bc771c..35c2942f0 100644 --- a/docs/self-hosting.md +++ b/docs/self-hosting.md @@ -16,9 +16,10 @@ of hosted infrastructure and account management. ## 1. Prerequisites -- **Node.js ≥ 20** (`node -v`). -- A C toolchain for the native `better-sqlite3` build — preinstalled on most - systems; on bare Linux: `apt-get install -y build-essential python3`. +- **Node.js ≥ 20** (`node -v`) and a C toolchain for the native + `better-sqlite3` build (on bare Linux: `apt-get install -y build-essential + python3`) — **or** the [standalone binary](#standalone-binary-no-nodejs), + which needs neither. - A writable directory for the SQLite file and uploaded files. - An open TCP port (default `8787`). @@ -42,6 +43,32 @@ npm install -g @relaycast/engine relaycast-engine --db ./relaycast.db --port 8787 ``` +### Standalone binary (no Node.js) + +Each GitHub release attaches a single-file `relaycast-engine` executable per +platform, so a machine needs no Node.js, npm, compiler, or native addon: +`relaycast-engine---` (`.exe` on Windows) for `linux-x64`, +`linux-arm64`, `linux-musl-x64`, `linux-musl-arm64`, `darwin-x64`, +`darwin-arm64`, and `windows-x64`, each with a `.sha256` file. + +```bash +V= # a release that carries binaries +T=linux-x64 # your platform +base=https://github.com/AgentWorkforce/relaycast/releases/download/v$V +curl -fsSLO "$base/relaycast-engine-$V-$T" -O "$base/relaycast-engine-$V-$T.sha256" +sha256sum -c "relaycast-engine-$V-$T.sha256" # macOS: shasum -a 256 -c +chmod +x "relaycast-engine-$V-$T" +./relaycast-engine-$V-$T --db ./relaycast.db --port 8787 +``` + +It takes the same flags and environment variables as the npm bin, and +`--version` prints the engine version. The binary embeds Bun and uses its +built-in SQLite. glibc builds need glibc 2.17 or newer; the musl builds need +`libstdc++` (`apk add libstdc++` on Alpine). The macOS and Windows binaries are +not yet signed by a developer certificate, so a browser download may need +`xattr -d com.apple.quarantine ` on macOS or a SmartScreen confirmation on +Windows (`curl` downloads are not quarantined). + On start it migrates the database (idempotent), then serves on the port and prints a hint for creating your first workspace: diff --git a/package-lock.json b/package-lock.json index a68328513..18c428704 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5954,6 +5954,16 @@ "@types/node": "*" } }, + "node_modules/@types/bun": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/@types/bun/-/bun-1.4.4.tgz", + "integrity": "sha512-9wkRDXUXK124pEnAutE6n/l18j+i9GC5IAHK8TmX/2nlySETmky0OU6yL5U8pW1AoKh++AN9auDCr647BfurjQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "bun-types": "1.4.4" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -7038,6 +7048,17 @@ "dev": true, "license": "MIT" }, + "node_modules/bun-types": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/bun-types/-/bun-types-1.4.4.tgz", + "integrity": "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "undici-types": "*" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -16691,6 +16712,7 @@ "devDependencies": { "@identities-ai/ratify-protocol": "1.0.0-alpha.16", "@types/better-sqlite3": "^7.6.11", + "@types/bun": "^1.4.4", "@types/node": "^25.2.1", "@types/ws": "^8.5.13", "drizzle-kit": "^0.31.9", diff --git a/packages/engine/CHANGELOG.md b/packages/engine/CHANGELOG.md index 77c4e97aa..8dce9e2de 100644 --- a/packages/engine/CHANGELOG.md +++ b/packages/engine/CHANGELOG.md @@ -14,6 +14,9 @@ and this project follows [Semantic Versioning](https://semver.org/spec/v2.0.0.ht - `PLAN_LIMITS.trial` and `PlanLimits.lifetime_messages`: a one-time message allowance enforced on message sends by the new `checkMessageAllowance` middleware; only successful fresh sends are counted (idempotent replays neither count nor get refused), under the never-expiring `usage::messages:lifetime` key (`lifetimeUsageCounterKey`, `getLifetimeUsageMetric`). - Optional `EntitlementsProvider.getUpgradeGuidance(workspace, metric)` hook; its URL and action are returned on `plan_limit_exceeded` as `upgrade_url` / `upgrade_action`. +- Standalone single-file `relaycast-engine` builds (`bun build --compile`, `bun:sqlite`) with the same flags and behaviour as the npm bin; the Node adapter now opens SQLite through a small driver seam, and better-sqlite3 remains the Node driver. +- `relaycast-engine --version` / `-v` prints the engine version. +- Outbound webhook DNS pinning also holds when the engine runs under Bun, whose `undici` shim ignores custom dispatchers. ### Fixed diff --git a/packages/engine/package.json b/packages/engine/package.json index e88bf16a3..a6ef0b9da 100644 --- a/packages/engine/package.json +++ b/packages/engine/package.json @@ -53,9 +53,12 @@ ], "scripts": { "build": "tsc && node -e \"require('fs').cpSync('src/db/migrations','dist/db/migrations',{recursive:true})\"", + "build:standalone": "bun standalone/build.ts", "test": "vitest run", + "test:bun": "bun --bun x vitest run", "lint": "eslint src/", - "typecheck": "tsc --noEmit -p tsconfig.typecheck.json" + "typecheck": "tsc --noEmit -p tsconfig.typecheck.json", + "typecheck:standalone": "tsc -p standalone/tsconfig.json" }, "dependencies": { "@hono/node-server": "^1.13.7", @@ -71,6 +74,7 @@ "devDependencies": { "@identities-ai/ratify-protocol": "1.0.0-alpha.16", "@types/better-sqlite3": "^7.6.11", + "@types/bun": "^1.4.4", "@types/node": "^25.2.1", "@types/ws": "^8.5.13", "drizzle-kit": "^0.31.9", diff --git a/packages/engine/src/__tests__/conformance/agentRoster.test.ts b/packages/engine/src/__tests__/conformance/agentRoster.test.ts index 2a0737d4f..9701d6d6d 100644 --- a/packages/engine/src/__tests__/conformance/agentRoster.test.ts +++ b/packages/engine/src/__tests__/conformance/agentRoster.test.ts @@ -2,6 +2,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { AGENT_LIVENESS_TTL_MS, AGENT_ROSTER_RECENCY_MS, RELEASED_AGENT_STATUS, listAgents } from '../../engine/agent.js'; import { createWorkspace, makeNodeStack, type TestStack } from './harness.js'; +// bun:sqlite (the standalone binary's driver, under `npm run test:bun`) has no +// user-defined SQL functions, which this visit-counting probe relies on. +const userFunctions = !process.versions.bun; + // SQLite stores last_seen in whole seconds. Exercise both the exact TTL // boundary and fractional-second request times, which must expire that row. const NOW = 1_800_000_000_000; @@ -211,7 +215,7 @@ describe('agent roster recency and projection', () => { }); describe('agent roster row visits', () => { - it('skips released history and seeks live status ranges within one workspace', async () => { + it.skipIf(!userFunctions)('skips released history and seeks live status ranges within one workspace', async () => { const stack = makeNodeStack(); const { workspaceId, workspaceKey } = await createWorkspace(stack.app, 'roster-row-visits'); const other = await createWorkspace(stack.app, 'other-row-visits'); diff --git a/packages/engine/src/adapters/node/__tests__/pinned-https-fetch.test.ts b/packages/engine/src/adapters/node/__tests__/pinned-https-fetch.test.ts new file mode 100644 index 000000000..5956b9426 --- /dev/null +++ b/packages/engine/src/adapters/node/__tests__/pinned-https-fetch.test.ts @@ -0,0 +1,83 @@ +import { createServer, type Server, type Socket } from 'node:net'; +import type { LookupFunction } from 'node:net'; +import { afterEach, describe, expect, it } from 'vitest'; +import { createLookupPinnedHttpsFetch } from '../pinned-https-fetch.js'; + +const servers: Server[] = []; +afterEach(async () => { + await Promise.all(servers.splice(0).map((server) => new Promise((resolve) => server.close(resolve)))); +}); + +/** A raw TCP listener: enough to observe where the client's socket lands. */ +async function listen(onSocket: (socket: Socket) => void): Promise { + const server = createServer(onSocket); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); + return (server.address() as { port: number }).port; +} + +describe('createLookupPinnedHttpsFetch', () => { + it('resolves the hostname only through the pinned lookup and fails closed when it rejects', async () => { + const looked: string[] = []; + const deny: LookupFunction = (hostname, _options, callback) => { + looked.push(hostname); + callback(Object.assign(new Error(`denied ${hostname}`), { code: 'ENOTFOUND' }), '', 0); + }; + const pinnedFetch = createLookupPinnedHttpsFetch(deny); + await expect(pinnedFetch('https://hooks.example.test/deliver', { method: 'POST', body: '{}' })) + .rejects.toThrow('denied hooks.example.test'); + expect(looked).toEqual(['hooks.example.test']); + }); + + it('connects to the address the lookup returned, not the name in the URL', async () => { + let connections = 0; + const port = await listen((socket) => { + connections += 1; + socket.destroy(); + }); + const toLoopback: LookupFunction = (_hostname, options, callback) => { + if (options.all) callback(null, [{ address: '127.0.0.1', family: 4 }]); + else callback(null, '127.0.0.1', 4); + }; + const pinnedFetch = createLookupPinnedHttpsFetch(toLoopback); + // The TLS handshake fails against a raw TCP listener; reaching it at all + // proves the socket went to the pinned address. + await expect(pinnedFetch(`https://pinned.example.test:${port}/`)).rejects.toThrow(); + expect(connections).toBe(1); + }); + + it('rejects non-https URLs and already-aborted signals before connecting', async () => { + const neverCalled: LookupFunction = () => { + throw new Error('lookup must not run'); + }; + const pinnedFetch = createLookupPinnedHttpsFetch(neverCalled); + await expect(pinnedFetch('http://plain.example.test/')).rejects.toThrow(/only supports https/); + await expect(pinnedFetch('https://hooks.example.test/', { signal: AbortSignal.abort() })).rejects.toThrow(); + }); + + it('does not connect when the signal aborts while the request body is buffered', async () => { + let lookups = 0; + const counting: LookupFunction = (hostname, _options, callback) => { + lookups += 1; + callback(Object.assign(new Error(`denied ${hostname}`), { code: 'ENOTFOUND' }), '', 0); + }; + const pinnedFetch = createLookupPinnedHttpsFetch(counting); + const controller = new AbortController(); + const body = new ReadableStream({ + pull(stream) { + controller.abort(); + stream.enqueue(new TextEncoder().encode('{}')); + stream.close(); + }, + }); + await expect( + pinnedFetch('https://hooks.example.test/', { + method: 'POST', + body, + signal: controller.signal, + duplex: 'half', + } as RequestInit), + ).rejects.toThrow(); + expect(lookups).toBe(0); + }); +}); diff --git a/packages/engine/src/adapters/node/database.ts b/packages/engine/src/adapters/node/database.ts index c9b4de602..1421ccf4d 100644 --- a/packages/engine/src/adapters/node/database.ts +++ b/packages/engine/src/adapters/node/database.ts @@ -1,11 +1,8 @@ import { AsyncLocalStorage } from 'node:async_hooks'; -import { existsSync, readdirSync, readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import Database, { type RunResult } from 'better-sqlite3'; -import { drizzle } from 'drizzle-orm/better-sqlite3'; -import * as schema from '../../db/schema.js'; +import type { RunResult } from 'better-sqlite3'; import { planMigrations } from '../../db/migrationPlan.js'; +import { sqliteDriver, type SqliteConnection } from './sqlite-driver.js'; +import { loadMigrationSource } from './migration-source.js'; import type { EngineDb, TransactionCapability } from '../../ports/database.js'; /** @@ -17,8 +14,12 @@ export type NodeEngineDb = EngineDb & TransactionCapability; export interface SqliteDbHandle { db: NodeEngineDb; - /** The raw better-sqlite3 connection (for migrations, pragmas, backups). */ - sqlite: Database.Database; + /** + * The raw SQLite connection (for migrations, pragmas, backups). better-sqlite3 + * on Node; a better-sqlite3-compatible `bun:sqlite` facade in the standalone + * binary. + */ + sqlite: SqliteConnection; } /** @@ -31,9 +32,9 @@ export interface SqliteDbHandle { * is preserved, so {@link NodeEngineDb} stays precisely typed. */ export function getSqliteDb(path: string): SqliteDbHandle { - const sqlite = new Database(path); + const sqlite = sqliteDriver.open(path); configureSqlite(sqlite, { enableWal: true }); - const db = drizzle(sqlite, { schema }) as unknown as NodeEngineDb; + const db = sqliteDriver.drizzle(sqlite) as NodeEngineDb; // `:memory:` databases are connection-local: a transaction and a raw write // share the same connection, so there is no cross-connection lock contention @@ -67,7 +68,7 @@ function isThenable(value: unknown): value is PromiseLike { * `:memory:` transaction capability: interactive transactions on the shared * connection, serialized so concurrent callers don't interleave `BEGIN`s. */ -function attachMemoryTransaction(db: NodeEngineDb, sqlite: Database.Database): void { +function attachMemoryTransaction(db: NodeEngineDb, sqlite: SqliteConnection): void { let txTail: Promise = Promise.resolve(); db.withTransaction = (fn: (tx: EngineDb) => Promise): Promise => { const run = async (): Promise => { @@ -149,10 +150,10 @@ function withFileWriteGate(db: NodeEngineDb, path: string): NodeEngineDb { // detects this capability and prefers it over a batch. db.withTransaction = (fn: (tx: EngineDb) => Promise): Promise => runGated(async () => { - const txSqlite = new Database(path); + const txSqlite = sqliteDriver.open(path); try { configureSqlite(txSqlite); - const txDb = drizzle(txSqlite, { schema }) as unknown as EngineDb; + const txDb = sqliteDriver.drizzle(txSqlite) as EngineDb; txSqlite.exec('BEGIN IMMEDIATE'); try { const result = await fn(txDb); @@ -274,7 +275,7 @@ function gatePreparedStatement

(prepared: P, runGated: WriteGat } function configureSqlite( - sqlite: Database.Database, + sqlite: SqliteConnection, options: { enableWal?: boolean } = {}, ): void { if (options.enableWal) sqlite.pragma('journal_mode = WAL'); @@ -282,19 +283,6 @@ function configureSqlite( sqlite.pragma('busy_timeout = 5000'); } -function migrationsDir(): string { - // dist/adapters/node/database.js -> dist/db/migrations - const here = dirname(fileURLToPath(import.meta.url)); - const candidates = [ - join(here, '../../db/migrations'), - join(here, '../../../src/db/migrations'), - ]; - for (const dir of candidates) { - if (existsSync(dir)) return dir; - } - return candidates[0]; -} - /** * Apply the engine's SQL migrations to a fresh/old SQLite database, in order. * Tracks applied files in a `_engine_migrations` table so repeated boots are @@ -312,12 +300,8 @@ export function runMigrations(handle: SqliteDbHandle): { applied: string[] } { .map((r) => (r as { name: string }).name), ); - const dir = migrationsDir(); - const files = readdirSync(dir) - .filter((f) => f.endsWith('.sql')) - .sort(); - const supersessionsPath = join(dir, 'supersessions.json'); - const pending = planMigrations(files, done, JSON.parse(readFileSync(supersessionsPath, 'utf8'))); + const source = loadMigrationSource(); + const pending = planMigrations(source.files, done, source.supersessions); const applied: string[] = []; const insert = sqlite.prepare( @@ -330,7 +314,7 @@ export function runMigrations(handle: SqliteDbHandle): { applied: string[] } { // applied in one transaction so a crash mid-migration can't leave the schema // changed but unrecorded (which would replay and fail on the next boot). // drizzle migrations never contain their own BEGIN/COMMIT, so this won't nest. - const ddl = readFileSync(join(dir, file), 'utf8').replace(/-->\s*statement-breakpoint/g, ''); + const ddl = source.read(file).replace(/-->\s*statement-breakpoint/g, ''); const applyOne = sqlite.transaction(() => { sqlite.exec(ddl); insert.run(file, Math.floor(Date.now() / 1000)); diff --git a/packages/engine/src/adapters/node/migration-source.ts b/packages/engine/src/adapters/node/migration-source.ts new file mode 100644 index 000000000..dee05e262 --- /dev/null +++ b/packages/engine/src/adapters/node/migration-source.ts @@ -0,0 +1,41 @@ +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +/** The engine's SQL migrations: file names, their contents, and the supersession manifest. */ +export interface MigrationSource { + /** Migration file names (`NNNN_name.sql`). Order is not significant; the planner sorts. */ + files: string[]; + /** Read one migration file's SQL. */ + read(file: string): string; + /** Parsed `supersessions.json`. */ + supersessions: unknown; +} + +function migrationsDir(): string { + // dist/adapters/node/migration-source.js -> dist/db/migrations + const here = dirname(fileURLToPath(import.meta.url)); + const candidates = [ + join(here, '../../db/migrations'), + join(here, '../../../src/db/migrations'), + ]; + for (const dir of candidates) { + if (existsSync(dir)) return dir; + } + return candidates[0]; +} + +/** + * Load migrations from the package's `db/migrations` directory on disk. + * + * The standalone single-file build replaces this module at bundle time with one + * that serves the same files embedded in the binary. + */ +export function loadMigrationSource(): MigrationSource { + const dir = migrationsDir(); + return { + files: readdirSync(dir).filter((f) => f.endsWith('.sql')), + read: (file) => readFileSync(join(dir, file), 'utf8'), + supersessions: JSON.parse(readFileSync(join(dir, 'supersessions.json'), 'utf8')), + }; +} diff --git a/packages/engine/src/adapters/node/pinned-https-fetch.ts b/packages/engine/src/adapters/node/pinned-https-fetch.ts new file mode 100644 index 000000000..ff94d6237 --- /dev/null +++ b/packages/engine/src/adapters/node/pinned-https-fetch.ts @@ -0,0 +1,82 @@ +import { request as httpsRequest } from 'node:https'; +import type { LookupFunction } from 'node:net'; +import { Readable } from 'node:stream'; + +/** Statuses whose response must not carry a body (`new Response` rejects one). */ +const NULL_BODY_STATUSES = new Set([101, 204, 205, 304]); + +/** + * A minimal `fetch` over `node:https` whose connection resolves the hostname + * through `lookup`, so the socket opens to exactly the address `lookup` + * approved, while TLS still verifies the certificate against the URL hostname + * (SNI and the Host header keep the hostname). + * + * This exists for Bun: Bun replaces the `undici` package with a built-in shim + * whose `fetch` ignores `dispatcher`, so undici's `connect.lookup` pinning + * silently stops applying. Bun's `node:https` honours `lookup`. + * + * Redirects are never followed (callers use `redirect: 'manual'`); the 3xx + * response is returned as-is. + */ +export function createLookupPinnedHttpsFetch(lookup: LookupFunction): typeof globalThis.fetch { + return async (input, init) => { + const req = new Request(input, init); + const url = new URL(req.url); + if (url.protocol !== 'https:') throw new TypeError(`Pinned fetch only supports https: (${url.protocol})`); + const signal = init?.signal ?? req.signal; + signal?.throwIfAborted(); + + const body = req.body ? Buffer.from(await req.arrayBuffer()) : undefined; + // The signal may have aborted while the body was buffered. + signal?.throwIfAborted(); + const headers: Record = {}; + req.headers.forEach((value, key) => { + headers[key] = value; + }); + if (body && headers['content-length'] === undefined) headers['content-length'] = String(body.length); + // Response bodies are passed through undecoded, so ask for an unencoded one. + if (headers['accept-encoding'] === undefined) headers['accept-encoding'] = 'identity'; + + return new Promise((resolve, reject) => { + const client = httpsRequest( + { + protocol: 'https:', + hostname: url.hostname.replace(/^\[(.*)\]$/, '$1'), + port: url.port || 443, + path: `${url.pathname}${url.search}`, + method: req.method, + headers, + lookup, + agent: false, + }, + (res) => { + // This callback runs outside the executor, so a throw here (e.g. + // `new Response` refusing a status outside 200-599) would escape as + // an uncaught exception instead of rejecting the fetch. + try { + const responseHeaders = new Headers(); + for (const [key, value] of Object.entries(res.headers)) { + if (value === undefined) continue; + for (const item of Array.isArray(value) ? value : [value]) responseHeaders.append(key, item); + } + const status = res.statusCode ?? 502; + if (status < 200 || status > 599) throw new TypeError(`Unsupported response status ${status}`); + const responseBody = NULL_BODY_STATUSES.has(status) || req.method === 'HEAD' + ? null + : (Readable.toWeb(res) as ReadableStream); + if (responseBody === null) res.resume(); + resolve(new Response(responseBody, { status, statusText: res.statusMessage, headers: responseHeaders })); + } catch (err) { + res.destroy(); + reject(err); + } + }, + ); + const onAbort = () => client.destroy(signal?.reason instanceof Error ? signal.reason : new Error('The operation was aborted')); + signal?.addEventListener('abort', onAbort, { once: true }); + client.on('close', () => signal?.removeEventListener('abort', onAbort)); + client.on('error', reject); + client.end(body); + }); + }; +} diff --git a/packages/engine/src/adapters/node/sqlite-driver.ts b/packages/engine/src/adapters/node/sqlite-driver.ts new file mode 100644 index 000000000..9d0186e91 --- /dev/null +++ b/packages/engine/src/adapters/node/sqlite-driver.ts @@ -0,0 +1,35 @@ +import Database from 'better-sqlite3'; +import { drizzle } from 'drizzle-orm/better-sqlite3'; +import * as schema from '../../db/schema.js'; + +/** + * The raw SQLite connection surface the Node adapter uses for migrations, + * pragmas, transactions, and shutdown. It is better-sqlite3's `Database`; + * another driver supplies a facade that implements the subset the engine + * relies on (`exec`, `prepare`, `pragma`, `transaction`, `inTransaction`, + * `open`, `close`). + */ +export type SqliteConnection = Database.Database; + +/** + * The seam between the Node adapter and a concrete synchronous SQLite driver. + * + * This module is the default (better-sqlite3). The standalone single-file + * build replaces it at bundle time with a `bun:sqlite` implementation that + * exports the same `sqliteDriver` binding (see `packages/engine/standalone/`), + * so every engine code path stays shared. + */ +export interface SqliteDriver { + /** Driver name, for diagnostics. */ + readonly name: string; + /** Open (creating if absent) a database file, or `':memory:'`. */ + open(path: string): SqliteConnection; + /** Build the synchronous Drizzle handle over the engine schema for `sqlite`. */ + drizzle(sqlite: SqliteConnection): unknown; +} + +export const sqliteDriver: SqliteDriver = { + name: 'better-sqlite3', + open: (path) => new Database(path), + drizzle: (sqlite) => drizzle(sqlite, { schema }), +}; diff --git a/packages/engine/src/adapters/node/ssrf-fetch.ts b/packages/engine/src/adapters/node/ssrf-fetch.ts index fb4d5e711..936a55bf0 100644 --- a/packages/engine/src/adapters/node/ssrf-fetch.ts +++ b/packages/engine/src/adapters/node/ssrf-fetch.ts @@ -1,40 +1,52 @@ import { lookup } from 'node:dns'; +import type { LookupFunction } from 'node:net'; import { Agent, fetch as undiciFetch, type RequestInit as UndiciRequestInit } from 'undici'; import { isGlobalIpAddress, isSafeExternalUrl } from '../../lib/ssrf.js'; +import { createLookupPinnedHttpsFetch } from './pinned-https-fetch.js'; -const dispatcher = new Agent({ - connect: { - // This callback is the connection's DNS lookup, not a preflight lookup. - // Undici therefore opens the socket to the exact address classified here - // while preserving the URL hostname for Host/SNI/TLS verification. - lookup(hostname, options, callback) { - lookup(hostname, { all: true, verbatim: true }, (error, addresses) => { - if (error) { - callback(error, '', 0); - return; - } - const unsafe = addresses.find((entry) => !isGlobalIpAddress(entry.address)); - if (unsafe || addresses.length === 0) { - const rejection = Object.assign( - new Error(`Outbound webhook DNS resolved to a non-global address for ${hostname}`), - { code: 'ENOTFOUND' }, - ); - callback(rejection, '', 0); - return; - } - if (options.all) { - callback(null, addresses); - } else { - callback(null, addresses[0].address, addresses[0].family); - } - }); - }, - }, -}); +/** + * The connection's DNS lookup, not a preflight lookup: the transport opens the + * socket to the exact address classified here while preserving the URL + * hostname for Host/SNI/TLS verification. + */ +const pinnedGlobalLookup: LookupFunction = (hostname, options, callback) => { + lookup(hostname, { all: true, verbatim: true }, (error, addresses) => { + if (error) { + callback(error, '', 0); + return; + } + const unsafe = addresses.find((entry) => !isGlobalIpAddress(entry.address)); + if (unsafe || addresses.length === 0) { + const rejection = Object.assign( + new Error(`Outbound webhook DNS resolved to a non-global address for ${hostname}`), + { code: 'ENOTFOUND' }, + ); + callback(rejection, '', 0); + return; + } + if (options.all) { + callback(null, addresses); + } else { + callback(null, addresses[0].address, addresses[0].family); + } + }); +}; + +const dispatcher = new Agent({ connect: { lookup: pinnedGlobalLookup } }); + +/** + * Under Bun (the standalone binary, or the package run with `bun`), `undici` + * resolves to Bun's built-in shim, whose `fetch` ignores `dispatcher` — the + * pinned lookup above would silently stop applying. Bun's `node:https` + * honours `lookup`, so use it there; Node keeps undici. + */ +const pinnedTransport = ( + typeof process.versions.bun === 'string' ? createLookupPinnedHttpsFetch(pinnedGlobalLookup) : undiciFetch +) as typeof undiciFetch; /** HTTPS-only fetch whose connector validates and pins DNS answers at connect time. */ export function createNodeSafeWebhookFetch( - fetchImpl: typeof undiciFetch = undiciFetch, + fetchImpl: typeof undiciFetch = pinnedTransport, ): typeof globalThis.fetch { return async (input, init) => { const rawUrl = typeof input === 'string' diff --git a/packages/engine/src/bin/serve.ts b/packages/engine/src/bin/serve.ts index a34b54268..a8b182726 100644 --- a/packages/engine/src/bin/serve.ts +++ b/packages/engine/src/bin/serve.ts @@ -1,5 +1,6 @@ #!/usr/bin/env node import { startServer } from '../entrypoints/node.js'; +import { engineVersion } from './version.js'; const HELP = `relaycast-engine — run a self-hosted Relaycast server (Node + SQLite) @@ -11,6 +12,7 @@ Options: --port HTTP port (default: $PORT or 8787) --base-url Public origin for signed file URLs (default: http://localhost:) --env Environment label (default: production) + -v, --version Print the engine version -h, --help Show this help After start, create a workspace: @@ -23,6 +25,7 @@ interface ServeArgs { baseUrl?: string; environment: string; help: boolean; + version: boolean; } function parseArgs(argv: string[], env: NodeJS.ProcessEnv): ServeArgs { @@ -32,10 +35,12 @@ function parseArgs(argv: string[], env: NodeJS.ProcessEnv): ServeArgs { baseUrl: undefined, environment: env.RELAYCAST_ENV ?? 'production', help: false, + version: false, }; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === '--help' || arg === '-h') out.help = true; + else if (arg === '--version' || arg === '-v') out.version = true; else if (arg === '--db') out.db = argv[++i] ?? out.db; else if (arg === '--port') out.port = Number(argv[++i] ?? out.port); else if (arg === '--base-url') out.baseUrl = argv[++i]; @@ -50,6 +55,10 @@ async function main(): Promise { process.stdout.write(HELP); return; } + if (opts.version) { + process.stdout.write(`${engineVersion()}\n`); + return; + } if (!Number.isFinite(opts.port) || opts.port <= 0) { process.stderr.write('Invalid --port\n'); process.exitCode = 1; diff --git a/packages/engine/src/bin/version.ts b/packages/engine/src/bin/version.ts new file mode 100644 index 000000000..be0f3c90f --- /dev/null +++ b/packages/engine/src/bin/version.ts @@ -0,0 +1,14 @@ +import { readFileSync } from 'node:fs'; + +/** + * The engine package version, read from `package.json` next to the build. + * + * The standalone single-file build replaces this module at bundle time with one + * that inlines the version, since the binary carries no `package.json`. + */ +export function engineVersion(): string { + const pkg = JSON.parse(readFileSync(new URL('../../package.json', import.meta.url), 'utf8')) as { + version: string; + }; + return pkg.version; +} diff --git a/packages/engine/src/db/__tests__/nodeForeignKeyIndexes.test.ts b/packages/engine/src/db/__tests__/nodeForeignKeyIndexes.test.ts index 3bc8c6a08..505b116b6 100644 --- a/packages/engine/src/db/__tests__/nodeForeignKeyIndexes.test.ts +++ b/packages/engine/src/db/__tests__/nodeForeignKeyIndexes.test.ts @@ -2,6 +2,10 @@ import { readFileSync } from 'node:fs'; import { afterEach, describe, expect, it } from 'vitest'; import { getSqliteDb, runMigrations, type SqliteDbHandle } from '../../adapters/node/database.js'; +// bun:sqlite (the standalone binary's driver, under `npm run test:bun`) has no +// user-defined SQL functions, which this visit-counting probe relies on. +const userFunctions = !process.versions.bun; + const migration = readFileSync(new URL('../migrations/0051_node_foreign_key_indexes.sql', import.meta.url), 'utf8'); const keys = [ ['deliveries', 'location_node_id', 'idx_deliveries_location_node_fk'], @@ -54,7 +58,7 @@ function fixture(history = 10_000) { } describe('node deletion foreign-key work', () => { - it('visits only matching child rows instead of retained history, including empty-node probes', () => { + it.skipIf(!userFunctions)('visits only matching child rows instead of retained history, including empty-node probes', () => { const db = fixture(); // Replay the same probe before and after the additive migration. These are // SQLite's child-key equality probes for REFERENCES nodes(id), with a diff --git a/packages/engine/src/engine/__tests__/boundedDatabaseWork.test.ts b/packages/engine/src/engine/__tests__/boundedDatabaseWork.test.ts index a22b6afad..abaa294d5 100644 --- a/packages/engine/src/engine/__tests__/boundedDatabaseWork.test.ts +++ b/packages/engine/src/engine/__tests__/boundedDatabaseWork.test.ts @@ -8,6 +8,10 @@ import { deliverPendingToNode, fetchDueNodeDeliveryEvents } from '../delivery.js import { pruneExpired } from '../retention.js'; import { reconcileInventory } from '../node.js'; +// bun:sqlite (the standalone binary's driver, under `npm run test:bun`) has no +// user-defined SQL functions, which this visit-counting probe relies on. +const userFunctions = !process.versions.bun; + const handles: SqliteDbHandle[] = []; afterEach(() => { vi.restoreAllMocks(); for (const handle of handles.splice(0)) handle.sqlite.close(); }); @@ -115,7 +119,7 @@ describe('bounded database work with retained history', () => { expect(plan).toContain('SEARCH deliveries USING INDEX idx_deliveries_node_initial'); }); - it.each([100, 10_000])('counts redrive candidate visits through %i expired rows', async (history) => { + it.skipIf(!userFunctions).each([100, 10_000])('counts redrive candidate visits through %i expired rows', async (history) => { const f = fixture(0, history + 1); f.sqlite.prepare('UPDATE deliveries SET expires_at = 1 WHERE seq <= ?').run(history); let visits = 0; diff --git a/packages/engine/src/engine/__tests__/workspaceEvents.test.ts b/packages/engine/src/engine/__tests__/workspaceEvents.test.ts index d52b7d9b8..dfd16d15c 100644 --- a/packages/engine/src/engine/__tests__/workspaceEvents.test.ts +++ b/packages/engine/src/engine/__tests__/workspaceEvents.test.ts @@ -11,6 +11,10 @@ import { } from '../workspaceEvents.js'; import { pruneExpired } from '../retention.js'; +// bun:sqlite (the standalone binary's driver, under `npm run test:bun`) has no +// user-defined SQL functions, which this visit-counting probe relies on. +const userFunctions = !process.versions.bun; + const DAY_MS = 24 * 60 * 60 * 1000; function openDb(): SqliteDbHandle { @@ -100,7 +104,7 @@ describe('appendWorkspaceEvent', () => { expect(seqs).toEqual([2, 1, 3, 2]); }); - it('reads only the high-water row per workspace regardless of retained history', async () => { + it.skipIf(!userFunctions)('reads only the high-water row per workspace regardless of retained history', async () => { const { db, sqlite } = track(openDb()); sqlite.exec(` WITH RECURSIVE seqs(n) AS ( diff --git a/packages/engine/standalone/build.ts b/packages/engine/standalone/build.ts new file mode 100644 index 000000000..b2c3430d4 --- /dev/null +++ b/packages/engine/standalone/build.ts @@ -0,0 +1,137 @@ +#!/usr/bin/env bun +/** + * Build `relaycast-engine` as standalone single-file executables with + * `bun build --compile`: no Node, npm, compiler, or native addon needed at run + * time. + * + * The binary runs the same entry as the npm bin (`src/bin/serve.ts`). Three + * modules are swapped at bundle time; everything else is the shared engine: + * + * src/adapters/node/sqlite-driver.ts -> standalone/sqlite-driver.ts (bun:sqlite) + * src/adapters/node/migration-source.ts -> standalone/migration-source.ts (embedded SQL) + * src/bin/version.ts -> standalone/version.ts (inlined version) + * + * Usage: + * bun standalone/build.ts # every target + * bun standalone/build.ts --target linux-x64 # one target (repeatable) + * bun standalone/build.ts --target host # the machine running the build + * bun standalone/build.ts --outdir dist-standalone + * + * Output: `/relaycast-engine---[.exe]` plus a + * `sha256sum`-compatible `.sha256` file next to each binary. + * + * Workspace dependencies (`@relaycast/types`, `@relaycast/a2a`) resolve to their + * built `dist`, so build them first (`npx turbo build --filter=@relaycast/engine`). + */ +import { createHash } from 'node:crypto'; +import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { basename, dirname, join, relative, resolve } from 'node:path'; +import type { BunPlugin } from 'bun'; + +/** Asset platform name -> Bun compile target. x64 uses the baseline (pre-AVX2) CPU build. */ +export const TARGETS = { + 'linux-x64': 'bun-linux-x64-baseline', + 'linux-arm64': 'bun-linux-arm64', + 'linux-musl-x64': 'bun-linux-x64-baseline-musl', + 'linux-musl-arm64': 'bun-linux-arm64-musl', + 'darwin-x64': 'bun-darwin-x64-baseline', + 'darwin-arm64': 'bun-darwin-arm64', + 'windows-x64': 'bun-windows-x64-baseline', +} as const; + +export type TargetName = keyof typeof TARGETS; + +const engineRoot = resolve(import.meta.dir, '..'); +const srcRoot = join(engineRoot, 'src'); + +/** `src/`-relative module (without extension) -> standalone replacement. */ +const SWAPS: Record = { + 'adapters/node/sqlite-driver': join(import.meta.dir, 'sqlite-driver.ts'), + 'adapters/node/migration-source': join(import.meta.dir, 'migration-source.ts'), + 'bin/version': join(import.meta.dir, 'version.ts'), +}; + +function hostTarget(): TargetName { + const arch = process.arch === 'arm64' ? 'arm64' : 'x64'; + if (process.platform === 'darwin') return `darwin-${arch}`; + if (process.platform === 'win32') return 'windows-x64'; + // glibc exposes its version in the process report; musl does not. + const report = process.report?.getReport() as { header?: { glibcVersionRuntime?: string } } | undefined; + return report?.header?.glibcVersionRuntime ? `linux-${arch}` : `linux-musl-${arch}`; +} + +function standalonePlugin(swapped: Set): BunPlugin { + return { + name: 'relaycast-standalone', + setup(build) { + build.onResolve({ filter: /^\.{1,2}\/.*\.js$/ }, (args) => { + const target = resolve(dirname(args.importer), args.path).replace(/\.js$/, ''); + const key = relative(srcRoot, target).split('\\').join('/'); + const replacement = SWAPS[key]; + if (!replacement) return undefined; + swapped.add(key); + return { path: replacement }; + }); + // The native addon must never reach the binary; fail the build if it would. + build.onResolve({ filter: /^(better-sqlite3|drizzle-orm\/better-sqlite3)$/ }, (args) => { + throw new Error(`${args.path} imported from ${args.importer}; the standalone build must use bun:sqlite`); + }); + }, + }; +} + +export function assetName(version: string, target: TargetName): string { + return `relaycast-engine-${version}-${target}${target.startsWith('windows-') ? '.exe' : ''}`; +} + +async function buildTarget(target: TargetName, version: string, outdir: string): Promise { + const outfile = join(outdir, assetName(version, target)); + const swapped = new Set(); + const result = await Bun.build({ + entrypoints: [join(srcRoot, 'bin/serve.ts')], + compile: { target: TARGETS[target], outfile }, + minify: true, + sourcemap: 'none', + plugins: [standalonePlugin(swapped)], + }); + if (!result.success) { + for (const log of result.logs) console.error(log); + throw new Error(`build failed for ${target}`); + } + const missing = Object.keys(SWAPS).filter((key) => !swapped.has(key)); + if (missing.length > 0) throw new Error(`standalone swap not applied for: ${missing.join(', ')}`); + + const digest = createHash('sha256').update(readFileSync(outfile)).digest('hex'); + writeFileSync(`${outfile}.sha256`, `${digest} ${basename(outfile)}\n`); + return outfile; +} + +function parseArgs(argv: string[]): { targets: TargetName[]; outdir: string } { + const targets: TargetName[] = []; + let outdir = join(engineRoot, 'dist-standalone'); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--outdir') outdir = resolve(argv[++i] ?? outdir); + else if (arg === '--target') { + const name = argv[++i]; + const resolved = name === 'host' ? hostTarget() : name; + if (!resolved || !(resolved in TARGETS)) { + throw new Error(`Unknown --target ${name}; expected host or one of ${Object.keys(TARGETS).join(', ')}`); + } + targets.push(resolved as TargetName); + } else { + throw new Error(`Unknown argument ${arg}`); + } + } + return { targets: targets.length > 0 ? targets : (Object.keys(TARGETS) as TargetName[]), outdir }; +} + +if (import.meta.main) { + const { targets, outdir } = parseArgs(process.argv.slice(2)); + const { version } = JSON.parse(readFileSync(join(engineRoot, 'package.json'), 'utf8')) as { version: string }; + mkdirSync(outdir, { recursive: true }); + for (const target of targets) { + const outfile = await buildTarget(target, version, outdir); + console.log(`built ${relative(process.cwd(), outfile)}`); + } +} diff --git a/packages/engine/standalone/embed-migrations.ts b/packages/engine/standalone/embed-migrations.ts new file mode 100644 index 000000000..48e95b4e6 --- /dev/null +++ b/packages/engine/standalone/embed-migrations.ts @@ -0,0 +1,24 @@ +/** + * Bundle-time macro: read the engine's SQL migrations so the standalone binary + * carries them inline. Imported only with `{ type: 'macro' }`, so it runs inside + * `bun build` and never ships in the binary. + */ +import { readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export interface EmbeddedMigrations { + files: Record; + supersessions: unknown; +} + +export function embeddedMigrations(): EmbeddedMigrations { + const dir = join(import.meta.dir, '../src/db/migrations'); + const files: Record = {}; + for (const file of readdirSync(dir).filter((f) => f.endsWith('.sql')).sort()) { + files[file] = readFileSync(join(dir, file), 'utf8'); + } + return { + files, + supersessions: JSON.parse(readFileSync(join(dir, 'supersessions.json'), 'utf8')), + }; +} diff --git a/packages/engine/standalone/migration-source.ts b/packages/engine/standalone/migration-source.ts new file mode 100644 index 000000000..0a642c087 --- /dev/null +++ b/packages/engine/standalone/migration-source.ts @@ -0,0 +1,20 @@ +/** + * Standalone replacement for `src/adapters/node/migration-source.ts`: serves the + * migrations embedded in the binary at bundle time instead of reading the disk. + */ +import { embeddedMigrations } from './embed-migrations.js' with { type: 'macro' }; +import type { MigrationSource } from '../src/adapters/node/migration-source.js'; + +const bundle = embeddedMigrations(); + +export function loadMigrationSource(): MigrationSource { + return { + files: Object.keys(bundle.files), + read: (file) => { + const sql = bundle.files[file]; + if (sql === undefined) throw new Error(`Unknown embedded migration: ${file}`); + return sql; + }, + supersessions: bundle.supersessions, + }; +} diff --git a/packages/engine/standalone/smoke.mjs b/packages/engine/standalone/smoke.mjs new file mode 100644 index 000000000..daa793db2 --- /dev/null +++ b/packages/engine/standalone/smoke.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node +/** + * Smoke-test a `relaycast-engine` executable end to end. + * + * node standalone/smoke.mjs [--expect-version ] + * + * Uses only Node built-ins (Node >= 22 for the global WebSocket), so it runs on + * a bare CI runner. Checks `--version`, then starts the server on a fresh + * SQLite file and a free port and drives the HTTP and WebSocket API: health, + * workspace create, two agent registrations, a DM observed on the observer + * workspace stream (WebSocket upgrade path), and the recipient's inbox. Finally it restarts the server on the same + * file (migrations must be idempotent) and confirms the data is still there. + */ +import { spawn, execFileSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; + +const args = process.argv.slice(2); +const binary = args[0] && resolve(args[0]); +if (!binary) { + console.error('usage: smoke.mjs [--expect-version ]'); + process.exit(2); +} +const versionFlag = args.indexOf('--expect-version'); +const expectVersion = versionFlag === -1 ? undefined : args[versionFlag + 1]; + +function assert(condition, message) { + if (!condition) throw new Error(`smoke: ${message}`); +} + +function step(name) { + console.log(`- ${name}`); +} + +async function freePort() { + return new Promise((resolvePort, reject) => { + const server = createServer(); + server.unref(); + server.on('error', reject); + server.listen(0, '127.0.0.1', () => { + const { port } = server.address(); + server.close(() => resolvePort(port)); + }); + }); +} + +function start(db, port, cwd) { + const child = spawn(binary, ['--db', db, '--port', String(port)], { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let output = ''; + child.stdout.on('data', (d) => { output += d; }); + child.stderr.on('data', (d) => { output += d; }); + const exited = new Promise((resolveExit) => child.on('exit', (code, signal) => resolveExit({ code, signal }))); + return { child, exited, output: () => output }; +} + +async function waitHealthy(base, server) { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (server.child.exitCode !== null) throw new Error(`server exited early:\n${server.output()}`); + try { + const res = await fetch(`${base}/health`); + if (res.ok) return res.json(); + } catch { + // not listening yet + } + await new Promise((r) => setTimeout(r, 200)); + } + throw new Error(`server did not become healthy:\n${server.output()}`); +} + +async function stop(server) { + if (process.platform === 'win32') { + // No POSIX signals on Windows; a hard kill is the only option. + server.child.kill(); + await server.exited; + return; + } + server.child.kill('SIGTERM'); + const result = await Promise.race([ + server.exited, + new Promise((r) => setTimeout(() => r(null), 10_000)), + ]); + if (result === null) { + server.child.kill('SIGKILL'); + throw new Error(`server did not stop on SIGTERM:\n${server.output()}`); + } + assert(result.code === 0, `server exited with ${JSON.stringify(result)} on SIGTERM:\n${server.output()}`); +} + +async function api(base, method, path, token, body) { + const res = await fetch(`${base}${path}`, { + method, + headers: { + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(body ? { 'content-type': 'application/json' } : {}), + }, + ...(body ? { body: JSON.stringify(body) } : {}), + }); + const json = await res.json(); + assert(res.ok && json.ok, `${method} ${path} -> ${res.status} ${JSON.stringify(json)}`); + return json.data; +} + +function openStream(base, token) { + const ws = new WebSocket(`${base.replace(/^http/, 'ws')}/v1/ws?token=${encodeURIComponent(token)}`); + const events = []; + const waiters = []; + ws.addEventListener('message', (event) => { + let parsed; + try { parsed = JSON.parse(String(event.data)); } catch { return; } + events.push(parsed); + for (const waiter of waiters.splice(0)) waiter(); + }); + const opened = new Promise((resolveOpen, reject) => { + ws.addEventListener('open', () => resolveOpen(), { once: true }); + ws.addEventListener('error', () => reject(new Error('observer WebSocket failed to open')), { once: true }); + }); + async function waitFor(predicate, timeoutMs = 10_000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const found = events.find(predicate); + if (found) return found; + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error(`timed out waiting for stream event; saw ${JSON.stringify(events.map((e) => e.type))}`); + await new Promise((r) => { + waiters.push(r); + setTimeout(r, remaining); + }); + } + } + return { opened, waitFor, close: () => ws.close() }; +} + +const dir = mkdtempSync(join(tmpdir(), 'relaycast-engine-smoke-')); +const db = join(dir, 'relaycast.db'); +let server; +try { + step('--version'); + const printed = execFileSync(binary, ['--version'], { encoding: 'utf8' }).trim(); + assert(/^\d+\.\d+\.\d+/.test(printed), `--version printed ${JSON.stringify(printed)}`); + if (expectVersion) assert(printed === expectVersion, `--version printed ${printed}, expected ${expectVersion}`); + + const port = await freePort(); + const base = `http://127.0.0.1:${port}`; + server = start(db, port, dir); + + step('GET /health'); + const health = await waitHealthy(base, server); + assert(health.ok === true, `health ${JSON.stringify(health)}`); + + step('create workspace'); + const workspace = await api(base, 'POST', '/v1/workspaces', undefined, { name: 'smoke' }); + assert(workspace.api_key, 'workspace api_key missing'); + + step('register two agents'); + const alice = await api(base, 'POST', '/v1/agents', workspace.api_key, { name: 'alice', type: 'agent' }); + const bob = await api(base, 'POST', '/v1/agents', workspace.api_key, { name: 'bob', type: 'agent' }); + assert(alice.token && bob.token, 'agent token missing'); + + step('open the observer workspace stream'); + const observer = await api(base, 'POST', '/v1/observer-tokens', workspace.api_key, { + name: 'smoke-observer', + scopes: ['stream:read', 'dms:read', 'messages:read'], + filters: { include_dms: true }, + }); + const stream = openStream(base, observer.token); + await stream.opened; + + step('send a DM'); + const text = `hello from the standalone smoke test ${Date.now()}`; + await api(base, 'POST', '/v1/dm', alice.token, { to: 'bob', text, mode: 'wait' }); + + step('observe the DM on the stream'); + await stream.waitFor((event) => JSON.stringify(event).includes(text)); + stream.close(); + + step("read the recipient's inbox"); + const inbox = await api(base, 'GET', '/v1/inbox', bob.token); + const unread = inbox.unread_dms ?? []; + assert( + unread.some((dm) => dm.from === 'alice' && (dm.unread_count ?? 0) >= 1), + `inbox has no unread DM from alice: ${JSON.stringify(inbox)}`, + ); + + step('stop (SIGTERM) and restart on the same database'); + await stop(server); + server = start(db, port, dir); + await waitHealthy(base, server); + const agents = await api(base, 'GET', '/v1/agents', workspace.api_key); + const names = agents.map((agent) => agent.name).sort(); + assert(names.includes('alice') && names.includes('bob'), `agents after restart: ${JSON.stringify(names)}`); + await stop(server); + server = undefined; + + console.log(`ok: ${binary} (${printed})`); +} catch (err) { + if (server) console.error(server.output()); + console.error(err instanceof Error ? err.message : err); + process.exitCode = 1; +} finally { + if (server && server.child.exitCode === null) server.child.kill('SIGKILL'); + rmSync(dir, { recursive: true, force: true }); +} diff --git a/packages/engine/standalone/sqlite-driver.test.ts b/packages/engine/standalone/sqlite-driver.test.ts new file mode 100644 index 000000000..e53d2473c --- /dev/null +++ b/packages/engine/standalone/sqlite-driver.test.ts @@ -0,0 +1,229 @@ +/** + * Pins the better-sqlite3 behaviour the bun:sqlite facade must reproduce. + * Run with `bun test standalone` (bun:sqlite only exists in the Bun runtime). + */ +import { afterEach, describe, expect, it, mock } from 'bun:test'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { sql } from 'drizzle-orm'; +import * as bunDriver from './sqlite-driver.js'; +import { BunSqliteDatabase, splitSqlChunks, sqliteDriver } from './sqlite-driver.js'; + +// Apply the standalone build's driver swap, then load the shared adapter. +mock.module('../src/adapters/node/sqlite-driver.js', () => bunDriver); +const { getSqliteDb, runMigrations } = await import('../src/adapters/node/database.js'); + +const dirs: string[] = []; +function tempDb(): string { + const dir = mkdtempSync(join(tmpdir(), 'relaycast-bun-sqlite-')); + dirs.push(dir); + return join(dir, 'test.db'); +} +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +describe('splitSqlChunks', () => { + it('splits only on top-level semicolons', () => { + const chunks = splitSqlChunks( + `INSERT INTO t VALUES ('a;b', "c;d", [e;f], \`g;h\`); -- x; y\n/* z; */ SELECT 1; \n`, + ); + expect(chunks.map((c) => c.text)).toEqual([ + `INSERT INTO t VALUES ('a;b', "c;d", [e;f], \`g;h\`);`, + ` -- x; y\n/* z; */ SELECT 1;`, + ` \n`, + ]); + expect(chunks.map((c) => c.hasCode)).toEqual([true, true, false]); + }); + + it('handles doubled quotes inside literals', () => { + expect(splitSqlChunks(`SELECT 'it''s; fine'; SELECT 2`).map((c) => c.text)).toEqual([ + `SELECT 'it''s; fine';`, + ` SELECT 2`, + ]); + }); +}); + +describe('BunSqliteDatabase.exec', () => { + it('throws on a failing statement that is followed by more SQL (bun:sqlite swallows it)', () => { + const db = new BunSqliteDatabase(); + db.exec(`CREATE TABLE t (id TEXT PRIMARY KEY, o TEXT CHECK (o IN ('a')));`); + expect(() => db.exec(`\n INSERT INTO t VALUES ('1', 'zzz');\n`)).toThrow(/CHECK constraint failed/); + expect(() => db.exec(`INSERT INTO t VALUES ('2', 'zzz'); INSERT INTO t VALUES ('3', 'a');`)).toThrow( + /CHECK constraint failed/, + ); + // Stops at the first error, like sqlite3_exec / better-sqlite3. + expect(db.prepare('SELECT id FROM t').all()).toEqual([]); + }); + + it('runs trigger bodies containing semicolons as one statement', () => { + const db = new BunSqliteDatabase(); + db.exec(` + CREATE TABLE t (a); + CREATE TABLE log (b); + CREATE TRIGGER tr AFTER INSERT ON t BEGIN + INSERT INTO log VALUES (NEW.a); + INSERT INTO log VALUES (NEW.a * 10); + END; + INSERT INTO t VALUES (1); + `); + expect(db.prepare('SELECT b FROM log ORDER BY b').all()).toEqual([{ b: 1 }, { b: 10 }]); + }); + + it('reports an incomplete trailing statement', () => { + const db = new BunSqliteDatabase(); + db.exec('CREATE TABLE t (a)'); + expect(() => db.exec('CREATE TRIGGER tr AFTER INSERT ON t BEGIN SELECT 1;')).toThrow(/incomplete input/); + }); +}); + +describe('BunSqliteDatabase.prepare', () => { + it('rejects multiple statements instead of silently dropping the tail', () => { + const db = new BunSqliteDatabase(); + db.exec('CREATE TABLE t (a)'); + expect(() => db.prepare('INSERT INTO t VALUES (1); INSERT INTO t VALUES (2)')).toThrow(RangeError); + expect(db.prepare('SELECT count(*) AS n FROM t').get()).toEqual({ n: 0 }); + }); + + it('returns undefined from get() when there is no row', () => { + const db = new BunSqliteDatabase(); + expect(db.prepare('SELECT 1 WHERE 0').get()).toBeUndefined(); + }); + + it('binds named parameters without a sigil and rejects missing ones', () => { + const db = new BunSqliteDatabase(); + expect(db.prepare('SELECT $a + :b AS n').get({ a: 1, b: 2 })).toEqual({ n: 3 }); + expect(() => db.prepare('SELECT $a AS n').get({})).toThrow(); + }); + + it('reports the statement\'s own changes and insert rowid, not FK/trigger side effects', () => { + const db = new BunSqliteDatabase(); + db.exec(` + PRAGMA foreign_keys = ON; + CREATE TABLE p (id TEXT PRIMARY KEY); + CREATE TABLE c (id INTEGER PRIMARY KEY, p TEXT REFERENCES p(id) ON DELETE CASCADE); + CREATE TABLE log (id INTEGER PRIMARY KEY, x); + CREATE TRIGGER tr AFTER DELETE ON p BEGIN INSERT INTO log (x) VALUES (1); END; + `); + expect(db.prepare(`INSERT INTO p VALUES ('a')`).run()).toEqual({ changes: 1, lastInsertRowid: 1 }); + for (let i = 0; i < 3; i++) db.prepare(`INSERT INTO c (p) VALUES ('a')`).run(); + const deleted = db.prepare(`DELETE FROM p WHERE id = 'a'`).run(); + expect(deleted.changes).toBe(1); + expect(db.prepare(`DELETE FROM p WHERE id = 'missing'`).run().changes).toBe(0); + expect(db.prepare('SELECT count(*) AS n FROM c').get()).toEqual({ n: 0 }); + }); + + it('supports raw mode and values()', () => { + const db = new BunSqliteDatabase(); + const stmt = db.prepare('SELECT 1 AS a, 2 AS b'); + expect(stmt.values()).toEqual([[1, 2]]); + expect(stmt.all()).toEqual([{ a: 1, b: 2 }]); + expect(stmt.raw().all()).toEqual([[1, 2]]); + expect(stmt.raw(false).get()).toEqual({ a: 1, b: 2 }); + }); + + it('binds BigInt and reads int64 as a number, like better-sqlite3 without safeIntegers', () => { + const db = new BunSqliteDatabase(); + db.exec('CREATE TABLE t (n INTEGER)'); + db.prepare('INSERT INTO t VALUES (?)').run(2n ** 53n + 1n); + db.prepare('INSERT INTO t VALUES (?)').run(1_700_000_000_000); + expect(db.prepare('SELECT n FROM t').all()).toEqual([{ n: 2 ** 53 }, { n: 1_700_000_000_000 }]); + }); + + it('surfaces extended constraint codes the engine matches on', () => { + const db = new BunSqliteDatabase(); + db.exec('CREATE TABLE t (id TEXT PRIMARY KEY)'); + db.prepare(`INSERT INTO t VALUES ('a')`).run(); + let caught: unknown; + try { + db.prepare(`INSERT INTO t VALUES ('a')`).run(); + } catch (err) { + caught = err; + } + expect(caught).toMatchObject({ code: 'SQLITE_CONSTRAINT_PRIMARYKEY' }); + expect((caught as Error).message).toContain('UNIQUE constraint failed: t.id'); + }); +}); + +describe('BunSqliteDatabase connection surface', () => { + it('applies WAL, foreign keys, and busy timeout pragmas on a file database', () => { + const db = new BunSqliteDatabase(tempDb()); + expect(db.pragma('journal_mode = WAL')).toEqual([{ journal_mode: 'wal' }]); + db.pragma('foreign_keys = ON'); + db.pragma('busy_timeout = 5000'); + expect(db.pragma('foreign_keys', { simple: true })).toBe(1); + expect(db.pragma('busy_timeout', { simple: true })).toBe(5000); + db.close(); + }); + + it('tracks transactions and rolls back a throwing transaction function', () => { + const db = new BunSqliteDatabase(); + db.exec('CREATE TABLE t (a)'); + expect(db.inTransaction).toBe(false); + db.exec('BEGIN IMMEDIATE'); + expect(db.inTransaction).toBe(true); + db.exec('ROLLBACK'); + const tx = db.transaction(() => { + db.prepare('INSERT INTO t VALUES (1)').run(); + throw new Error('boom'); + }); + expect(() => tx()).toThrow('boom'); + expect(db.prepare('SELECT count(*) AS n FROM t').get()).toEqual({ n: 0 }); + }); + + it('reports open/closed and closes idempotently', () => { + const db = new BunSqliteDatabase(); + expect(db.open).toBe(true); + db.close(); + db.close(); + expect(db.open).toBe(false); + }); + + it('finalizes outstanding statements on close, like better-sqlite3', () => { + const db = new BunSqliteDatabase(tempDb()); + db.exec('CREATE TABLE t (id INTEGER PRIMARY KEY)'); + expect(db.pragma('journal_mode', { simple: true })).toBeDefined(); + const kept = db.prepare('SELECT count(*) AS n FROM t'); + const done = db.prepare('SELECT 1 AS one'); + expect(kept.get()).toEqual({ n: 0 }); + done.finalize(); + db.close(); + // The facade finalized it, so the connection is really closed. + expect(() => kept.get()).toThrow(); + // Finalizing after close (as drizzle may) is a no-op, not an error. + expect(() => kept.finalize()).not.toThrow(); + expect(() => done.finalize()).not.toThrow(); + }); + + it('backs up to a file that reopens with the same rows', async () => { + const path = tempDb(); + const db = new BunSqliteDatabase(path); + db.pragma('journal_mode = WAL'); + db.exec(`CREATE TABLE t (a); INSERT INTO t VALUES (42);`); + const copy = `${path}.bak`; + await db.backup(copy); + expect(new BunSqliteDatabase(copy).prepare('SELECT a FROM t').get()).toEqual({ a: 42 }); + }); +}); + +describe('engine adapter over bun:sqlite', () => { + it('uses the bun driver for the swapped seam', () => { + expect(sqliteDriver.name).toBe('bun:sqlite'); + }); + + it('migrates a file database idempotently and serializes file-backed transactions', async () => { + const path = tempDb(); + const handle = getSqliteDb(path); + expect(handle.sqlite).toBeInstanceOf(BunSqliteDatabase); + const first = runMigrations(handle); + expect(first.applied.length).toBeGreaterThan(50); + expect(runMigrations(handle).applied).toEqual([]); + expect(handle.sqlite.pragma('foreign_key_check')).toEqual([]); + + const { db } = handle; + const result = await db.withTransaction((tx) => tx.all<{ n: number }>(sql`SELECT 7 AS n`)); + expect(result).toEqual([{ n: 7 }]); + handle.sqlite.close(); + }); +}); diff --git a/packages/engine/standalone/sqlite-driver.ts b/packages/engine/standalone/sqlite-driver.ts new file mode 100644 index 000000000..f6b38654a --- /dev/null +++ b/packages/engine/standalone/sqlite-driver.ts @@ -0,0 +1,331 @@ +/** + * `bun:sqlite` implementation of the Node adapter's SQLite driver seam. + * + * The standalone build (`standalone/build.ts`) swaps this module in for + * `src/adapters/node/sqlite-driver.ts`, so the compiled binary needs no native + * addon. {@link BunSqliteDatabase} is a facade that gives `bun:sqlite` the + * better-sqlite3 surface the adapter and its tests use, normalizing the + * differences that matter: + * + * - `stmt.get()` with no row returns `undefined` (bun:sqlite returns `null`). + * - `stmt.run()` reports the statement's own `changes` / `lastInsertRowid` + * (bun:sqlite includes rows written by FK actions and triggers). + * - Named parameters bind without a sigil prefix (`strict: true`), and a + * missing parameter throws, as in better-sqlite3. + * - `pragma(source, { simple })` exists (bun:sqlite has no `pragma()`). + * - `open` reflects whether the connection is still open. + * - `backup(path)` writes a consistent snapshot via `serialize()`. + * - `exec(sql)` runs a multi-statement script one statement at a time and stops + * at the first error. bun:sqlite's own `exec` silently swallows the error of + * any statement that is not the last one (trailing whitespace counts as a + * following statement), and its `prepare` silently ignores everything after + * the first statement; better-sqlite3 throws in both cases. Without this a + * failing migration statement would be skipped and the migration recorded as + * applied. + */ +import { writeFileSync } from 'node:fs'; +import { Database, type Statement } from 'bun:sqlite'; +import type { DrizzleConfig } from 'drizzle-orm'; +import { drizzle } from 'drizzle-orm/bun-sqlite'; +import * as schema from '../src/db/schema.js'; +import type { SqliteConnection, SqliteDriver } from '../src/adapters/node/sqlite-driver.js'; + +type Params = unknown[]; + +interface SqlChunk { + text: string; + /** False when the chunk is only whitespace and comments. */ + hasCode: boolean; +} + +/** + * Split a SQL script after every `;` that sits outside string literals, quoted + * identifiers, and comments. A chunk may still be an incomplete statement (a + * `CREATE TRIGGER ... BEGIN ...;` body); the caller detects that via SQLite's + * own "incomplete input" error and joins chunks until the statement compiles. + */ +export function splitSqlChunks(sql: string): SqlChunk[] { + const chunks: SqlChunk[] = []; + let start = 0; + let hasCode = false; + let i = 0; + const closeOf: Record = { "'": "'", '"': '"', '`': '`', '[': ']' }; + while (i < sql.length) { + const ch = sql[i]; + if (ch === '-' && sql[i + 1] === '-') { + const end = sql.indexOf('\n', i + 2); + i = end === -1 ? sql.length : end + 1; + } else if (ch === '/' && sql[i + 1] === '*') { + const end = sql.indexOf('*/', i + 2); + i = end === -1 ? sql.length : end + 2; + } else if (ch in closeOf) { + // A doubled quote inside a literal re-enters the same state on the next + // iteration, so escaped quotes need no special case. + const end = sql.indexOf(closeOf[ch], i + 1); + i = end === -1 ? sql.length : end + 1; + hasCode = true; + } else if (ch === ';') { + i += 1; + chunks.push({ text: sql.slice(start, i), hasCode: true }); + start = i; + hasCode = false; + } else { + if (!/\s/.test(ch)) hasCode = true; + i += 1; + } + } + if (start < sql.length) chunks.push({ text: sql.slice(start), hasCode }); + return chunks; +} + +function isIncompleteInput(err: unknown): boolean { + return err instanceof Error && /incomplete input/i.test(err.message); +} + +class BunSqliteStatement { + #raw = false; + + constructor( + private readonly stmt: Statement, + private readonly db: Database, + private readonly live?: LiveStatements, + ) { + live?.add(this); + } + + run(...params: Params): { changes: number; lastInsertRowid: number | bigint } { + const result = this.stmt.run(...(params as never[])); + // bun:sqlite reports the `total_changes()` delta, which also counts rows + // touched by FK actions and triggers, and a trigger's insert rowid. + // better-sqlite3 reports the statement's own `changes()` and + // `last_insert_rowid()`. + const own = this.db + .query('SELECT changes() AS changes, last_insert_rowid() AS lastInsertRowid') + .get() as { changes: number; lastInsertRowid: number }; + return { changes: result.changes === 0 ? 0 : own.changes, lastInsertRowid: own.lastInsertRowid }; + } + + get(...params: Params): unknown { + if (this.#raw) return this.stmt.values(...(params as never[]))[0]; + const row = this.stmt.get(...(params as never[])); + return row === null ? undefined : row; + } + + all(...params: Params): unknown[] { + return this.#raw ? this.stmt.values(...(params as never[])) : this.stmt.all(...(params as never[])); + } + + /** better-sqlite3's raw mode: rows come back as value arrays. */ + raw(toggle = true): this { + this.#raw = toggle; + return this; + } + + /** Row value arrays (drizzle's bun-sqlite session calls this); runs through `all` in raw mode, as better-sqlite3 does. */ + values(...params: Params): unknown[][] { + const previous = this.#raw; + this.#raw = true; + try { + return this.all(...params) as unknown[][]; + } finally { + this.#raw = previous; + } + } + + iterate(...params: Params): IterableIterator { + return this.stmt.iterate(...(params as never[])); + } + + finalize(): void { + if (this.live && !this.live.delete(this)) return; + this.stmt.finalize(); + } + + /** Finalize without consulting tracking (the connection is draining it). */ + finalizeNative(): void { + this.stmt.finalize(); + } +} + +/** + * Weakly tracks a connection's prepared statements so `close()` can finalize + * the ones still reachable without keeping completed queries alive on a + * long-lived connection: a collected statement is finalized by bun:sqlite and + * its entry is dropped here. + */ +class LiveStatements { + readonly #refOf = new WeakMap>(); + readonly #byRef = new Set>(); + readonly #registry = new FinalizationRegistry>((ref) => this.#byRef.delete(ref)); + + add(stmt: BunSqliteStatement): void { + const ref = new WeakRef(stmt); + this.#refOf.set(stmt, ref); + this.#byRef.add(ref); + this.#registry.register(stmt, ref, ref); + } + + /** Stop tracking `stmt`; false when it was already released. */ + delete(stmt: BunSqliteStatement): boolean { + const ref = this.#refOf.get(stmt); + if (!ref || !this.#byRef.delete(ref)) return false; + this.#refOf.delete(stmt); + this.#registry.unregister(ref); + return true; + } + + /** Statements still reachable, removed from tracking. */ + drain(): BunSqliteStatement[] { + const live: BunSqliteStatement[] = []; + for (const ref of this.#byRef) { + const stmt = ref.deref(); + this.#registry.unregister(ref); + if (stmt) { + this.#refOf.delete(stmt); + live.push(stmt); + } + } + this.#byRef.clear(); + return live; + } +} + +export class BunSqliteDatabase { + readonly native: Database; + readonly name: string; + readonly memory: boolean; + #open = true; + /** + * Statements from `prepare()`. bun:sqlite closes with `sqlite3_close_v2`, so a + * connection with unfinalized statements stays open until they are collected; + * better-sqlite3 finalizes them on `close()`, and so does this facade. + */ + readonly #live = new LiveStatements(); + + constructor(path = ':memory:') { + this.native = new Database(path, { create: true, strict: true }); + this.name = path; + this.memory = path === ':memory:' || path === ''; + } + + get open(): boolean { + return this.#open; + } + + get inTransaction(): boolean { + return this.native.inTransaction; + } + + exec(sql: string): this { + for (const stmt of this.#statements(sql)) { + try { + stmt.run(); + } finally { + stmt.finalize(); + } + } + return this; + } + + /** + * Compile `sql` one complete statement at a time. Each statement is yielded + * before the next is compiled, so DDL earlier in a script is visible to later + * statements when the caller runs each as it arrives. + */ + *#statements(sql: string): Generator { + let pending = ''; + let pendingHasCode = false; + for (const chunk of splitSqlChunks(sql)) { + pending += chunk.text; + pendingHasCode ||= chunk.hasCode; + if (!pendingHasCode) { + pending = ''; + continue; + } + let stmt: Statement; + try { + stmt = this.native.prepare(pending); + } catch (err) { + if (isIncompleteInput(err)) continue; + throw err; + } + yield stmt; + pending = ''; + pendingHasCode = false; + } + // A trailing statement that never completed: let SQLite report it. + if (pendingHasCode) yield this.native.prepare(pending); + } + + prepare(sql: string): BunSqliteStatement { + if (splitSqlChunks(sql).filter((chunk) => chunk.hasCode).length > 1) { + // Possibly several statements (or one trigger with `;` in its body). + // bun:sqlite would silently compile only the first; reject like + // better-sqlite3 instead. + const statements = [...this.#statements(sql)]; + for (const stmt of statements.slice(1)) stmt.finalize(); + if (statements.length > 1) { + statements[0].finalize(); + throw new RangeError('The supplied SQL string contains more than one statement'); + } + return new BunSqliteStatement(statements[0], this.native, this.#live); + } + return new BunSqliteStatement(this.native.prepare(sql), this.native, this.#live); + } + + pragma(source: string, options: { simple?: boolean } = {}): unknown { + const stmt = this.native.prepare(`PRAGMA ${source}`); + let rows: Record[]; + try { + rows = stmt.all() as Record[]; + } finally { + stmt.finalize(); + } + if (!options.simple) return rows; + const first = rows[0]; + return first === undefined ? undefined : Object.values(first)[0]; + } + + transaction unknown>(fn: F): F { + return this.native.transaction(fn) as unknown as F; + } + + function(name: string): never { + throw new Error(`bun:sqlite does not support user-defined SQL functions (${name})`); + } + + async backup(destination: string): Promise<{ totalPages: number; remainingPages: number }> { + writeFileSync(destination, this.native.serialize()); + return { totalPages: 0, remainingPages: 0 }; + } + + close(): this { + if (this.#open) { + for (const stmt of this.#live.drain()) { + try { + stmt.finalizeNative(); + } catch { + // Already finalized natively; closing must not fail over it. + } + } + this.native.close(); + this.#open = false; + } + return this; + } +} + +/** + * Drizzle runs over the facade rather than the native handle, so its + * statements get the same normalization (and the same `prepare` seam) as + * direct callers. + */ +export function drizzleOver(sqlite: unknown, config: DrizzleConfig> = {}) { + return drizzle(sqlite as Database, config); +} + +export const sqliteDriver: SqliteDriver = { + name: 'bun:sqlite', + open: (path) => new BunSqliteDatabase(path) as unknown as SqliteConnection, + drizzle: (sqlite) => drizzleOver(sqlite, { schema }), +}; diff --git a/packages/engine/standalone/ssrf-guard.test.ts b/packages/engine/standalone/ssrf-guard.test.ts new file mode 100644 index 000000000..0e28b27da --- /dev/null +++ b/packages/engine/standalone/ssrf-guard.test.ts @@ -0,0 +1,32 @@ +/** + * Under Bun, `undici` is Bun's shim and ignores `dispatcher`, so the engine's + * connect-time DNS pinning must come from the node:https transport instead. + * Run with `bun test standalone`. + */ +import { afterAll, expect, it, mock } from 'bun:test'; +import * as dns from 'node:dns'; +import { createServer } from 'node:net'; + +// Every name resolves to loopback, as a hostile public DNS record would. +mock.module('node:dns', () => ({ + ...dns, + lookup: (_host: string, _opts: unknown, cb: (e: null, a: { address: string; family: number }[]) => void) => + cb(null, [{ address: '127.0.0.1', family: 4 }]), +})); +const { nodeSafeWebhookFetch } = await import('../src/adapters/node/ssrf-fetch.js'); + +let connections = 0; +const server = createServer((socket) => { + connections += 1; + socket.destroy(); +}); +await new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve())); +const { port } = server.address() as { port: number }; +afterAll(() => server.close()); + +it('refuses a webhook host that resolves to a private address before connecting', async () => { + await expect( + nodeSafeWebhookFetch(`https://hooks.example.com:${port}/deliver`, { method: 'POST', body: '{}' }), + ).rejects.toThrow('non-global address'); + expect(connections).toBe(0); +}); diff --git a/packages/engine/standalone/test-shims/better-sqlite3.ts b/packages/engine/standalone/test-shims/better-sqlite3.ts new file mode 100644 index 000000000..401400b28 --- /dev/null +++ b/packages/engine/standalone/test-shims/better-sqlite3.ts @@ -0,0 +1,9 @@ +/** + * Test-only: under `npm run test:bun` (vitest in the Bun runtime) direct + * `better-sqlite3` imports in the suite resolve here, so tests that open their + * own connection exercise bun:sqlite through the same facade the standalone + * binary uses. + */ +import { BunSqliteDatabase } from '../sqlite-driver.js'; + +export default BunSqliteDatabase; diff --git a/packages/engine/standalone/test-shims/drizzle-better-sqlite3.ts b/packages/engine/standalone/test-shims/drizzle-better-sqlite3.ts new file mode 100644 index 000000000..61bf711f7 --- /dev/null +++ b/packages/engine/standalone/test-shims/drizzle-better-sqlite3.ts @@ -0,0 +1,2 @@ +/** Test-only: `drizzle-orm/better-sqlite3` -> `drizzle-orm/bun-sqlite` over the bun:sqlite facade. */ +export { drizzleOver as drizzle } from '../sqlite-driver.js'; diff --git a/packages/engine/standalone/tsconfig.json b/packages/engine/standalone/tsconfig.json new file mode 100644 index 000000000..c4f4af994 --- /dev/null +++ b/packages/engine/standalone/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "noEmit": true, + "composite": false, + "declaration": false, + "declarationMap": false, + "rootDir": "..", + "types": ["bun"] + }, + "include": ["."], + // Tests import Node-typed src/ modules (e.g. fetch signatures) that do not + // match Bun's lib types; `bun test` transpiles and runs them instead. + "exclude": ["*.test.ts"] +} diff --git a/packages/engine/standalone/version.ts b/packages/engine/standalone/version.ts new file mode 100644 index 000000000..416508806 --- /dev/null +++ b/packages/engine/standalone/version.ts @@ -0,0 +1,6 @@ +/** Standalone replacement for `src/bin/version.ts`: the version is inlined at bundle time. */ +import pkg from '../package.json' with { type: 'json' }; + +export function engineVersion(): string { + return pkg.version; +} diff --git a/packages/engine/vitest.config.ts b/packages/engine/vitest.config.ts index a32c9ef15..68cacb86f 100644 --- a/packages/engine/vitest.config.ts +++ b/packages/engine/vitest.config.ts @@ -1,10 +1,35 @@ +import { fileURLToPath } from 'node:url'; import { defineConfig } from 'vitest/config'; +// Under the Bun runtime (`npm run test:bun`), run the suite against bun:sqlite — +// the driver the standalone binary ships — instead of the better-sqlite3 addon. +const isBun = typeof process.versions.bun === 'string'; +const standalone = (path: string) => fileURLToPath(new URL(`./standalone/${path}`, import.meta.url)); + export default defineConfig({ + resolve: { + alias: isBun + ? [ + { find: /^\.\/sqlite-driver\.js$/, replacement: standalone('sqlite-driver.ts') }, + { find: /^better-sqlite3$/, replacement: standalone('test-shims/better-sqlite3.ts') }, + { find: /^drizzle-orm\/better-sqlite3$/, replacement: standalone('test-shims/drizzle-better-sqlite3.ts') }, + ] + : [], + }, test: { include: ['src/**/*.test.ts'], + // These spawn the repo's Node maintenance scripts (better-sqlite3) as child + // processes; they are not part of the standalone binary. + exclude: isBun + ? ['src/__tests__/conformance/agentRetentionCli.test.ts', 'src/db/__tests__/compactMigrations.test.ts'] + : [], environment: 'node', - testTimeout: 20_000, - hookTimeout: 20_000, + // vitest's module runner is slower under Bun; the heavier conformance + // fixtures need more headroom there. + testTimeout: isBun ? 60_000 : 20_000, + hookTimeout: isBun ? 60_000 : 20_000, + // zod's `export { z }` namespace re-export resolves to undefined through + // vitest's externalized import path under Bun; inlining it avoids that. + server: { deps: { inline: isBun ? ['zod'] : [] } }, }, });