From 25bbb7186c040cc04b34cb21a6c895a30edae870 Mon Sep 17 00:00:00 2001 From: Jalil Date: Wed, 28 Oct 2026 13:04:09 -0700 Subject: [PATCH 01/76] feat: add the control plane, policy engine, scheduler and stores differently-server exposes the v1 API with scoped API keys, organizations and projects, atomic quota admission and activation, fair queueing and policy-driven placement that keeps external providers fail-closed. The scheduler provisions idempotently, retries safely and reconciles orphans and lost instances. State lives in Postgres (memory store for dev), with a tamper-evident audit chain, a trust report, usage estimates, Prometheus metrics and a built-in dashboard. Adapters cover Firecracker pools, Daytona, Freestyle, remote plugins and a dev-only host adapter. Signed-off-by: Jalil --- Makefile | 60 ++ api/embed.go | 9 + api/openapi.yaml | 569 +++++++++++ cmd/differently-server/main.go | 241 +++++ go.mod | 32 + go.sum | 68 ++ internal/adapters/daytona/daytona.go | 405 ++++++++ internal/adapters/daytona/daytona_test.go | 168 ++++ internal/adapters/devhost/devhost.go | 246 +++++ internal/adapters/devhost/devhost_test.go | 59 ++ internal/adapters/fake/fake.go | 343 +++++++ internal/adapters/firecracker/pool.go | 349 +++++++ internal/adapters/firecracker/pool_test.go | 75 ++ internal/adapters/freestyle/freestyle.go | 402 ++++++++ internal/adapters/freestyle/freestyle_test.go | 140 +++ internal/adapters/httpx/httpx.go | 174 ++++ internal/audit/audit.go | 71 ++ internal/audit/audit_test.go | 37 + internal/auditlog/auditlog.go | 66 ++ internal/auth/auth.go | 167 ++++ internal/auth/auth_test.go | 69 ++ internal/config/config.go | 372 ++++++++ internal/config/config_test.go | 64 ++ internal/dashboard/dashboard.go | 25 + internal/dashboard/static/app.js | 110 +++ internal/dashboard/static/index.html | 29 + internal/dashboard/static/style.css | 23 + internal/guest/client.go | 154 +++ internal/guest/guest.go | 369 ++++++++ internal/guest/guest_test.go | 85 ++ internal/guest/proc_other.go | 7 + internal/guest/proc_unix.go | 20 + internal/id/id.go | 73 ++ internal/id/id_test.go | 24 + internal/metrics/metrics.go | 48 + internal/policy/policy.go | 334 +++++++ internal/policy/policy_test.go | 147 +++ internal/pools/pools.go | 274 ++++++ internal/scheduler/scheduler.go | 603 ++++++++++++ internal/server/admin.go | 495 ++++++++++ internal/server/routes_test.go | 40 + internal/server/sandboxes.go | 880 ++++++++++++++++++ internal/server/server.go | 374 ++++++++ internal/server/server_test.go | 467 ++++++++++ internal/server/tenancy.go | 504 ++++++++++ internal/store/memory.go | 576 ++++++++++++ internal/store/memory_test.go | 12 + .../store/postgres/migrations/0001_init.sql | 75 ++ internal/store/postgres/postgres.go | 709 ++++++++++++++ internal/store/postgres/postgres_test.go | 34 + internal/store/store.go | 300 ++++++ internal/store/storetest/storetest.go | 254 +++++ internal/version/version.go | 9 + internal/wiring/wiring.go | 93 ++ pkg/adapter/adapter.go | 295 ++++++ pkg/adapter/adapterhttp/adapterhttp.go | 576 ++++++++++++ pkg/api/errors.go | 96 ++ pkg/api/types.go | 455 +++++++++ 58 files changed, 12755 insertions(+) create mode 100644 Makefile create mode 100644 api/embed.go create mode 100644 api/openapi.yaml create mode 100644 cmd/differently-server/main.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/adapters/daytona/daytona.go create mode 100644 internal/adapters/daytona/daytona_test.go create mode 100644 internal/adapters/devhost/devhost.go create mode 100644 internal/adapters/devhost/devhost_test.go create mode 100644 internal/adapters/fake/fake.go create mode 100644 internal/adapters/firecracker/pool.go create mode 100644 internal/adapters/firecracker/pool_test.go create mode 100644 internal/adapters/freestyle/freestyle.go create mode 100644 internal/adapters/freestyle/freestyle_test.go create mode 100644 internal/adapters/httpx/httpx.go create mode 100644 internal/audit/audit.go create mode 100644 internal/audit/audit_test.go create mode 100644 internal/auditlog/auditlog.go create mode 100644 internal/auth/auth.go create mode 100644 internal/auth/auth_test.go create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/dashboard/dashboard.go create mode 100644 internal/dashboard/static/app.js create mode 100644 internal/dashboard/static/index.html create mode 100644 internal/dashboard/static/style.css create mode 100644 internal/guest/client.go create mode 100644 internal/guest/guest.go create mode 100644 internal/guest/guest_test.go create mode 100644 internal/guest/proc_other.go create mode 100644 internal/guest/proc_unix.go create mode 100644 internal/id/id.go create mode 100644 internal/id/id_test.go create mode 100644 internal/metrics/metrics.go create mode 100644 internal/policy/policy.go create mode 100644 internal/policy/policy_test.go create mode 100644 internal/pools/pools.go create mode 100644 internal/scheduler/scheduler.go create mode 100644 internal/server/admin.go create mode 100644 internal/server/routes_test.go create mode 100644 internal/server/sandboxes.go create mode 100644 internal/server/server.go create mode 100644 internal/server/server_test.go create mode 100644 internal/server/tenancy.go create mode 100644 internal/store/memory.go create mode 100644 internal/store/memory_test.go create mode 100644 internal/store/postgres/migrations/0001_init.sql create mode 100644 internal/store/postgres/postgres.go create mode 100644 internal/store/postgres/postgres_test.go create mode 100644 internal/store/store.go create mode 100644 internal/store/storetest/storetest.go create mode 100644 internal/version/version.go create mode 100644 internal/wiring/wiring.go create mode 100644 pkg/adapter/adapter.go create mode 100644 pkg/adapter/adapterhttp/adapterhttp.go create mode 100644 pkg/api/errors.go create mode 100644 pkg/api/types.go diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5123ea3 --- /dev/null +++ b/Makefile @@ -0,0 +1,60 @@ +# Differently build and verification targets. `make verify` runs what CI runs. + +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo 0.1.0-dev) +COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +LDFLAGS := -s -w -X github.com/different-ai/differently/internal/version.Version=$(VERSION) -X github.com/different-ai/differently/internal/version.Commit=$(COMMIT) +BINS := differently-server differently-agent differently-guest differently +PG_PORT ?= 55432 + +.PHONY: build build-linux test test-postgres fmt-check vet sdk-test helm-lint verify dev images clean + +build: + @mkdir -p bin + CGO_ENABLED=0 go build -trimpath -ldflags "$(LDFLAGS)" -o bin/ $(addprefix ./cmd/,$(BINS)) + +build-linux: + @for arch in amd64 arm64; do \ + mkdir -p bin/linux-$$arch; \ + CGO_ENABLED=0 GOOS=linux GOARCH=$$arch go build -trimpath -ldflags "$(LDFLAGS)" -o bin/linux-$$arch/ $(addprefix ./cmd/,$(BINS)) || exit 1; \ + done + +fmt-check: + @out=$$(gofmt -l cmd internal pkg api); if [ -n "$$out" ]; then echo "gofmt needed:"; echo "$$out"; exit 1; fi + +vet: + go vet ./... + +test: + go test -race -count=1 ./... + +# Runs the store conformance suite against a disposable Postgres container. +test-postgres: + @docker rm -f differently-pg-test >/dev/null 2>&1 || true + docker run -d --rm --name differently-pg-test -p $(PG_PORT):5432 -e POSTGRES_PASSWORD=test postgres:17-alpine >/dev/null + @for i in $$(seq 1 30); do docker exec differently-pg-test pg_isready -U postgres >/dev/null 2>&1 && break; sleep 1; done + DIFFERENTLY_TEST_POSTGRES_URL=postgres://postgres:test@localhost:$(PG_PORT)/postgres?sslmode=disable go test -count=1 ./internal/store/postgres/; \ + status=$$?; docker rm -f differently-pg-test >/dev/null; exit $$status + +sdk-test: + cd sdk/typescript && npm ci --no-audit --no-fund && npm test + cd sdk/python && python3 -m unittest discover -s tests + +helm-lint: + helm lint deploy/helm/differently + @for f in deploy/helm/differently/ci/*.yaml; do \ + echo "helm template with $$f"; \ + helm template differently deploy/helm/differently -f $$f >/dev/null || exit 1; \ + done + +verify: fmt-check vet test sdk-test helm-lint + +dev: build + ./bin/differently-server --dev + +images: + docker build -f build/server.Dockerfile -t ghcr.io/different-ai/differently-server:$(VERSION) . + docker build -f build/agent.Dockerfile -t ghcr.io/different-ai/differently-agent:$(VERSION) . + docker build -f build/cli.Dockerfile -t ghcr.io/different-ai/differently-cli:$(VERSION) . + +clean: + rm -rf bin dist diff --git a/api/embed.go b/api/embed.go new file mode 100644 index 0000000..3849472 --- /dev/null +++ b/api/embed.go @@ -0,0 +1,9 @@ +// Package apispec embeds the OpenAPI description of the Differently API. +package apispec + +import _ "embed" + +// OpenAPI is the OpenAPI 3.1 document served at /openapi.yaml. +// +//go:embed openapi.yaml +var OpenAPI []byte diff --git a/api/openapi.yaml b/api/openapi.yaml new file mode 100644 index 0000000..05be540 --- /dev/null +++ b/api/openapi.yaml @@ -0,0 +1,569 @@ +openapi: 3.1.0 +info: + title: Differently API + version: v1 + summary: Self-hosted sandbox orchestration. Your infrastructure, your rules, one API. + description: | + One vendor-neutral API for isolated compute environments on infrastructure you + own, with optional, policy-gated use of external providers. + + * Authenticate with `Authorization: Bearer dly__`. + * Errors use `{"error": {"code", "message", "details", "requestId"}}`. Branch on `code`. + * Lists return `{"items": [...], "nextCursor": "..."}`. + * Backends are never named by applications: placement is decided by operator, + organization and project policies, which a request can only narrow. + license: + name: Apache-2.0 + identifier: Apache-2.0 +servers: + - url: http://localhost:8080 + description: Local development server (differently-server --dev) +security: + - bearer: [] +tags: + - name: Sandboxes + - name: Exec and files + - name: Lifecycle + - name: Tenancy + - name: Policy + - name: Usage and audit + - name: Admin + - name: System + +paths: + /healthz: + get: { tags: [System], security: [], summary: Liveness, responses: { "200": { description: OK } } } + /readyz: + get: { tags: [System], security: [], summary: Readiness (store reachable), responses: { "200": { description: Ready }, "503": { $ref: "#/components/responses/Error" } } } + /version: + get: + tags: [System] + security: [] + summary: Build and installation identity + responses: { "200": { description: Version, content: { application/json: { schema: { $ref: "#/components/schemas/VersionInfo" } } } } } + /metrics: + get: { tags: [System], security: [], summary: Prometheus metrics, responses: { "200": { description: Prometheus text format } } } + /v1/me: + get: + tags: [Tenancy] + summary: Describe the calling key + responses: { "200": { description: Caller, content: { application/json: { schema: { $ref: "#/components/schemas/Me" } } } } } + + /v1/sandboxes: + post: + tags: [Sandboxes] + summary: Create a sandbox + description: | + Admits the request against quota and queues it (202). The scheduler places it on + an eligible pool. Returns 422 immediately when no permitted pool could ever satisfy + it (`no_eligible_pool`, `unsupported_capability`). Requires `sandboxes:write`. + parameters: + - { name: Idempotency-Key, in: header, schema: { type: string }, description: Replaying the same key within a project returns the original sandbox (200). } + requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/CreateSandboxRequest" } } } } + responses: + "202": { description: Queued, content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } } + "200": { description: Idempotent replay, content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } } + "422": { $ref: "#/components/responses/Error" } + "429": { $ref: "#/components/responses/Error" } + get: + tags: [Sandboxes] + summary: List sandboxes (newest first) + parameters: + - { name: status, in: query, schema: { type: string }, description: Comma-separated statuses } + - { name: projectId, in: query, schema: { type: string } } + - { name: pool, in: query, schema: { type: string } } + - { name: label, in: query, schema: { type: string }, description: "key=value; repeatable" } + - { $ref: "#/components/parameters/Cursor" } + - { $ref: "#/components/parameters/Limit" } + responses: { "200": { description: Page, content: { application/json: { schema: { $ref: "#/components/schemas/SandboxList" } } } } } + /v1/sandboxes/{id}: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + get: + tags: [Sandboxes] + summary: Get a sandbox + responses: { "200": { description: Sandbox, content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } }, "404": { $ref: "#/components/responses/Error" } } + patch: + tags: [Sandboxes] + summary: Update labels or TTL + requestBody: { content: { application/json: { schema: { $ref: "#/components/schemas/UpdateSandboxRequest" } } } } + responses: { "200": { description: Updated, content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } } } + delete: + tags: [Sandboxes] + summary: Delete (terminate) a sandbox + description: Idempotent. Queued sandboxes terminate immediately; others move to `terminating`. + responses: { "202": { description: Terminating, content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } } } + /v1/sandboxes/{id}/wait: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + get: + tags: [Sandboxes] + summary: Long-poll until a status is reached + parameters: + - { name: status, in: query, schema: { $ref: "#/components/schemas/SandboxStatus" }, description: Defaults to running } + - { name: timeoutSeconds, in: query, schema: { type: integer, minimum: 1, maximum: 300, default: 60 } } + responses: { "200": { description: Current sandbox (check status), content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } } } + /v1/sandboxes/{id}/placement: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + get: + tags: [Sandboxes] + summary: Explain where the sandbox runs and why + description: Includes every pool evaluated with reasons. For queued sandboxes, `current` is a fresh dry-run decision. + responses: { "200": { description: Explanation, content: { application/json: { schema: { $ref: "#/components/schemas/PlacementExplanation" } } } } } + /v1/sandboxes/{id}/exec: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: + tags: [Exec and files] + summary: Run a command + description: | + Requires `sandboxes:exec` and a running sandbox. With `?stream=1` (or + `Accept: text/event-stream`) the response is server-sent events named + `stdout`, `stderr`, `exit` and `error`, each carrying an ExecEvent as JSON data. + parameters: [{ name: stream, in: query, schema: { type: string } }] + requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/ExecRequest" } } } } + responses: + "200": + description: Result, or an event stream + content: + application/json: { schema: { $ref: "#/components/schemas/ExecResult" } } + text/event-stream: { schema: { type: string } } + "409": { $ref: "#/components/responses/Error" } + /v1/sandboxes/{id}/files: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + get: + tags: [Exec and files] + summary: List a directory + parameters: [{ $ref: "#/components/parameters/Path" }] + responses: { "200": { description: Entries, content: { application/json: { schema: { $ref: "#/components/schemas/FileList" } } } }, "501": { $ref: "#/components/responses/Error" } } + /v1/sandboxes/{id}/files/content: + parameters: [{ $ref: "#/components/parameters/SandboxID" }, { $ref: "#/components/parameters/Path" }] + get: + tags: [Exec and files] + summary: Download a file + responses: { "200": { description: File bytes, content: { application/octet-stream: { schema: { type: string, format: binary } } } } } + put: + tags: [Exec and files] + summary: Upload a file (max 100 MiB), creating parent directories + requestBody: { required: true, content: { application/octet-stream: { schema: { type: string, format: binary } } } } + responses: { "204": { description: Written } } + /v1/sandboxes/{id}/stop: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: { tags: [Lifecycle], summary: Stop (keep disk, release CPU and memory), responses: { "200": { $ref: "#/components/responses/SandboxOK" }, "501": { $ref: "#/components/responses/Error" } } } + /v1/sandboxes/{id}/start: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: { tags: [Lifecycle], summary: Start a stopped sandbox from its disk, responses: { "200": { $ref: "#/components/responses/SandboxOK" }, "501": { $ref: "#/components/responses/Error" } } } + /v1/sandboxes/{id}/pause: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: { tags: [Lifecycle], summary: Pause (freeze vCPUs; memory stays resident), responses: { "200": { $ref: "#/components/responses/SandboxOK" }, "501": { $ref: "#/components/responses/Error" } } } + /v1/sandboxes/{id}/resume: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: { tags: [Lifecycle], summary: Resume a paused sandbox, responses: { "200": { $ref: "#/components/responses/SandboxOK" }, "501": { $ref: "#/components/responses/Error" } } } + /v1/sandboxes/{id}/snapshots: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: + tags: [Lifecycle] + summary: Snapshot a sandbox + description: Snapshots live on the pool (and, for Firecracker, the node) that took them; restoring is only possible there. + requestBody: { content: { application/json: { schema: { $ref: "#/components/schemas/CreateSnapshotRequest" } } } } + responses: { "201": { description: Snapshot, content: { application/json: { schema: { $ref: "#/components/schemas/Snapshot" } } } }, "501": { $ref: "#/components/responses/Error" } } + /v1/sandboxes/{id}/fork: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: + tags: [Lifecycle] + summary: Snapshot and start a copy on the same pool + responses: { "202": { description: Queued child, content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } } } + /v1/sandboxes/{id}/ports: + parameters: [{ $ref: "#/components/parameters/SandboxID" }] + post: + tags: [Lifecycle] + summary: Expose a service port (backend-dependent) + requestBody: { required: true, content: { application/json: { schema: { type: object, required: [port], properties: { port: { type: integer, minimum: 1, maximum: 65535 } } } } } } + responses: { "200": { description: URL, content: { application/json: { schema: { $ref: "#/components/schemas/PortURL" } } } }, "501": { $ref: "#/components/responses/Error" } } + /v1/snapshots: + get: + tags: [Lifecycle] + summary: List snapshots + responses: { "200": { description: Snapshots, content: { application/json: { schema: { type: object, properties: { items: { type: array, items: { $ref: "#/components/schemas/Snapshot" } } } } } } } } + /v1/snapshots/{id}: + parameters: [{ name: id, in: path, required: true, schema: { type: string } }] + get: { tags: [Lifecycle], summary: Get a snapshot, responses: { "200": { description: Snapshot, content: { application/json: { schema: { $ref: "#/components/schemas/Snapshot" } } } } } } + delete: { tags: [Lifecycle], summary: Delete a snapshot, responses: { "204": { description: Deleted } } } + + /v1/projects: + get: { tags: [Tenancy], summary: List projects, responses: { "200": { description: Projects, content: { application/json: { schema: { type: object, properties: { items: { type: array, items: { $ref: "#/components/schemas/Project" } } } } } } } } } + post: + tags: [Tenancy] + summary: Create a project (projects:write) + requestBody: { required: true, content: { application/json: { schema: { type: object, required: [name], properties: { name: { type: string }, orgId: { type: string }, quota: { $ref: "#/components/schemas/Quota" } } } } } } + responses: { "201": { description: Project, content: { application/json: { schema: { $ref: "#/components/schemas/Project" } } } } } + /v1/projects/{id}: + parameters: [{ name: id, in: path, required: true, schema: { type: string } }] + get: { tags: [Tenancy], summary: Get a project, responses: { "200": { description: Project, content: { application/json: { schema: { $ref: "#/components/schemas/Project" } } } } } } + delete: { tags: [Tenancy], summary: Delete an empty project, responses: { "204": { description: Deleted }, "409": { $ref: "#/components/responses/Error" } } } + /v1/api-keys: + get: { tags: [Tenancy], summary: List keys (keys:write), responses: { "200": { description: Keys, content: { application/json: { schema: { type: object, properties: { items: { type: array, items: { $ref: "#/components/schemas/APIKey" } } } } } } } } } + post: + tags: [Tenancy] + summary: Create a scoped key + description: A key can only grant scopes it holds, within its own organization and project. The token is returned once. + requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/CreateAPIKeyRequest" } } } } + responses: { "201": { description: Key and one-time token, content: { application/json: { schema: { $ref: "#/components/schemas/CreateAPIKeyResponse" } } } }, "403": { $ref: "#/components/responses/Error" } } + /v1/api-keys/{id}: + delete: { tags: [Tenancy], summary: Revoke a key, parameters: [{ name: id, in: path, required: true, schema: { type: string } }], responses: { "204": { description: Revoked } } } + /v1/quotas: + get: + tags: [Usage and audit] + summary: Quota limits and current consumption (usage:read) + parameters: [{ name: projectId, in: query, schema: { type: string } }, { name: orgId, in: query, schema: { type: string }, description: Operator keys only }] + responses: { "200": { description: Report, content: { application/json: { schema: { $ref: "#/components/schemas/QuotaReport" } } } } } + /v1/usage: + get: + tags: [Usage and audit] + summary: Estimated usage over a window (usage:read) + parameters: + - { name: from, in: query, schema: { type: string, format: date-time } } + - { name: to, in: query, schema: { type: string, format: date-time } } + - { name: groupBy, in: query, schema: { type: string, enum: [pool, project, org, trust], default: pool } } + responses: { "200": { description: Report, content: { application/json: { schema: { $ref: "#/components/schemas/UsageReport" } } } } } + /v1/audit-events: + get: + tags: [Usage and audit] + summary: List audit events in sequence order (audit:read) + parameters: + - { name: afterSeq, in: query, schema: { type: integer } } + - { name: resource, in: query, schema: { type: string } } + - { name: action, in: query, schema: { type: string } } + - { $ref: "#/components/parameters/Limit" } + responses: { "200": { description: Events, content: { application/json: { schema: { type: object, properties: { items: { type: array, items: { $ref: "#/components/schemas/AuditEvent" } }, nextCursor: { type: string } } } } } } } + /v1/policies: + get: { tags: [Policy], summary: List visible placement policies, responses: { "200": { description: Policies, content: { application/json: { schema: { type: object, properties: { items: { type: array, items: { $ref: "#/components/schemas/PlacementPolicy" } } } } } } } } } + /v1/policies/{scope}: + parameters: [{ name: scope, in: path, required: true, schema: { type: string }, description: "global (operator), org: or project:" }] + put: + tags: [Policy] + summary: Set a placement policy + requestBody: { required: true, content: { application/json: { schema: { $ref: "#/components/schemas/PlacementPolicy" } } } } + responses: { "200": { description: Stored, content: { application/json: { schema: { $ref: "#/components/schemas/PlacementPolicy" } } } }, "403": { $ref: "#/components/responses/Error" } } + delete: { tags: [Policy], summary: Remove a policy, responses: { "204": { description: Removed } } } + /v1/capabilities: + get: { tags: [Policy], summary: Pools as tenants see them (no config or credentials), responses: { "200": { description: Pools } } } + /v1/profiles: + get: { tags: [Policy], summary: Resource profiles, responses: { "200": { description: Profiles } } } + /v1/templates: + get: { tags: [Policy], summary: Image templates, responses: { "200": { description: Templates } } } + /v1/classifications: + get: { tags: [Policy], summary: Data classifications, responses: { "200": { description: Classifications } } } + + /v1/admin/organizations: + get: { tags: [Admin], summary: List organizations, responses: { "200": { description: Organizations } } } + post: + tags: [Admin] + summary: Create an organization (optionally with a default project) + requestBody: { required: true, content: { application/json: { schema: { type: object, required: [name], properties: { name: { type: string }, quota: { $ref: "#/components/schemas/Quota" }, defaultProject: { type: string } } } } } } + responses: { "201": { description: Created } } + /v1/admin/organizations/{id}: + parameters: [{ name: id, in: path, required: true, schema: { type: string } }] + get: { tags: [Admin], summary: Get an organization, responses: { "200": { description: Organization } } } + patch: { tags: [Admin], summary: Rename or set quota, responses: { "200": { description: Organization } } } + /v1/admin/organizations/{id}/suspend: + post: + tags: [Admin] + summary: Suspend a tenant (optionally terminating its sandboxes) + parameters: [{ name: id, in: path, required: true, schema: { type: string } }] + requestBody: { content: { application/json: { schema: { type: object, properties: { terminate: { type: boolean }, reason: { type: string } } } } } } + responses: { "200": { description: Suspended } } + /v1/admin/organizations/{id}/resume: + post: { tags: [Admin], summary: Lift a suspension, parameters: [{ name: id, in: path, required: true, schema: { type: string } }], responses: { "200": { description: Resumed } } } + /v1/admin/projects/{id}: + patch: { tags: [Admin], summary: Set a project quota, parameters: [{ name: id, in: path, required: true, schema: { type: string } }], responses: { "200": { description: Project } } } + /v1/admin/pools: + get: { tags: [Admin], summary: List pools with health and capacity, responses: { "200": { description: Pools } } } + post: + tags: [Admin] + summary: Register a pool at runtime + description: 'Credentials are references (`{"env": NAME}` or `{"file": PATH}`) resolved on the server, never values.' + responses: { "201": { description: Pool } } + /v1/admin/pools/{name}: + parameters: [{ name: name, in: path, required: true, schema: { type: string } }] + get: { tags: [Admin], summary: Get a pool, responses: { "200": { description: Pool } } } + patch: { tags: [Admin], summary: Enable/disable, set concurrency, cost model or region, responses: { "200": { description: Pool } } } + delete: { tags: [Admin], summary: Remove an API-registered pool with no sandboxes, responses: { "204": { description: Removed } } } + /v1/admin/pools/{name}/drain: + post: { tags: [Admin], summary: Stop new placements on a pool, parameters: [{ name: name, in: path, required: true, schema: { type: string } }], responses: { "200": { description: Pool } } } + /v1/admin/pools/{name}/undrain: + post: { tags: [Admin], summary: Resume placements on a pool, parameters: [{ name: name, in: path, required: true, schema: { type: string } }], responses: { "200": { description: Pool } } } + /v1/admin/agents: + get: { tags: [Admin], summary: List node agents, responses: { "200": { description: Agents } } } + /v1/admin/sandboxes/{id}/terminate: + post: { tags: [Admin], summary: Terminate any tenant's sandbox, parameters: [{ $ref: "#/components/parameters/SandboxID" }], responses: { "202": { description: Terminating } } } + /v1/admin/trust-report: + get: { tags: [Admin], summary: Where work may go, where it went, and what cannot be proven, responses: { "200": { description: Trust report } } } + /v1/admin/audit/verify: + get: { tags: [Admin], summary: Verify the audit hash chain, responses: { "200": { description: Result } } } + /v1/admin/reconcile: + post: { tags: [Admin], summary: Run the reconciler now, responses: { "200": { description: Done } } } + /internal/v1/agents/heartbeat: + post: + tags: [System] + security: [{ agentToken: [] }] + summary: Node agent registration and capacity heartbeat + responses: { "200": { description: Registered } } + +components: + securitySchemes: + bearer: { type: http, scheme: bearer, bearerFormat: "dly__" } + agentToken: { type: http, scheme: bearer, description: Shared node-agent token } + parameters: + SandboxID: { name: id, in: path, required: true, schema: { type: string, example: sbx_01j9z3k6c8x2m4n6p8q0r2s4t6 } } + Path: { name: path, in: query, required: true, schema: { type: string } } + Cursor: { name: cursor, in: query, schema: { type: string } } + Limit: { name: limit, in: query, schema: { type: integer, minimum: 1, maximum: 500, default: 50 } } + responses: + Error: + description: Structured error + content: { application/json: { schema: { $ref: "#/components/schemas/ErrorEnvelope" } } } + SandboxOK: + description: Sandbox + content: { application/json: { schema: { $ref: "#/components/schemas/Sandbox" } } } + schemas: + ErrorEnvelope: + type: object + properties: + error: + type: object + required: [code, message] + properties: + code: + type: string + enum: [invalid_request, unauthenticated, permission_denied, not_found, conflict, quota_exceeded, no_eligible_pool, unsupported_capability, unsupported_operation, provider_unavailable, invalid_state, rate_limited, suspended, internal] + message: { type: string } + details: { type: object, additionalProperties: { type: string } } + requestId: { type: string } + VersionInfo: + type: object + properties: { version: { type: string }, commit: { type: string }, apiVersion: { type: string }, installation: { type: string } } + Me: + type: object + properties: { keyId: { type: string }, orgId: { type: string }, projectId: { type: string }, scopes: { type: array, items: { type: string } }, operator: { type: boolean } } + SandboxStatus: + type: string + enum: [queued, provisioning, running, stopped, paused, terminating, terminated, failed] + Resources: + type: object + properties: { cpu: { type: integer, minimum: 1 }, memoryMiB: { type: integer, minimum: 128 }, diskGiB: { type: integer, minimum: 0 } } + Placement: + type: object + description: Request-level preference; can only narrow policy. + properties: + strategy: { type: string, enum: [private-first, cost, spread] } + allowedPools: { type: array, items: { type: string } } + regions: { type: array, items: { type: string } } + external: { type: string, enum: [never, overflow, allowed] } + Capability: + type: string + enum: [exec, exec.stream, files, lifecycle.stop-start, lifecycle.pause-resume, snapshots.disk, snapshots.memory, fork, persistence.disk, ports.expose, ssh, pty, network.egress-control, gpu, arch.amd64, arch.arm64, isolation.microvm, isolation.container] + CreateSandboxRequest: + type: object + properties: + projectId: { type: string, description: Required for organization and operator keys } + template: { type: string } + profile: { type: string, description: Named resource preset, e.g. small } + resources: { $ref: "#/components/schemas/Resources" } + capabilities: { type: array, items: { $ref: "#/components/schemas/Capability" } } + classification: { type: string, description: "Operator-defined, e.g. private-only, internal, public" } + placement: { $ref: "#/components/schemas/Placement" } + ttlSeconds: { type: integer } + env: { type: object, additionalProperties: { type: string }, description: Write-only; never returned } + labels: { type: object, additionalProperties: { type: string } } + priority: { type: integer } + fromSnapshot: { type: string } + UpdateSandboxRequest: + type: object + properties: { labels: { type: object, additionalProperties: { type: string } }, ttlSeconds: { type: integer } } + Sandbox: + type: object + required: [id, orgId, projectId, status, resources, classification, createdAt] + properties: + id: { type: string } + orgId: { type: string } + projectId: { type: string } + status: { $ref: "#/components/schemas/SandboxStatus" } + statusReason: { type: string } + template: { type: string } + profile: { type: string } + resources: { $ref: "#/components/schemas/Resources" } + capabilities: { type: array, items: { type: string } } + classification: { type: string } + placement: { $ref: "#/components/schemas/Placement" } + pool: { type: string } + adapter: { type: string } + trust: { type: string, enum: [private, external] } + region: { type: string } + labels: { type: object, additionalProperties: { type: string } } + envKeys: { type: array, items: { type: string } } + priority: { type: integer } + ttlSeconds: { type: integer } + expiresAt: { type: string, format: date-time } + fromSnapshot: { type: string } + attempts: { type: integer } + createdBy: { type: string } + createdAt: { type: string, format: date-time } + updatedAt: { type: string, format: date-time } + startedAt: { type: string, format: date-time } + endedAt: { type: string, format: date-time } + decision: { $ref: "#/components/schemas/PlacementDecision" } + SandboxList: + type: object + properties: { items: { type: array, items: { $ref: "#/components/schemas/Sandbox" } }, nextCursor: { type: string } } + PlacementDecision: + type: object + properties: + pool: { type: string } + strategy: { type: string } + external: { type: string } + outcome: { type: string, enum: [placed, waiting, rejected] } + reason: { type: string } + policies: { type: array, items: { type: string } } + evaluatedAt: { type: string, format: date-time } + candidates: + type: array + items: + type: object + properties: + pool: { type: string } + trust: { type: string } + eligible: { type: boolean } + hasCapacity: { type: boolean } + estimatedCostPerHour: { type: number } + rank: { type: integer } + reasons: { type: array, items: { type: string } } + PlacementExplanation: + type: object + properties: + sandboxId: { type: string } + status: { $ref: "#/components/schemas/SandboxStatus" } + pool: { type: string } + trust: { type: string } + decision: { $ref: "#/components/schemas/PlacementDecision" } + current: { $ref: "#/components/schemas/PlacementDecision" } + ExecRequest: + type: object + description: Set `command` (run by /bin/sh -c) or `args` (executed directly). + properties: + command: { type: string } + args: { type: array, items: { type: string } } + env: { type: object, additionalProperties: { type: string } } + cwd: { type: string } + stdin: { type: string } + timeoutSeconds: { type: integer } + ExecResult: + type: object + properties: { exitCode: { type: integer }, stdout: { type: string }, stderr: { type: string }, durationMs: { type: integer }, truncated: { type: boolean } } + ExecEvent: + type: object + properties: { stream: { type: string, enum: [stdout, stderr] }, data: { type: string }, exitCode: { type: integer }, error: { type: string } } + FileInfo: + type: object + properties: { path: { type: string }, name: { type: string }, size: { type: integer }, mode: { type: string }, isDir: { type: boolean }, modTime: { type: string, format: date-time } } + FileList: + type: object + properties: { items: { type: array, items: { $ref: "#/components/schemas/FileInfo" } } } + CreateSnapshotRequest: + type: object + properties: { kind: { type: string, enum: [disk, memory], default: disk } } + Snapshot: + type: object + properties: { id: { type: string }, sandboxId: { type: string }, orgId: { type: string }, projectId: { type: string }, pool: { type: string }, kind: { type: string }, status: { type: string }, sizeBytes: { type: integer }, createdAt: { type: string, format: date-time }, createdBy: { type: string } } + PortURL: + type: object + properties: { port: { type: integer }, url: { type: string } } + Quota: + type: object + description: Zero means unlimited at that level. + properties: + maxActiveSandboxes: { type: integer } + maxQueued: { type: integer } + maxCPU: { type: integer } + maxMemoryMiB: { type: integer } + maxDiskGiB: { type: integer } + maxTTLSeconds: { type: integer } + maxSnapshots: { type: integer } + poolBudgetsPerHour: { type: object, additionalProperties: { type: number } } + QuotaUsage: + type: object + properties: { activeSandboxes: { type: integer }, queued: { type: integer }, cpu: { type: integer }, memoryMiB: { type: integer }, diskGiB: { type: integer }, snapshots: { type: integer } } + QuotaReport: + type: object + properties: + orgId: { type: string } + projectId: { type: string } + orgQuota: { $ref: "#/components/schemas/Quota" } + orgUsage: { $ref: "#/components/schemas/QuotaUsage" } + projectQuota: { $ref: "#/components/schemas/Quota" } + projectUsage: { $ref: "#/components/schemas/QuotaUsage" } + Project: + type: object + properties: { id: { type: string }, orgId: { type: string }, name: { type: string }, quota: { $ref: "#/components/schemas/Quota" }, createdAt: { type: string, format: date-time } } + APIKey: + type: object + properties: + id: { type: string } + name: { type: string } + orgId: { type: string } + projectId: { type: string } + scopes: { type: array, items: { type: string } } + createdBy: { type: string } + createdAt: { type: string, format: date-time } + expiresAt: { type: string, format: date-time } + lastUsedAt: { type: string, format: date-time } + revoked: { type: boolean } + CreateAPIKeyRequest: + type: object + required: [name, scopes] + properties: + name: { type: string } + orgId: { type: string } + projectId: { type: string } + scopes: { type: array, items: { type: string, enum: [admin, "org:admin", "sandboxes:read", "sandboxes:write", "sandboxes:exec", "projects:write", "keys:write", "usage:read", "audit:read", "policies:write"] } } + ttlHours: { type: integer } + CreateAPIKeyResponse: + allOf: + - { $ref: "#/components/schemas/APIKey" } + - { type: object, properties: { token: { type: string, description: Shown once } } } + PlacementPolicy: + type: object + properties: + scope: { type: string } + allowedPools: { type: array, items: { type: string } } + deniedPools: { type: array, items: { type: string } } + allowedRegions: { type: array, items: { type: string } } + external: { type: string, enum: [never, overflow, allowed] } + externalPools: { type: array, items: { type: string }, description: External pools explicitly approved; none are approved by default } + strategy: { type: string, enum: [private-first, cost, spread] } + maxCostPerHour: { type: number } + maxQueueSeconds: { type: integer } + UsageReport: + type: object + properties: + from: { type: string, format: date-time } + to: { type: string, format: date-time } + groupBy: { type: string } + note: { type: string } + lines: { type: array, items: { $ref: "#/components/schemas/UsageLine" } } + total: { $ref: "#/components/schemas/UsageLine" } + UsageLine: + type: object + properties: { key: { type: string }, sandboxes: { type: integer }, cpuSeconds: { type: number }, memoryGiBSeconds: { type: number }, runSeconds: { type: number }, queueSeconds: { type: number }, estimatedCost: { type: number } } + AuditEvent: + type: object + properties: + seq: { type: integer } + time: { type: string, format: date-time } + actor: { type: string } + orgId: { type: string } + projectId: { type: string } + action: { type: string } + resource: { type: string } + outcome: { type: string } + details: { type: object, additionalProperties: { type: string } } + requestId: { type: string } + prevHash: { type: string } + hash: { type: string } diff --git a/cmd/differently-server/main.go b/cmd/differently-server/main.go new file mode 100644 index 0000000..db06cf8 --- /dev/null +++ b/cmd/differently-server/main.go @@ -0,0 +1,241 @@ +// Command differently-server runs the Differently control plane: the API, +// scheduler, reconciler and dashboard. +// +// differently-server [--config FILE] [--dev] serve +// differently-server migrate apply database migrations +// differently-server preflight validate config and environment +// differently-server token print a new random API token +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/different-ai/differently/internal/auditlog" + "github.com/different-ai/differently/internal/auth" + "github.com/different-ai/differently/internal/config" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/pools" + "github.com/different-ai/differently/internal/scheduler" + "github.com/different-ai/differently/internal/server" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/internal/store/postgres" + "github.com/different-ai/differently/internal/version" + "github.com/different-ai/differently/internal/wiring" +) + +func main() { + cmd := "serve" + args := os.Args[1:] + if len(args) > 0 && args[0] != "" && args[0][0] != '-' { + cmd, args = args[0], args[1:] + } + fs := flag.NewFlagSet("differently-server", flag.ExitOnError) + cfgPath := fs.String("config", os.Getenv("DIFFERENTLY_CONFIG"), "path to the YAML config file") + dev := fs.Bool("dev", false, "zero-setup local mode: memory store and a devhost pool with NO isolation") + listen := fs.String("listen", "", "override the listen address") + dataDir := fs.String("data-dir", "", "override the data directory") + _ = fs.Parse(args) + + var err error + switch cmd { + case "serve": + err = serve(*cfgPath, *dev, *listen, *dataDir) + case "migrate": + err = migrate(*cfgPath) + case "preflight": + err = preflight(*cfgPath) + case "token": + _, token, _ := auth.NewToken() + fmt.Println(token) + case "version": + fmt.Println(version.Version, version.Commit) + default: + err = fmt.Errorf("unknown command %q (serve, migrate, preflight, token, version)", cmd) + } + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func loadConfig(path string, dev bool, dataDir string) (config.Config, error) { + if path == "" { + if !dev { + return config.Config{}, errors.New("--config is required (or --dev for local development)") + } + if dataDir == "" { + home, _ := os.UserHomeDir() + dataDir = filepath.Join(home, ".differently", "dev") + } + return config.DevDefault(dataDir), nil + } + cfg, err := config.Load(path) + if err != nil { + return cfg, err + } + if dev { + cfg.Dev = true + } + if dataDir != "" { + cfg.DataDir = dataDir + } + return cfg, cfg.Validate() +} + +func openStore(ctx context.Context, cfg config.Config) (store.Store, error) { + if cfg.Store.Driver == "postgres" { + dsn := os.Getenv(cfg.Store.DSNEnv) + if dsn == "" { + return nil, fmt.Errorf("store.driver is postgres but %s is empty", cfg.Store.DSNEnv) + } + return postgres.Open(ctx, dsn) + } + return store.NewMemory(), nil +} + +func serve(cfgPath string, dev bool, listen, dataDir string) error { + cfg, err := loadConfig(cfgPath, dev, dataDir) + if err != nil { + return err + } + if listen != "" { + cfg.Listen = listen + } + var handler slog.Handler = slog.NewJSONHandler(os.Stderr, nil) + if cfg.Dev { + handler = slog.NewTextHandler(os.Stderr, nil) + } + log := slog.New(handler) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + st, err := openStore(ctx, cfg) + if err != nil { + return err + } + defer st.Close() + if pg, ok := st.(*postgres.Store); ok { + if err := pg.Migrate(ctx); err != nil { + return fmt.Errorf("migrate: %w", err) + } + } + installation, err := st.Installation(ctx) + if err != nil { + return err + } + rec, err := auditlog.New(st, log, cfg.Audit.File) + if err != nil { + return err + } + defer rec.Close() + + wiring.Register() + agentToken := os.Getenv(cfg.Auth.AgentTokenEnv) + if agentToken == "" && cfg.Dev { + agentToken = "dev-agent-" + id.Secret(24) + } + pm := pools.NewManager(pools.Deps{ + Store: st, AgentToken: agentToken, AgentTimeout: time.Duration(cfg.Scheduler.AgentTimeoutSeconds) * time.Second, + Dev: cfg.Dev, DataDir: cfg.DataDir, Logger: log, + }) + sch := scheduler.New(st, pm, cfg, installation, rec, log) + srv := server.New(cfg, st, pm, sch, rec, log, installation, agentToken) + if err := srv.Bootstrap(ctx); err != nil { + return fmt.Errorf("bootstrap: %w", err) + } + go sch.Run(ctx) + + httpSrv := &http.Server{Addr: cfg.Listen, Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 120 * time.Second} + errc := make(chan error, 1) + go func() { errc <- httpSrv.ListenAndServe() }() + log.Info("differently-server started", "listen", cfg.Listen, "version", version.Version, "installation", installation, "store", cfg.Store.Driver, "dev", cfg.Dev) + if cfg.Dev { + log.Warn("DEV MODE: the devhost pool runs commands directly on this machine with no isolation", + "operatorTokenFile", filepath.Join(cfg.DataDir, "token"), "dashboard", "http://localhost"+cfg.Listen+"/dashboard/") + } + select { + case err := <-errc: + if !errors.Is(err, http.ErrServerClosed) { + return err + } + case <-ctx.Done(): + } + shutdown, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + log.Info("shutting down") + return httpSrv.Shutdown(shutdown) +} + +func migrate(cfgPath string) error { + cfg, err := loadConfig(cfgPath, false, "") + if err != nil { + return err + } + if cfg.Store.Driver != "postgres" { + fmt.Println("store.driver is memory: nothing to migrate") + return nil + } + ctx := context.Background() + st, err := openStore(ctx, cfg) + if err != nil { + return err + } + defer st.Close() + if err := st.(*postgres.Store).Migrate(ctx); err != nil { + return err + } + fmt.Println("migrations applied") + return nil +} + +// preflight validates the config and the secrets it references. +func preflight(cfgPath string) error { + cfg, err := loadConfig(cfgPath, false, "") + if err != nil { + return err + } + var problems []error + check := func(ok bool, format string, args ...any) { + status := "ok " + if !ok { + status = "FAIL" + problems = append(problems, fmt.Errorf(format, args...)) + } + fmt.Printf("[%s] %s\n", status, fmt.Sprintf(format, args...)) + } + token := os.Getenv(cfg.Auth.BootstrapTokenEnv) + _, _, valid := auth.ParseToken(token) + check(token == "" || valid, "bootstrap token %s is a valid dly_ token (or unset)", cfg.Auth.BootstrapTokenEnv) + hasFirecracker := false + for _, p := range cfg.Pools { + hasFirecracker = hasFirecracker || p.Adapter == "firecracker" + if p.Credentials.String() != "" { + _, err := p.Credentials.Resolve() + check(err == nil, "pool %s credential %s resolves", p.Name, p.Credentials.String()) + } + } + if hasFirecracker { + check(os.Getenv(cfg.Auth.AgentTokenEnv) != "", "agent token %s is set for firecracker pools", cfg.Auth.AgentTokenEnv) + } + if cfg.Store.Driver == "postgres" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + st, err := openStore(ctx, cfg) + check(err == nil, "postgres reachable via %s", cfg.Store.DSNEnv) + if err == nil { + _ = st.Close() + } + } + check(cfg.Policy.External == "never" || len(cfg.Policy.ExternalPools) > 0, "external placement is off, or names approved externalPools") + return errors.Join(problems...) +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c6034e5 --- /dev/null +++ b/go.mod @@ -0,0 +1,32 @@ +module github.com/different-ai/differently + +go 1.26.0 + +require ( + github.com/jackc/pgx/v5 v5.11.0 + github.com/mdlayher/vsock v1.3.0 + github.com/prometheus/client_golang v1.25.0 + github.com/spf13/cobra v1.10.2 + sigs.k8s.io/yaml v1.6.0 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mdlayher/socket v0.6.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.3 // indirect + github.com/prometheus/common v0.72.0 // indirect + github.com/prometheus/procfs v0.22.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..61027ca --- /dev/null +++ b/go.sum @@ -0,0 +1,68 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mdlayher/socket v0.6.0 h1:ScZPaAGyO1icQnbFrhPM8mnXyMu9qukC1K4ZoM2IQKU= +github.com/mdlayher/socket v0.6.0/go.mod h1:q7vozUAnxSqnjHc12Fik5yUKIzfZ8ITCfMkhOtE9z18= +github.com/mdlayher/vsock v1.3.0 h1:bqQfZ1OznI03y6YiXp2sze05RVdzLn/zsfjnjd4+ivI= +github.com/mdlayher/vsock v1.3.0/go.mod h1:WsuksavOvwCnV5UqGHUkvAvCy+Dqy81y4goKQTzxxNY= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0= +github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk= +github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo= +github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM= +github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4= +github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk= +github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics= +github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/adapters/daytona/daytona.go b/internal/adapters/daytona/daytona.go new file mode 100644 index 0000000..cfd9719 --- /dev/null +++ b/internal/adapters/daytona/daytona.go @@ -0,0 +1,405 @@ +// Package daytona adapts Daytona's hosted (or customer-region) sandbox API. +// +// Daytona runs the workload and its control plane, so pools built on this +// adapter are always TrustExternal. The control plane API creates and manages +// sandboxes; each sandbox's toolbox API (at its toolboxProxyUrl) runs commands +// and moves files. Ownership is tracked with Differently labels, which also +// makes Create idempotent. +// +// Verified against Daytona's published OpenAPI specs (2026-10-28), not yet +// against a live account. Known semantic differences: exec output combines +// stdout and stderr; stdin is not supported; snapshots are filesystem-only +// unless the sandbox is a VM class. +package daytona + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "sync" + "time" + + "github.com/different-ai/differently/internal/adapters/httpx" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// DefaultAPIURL is Daytona's hosted control plane. +const DefaultAPIURL = "https://app.daytona.io/api" + +// Options configures the adapter. Pool config keys: apiURL, target, snapshot, +// class (container|linux-vm), autoStopMinutes. +type Options struct { + APIURL string + APIKey string + Target string + Snapshot string + Class string + ReadyTimeout time.Duration +} + +// FromConfig maps pool config to options. +func FromConfig(cfg map[string]string, apiKey string) Options { + o := Options{APIURL: cfg["apiURL"], APIKey: apiKey, Target: cfg["target"], Snapshot: cfg["snapshot"], Class: cfg["class"]} + if o.APIURL == "" { + o.APIURL = DefaultAPIURL + } + if o.Class == "" { + o.Class = "container" + } + return o +} + +// Adapter talks to Daytona. +type Adapter struct { + opts Options + c *httpx.Client + + mu sync.Mutex + toolbox map[string]string // sandbox id -> toolbox base URL +} + +// New returns a Daytona adapter. +func New(o Options) *Adapter { + if o.ReadyTimeout == 0 { + o.ReadyTimeout = 3 * time.Minute + } + return &Adapter{opts: o, c: httpx.New("daytona", o.APIURL, o.APIKey), toolbox: map[string]string{}} +} + +// Endpoint is reported in the trust report. +func (a *Adapter) Endpoint() string { return a.opts.APIURL } + +type sandbox struct { + ID string `json:"id"` + State string `json:"state"` + Target string `json:"target"` + Labels map[string]string `json:"labels"` + ToolboxProxyURL string `json:"toolboxProxyUrl"` + ErrorReason string `json:"errorReason"` + Snapshot string `json:"snapshot"` +} + +func (a *Adapter) Info() adapter.Info { + iso := "container (Daytona-managed)" + if a.opts.Class == "linux-vm" { + iso = "virtual machine (Daytona-managed)" + } + return adapter.Info{Kind: "daytona", Version: "1", Trust: api.TrustExternal, Isolation: iso, Protocol: adapter.ProtocolVersion, Description: "Daytona sandboxes at " + a.opts.APIURL} +} + +func (a *Adapter) Capabilities(context.Context) (adapter.Capabilities, error) { + f := []string{adapter.FeatureExec, adapter.FeatureFiles, adapter.FeatureStopStart, adapter.FeaturePersistDisk, adapter.FeatureSnapshotDisk, adapter.FeaturePorts, adapter.FeatureArchAMD64} + if a.opts.Class == "linux-vm" { + f = append(f, adapter.FeaturePauseResume, adapter.FeatureSnapshotMem, adapter.FeatureMicroVM) + } else { + f = append(f, adapter.FeatureContainer) + } + var regions []string + if a.opts.Target != "" { + regions = []string{a.opts.Target} + } + // Daytona's default per-sandbox maximum is 4 vCPU / 8 GiB / 10 GiB. + return adapter.Capabilities{Features: f, MaxCPU: 4, MaxMemoryMiB: 8192, MaxDiskGiB: 10, Regions: regions}, nil +} + +func (a *Adapter) Capacity(context.Context) (adapter.Capacity, error) { return adapter.Capacity{}, nil } + +func (a *Adapter) Health(ctx context.Context) error { + return a.c.JSON(ctx, http.MethodGet, "/sandbox?limit=1", nil, nil) +} + +func state(s sandbox) string { + switch s.State { + case "started": + return adapter.StateRunning + case "stopped", "archived": + return adapter.StateStopped + case "paused": + return adapter.StatePaused + case "error", "build_failed": + return adapter.StateError + } + return adapter.StatePending +} + +func (a *Adapter) instance(s sandbox) adapter.Instance { + a.mu.Lock() + if s.ToolboxProxyURL != "" { + a.toolbox[s.ID] = s.ToolboxProxyURL + } + a.mu.Unlock() + return adapter.Instance{Ref: s.ID, SandboxID: s.Labels[adapter.LabelSandboxID], State: state(s), Region: s.Target, Message: s.ErrorReason} +} + +func (a *Adapter) find(ctx context.Context, labels map[string]string) ([]sandbox, error) { + q, _ := json.Marshal(labels) + var out []sandbox + cursor := "" + for { + path := "/sandbox?limit=200&labels=" + url.QueryEscape(string(q)) + if cursor != "" { + path += "&cursor=" + url.QueryEscape(cursor) + } + var page struct { + Items []sandbox `json:"items"` + NextCursor *string `json:"nextCursor"` + } + if err := a.c.JSON(ctx, http.MethodGet, path, nil, &page); err != nil { + return nil, err + } + out = append(out, page.Items...) + if page.NextCursor == nil || *page.NextCursor == "" { + return out, nil + } + cursor = *page.NextCursor + } +} + +func (a *Adapter) Create(ctx context.Context, req adapter.CreateRequest) (adapter.Instance, error) { + existing, err := a.find(ctx, map[string]string{adapter.LabelSandboxID: req.ID}) + if err != nil { + return adapter.Instance{}, err + } + var s sandbox + if len(existing) > 0 { + s = existing[0] + } else { + snapshot := a.opts.Snapshot + if req.Template != "" { + snapshot = req.Template + } + if req.FromSnapshot != "" { + snapshot = req.FromSnapshot + } + body := map[string]any{ + "name": req.ID, + "labels": adapter.OwnershipLabels(req), + "env": req.Env, + "autoStopInterval": 0, // Differently owns TTL and lifecycle + } + if snapshot != "" { + body["snapshot"] = snapshot + } + if a.opts.Target != "" { + body["target"] = a.opts.Target + } + if req.TTLSeconds > 0 { + // Backstop in case this control plane is lost: the provider deletes too. + body["ttlMinutes"] = (req.TTLSeconds + 59) / 60 + } + if err := a.c.JSON(ctx, http.MethodPost, "/sandbox", body, &s); err != nil { + return adapter.Instance{}, err + } + } + err = httpx.Poll(ctx, time.Second, a.opts.ReadyTimeout, func() (bool, error) { + if state(s) == adapter.StateRunning { + return true, nil + } + if state(s) == adapter.StateError { + return false, adapter.Unavailable(fmt.Errorf("daytona sandbox %s failed: %s", s.ID, s.ErrorReason)) + } + return false, a.c.JSON(ctx, http.MethodGet, "/sandbox/"+url.PathEscape(s.ID), nil, &s) + }) + if err != nil { + return adapter.Instance{}, err + } + return a.instance(s), nil +} + +func (a *Adapter) Get(ctx context.Context, ref string) (adapter.Instance, error) { + var s sandbox + if err := a.c.JSON(ctx, http.MethodGet, "/sandbox/"+url.PathEscape(ref), nil, &s); err != nil { + return adapter.Instance{}, err + } + if s.State == "destroyed" || s.State == "destroying" { + return adapter.Instance{}, adapter.NotFound(ref) + } + return a.instance(s), nil +} + +func (a *Adapter) List(ctx context.Context, installation string) ([]adapter.Instance, error) { + items, err := a.find(ctx, map[string]string{adapter.LabelInstallation: installation}) + if err != nil { + return nil, err + } + var out []adapter.Instance + for _, s := range items { + if s.State != "destroyed" && s.State != "destroying" { + out = append(out, a.instance(s)) + } + } + return out, nil +} + +func (a *Adapter) Delete(ctx context.Context, ref string) error { + err := a.c.JSON(ctx, http.MethodDelete, "/sandbox/"+url.PathEscape(ref), nil, nil) + if api.IsCode(err, api.CodeNotFound) { + return nil + } + return err +} + +func (a *Adapter) toolboxURL(ctx context.Context, ref string) (string, error) { + a.mu.Lock() + base, ok := a.toolbox[ref] + a.mu.Unlock() + if !ok { + if _, err := a.Get(ctx, ref); err != nil { + return "", err + } + a.mu.Lock() + base = a.toolbox[ref] + a.mu.Unlock() + } + if base == "" { + return "", adapter.Unavailable(fmt.Errorf("daytona did not report a toolbox URL for %s", ref)) + } + return base + "/" + url.PathEscape(ref), nil +} + +func (a *Adapter) Exec(ctx context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) { + if req.Stdin != "" { + return api.ExecResult{}, api.Errorf(api.CodeUnsupportedOperation, "the daytona backend does not support exec stdin").With("operation", "exec.stdin") + } + base, err := a.toolboxURL(ctx, ref) + if err != nil { + return api.ExecResult{}, err + } + timeout := req.TimeoutSeconds + if timeout == 0 { + timeout = 600 // the provider default is short; always send one + } + body := map[string]any{"command": httpx.CommandLine(api.ExecRequest{Command: req.Command, Args: req.Args}), "timeout": timeout} + if req.Cwd != "" { + body["cwd"] = req.Cwd + } + if len(req.Env) > 0 { + body["envs"] = req.Env + } + var out struct { + ExitCode int `json:"exitCode"` + Result string `json:"result"` + } + start := time.Now() + if err := a.c.JSON(ctx, http.MethodPost, base+"/process/execute", body, &out); err != nil { + return api.ExecResult{}, err + } + // Daytona combines stdout and stderr into one buffer. + return api.ExecResult{ExitCode: out.ExitCode, Stdout: out.Result, DurationMs: time.Since(start).Milliseconds()}, nil +} + +func (a *Adapter) ReadFile(ctx context.Context, ref, path string) ([]byte, error) { + base, err := a.toolboxURL(ctx, ref) + if err != nil { + return nil, err + } + resp, err := a.c.Do(ctx, http.MethodGet, base+"/files/download?path="+url.QueryEscape(path), nil, "", nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, 100<<20)) +} + +func (a *Adapter) WriteFile(ctx context.Context, ref, path string, data []byte) error { + base, err := a.toolboxURL(ctx, ref) + if err != nil { + return err + } + resp, err := a.c.Do(ctx, http.MethodPost, base+"/files/upload-v2?path="+url.QueryEscape(path), bytes.NewReader(data), "application/octet-stream", nil) + if err != nil { + return err + } + return resp.Body.Close() +} + +func (a *Adapter) ListFiles(ctx context.Context, ref, path string) ([]api.FileInfo, error) { + base, err := a.toolboxURL(ctx, ref) + if err != nil { + return nil, err + } + var items []struct { + Name string `json:"name"` + Path string `json:"path"` + Size int64 `json:"size"` + IsDir bool `json:"isDir"` + Mode string `json:"mode"` + ModifiedAt string `json:"modifiedAt"` + } + if err := a.c.JSON(ctx, http.MethodGet, base+"/files?path="+url.QueryEscape(path), nil, &items); err != nil { + return nil, err + } + out := make([]api.FileInfo, 0, len(items)) + for _, it := range items { + mt, _ := time.Parse(time.RFC3339, it.ModifiedAt) + out = append(out, api.FileInfo{Path: it.Path, Name: it.Name, Size: it.Size, IsDir: it.IsDir, Mode: it.Mode, ModTime: mt}) + } + return out, nil +} + +func (a *Adapter) action(ctx context.Context, ref, action string) error { + return a.c.JSON(ctx, http.MethodPost, "/sandbox/"+url.PathEscape(ref)+"/"+action, nil, nil) +} + +func (a *Adapter) Stop(ctx context.Context, ref string) error { return a.action(ctx, ref, "stop") } +func (a *Adapter) Start(ctx context.Context, ref string) error { return a.action(ctx, ref, "start") } + +// Pause is only meaningful for VM-class sandboxes; Capabilities hides it for +// containers so placement never routes pause-requiring work here. +func (a *Adapter) Pause(ctx context.Context, ref string) error { return a.action(ctx, ref, "pause") } +func (a *Adapter) Resume(ctx context.Context, ref string) error { return a.action(ctx, ref, "start") } + +func (a *Adapter) Snapshot(ctx context.Context, ref, kind string) (adapter.Snapshot, error) { + if kind == "memory" && a.opts.Class != "linux-vm" { + return adapter.Snapshot{}, adapter.Unsupported("snapshots.memory") + } + name := "dly-" + ref + "-" + strconv.FormatInt(time.Now().Unix(), 36) + var s sandbox + body := map[string]any{"name": name, "includeMemory": kind == "memory"} + if err := a.c.JSON(ctx, http.MethodPost, "/sandbox/"+url.PathEscape(ref)+"/snapshot", body, &s); err != nil { + return adapter.Snapshot{}, err + } + // Snapshotting is asynchronous; wait until the source settles. + err := httpx.Poll(ctx, time.Second, a.opts.ReadyTimeout, func() (bool, error) { + if s.State != "snapshotting" { + return true, nil + } + return false, a.c.JSON(ctx, http.MethodGet, "/sandbox/"+url.PathEscape(ref), nil, &s) + }) + if kind == "" { + kind = "disk" + } + return adapter.Snapshot{Ref: name, Kind: kind}, err +} + +func (a *Adapter) DeleteSnapshot(ctx context.Context, ref string) error { + err := a.c.JSON(ctx, http.MethodDelete, "/snapshots/"+url.PathEscape(ref), nil, nil) + if api.IsCode(err, api.CodeNotFound) { + return nil + } + return err +} + +// ExposePort returns a signed preview URL valid for one hour. +func (a *Adapter) ExposePort(ctx context.Context, ref string, port int) (string, error) { + var out struct { + URL string `json:"url"` + } + err := a.c.JSON(ctx, http.MethodGet, fmt.Sprintf("/sandbox/%s/ports/%d/signed-preview-url?expiresInSeconds=3600", url.PathEscape(ref), port), nil, &out) + return out.URL, err +} + +var ( + _ adapter.ExecutionAdapter = (*Adapter)(nil) + _ adapter.FileSystem = (*Adapter)(nil) + _ adapter.Lifecycle = (*Adapter)(nil) + _ adapter.Pauser = (*Adapter)(nil) + _ adapter.Snapshotter = (*Adapter)(nil) + _ adapter.PortExposer = (*Adapter)(nil) +) diff --git a/internal/adapters/daytona/daytona_test.go b/internal/adapters/daytona/daytona_test.go new file mode 100644 index 0000000..c5379c2 --- /dev/null +++ b/internal/adapters/daytona/daytona_test.go @@ -0,0 +1,168 @@ +package daytona + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// fakeDaytona mimics the documented control-plane and toolbox shapes. +type fakeDaytona struct { + mu sync.Mutex + creates int + boxes map[string]map[string]any + files map[string][]byte + srv *httptest.Server +} + +func newFake(t *testing.T) *fakeDaytona { + f := &fakeDaytona{boxes: map[string]map[string]any{}, files: map[string][]byte{}} + mux := http.NewServeMux() + mux.HandleFunc("POST /api/sandbox", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + f.mu.Lock() + f.creates++ + id := "dt-" + body["name"].(string) + box := map[string]any{"id": id, "state": "creating", "labels": body["labels"], "target": "us", "toolboxProxyUrl": f.srv.URL + "/toolbox"} + f.boxes[id] = box + f.mu.Unlock() + json.NewEncoder(w).Encode(box) + }) + mux.HandleFunc("GET /api/sandbox", func(w http.ResponseWriter, r *http.Request) { + var want map[string]string + _ = json.Unmarshal([]byte(r.URL.Query().Get("labels")), &want) + f.mu.Lock() + defer f.mu.Unlock() + items := []any{} + for _, b := range f.boxes { + labels, _ := b["labels"].(map[string]any) + match := true + for k, v := range want { + if labels[k] != v { + match = false + } + } + if match { + items = append(items, b) + } + } + json.NewEncoder(w).Encode(map[string]any{"items": items, "nextCursor": nil}) + }) + mux.HandleFunc("GET /api/sandbox/{id}", func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + b, ok := f.boxes[r.PathValue("id")] + if !ok { + w.WriteHeader(404) + json.NewEncoder(w).Encode(map[string]any{"statusCode": 404, "error": "Not Found", "message": "Sandbox not found"}) + return + } + b["state"] = "started" // becomes ready on first poll + json.NewEncoder(w).Encode(b) + }) + mux.HandleFunc("DELETE /api/sandbox/{id}", func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + if _, ok := f.boxes[r.PathValue("id")]; !ok { + w.WriteHeader(404) + return + } + delete(f.boxes, r.PathValue("id")) + w.Write([]byte(`{}`)) + }) + mux.HandleFunc("POST /toolbox/{id}/process/execute", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + if body["timeout"] == nil { + t.Error("timeout must always be sent") + } + json.NewEncoder(w).Encode(map[string]any{"exitCode": 0, "result": "ran: " + body["command"].(string)}) + }) + mux.HandleFunc("POST /toolbox/{id}/files/upload-v2", func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + f.mu.Lock() + f.files[r.URL.Query().Get("path")] = b + f.mu.Unlock() + w.Write([]byte(`{}`)) + }) + mux.HandleFunc("GET /toolbox/{id}/files/download", func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + w.Write(f.files[r.URL.Query().Get("path")]) + }) + f.srv = httptest.NewServer(mux) + t.Cleanup(f.srv.Close) + return f +} + +func TestDaytonaAdapter(t *testing.T) { + f := newFake(t) + a := New(Options{APIURL: f.srv.URL + "/api", APIKey: "k", Snapshot: "daytona-small"}) + ctx := context.Background() + req := adapter.CreateRequest{ID: "sbx_1", Installation: "inst_1", Resources: api.Resources{CPU: 1, MemoryMiB: 1024}} + inst, err := a.Create(ctx, req) + if err != nil { + t.Fatal(err) + } + if inst.State != adapter.StateRunning || inst.SandboxID != "sbx_1" { + t.Fatalf("instance %+v", inst) + } + if _, err := a.Create(ctx, req); err != nil || f.creates != 1 { + t.Fatalf("replayed create made %d sandboxes (%v)", f.creates, err) + } + res, err := a.Exec(ctx, inst.Ref, api.ExecRequest{Args: []string{"echo", "hi there"}, Cwd: "/tmp"}) + if err != nil || !strings.Contains(res.Stdout, `echo 'hi there'`) { + t.Fatalf("exec %+v %v", res, err) + } + if _, err := a.Exec(ctx, inst.Ref, api.ExecRequest{Command: "cat", Stdin: "x"}); !api.IsCode(err, api.CodeUnsupportedOperation) { + t.Fatalf("stdin should be unsupported: %v", err) + } + if err := a.WriteFile(ctx, inst.Ref, "/home/a.txt", []byte("v")); err != nil { + t.Fatal(err) + } + if b, err := a.ReadFile(ctx, inst.Ref, "/home/a.txt"); err != nil || string(b) != "v" { + t.Fatalf("read %q %v", b, err) + } + list, err := a.List(ctx, "inst_1") + if err != nil || len(list) != 1 { + t.Fatalf("list %v %v", list, err) + } + if err := a.Delete(ctx, inst.Ref); err != nil { + t.Fatal(err) + } + if err := a.Delete(ctx, inst.Ref); err != nil { + t.Fatalf("second delete must succeed: %v", err) + } + caps, _ := a.Capabilities(ctx) + if caps.Has(adapter.FeaturePauseResume) { + t.Fatal("container class must not advertise pause") + } +} + +func TestDaytonaErrorMapping(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.Contains(r.URL.RawQuery, "limit=1") { + w.WriteHeader(401) + return + } + w.WriteHeader(429) + json.NewEncoder(w).Encode(map[string]any{"statusCode": 429, "message": "Rate limit exceeded"}) + })) + defer srv.Close() + a := New(Options{APIURL: srv.URL + "/api", APIKey: "k"}) + if err := a.Health(context.Background()); !api.IsCode(err, api.CodeProviderUnavailable) || strings.Contains(err.Error(), "k") && strings.Contains(err.Error(), "Bearer") { + t.Fatalf("401 mapping: %v", err) + } + if _, err := a.Create(context.Background(), adapter.CreateRequest{ID: "sbx_2"}); !api.IsCode(err, adapter.CodeInsufficientCapacity) { + t.Fatalf("429 should requeue as insufficient capacity: %v", err) + } +} diff --git a/internal/adapters/devhost/devhost.go b/internal/adapters/devhost/devhost.go new file mode 100644 index 0000000..f1deeeb --- /dev/null +++ b/internal/adapters/devhost/devhost.go @@ -0,0 +1,246 @@ +// Package devhost runs sandboxes as plain directories on the control-plane +// host. Commands execute as the server's user with NO isolation. It exists so +// the full API, CLI and SDK flow can be exercised on a laptop; the server only +// enables it in --dev mode and its pools are labelled isolation.none. +package devhost + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sync" + + "github.com/different-ai/differently/internal/guest" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// Adapter manages sandbox directories under Root. +type Adapter struct { + Root string + mu sync.Mutex +} + +type meta struct { + adapter.Instance + Installation string `json:"installation"` + Env map[string]string `json:"env,omitempty"` +} + +// New returns a devhost adapter rooted at root. +func New(root string) (*Adapter, error) { + for _, d := range []string{"sandboxes", "snapshots"} { + if err := os.MkdirAll(filepath.Join(root, d), 0o755); err != nil { + return nil, err + } + } + return &Adapter{Root: root}, nil +} + +func (a *Adapter) dir(ref string) string { + return filepath.Join(a.Root, "sandboxes", filepath.Base(ref)) +} +func (a *Adapter) fsDir(ref string) string { return filepath.Join(a.dir(ref), "fs") } + +func (a *Adapter) load(ref string) (meta, error) { + var m meta + b, err := os.ReadFile(filepath.Join(a.dir(ref), "meta.json")) + if errors.Is(err, fs.ErrNotExist) { + return m, adapter.NotFound(ref) + } + if err != nil { + return m, err + } + return m, json.Unmarshal(b, &m) +} + +func (a *Adapter) save(m meta) error { + b, _ := json.Marshal(m) + return os.WriteFile(filepath.Join(a.dir(m.Ref), "meta.json"), b, 0o600) +} + +func (a *Adapter) Info() adapter.Info { + return adapter.Info{Kind: "devhost", Version: "1", Trust: api.TrustPrivate, Isolation: "none", Description: "development only: runs commands on the host", Protocol: adapter.ProtocolVersion} +} + +func (a *Adapter) Capabilities(context.Context) (adapter.Capabilities, error) { + return adapter.Capabilities{Features: []string{ + adapter.FeatureExec, adapter.FeatureExecStream, adapter.FeatureFiles, adapter.FeatureStopStart, + adapter.FeaturePersistDisk, adapter.FeatureSnapshotDisk, adapter.FeatureFork, adapter.FeatureNoIsolation, + }}, nil +} + +func (a *Adapter) Capacity(context.Context) (adapter.Capacity, error) { return adapter.Capacity{}, nil } + +func (a *Adapter) Health(context.Context) error { + _, err := os.Stat(a.Root) + return err +} + +func (a *Adapter) Create(_ context.Context, req adapter.CreateRequest) (adapter.Instance, error) { + a.mu.Lock() + defer a.mu.Unlock() + ref := req.ID // the sandbox ID is the idempotency key and the ref + if m, err := a.load(ref); err == nil { + return m.Instance, nil + } + if err := os.MkdirAll(a.fsDir(ref), 0o755); err != nil { + return adapter.Instance{}, err + } + if req.FromSnapshot != "" { + src := filepath.Join(a.Root, "snapshots", filepath.Base(req.FromSnapshot)) + if err := copyTree(src, a.fsDir(ref)); err != nil { + _ = os.RemoveAll(a.dir(ref)) + return adapter.Instance{}, err + } + } + m := meta{Instance: adapter.Instance{Ref: ref, SandboxID: req.ID, State: adapter.StateRunning}, Installation: req.Installation, Env: req.Env} + return m.Instance, a.save(m) +} + +func (a *Adapter) Get(_ context.Context, ref string) (adapter.Instance, error) { + m, err := a.load(ref) + return m.Instance, err +} + +func (a *Adapter) List(_ context.Context, installation string) ([]adapter.Instance, error) { + entries, err := os.ReadDir(filepath.Join(a.Root, "sandboxes")) + if err != nil { + return nil, err + } + var out []adapter.Instance + for _, e := range entries { + m, err := a.load(e.Name()) + if err == nil && (installation == "" || m.Installation == installation) { + out = append(out, m.Instance) + } + } + return out, nil +} + +func (a *Adapter) Delete(_ context.Context, ref string) error { + return os.RemoveAll(a.dir(ref)) +} + +func (a *Adapter) runner(ref string) (*guest.Runner, error) { + m, err := a.load(ref) + if err != nil { + return nil, err + } + if m.State != adapter.StateRunning { + return nil, api.Errorf(api.CodeInvalidState, "sandbox is %s", m.State) + } + return &guest.Runner{Root: a.fsDir(ref), BaseEnv: m.Env}, nil +} + +func (a *Adapter) Exec(ctx context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) { + r, err := a.runner(ref) + if err != nil { + return api.ExecResult{}, err + } + return r.Exec(ctx, req) +} + +func (a *Adapter) ExecStream(ctx context.Context, ref string, req api.ExecRequest, emit func(api.ExecEvent) error) (int, error) { + r, err := a.runner(ref) + if err != nil { + return -1, err + } + return r.ExecStream(ctx, req, emit) +} + +func (a *Adapter) ReadFile(_ context.Context, ref, p string) ([]byte, error) { + r, err := a.runner(ref) + if err != nil { + return nil, err + } + return r.ReadFile(p) +} + +func (a *Adapter) WriteFile(_ context.Context, ref, p string, data []byte) error { + r, err := a.runner(ref) + if err != nil { + return err + } + return r.WriteFile(p, data) +} + +func (a *Adapter) ListFiles(_ context.Context, ref, p string) ([]api.FileInfo, error) { + r, err := a.runner(ref) + if err != nil { + return nil, err + } + return r.ListFiles(p) +} + +func (a *Adapter) setState(ref, state string) error { + a.mu.Lock() + defer a.mu.Unlock() + m, err := a.load(ref) + if err != nil { + return err + } + m.State = state + return a.save(m) +} + +func (a *Adapter) Stop(_ context.Context, ref string) error { + return a.setState(ref, adapter.StateStopped) +} + +func (a *Adapter) Start(_ context.Context, ref string) error { + return a.setState(ref, adapter.StateRunning) +} + +func (a *Adapter) Snapshot(_ context.Context, ref, kind string) (adapter.Snapshot, error) { + if kind == "memory" { + return adapter.Snapshot{}, adapter.Unsupported("snapshots.memory") + } + if _, err := a.load(ref); err != nil { + return adapter.Snapshot{}, err + } + snap := id.New("snap") + dst := filepath.Join(a.Root, "snapshots", snap) + if err := copyTree(a.fsDir(ref), dst); err != nil { + return adapter.Snapshot{}, err + } + return adapter.Snapshot{Ref: snap, Kind: "disk", SizeBytes: treeSize(dst)}, nil +} + +func (a *Adapter) DeleteSnapshot(_ context.Context, ref string) error { + return os.RemoveAll(filepath.Join(a.Root, "snapshots", filepath.Base(ref))) +} + +func copyTree(src, dst string) error { + if err := os.MkdirAll(dst, 0o755); err != nil { + return err + } + // cp -a preserves modes and symlinks without following them. + return exec.Command("cp", "-a", src+"/.", dst).Run() +} + +func treeSize(dir string) int64 { + var n int64 + _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err == nil && !d.IsDir() { + if info, err := d.Info(); err == nil { + n += info.Size() + } + } + return nil + }) + return n +} + +var ( + _ adapter.ExecutionAdapter = (*Adapter)(nil) + _ adapter.Streamer = (*Adapter)(nil) + _ adapter.FileSystem = (*Adapter)(nil) + _ adapter.Lifecycle = (*Adapter)(nil) + _ adapter.Snapshotter = (*Adapter)(nil) +) diff --git a/internal/adapters/devhost/devhost_test.go b/internal/adapters/devhost/devhost_test.go new file mode 100644 index 0000000..3d2064c --- /dev/null +++ b/internal/adapters/devhost/devhost_test.go @@ -0,0 +1,59 @@ +package devhost + +import ( + "context" + "strings" + "testing" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +func TestLifecycleAndSnapshotRestore(t *testing.T) { + a, err := New(t.TempDir()) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + inst, err := a.Create(ctx, adapter.CreateRequest{ID: "sbx_1", Installation: "inst", Env: map[string]string{"NAME": "world"}}) + if err != nil { + t.Fatal(err) + } + again, _ := a.Create(ctx, adapter.CreateRequest{ID: "sbx_1", Installation: "inst"}) + if again.Ref != inst.Ref { + t.Fatal("create is not idempotent") + } + res, err := a.Exec(ctx, inst.Ref, api.ExecRequest{Command: "echo hello $NAME > greeting && cat greeting"}) + if err != nil || strings.TrimSpace(res.Stdout) != "hello world" { + t.Fatalf("exec %+v %v", res, err) + } + if err := a.Stop(ctx, inst.Ref); err != nil { + t.Fatal(err) + } + if _, err := a.Exec(ctx, inst.Ref, api.ExecRequest{Command: "true"}); !api.IsCode(err, api.CodeInvalidState) { + t.Fatalf("exec on stopped sandbox: %v", err) + } + if err := a.Start(ctx, inst.Ref); err != nil { + t.Fatal(err) + } + snap, err := a.Snapshot(ctx, inst.Ref, "disk") + if err != nil { + t.Fatal(err) + } + fork, err := a.Create(ctx, adapter.CreateRequest{ID: "sbx_2", Installation: "inst", FromSnapshot: snap.Ref}) + if err != nil { + t.Fatal(err) + } + data, err := a.ReadFile(ctx, fork.Ref, "/greeting") + if err != nil || strings.TrimSpace(string(data)) != "hello world" { + t.Fatalf("fork lost disk state: %q %v", data, err) + } + items, _ := a.List(ctx, "inst") + if len(items) != 2 { + t.Fatalf("list %v", items) + } + _ = a.Delete(ctx, inst.Ref) + if _, err := a.Get(ctx, inst.Ref); !api.IsCode(err, api.CodeNotFound) { + t.Fatalf("deleted sandbox still present: %v", err) + } +} diff --git a/internal/adapters/fake/fake.go b/internal/adapters/fake/fake.go new file mode 100644 index 0000000..37876b4 --- /dev/null +++ b/internal/adapters/fake/fake.go @@ -0,0 +1,343 @@ +// Package fake is an in-memory execution adapter for tests and demos. It +// implements every optional interface and supports failure injection. +package fake + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// Options configures a fake adapter. +type Options struct { + Trust string + Features []string + // Capacity bounds total CPU and memory when non-zero. + TotalCPU int + TotalMemoryMiB int + Region string + // ExecFunc overrides command results. + ExecFunc func(req api.ExecRequest) api.ExecResult +} + +// DefaultFeatures is everything the fake implements. +var DefaultFeatures = []string{ + adapter.FeatureExec, adapter.FeatureExecStream, adapter.FeatureFiles, adapter.FeatureStopStart, + adapter.FeaturePersistDisk, adapter.FeaturePauseResume, adapter.FeatureSnapshotDisk, + adapter.FeatureSnapshotMem, adapter.FeatureFork, adapter.FeaturePorts, adapter.FeatureMicroVM, +} + +type instance struct { + adapter.Instance + installation string + resources api.Resources + files map[string][]byte +} + +// Adapter is the fake backend. +type Adapter struct { + opts Options + + mu sync.Mutex + instances map[string]*instance // by ref + byID map[string]string // sandbox ID -> ref + snapshots map[string]map[string][]byte + // FailCreates makes the next N Create calls fail as unavailable. + FailCreates int + creates int +} + +// New returns a fake adapter. +func New(opts Options) *Adapter { + if opts.Trust == "" { + opts.Trust = api.TrustPrivate + } + if opts.Features == nil { + opts.Features = DefaultFeatures + } + return &Adapter{opts: opts, instances: map[string]*instance{}, byID: map[string]string{}, snapshots: map[string]map[string][]byte{}} +} + +func (a *Adapter) Info() adapter.Info { + return adapter.Info{Kind: "fake", Version: "1", Trust: a.opts.Trust, Isolation: "none (simulated)", Protocol: adapter.ProtocolVersion} +} + +func (a *Adapter) Capabilities(context.Context) (adapter.Capabilities, error) { + return adapter.Capabilities{Features: a.opts.Features, MaxCPU: a.opts.TotalCPU, MaxMemoryMiB: a.opts.TotalMemoryMiB}, nil +} + +func (a *Adapter) Capacity(context.Context) (adapter.Capacity, error) { + if a.opts.TotalCPU == 0 { + return adapter.Capacity{}, nil + } + a.mu.Lock() + defer a.mu.Unlock() + c := adapter.Capacity{Bounded: true, TotalCPU: a.opts.TotalCPU, TotalMemoryMiB: a.opts.TotalMemoryMiB, Nodes: 1} + c.FreeCPU, c.FreeMemoryMiB = c.TotalCPU, c.TotalMemoryMiB + for _, in := range a.instances { + if in.State == adapter.StateRunning || in.State == adapter.StatePaused { + c.FreeCPU -= in.resources.CPU + c.FreeMemoryMiB -= in.resources.MemoryMiB + } + } + return c, nil +} + +func (a *Adapter) Health(context.Context) error { return nil } + +// Count returns how many instances exist. +func (a *Adapter) Count() int { + a.mu.Lock() + defer a.mu.Unlock() + return len(a.instances) +} + +// Forget deletes an instance behind the control plane's back. +func (a *Adapter) Forget(ref string) { + a.mu.Lock() + defer a.mu.Unlock() + if in, ok := a.instances[ref]; ok { + delete(a.byID, in.SandboxID) + delete(a.instances, ref) + } +} + +// Inject creates an instance the control plane does not know (an orphan). +func (a *Adapter) Inject(sandboxID, installation string) string { + a.mu.Lock() + defer a.mu.Unlock() + ref := id.New("fake") + a.instances[ref] = &instance{Instance: adapter.Instance{Ref: ref, SandboxID: sandboxID, State: adapter.StateRunning}, installation: installation, files: map[string][]byte{}} + a.byID[sandboxID] = ref + return ref +} + +func (a *Adapter) Create(_ context.Context, req adapter.CreateRequest) (adapter.Instance, error) { + a.mu.Lock() + defer a.mu.Unlock() + if ref, ok := a.byID[req.ID]; ok { + return a.instances[ref].Instance, nil + } + if a.FailCreates > 0 { + a.FailCreates-- + return adapter.Instance{}, adapter.Unavailable(fmt.Errorf("injected create failure")) + } + if a.opts.TotalCPU > 0 { + used := 0 + for _, in := range a.instances { + used += in.resources.CPU + } + if used+req.Resources.CPU > a.opts.TotalCPU { + return adapter.Instance{}, adapter.InsufficientCapacity("fake pool full") + } + } + files := map[string][]byte{} + if req.FromSnapshot != "" { + snap, ok := a.snapshots[req.FromSnapshot] + if !ok { + return adapter.Instance{}, adapter.NotFound(req.FromSnapshot) + } + for k, v := range snap { + files[k] = append([]byte(nil), v...) + } + } + a.creates++ + ref := id.New("fake") + in := &instance{Instance: adapter.Instance{Ref: ref, SandboxID: req.ID, State: adapter.StateRunning, Region: a.opts.Region}, installation: req.Installation, resources: req.Resources, files: files} + a.instances[ref] = in + a.byID[req.ID] = ref + return in.Instance, nil +} + +func (a *Adapter) get(ref string) (*instance, error) { + in, ok := a.instances[ref] + if !ok { + return nil, adapter.NotFound(ref) + } + return in, nil +} + +func (a *Adapter) Get(_ context.Context, ref string) (adapter.Instance, error) { + a.mu.Lock() + defer a.mu.Unlock() + in, err := a.get(ref) + if err != nil { + return adapter.Instance{}, err + } + return in.Instance, nil +} + +func (a *Adapter) List(_ context.Context, installation string) ([]adapter.Instance, error) { + a.mu.Lock() + defer a.mu.Unlock() + var out []adapter.Instance + for _, in := range a.instances { + if installation == "" || in.installation == installation { + out = append(out, in.Instance) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Ref < out[j].Ref }) + return out, nil +} + +func (a *Adapter) Delete(_ context.Context, ref string) error { + a.mu.Lock() + defer a.mu.Unlock() + if in, ok := a.instances[ref]; ok { + delete(a.byID, in.SandboxID) + delete(a.instances, ref) + } + return nil +} + +func (a *Adapter) running(ref string) (*instance, error) { + in, err := a.get(ref) + if err != nil { + return nil, err + } + if in.State != adapter.StateRunning { + return nil, api.Errorf(api.CodeInvalidState, "instance is %s", in.State) + } + return in, nil +} + +func (a *Adapter) Exec(_ context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) { + a.mu.Lock() + defer a.mu.Unlock() + if _, err := a.running(ref); err != nil { + return api.ExecResult{}, err + } + if a.opts.ExecFunc != nil { + return a.opts.ExecFunc(req), nil + } + cmd := req.Command + if cmd == "" { + cmd = fmt.Sprint(req.Args) + } + return api.ExecResult{Stdout: "fake: " + cmd + "\n"}, nil +} + +func (a *Adapter) ExecStream(ctx context.Context, ref string, req api.ExecRequest, emit func(api.ExecEvent) error) (int, error) { + res, err := a.Exec(ctx, ref, req) + if err != nil { + return -1, err + } + if err := emit(api.ExecEvent{Stream: "stdout", Data: res.Stdout}); err != nil { + return -1, err + } + return res.ExitCode, nil +} + +func (a *Adapter) ReadFile(_ context.Context, ref, path string) ([]byte, error) { + a.mu.Lock() + defer a.mu.Unlock() + in, err := a.running(ref) + if err != nil { + return nil, err + } + data, ok := in.files[path] + if !ok { + return nil, api.Errorf(api.CodeNotFound, "%s does not exist", path) + } + return append([]byte(nil), data...), nil +} + +func (a *Adapter) WriteFile(_ context.Context, ref, path string, data []byte) error { + a.mu.Lock() + defer a.mu.Unlock() + in, err := a.running(ref) + if err != nil { + return err + } + in.files[path] = append([]byte(nil), data...) + return nil +} + +func (a *Adapter) ListFiles(_ context.Context, ref, _ string) ([]api.FileInfo, error) { + a.mu.Lock() + defer a.mu.Unlock() + in, err := a.running(ref) + if err != nil { + return nil, err + } + var out []api.FileInfo + for p, d := range in.files { + out = append(out, api.FileInfo{Path: p, Name: p, Size: int64(len(d)), Mode: "-rw-r--r--"}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out, nil +} + +func (a *Adapter) setState(ref, from, to string) error { + a.mu.Lock() + defer a.mu.Unlock() + in, err := a.get(ref) + if err != nil { + return err + } + if in.State != from && in.State != to { + return api.Errorf(api.CodeInvalidState, "instance is %s", in.State) + } + in.State = to + return nil +} + +func (a *Adapter) Stop(_ context.Context, ref string) error { + return a.setState(ref, adapter.StateRunning, adapter.StateStopped) +} +func (a *Adapter) Start(_ context.Context, ref string) error { + return a.setState(ref, adapter.StateStopped, adapter.StateRunning) +} +func (a *Adapter) Pause(_ context.Context, ref string) error { + return a.setState(ref, adapter.StateRunning, adapter.StatePaused) +} +func (a *Adapter) Resume(_ context.Context, ref string) error { + return a.setState(ref, adapter.StatePaused, adapter.StateRunning) +} + +func (a *Adapter) Snapshot(_ context.Context, ref, kind string) (adapter.Snapshot, error) { + a.mu.Lock() + defer a.mu.Unlock() + in, err := a.get(ref) + if err != nil { + return adapter.Snapshot{}, err + } + snapRef := id.New("fakesnap") + files := map[string][]byte{} + size := int64(0) + for k, v := range in.files { + files[k] = append([]byte(nil), v...) + size += int64(len(v)) + } + a.snapshots[snapRef] = files + if kind == "" { + kind = "disk" + } + return adapter.Snapshot{Ref: snapRef, Kind: kind, SizeBytes: size}, nil +} + +func (a *Adapter) DeleteSnapshot(_ context.Context, ref string) error { + a.mu.Lock() + defer a.mu.Unlock() + delete(a.snapshots, ref) + return nil +} + +func (a *Adapter) ExposePort(_ context.Context, ref string, port int) (string, error) { + return fmt.Sprintf("https://%d-%s.fake.invalid", port, ref), nil +} + +var ( + _ adapter.ExecutionAdapter = (*Adapter)(nil) + _ adapter.Streamer = (*Adapter)(nil) + _ adapter.FileSystem = (*Adapter)(nil) + _ adapter.Lifecycle = (*Adapter)(nil) + _ adapter.Pauser = (*Adapter)(nil) + _ adapter.Snapshotter = (*Adapter)(nil) + _ adapter.PortExposer = (*Adapter)(nil) +) diff --git a/internal/adapters/firecracker/pool.go b/internal/adapters/firecracker/pool.go new file mode 100644 index 0000000..7f80885 --- /dev/null +++ b/internal/adapters/firecracker/pool.go @@ -0,0 +1,349 @@ +// Package firecracker is the control-plane side of the direct Firecracker +// backend. A pool is the set of node agents (differently-agent) that +// registered under the pool's name; each agent manages microVMs on one KVM +// host. The pool places a sandbox on the agent with the most free memory that +// fits it and addresses instances as ":". +package firecracker + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/adapter/adapterhttp" + "github.com/different-ai/differently/pkg/api" +) + +// PartitionGrace is how long a silent agent is presumed partitioned rather +// than lost. +const PartitionGrace = 10 * time.Minute + +// Pool aggregates node agents. +type Pool struct { + Name string + Store store.Store + Token string + Timeout time.Duration + + mu sync.Mutex + clients map[string]*adapterhttp.Client +} + +// New returns a pool backed by agents registered in st. +func New(name string, st store.Store, token string, timeout time.Duration) *Pool { + if timeout == 0 { + timeout = 30 * time.Second + } + return &Pool{Name: name, Store: st, Token: token, Timeout: timeout, clients: map[string]*adapterhttp.Client{}} +} + +func (p *Pool) agents(ctx context.Context) ([]api.Agent, error) { + all, err := p.Store.ListAgents(ctx, p.Name) + if err != nil { + return nil, err + } + cutoff := time.Now().Add(-p.Timeout) + var out []api.Agent + for _, a := range all { + if a.LastSeen.After(cutoff) { + out = append(out, a) + } + } + return out, nil +} + +func (p *Pool) client(a api.Agent) *adapterhttp.Client { + p.mu.Lock() + defer p.mu.Unlock() + c, ok := p.clients[a.ID] + if !ok || c.BaseURL != strings.TrimRight(a.Endpoint, "/") { + c = adapterhttp.NewClient(a.Endpoint, p.Token) + p.clients[a.ID] = c + } + return c +} + +func (p *Pool) route(ctx context.Context, ref string) (*adapterhttp.Client, string, error) { + agentID, vm, ok := strings.Cut(ref, ":") + if !ok { + return nil, "", adapter.NotFound(ref) + } + agents, err := p.Store.ListAgents(ctx, p.Name) + if err != nil { + return nil, "", err + } + for _, a := range agents { + if a.ID == agentID { + return p.client(a), vm, nil + } + } + return nil, "", adapter.Unavailable(fmt.Errorf("agent %s is not registered", agentID)) +} + +func (p *Pool) Info() adapter.Info { + return adapter.Info{Kind: "firecracker", Version: "1", Trust: api.TrustPrivate, Isolation: "firecracker microVM (KVM)", Protocol: adapter.ProtocolVersion} +} + +// Capabilities returns the features every live agent supports and the union +// of their templates. +func (p *Pool) Capabilities(ctx context.Context) (adapter.Capabilities, error) { + agents, err := p.agents(ctx) + if err != nil { + return adapter.Capabilities{}, err + } + var caps adapter.Capabilities + for i, a := range agents { + if i == 0 { + caps.Features = slices.Clone(a.Capabilities) + } else { + caps.Features = slices.DeleteFunc(caps.Features, func(f string) bool { return !slices.Contains(a.Capabilities, f) }) + } + for _, t := range a.Templates { + if !slices.Contains(caps.Templates, t) { + caps.Templates = append(caps.Templates, t) + } + } + caps.MaxCPU = max(caps.MaxCPU, a.TotalCPU) + caps.MaxMemoryMiB = max(caps.MaxMemoryMiB, a.TotalMemoryMiB) + } + sort.Strings(caps.Templates) + return caps, nil +} + +func (p *Pool) Capacity(ctx context.Context) (adapter.Capacity, error) { + agents, err := p.agents(ctx) + if err != nil { + return adapter.Capacity{}, err + } + c := adapter.Capacity{Bounded: true, Nodes: len(agents)} + for _, a := range agents { + c.TotalCPU += a.TotalCPU + c.FreeCPU += a.FreeCPU + c.TotalMemoryMiB += a.TotalMemoryMiB + c.FreeMemoryMiB += a.FreeMemoryMiB + if a.FreeMemoryMiB > c.LargestFreeMemoryMiB { + c.LargestFreeMemoryMiB, c.LargestFreeCPU = a.FreeMemoryMiB, a.FreeCPU + } + } + return c, nil +} + +func (p *Pool) Health(ctx context.Context) error { + agents, err := p.agents(ctx) + if err != nil { + return err + } + if len(agents) == 0 { + return errors.New("no node agents have reported in; check the differently-agent DaemonSet and /dev/kvm on nodes") + } + return nil +} + +func (p *Pool) Create(ctx context.Context, req adapter.CreateRequest) (adapter.Instance, error) { + agents, err := p.agents(ctx) + if err != nil { + return adapter.Instance{}, err + } + // Idempotency: an agent may already hold this sandbox from an earlier + // attempt whose response was lost. + for _, a := range agents { + if inst, err := p.client(a).Get(ctx, req.ID); err == nil { + inst.Ref = a.ID + ":" + inst.Ref + return inst, nil + } + } + // Snapshots live on one agent's disk, so a restore must land there. + pinned := "" + if req.FromSnapshot != "" { + var snap string + pinned, snap, _ = strings.Cut(req.FromSnapshot, ":") + req.FromSnapshot = snap + } + var fits []api.Agent + for _, a := range agents { + if pinned != "" && a.ID != pinned { + continue + } + if a.FreeCPU >= req.Resources.CPU && a.FreeMemoryMiB >= req.Resources.MemoryMiB && + (req.Template == "" || len(a.Templates) == 0 || slices.Contains(a.Templates, req.Template)) { + fits = append(fits, a) + } + } + if len(fits) == 0 { + return adapter.Instance{}, adapter.InsufficientCapacity("no node agent in pool " + p.Name + " can fit the sandbox") + } + sort.Slice(fits, func(i, j int) bool { + if fits[i].FreeMemoryMiB != fits[j].FreeMemoryMiB { + return fits[i].FreeMemoryMiB > fits[j].FreeMemoryMiB + } + return fits[i].ID < fits[j].ID + }) + var lastErr error + for _, a := range fits { + inst, err := p.client(a).Create(ctx, req) + if err == nil { + inst.Ref = a.ID + ":" + inst.Ref + return inst, nil + } + lastErr = err + if !api.IsCode(err, adapter.CodeInsufficientCapacity) { + break + } + } + return adapter.Instance{}, lastErr +} + +func (p *Pool) Get(ctx context.Context, ref string) (adapter.Instance, error) { + c, vm, err := p.route(ctx, ref) + if err != nil { + return adapter.Instance{}, err + } + inst, err := c.Get(ctx, vm) + inst.Ref = ref + return inst, err +} + +func (p *Pool) List(ctx context.Context, installation string) ([]adapter.Instance, error) { + // An agent that recently stopped heartbeating may only be partitioned. + // Refuse to report a partial view so the reconciler neither fails its + // sandboxes nor later deletes its VMs as orphans; after PartitionGrace + // the agent is treated as gone. + all, err := p.Store.ListAgents(ctx, p.Name) + if err != nil { + return nil, err + } + now := time.Now() + for _, a := range all { + if age := now.Sub(a.LastSeen); age > p.Timeout && age < PartitionGrace { + return nil, adapter.Unavailable(fmt.Errorf("agent %s missed heartbeats for %s; reconciliation of pool %s is paused", a.ID, age.Round(time.Second), p.Name)) + } + } + agents, err := p.agents(ctx) + if err != nil { + return nil, err + } + var out []adapter.Instance + for _, a := range agents { + items, err := p.client(a).List(ctx, installation) + if err != nil { + return nil, err // a partial list would make the reconciler think VMs vanished + } + for _, it := range items { + it.Ref = a.ID + ":" + it.Ref + out = append(out, it) + } + } + return out, nil +} + +func (p *Pool) Delete(ctx context.Context, ref string) error { + c, vm, err := p.route(ctx, ref) + if api.IsCode(err, api.CodeNotFound) { + return nil + } + if err != nil { + return err + } + return c.Delete(ctx, vm) +} + +func (p *Pool) Exec(ctx context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) { + c, vm, err := p.route(ctx, ref) + if err != nil { + return api.ExecResult{}, err + } + return c.Exec(ctx, vm, req) +} + +func (p *Pool) ExecStream(ctx context.Context, ref string, req api.ExecRequest, emit func(api.ExecEvent) error) (int, error) { + c, vm, err := p.route(ctx, ref) + if err != nil { + return -1, err + } + return c.ExecStream(ctx, vm, req, emit) +} + +func (p *Pool) ReadFile(ctx context.Context, ref, path string) ([]byte, error) { + c, vm, err := p.route(ctx, ref) + if err != nil { + return nil, err + } + return c.ReadFile(ctx, vm, path) +} + +func (p *Pool) WriteFile(ctx context.Context, ref, path string, data []byte) error { + c, vm, err := p.route(ctx, ref) + if err != nil { + return err + } + return c.WriteFile(ctx, vm, path, data) +} + +func (p *Pool) ListFiles(ctx context.Context, ref, path string) ([]api.FileInfo, error) { + c, vm, err := p.route(ctx, ref) + if err != nil { + return nil, err + } + return c.ListFiles(ctx, vm, path) +} + +func (p *Pool) call(ctx context.Context, ref string, fn func(*adapterhttp.Client, string) error) error { + c, vm, err := p.route(ctx, ref) + if err != nil { + return err + } + return fn(c, vm) +} + +func (p *Pool) Stop(ctx context.Context, ref string) error { + return p.call(ctx, ref, func(c *adapterhttp.Client, vm string) error { return c.Stop(ctx, vm) }) +} + +func (p *Pool) Start(ctx context.Context, ref string) error { + return p.call(ctx, ref, func(c *adapterhttp.Client, vm string) error { return c.Start(ctx, vm) }) +} + +func (p *Pool) Pause(ctx context.Context, ref string) error { + return p.call(ctx, ref, func(c *adapterhttp.Client, vm string) error { return c.Pause(ctx, vm) }) +} + +func (p *Pool) Resume(ctx context.Context, ref string) error { + return p.call(ctx, ref, func(c *adapterhttp.Client, vm string) error { return c.Resume(ctx, vm) }) +} + +// Snapshot captures on the VM's agent. Snapshot refs are agent-scoped +// (":"); restoring places the new VM on the same agent. +func (p *Pool) Snapshot(ctx context.Context, ref, kind string) (adapter.Snapshot, error) { + agentID, _, _ := strings.Cut(ref, ":") + c, vm, err := p.route(ctx, ref) + if err != nil { + return adapter.Snapshot{}, err + } + snap, err := c.Snapshot(ctx, vm, kind) + snap.Ref = agentID + ":" + snap.Ref + return snap, err +} + +func (p *Pool) DeleteSnapshot(ctx context.Context, ref string) error { + c, snap, err := p.route(ctx, ref) + if err != nil { + return err + } + return c.DeleteSnapshot(ctx, snap) +} + +var ( + _ adapter.ExecutionAdapter = (*Pool)(nil) + _ adapter.Streamer = (*Pool)(nil) + _ adapter.FileSystem = (*Pool)(nil) + _ adapter.Lifecycle = (*Pool)(nil) + _ adapter.Pauser = (*Pool)(nil) + _ adapter.Snapshotter = (*Pool)(nil) +) diff --git a/internal/adapters/firecracker/pool_test.go b/internal/adapters/firecracker/pool_test.go new file mode 100644 index 0000000..f9e2825 --- /dev/null +++ b/internal/adapters/firecracker/pool_test.go @@ -0,0 +1,75 @@ +package firecracker + +import ( + "context" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/different-ai/differently/internal/adapters/fake" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/adapter/adapterhttp" + "github.com/different-ai/differently/pkg/api" +) + +// Two "nodes" speak the agent protocol; the pool aggregates and routes. +func TestPoolPlacesAndRoutesAcrossAgents(t *testing.T) { + st := store.NewMemory() + ctx := context.Background() + nodes := map[string]*fake.Adapter{ + "node-a": fake.New(fake.Options{TotalCPU: 4, TotalMemoryMiB: 4096}), + "node-b": fake.New(fake.Options{TotalCPU: 8, TotalMemoryMiB: 16384}), + } + for name, a := range nodes { + srv := httptest.NewServer(adapterhttp.Handler(a, "tok")) + t.Cleanup(srv.Close) + c, _ := a.Capacity(ctx) + _ = st.UpsertAgent(ctx, api.Agent{ID: name, Pool: "fc", Endpoint: srv.URL, TotalCPU: c.TotalCPU, FreeCPU: c.FreeCPU, + TotalMemoryMiB: c.TotalMemoryMiB, FreeMemoryMiB: c.FreeMemoryMiB, Capabilities: fake.DefaultFeatures, LastSeen: time.Now()}) + } + _ = st.UpsertAgent(ctx, api.Agent{ID: "stale", Pool: "fc", Endpoint: "http://127.0.0.1:1", FreeCPU: 64, FreeMemoryMiB: 1 << 20, LastSeen: time.Now().Add(-time.Hour)}) + p := New("fc", st, "tok", 30*time.Second) + + if err := p.Health(ctx); err != nil { + t.Fatal(err) + } + capa, _ := p.Capacity(ctx) + if capa.Nodes != 2 || capa.TotalCPU != 12 || capa.LargestFreeMemoryMiB != 16384 { + t.Fatalf("stale agent counted or totals wrong: %+v", capa) + } + inst, err := p.Create(ctx, adapter.CreateRequest{ID: "sbx_1", Installation: "i", Resources: api.Resources{CPU: 2, MemoryMiB: 2048}}) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(inst.Ref, "node-b:") { + t.Fatalf("expected the roomier node, got %s", inst.Ref) + } + if res, err := p.Exec(ctx, inst.Ref, api.ExecRequest{Command: "uname"}); err != nil || !strings.Contains(res.Stdout, "uname") { + t.Fatalf("routed exec %+v %v", res, err) + } + snap, err := p.Snapshot(ctx, inst.Ref, "disk") + if err != nil || !strings.HasPrefix(snap.Ref, "node-b:") { + t.Fatalf("snapshot %+v %v", snap, err) + } + restored, err := p.Create(ctx, adapter.CreateRequest{ID: "sbx_2", Installation: "i", Resources: api.Resources{CPU: 1, MemoryMiB: 512}, FromSnapshot: snap.Ref}) + if err != nil || !strings.HasPrefix(restored.Ref, "node-b:") { + t.Fatalf("restore must stay on the snapshot's node: %+v %v", restored, err) + } + list, err := p.List(ctx, "i") + if err != nil || len(list) != 2 { + t.Fatalf("list %v %v", list, err) + } + if _, err := p.Create(ctx, adapter.CreateRequest{ID: "sbx_3", Resources: api.Resources{CPU: 32, MemoryMiB: 1024}}); !api.IsCode(err, adapter.CodeInsufficientCapacity) { + t.Fatalf("oversized create: %v", err) + } + if err := p.Delete(ctx, inst.Ref); err != nil || nodes["node-b"].Count() != 1 { + t.Fatalf("delete %v count %d", err, nodes["node-b"].Count()) + } + // A node that only just went quiet may be partitioned: no partial lists. + _ = st.UpsertAgent(ctx, api.Agent{ID: "node-c", Pool: "fc", Endpoint: "http://127.0.0.1:1", LastSeen: time.Now().Add(-2 * time.Minute)}) + if _, err := p.List(ctx, "i"); !api.IsCode(err, api.CodeProviderUnavailable) { + t.Fatalf("list during a possible partition must fail, got %v", err) + } +} diff --git a/internal/adapters/freestyle/freestyle.go b/internal/adapters/freestyle/freestyle.go new file mode 100644 index 0000000..1a1e430 --- /dev/null +++ b/internal/adapters/freestyle/freestyle.go @@ -0,0 +1,402 @@ +// Package freestyle adapts Freestyle's hosted microVM API (/v5). +// +// Freestyle is hosted-only, so pools on this adapter are always +// TrustExternal. Notable differences from other backends, handled here: +// +// - create has no resource or env fields: VMs boot at their snapshot's size +// and are grown with /resize; sandbox env is written to a 0600 file inside +// the VM and sourced by every exec; +// - there is no stop endpoint: stop runs poweroff and waits for "stopped"; +// - snapshots always capture memory and disk; +// - networking is deny-all unless the pool sets config.egress=public. +// +// Verified against Freestyle's published OpenAPI spec (2026-10-28), not yet +// against a live account. +package freestyle + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "github.com/different-ai/differently/internal/adapters/httpx" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// DefaultAPIURL is Freestyle's API. +const DefaultAPIURL = "https://api.freestyle.sh" + +// Metadata keys. Freestyle metadata keys are short and unprefixed. +const ( + metaSandbox = "dly-sandbox" + metaInstallation = "dly-installation" + envFile = "/etc/differently/env" +) + +// Options configures the adapter. Pool config keys: apiURL, snapshot, +// egress (none|public), domain (for exposed ports, default style.dev). +type Options struct { + APIURL string + APIKey string + Snapshot string + Egress string + Domain string + ReadyTimeout time.Duration +} + +// FromConfig maps pool config to options. +func FromConfig(cfg map[string]string, apiKey string) Options { + o := Options{APIURL: cfg["apiURL"], APIKey: apiKey, Snapshot: cfg["snapshot"], Egress: cfg["egress"], Domain: cfg["domain"]} + if o.APIURL == "" { + o.APIURL = DefaultAPIURL + } + if o.Domain == "" { + o.Domain = "style.dev" + } + return o +} + +// Adapter talks to Freestyle. +type Adapter struct { + opts Options + c *httpx.Client +} + +// New returns a Freestyle adapter. +func New(o Options) *Adapter { + if o.ReadyTimeout == 0 { + o.ReadyTimeout = 3 * time.Minute + } + return &Adapter{opts: o, c: httpx.New("freestyle", o.APIURL, o.APIKey)} +} + +// Endpoint is reported in the trust report. +func (a *Adapter) Endpoint() string { return a.opts.APIURL } + +type vm struct { + ID string `json:"id"` + State string `json:"state"` + Slug string `json:"slug"` + Metadata map[string]string `json:"metadata"` + Resources struct { + CPU int `json:"cpu"` + Memory int `json:"memory"` + Storage int `json:"storage"` + } `json:"resources"` +} + +func (a *Adapter) Info() adapter.Info { + return adapter.Info{Kind: "freestyle", Version: "1", Trust: api.TrustExternal, Isolation: "hardware-virtualized microVM (Freestyle-managed)", Protocol: adapter.ProtocolVersion, Description: "Freestyle VMs at " + a.opts.APIURL} +} + +func (a *Adapter) Capabilities(context.Context) (adapter.Capabilities, error) { + return adapter.Capabilities{Features: []string{ + adapter.FeatureExec, adapter.FeatureFiles, adapter.FeatureStopStart, adapter.FeaturePersistDisk, + adapter.FeaturePauseResume, adapter.FeatureSnapshotMem, adapter.FeatureSnapshotDisk, adapter.FeatureFork, + adapter.FeaturePorts, adapter.FeatureEgressControl, adapter.FeatureMicroVM, adapter.FeatureArchAMD64, + }, MaxCPU: 32, MaxMemoryMiB: 65536, MaxDiskGiB: 256}, nil +} + +func (a *Adapter) Capacity(context.Context) (adapter.Capacity, error) { return adapter.Capacity{}, nil } + +func (a *Adapter) Health(ctx context.Context) error { + return a.c.JSON(ctx, http.MethodGet, "/v5/vms?limit=1", nil, nil) +} + +// slug derives Freestyle's unique per-account name from the sandbox ID, which +// makes create idempotent: a replay conflicts and resolves to the same VM. +func slug(sandboxID string) string { return strings.ReplaceAll(strings.ToLower(sandboxID), "_", "-") } + +func state(v vm) string { + switch v.State { + case "running": + return adapter.StateRunning + case "paused": + return adapter.StatePaused + case "stopped": + return adapter.StateStopped + } + return adapter.StatePending +} + +func inst(v vm) adapter.Instance { + return adapter.Instance{Ref: v.ID, SandboxID: v.Metadata[metaSandbox], State: state(v)} +} + +func (a *Adapter) get(ctx context.Context, ref string) (vm, error) { + var v vm + return v, a.c.JSON(ctx, http.MethodGet, "/v5/vms/"+url.PathEscape(ref), nil, &v) +} + +func (a *Adapter) waitState(ctx context.Context, ref, want string) (vm, error) { + var v vm + err := httpx.Poll(ctx, time.Second, a.opts.ReadyTimeout, func() (bool, error) { + var err error + v, err = a.get(ctx, ref) + return err == nil && v.State == want, err + }) + return v, err +} + +func (a *Adapter) Create(ctx context.Context, req adapter.CreateRequest) (adapter.Instance, error) { + v, err := a.get(ctx, slug(req.ID)) + if api.IsCode(err, api.CodeNotFound) { + rules := []map[string]any{} + if a.opts.Egress == "public" { + rules = append(rules, map[string]any{"action": "allow", "source": map[string]any{}, "destination": map[string]any{"public": true}, "description": "egress allowed by Differently pool config"}) + } + body := map[string]any{ + "slug": slug(req.ID), + "firewall": map[string]any{"rules": rules}, + "metadata": map[string]string{metaSandbox: req.ID, metaInstallation: req.Installation}, + } + snapshot := a.opts.Snapshot + if req.Template != "" { + snapshot = req.Template + } + if req.FromSnapshot != "" { + snapshot = req.FromSnapshot + } + if snapshot != "" { + body["snapshotId"] = snapshot + } + if req.TTLSeconds > 0 { + body["ttlSeconds"] = req.TTLSeconds // provider-side backstop + } + err = a.c.JSON(ctx, http.MethodPost, "/v5/vms", body, &v) + if api.IsCode(err, api.CodeInvalidState) { + v, err = a.get(ctx, slug(req.ID)) // lost the race with a replay + } + } + if err != nil { + return adapter.Instance{}, err + } + if v, err = a.waitState(ctx, v.ID, "running"); err != nil { + return adapter.Instance{}, err + } + // Grow to the requested envelope; resize never shrinks. + need := map[string]int{} + if req.Resources.CPU > v.Resources.CPU { + need["cpu"] = req.Resources.CPU + } + if req.Resources.MemoryMiB > v.Resources.Memory { + need["memory"] = req.Resources.MemoryMiB + } + if d := req.Resources.DiskGiB * 1024; d > v.Resources.Storage { + need["storage"] = d + } + if len(need) > 0 { + if err := a.c.JSON(ctx, http.MethodPost, "/v5/vms/"+url.PathEscape(v.ID)+"/resize", need, &v); err != nil { + return adapter.Instance{}, err + } + } + if len(req.Env) > 0 { + if err := a.WriteFile(ctx, v.ID, envFile, envFileContent(req.Env)); err != nil { + return adapter.Instance{}, err + } + } + return inst(v), nil +} + +func envFileContent(env map[string]string) []byte { + keys := make([]string, 0, len(env)) + for k := range env { + keys = append(keys, k) + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k + "=" + httpx.ShellQuote([]string{env[k]}) + "\n") + } + return []byte(b.String()) +} + +func (a *Adapter) Get(ctx context.Context, ref string) (adapter.Instance, error) { + v, err := a.get(ctx, ref) + return inst(v), err +} + +func (a *Adapter) List(ctx context.Context, installation string) ([]adapter.Instance, error) { + var out []adapter.Instance + for offset := 0; ; { + var page struct { + VMs []vm `json:"vms"` + TotalCount int `json:"totalCount"` + } + path := fmt.Sprintf("/v5/vms?limit=100&offset=%d&metadata=%s", offset, url.QueryEscape(metaInstallation+":"+installation)) + if err := a.c.JSON(ctx, http.MethodGet, path, nil, &page); err != nil { + return nil, err + } + for _, v := range page.VMs { + out = append(out, inst(v)) + } + offset += len(page.VMs) + if len(page.VMs) == 0 || offset >= page.TotalCount { + return out, nil + } + } +} + +func (a *Adapter) Delete(ctx context.Context, ref string) error { + err := a.c.JSON(ctx, http.MethodDelete, "/v5/vms/"+url.PathEscape(ref), nil, nil) + if api.IsCode(err, api.CodeNotFound) { + return nil + } + return err +} + +func (a *Adapter) Exec(ctx context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) { + timeout := req.TimeoutSeconds * 1000 + if timeout <= 0 || timeout > 300_000 { + timeout = 300_000 // inline exec cap + } + body := map[string]any{ + "command": "set -a; [ -r " + envFile + " ] && . " + envFile + "; set +a; " + httpx.CommandLine(req), + "timeoutMs": timeout, + } + if len(req.Env) > 0 { + body["env"] = req.Env + } + if req.Stdin != "" { + body["stdin"] = base64.StdEncoding.EncodeToString([]byte(req.Stdin)) + } + var out struct { + StatusCode *int `json:"statusCode"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + } + start := time.Now() + if err := a.c.JSON(ctx, http.MethodPost, "/v5/vms/"+url.PathEscape(ref)+"/exec-await", body, &out); err != nil { + return api.ExecResult{}, err + } + code := 124 // statusCode is absent when the command was killed by timeout + if out.StatusCode != nil { + code = *out.StatusCode + } + return api.ExecResult{ExitCode: code, Stdout: out.Stdout, Stderr: out.Stderr, DurationMs: time.Since(start).Milliseconds()}, nil +} + +func absPath(p string) string { + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + return p +} + +func (a *Adapter) ReadFile(ctx context.Context, ref, path string) ([]byte, error) { + resp, err := a.c.Do(ctx, http.MethodGet, "/v5/vms/"+url.PathEscape(ref)+"/fs/read?path="+url.QueryEscape(absPath(path)), nil, "", nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, 100<<20)) +} + +func (a *Adapter) WriteFile(ctx context.Context, ref, path string, data []byte) error { + dir := absPath(path) + if i := strings.LastIndex(dir, "/"); i > 0 { + if err := a.c.JSON(ctx, http.MethodPost, "/v5/vms/"+url.PathEscape(ref)+"/fs/mkdir", map[string]string{"path": dir[:i]}, nil); err != nil { + return err + } + } + resp, err := a.c.Do(ctx, http.MethodPut, "/v5/vms/"+url.PathEscape(ref)+"/fs/write?mode=0600&path="+url.QueryEscape(absPath(path)), bytes.NewReader(data), "application/octet-stream", nil) + if err != nil { + return err + } + return resp.Body.Close() +} + +func (a *Adapter) ListFiles(ctx context.Context, ref, path string) ([]api.FileInfo, error) { + var out struct { + Entries []struct { + Name string `json:"name"` + Kind string `json:"kind"` + } `json:"entries"` + } + dir := absPath(path) + if err := a.c.JSON(ctx, http.MethodGet, "/v5/vms/"+url.PathEscape(ref)+"/fs/dir?path="+url.QueryEscape(dir), nil, &out); err != nil { + return nil, err + } + items := make([]api.FileInfo, 0, len(out.Entries)) + for _, e := range out.Entries { + items = append(items, api.FileInfo{Path: strings.TrimRight(dir, "/") + "/" + e.Name, Name: e.Name, IsDir: e.Kind == "directory"}) + } + return items, nil +} + +// Stop powers the VM off from inside (Freestyle has no stop endpoint). +func (a *Adapter) Stop(ctx context.Context, ref string) error { + // The exec usually fails when the guest disconnects; the state poll decides. + _, _ = a.Exec(ctx, ref, api.ExecRequest{Command: "poweroff", TimeoutSeconds: 10}) + _, err := a.waitState(ctx, ref, "stopped") + return err +} + +func (a *Adapter) Start(ctx context.Context, ref string) error { + if err := a.c.JSON(ctx, http.MethodPost, "/v5/vms/"+url.PathEscape(ref)+"/start", nil, nil); err != nil { + return err + } + _, err := a.waitState(ctx, ref, "running") + return err +} + +func (a *Adapter) Pause(ctx context.Context, ref string) error { + if err := a.c.JSON(ctx, http.MethodPost, "/v5/vms/"+url.PathEscape(ref)+"/pause", nil, nil); err != nil { + return err + } + _, err := a.waitState(ctx, ref, "paused") + return err +} + +func (a *Adapter) Resume(ctx context.Context, ref string) error { return a.Start(ctx, ref) } + +// Snapshot always captures memory and disk on Freestyle. +func (a *Adapter) Snapshot(ctx context.Context, ref, _ string) (adapter.Snapshot, error) { + var out struct { + SnapshotID string `json:"snapshotId"` + } + if err := a.c.JSON(ctx, http.MethodPost, "/v5/vms/"+url.PathEscape(ref)+"/snapshot", map[string]any{}, &out); err != nil { + return adapter.Snapshot{}, err + } + return adapter.Snapshot{Ref: out.SnapshotID, Kind: "memory"}, nil +} + +func (a *Adapter) DeleteSnapshot(ctx context.Context, ref string) error { + err := a.c.JSON(ctx, http.MethodDelete, "/v5/snapshots/"+url.PathEscape(ref), nil, nil) + if api.IsCode(err, api.CodeNotFound) { + return nil + } + return err +} + +// ExposePort publishes a port on a *.style.dev hostname. The URL is public: +// anyone with it can reach the port. +func (a *Adapter) ExposePort(ctx context.Context, ref string, port int) (string, error) { + v, err := a.get(ctx, ref) + if err != nil { + return "", err + } + domain := fmt.Sprintf("%d-%s.%s", port, v.Slug, a.opts.Domain) + body := map[string]any{"action": "allow", "domain": domain, "source": map[string]any{"public": true}, "destination": map[string]any{"vmId": v.ID, "port": port}} + if err := a.c.JSON(ctx, http.MethodPost, "/v5/tls", body, nil); err != nil && !api.IsCode(err, api.CodeInvalidState) { + return "", err + } + return "https://" + domain, nil +} + +var ( + _ adapter.ExecutionAdapter = (*Adapter)(nil) + _ adapter.FileSystem = (*Adapter)(nil) + _ adapter.Lifecycle = (*Adapter)(nil) + _ adapter.Pauser = (*Adapter)(nil) + _ adapter.Snapshotter = (*Adapter)(nil) + _ adapter.PortExposer = (*Adapter)(nil) +) diff --git a/internal/adapters/freestyle/freestyle_test.go b/internal/adapters/freestyle/freestyle_test.go new file mode 100644 index 0000000..6de10bb --- /dev/null +++ b/internal/adapters/freestyle/freestyle_test.go @@ -0,0 +1,140 @@ +package freestyle + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +func TestFreestyleAdapter(t *testing.T) { + var mu sync.Mutex + vms := map[string]map[string]any{} + files := map[string]string{} + var resized map[string]int + var execBody map[string]any + mux := http.NewServeMux() + notFound := func(w http.ResponseWriter) { + w.WriteHeader(404) + json.NewEncoder(w).Encode(map[string]string{"code": "NOT_FOUND", "message": "no vm"}) + } + lookup := func(key string) map[string]any { + for _, v := range vms { + if v["id"] == key || v["slug"] == key { + return v + } + } + return nil + } + mux.HandleFunc("POST /v5/vms", func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + _ = json.NewDecoder(r.Body).Decode(&body) + if _, ok := body["firewall"]; !ok { + w.WriteHeader(400) + return + } + mu.Lock() + defer mu.Unlock() + v := map[string]any{"id": "vm-1", "slug": body["slug"], "state": "starting", "metadata": body["metadata"], "resources": map[string]int{"cpu": 2, "memory": 4096, "storage": 16384}} + vms["vm-1"] = v + w.WriteHeader(201) + json.NewEncoder(w).Encode(v) + }) + mux.HandleFunc("GET /v5/vms/{id}", func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + v := lookup(r.PathValue("id")) + if v == nil { + notFound(w) + return + } + if v["state"] == "starting" { + v["state"] = "running" + } + json.NewEncoder(w).Encode(v) + }) + mux.HandleFunc("GET /v5/vms", func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + list := []any{} + for _, v := range vms { + if strings.HasSuffix(r.URL.Query().Get("metadata"), v["metadata"].(map[string]any)[metaInstallation].(string)) { + list = append(list, v) + } + } + json.NewEncoder(w).Encode(map[string]any{"vms": list, "totalCount": len(list)}) + }) + mux.HandleFunc("POST /v5/vms/{id}/resize", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&resized) + mu.Lock() + defer mu.Unlock() + json.NewEncoder(w).Encode(lookup(r.PathValue("id"))) + }) + mux.HandleFunc("POST /v5/vms/{id}/exec-await", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&execBody) + json.NewEncoder(w).Encode(map[string]any{"statusCode": 2, "stdout": "out", "stderr": "err"}) + }) + mux.HandleFunc("POST /v5/vms/{id}/fs/mkdir", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(204) }) + mux.HandleFunc("PUT /v5/vms/{id}/fs/write", func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + mu.Lock() + files[r.URL.Query().Get("path")] = string(b) + mu.Unlock() + w.WriteHeader(204) + }) + mux.HandleFunc("DELETE /v5/vms/{id}", func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + if lookup(r.PathValue("id")) == nil { + notFound(w) + return + } + delete(vms, r.PathValue("id")) + w.WriteHeader(204) + }) + srv := httptest.NewServer(mux) + defer srv.Close() + + a := New(Options{APIURL: srv.URL, APIKey: "k"}) + ctx := context.Background() + req := adapter.CreateRequest{ID: "sbx_ABC", Installation: "inst_1", Resources: api.Resources{CPU: 4, MemoryMiB: 2048, DiskGiB: 10}, Env: map[string]string{"TOKEN": "it's secret"}} + inst, err := a.Create(ctx, req) + if err != nil { + t.Fatal(err) + } + if inst.State != adapter.StateRunning || inst.SandboxID != "sbx_ABC" { + t.Fatalf("instance %+v", inst) + } + if resized["cpu"] != 4 || resized["memory"] != 0 || resized["storage"] != 0 { + t.Fatalf("resize should only grow cpu: %v", resized) + } + if got := files[envFile]; got != "TOKEN='it'\\''s secret'\n" { + t.Fatalf("env file %q", got) + } + if again, err := a.Create(ctx, req); err != nil || again.Ref != inst.Ref || len(vms) != 1 { + t.Fatalf("create not idempotent: %v %v", again, err) + } + res, err := a.Exec(ctx, inst.Ref, api.ExecRequest{Command: "make", Cwd: "/src"}) + if err != nil || res.ExitCode != 2 || res.Stderr != "err" { + t.Fatalf("exec %+v %v", res, err) + } + if cmd := execBody["command"].(string); !strings.Contains(cmd, ". "+envFile) || !strings.HasSuffix(cmd, "cd /src && make") { + t.Fatalf("command %q", cmd) + } + if list, err := a.List(ctx, "inst_1"); err != nil || len(list) != 1 { + t.Fatalf("list %v %v", list, err) + } + if err := a.Delete(ctx, inst.Ref); err != nil { + t.Fatal(err) + } + if err := a.Delete(ctx, inst.Ref); err != nil { + t.Fatalf("idempotent delete: %v", err) + } +} diff --git a/internal/adapters/httpx/httpx.go b/internal/adapters/httpx/httpx.go new file mode 100644 index 0000000..e19bfdb --- /dev/null +++ b/internal/adapters/httpx/httpx.go @@ -0,0 +1,174 @@ +// Package httpx is a small JSON-over-HTTP helper for provider adapters. It +// maps provider HTTP failures onto Differently error codes so the scheduler +// can tell transient capacity limits from outages. +package httpx + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// Client calls one provider API. +type Client struct { + BaseURL string + Token string + HTTP *http.Client + // Provider names the backend in error messages. + Provider string +} + +// New returns a client with a sane default timeout. +func New(provider, baseURL, token string) *Client { + return &Client{Provider: provider, BaseURL: strings.TrimRight(baseURL, "/"), Token: token, HTTP: &http.Client{Timeout: 2*time.Hour + 15*time.Minute}} +} + +// JSON sends a JSON request and decodes a JSON response into out (may be nil). +func (c *Client) JSON(ctx context.Context, method, url string, in, out any) error { + var body io.Reader + ct := "" + if in != nil { + b, err := json.Marshal(in) + if err != nil { + return err + } + body, ct = bytes.NewReader(b), "application/json" + } + resp, err := c.Do(ctx, method, url, body, ct, nil) + if err != nil { + return err + } + defer resp.Body.Close() + if out == nil || resp.StatusCode == http.StatusNoContent { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} + +// Do sends a request; url may be absolute or relative to BaseURL. Non-2xx +// responses become *api.Error. +func (c *Client) Do(ctx context.Context, method, url string, body io.Reader, contentType string, header http.Header) (*http.Response, error) { + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + url = c.BaseURL + url + } + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + for k, vs := range header { + for _, v := range vs { + req.Header.Add(k, v) + } + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + req.Header.Set("Accept", "application/json") + if c.Token != "" { + req.Header.Set("Authorization", "Bearer "+c.Token) + } + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, adapter.Unavailable(fmt.Errorf("%s: %w", c.Provider, err)) + } + if resp.StatusCode < 300 { + return resp, nil + } + defer resp.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + return nil, c.mapError(resp.StatusCode, raw) +} + +// mapError converts a provider error into a Differently error. The message +// keeps the provider's own text but never echoes request credentials. +func (c *Client) mapError(status int, raw []byte) error { + var body struct { + Code string `json:"code"` + Message any `json:"message"` + Error string `json:"error"` + } + _ = json.Unmarshal(raw, &body) + msg := fmt.Sprint(body.Message) + if body.Message == nil { + msg = body.Error + } + if msg == "" { + msg = http.StatusText(status) + } + msg = c.Provider + ": " + msg + if body.Code != "" { + msg += " (" + body.Code + ")" + } + e := &api.Error{Message: msg, Details: map[string]string{"provider": c.Provider, "providerStatus": fmt.Sprint(status)}} + switch { + case status == http.StatusNotFound: + e.Code = api.CodeNotFound + case status == http.StatusTooManyRequests: + e.Code = adapter.CodeInsufficientCapacity // provider plan or rate cap: retry later + case status == http.StatusConflict: + e.Code = api.CodeInvalidState + case status == http.StatusBadRequest || status == http.StatusUnprocessableEntity: + e.Code = api.CodeInvalidRequest + case status == http.StatusUnauthorized || status == http.StatusForbidden: + e.Code = api.CodeProviderUnavailable + e.Message = c.Provider + ": credentials were rejected; check the pool's credential reference" + default: + e.Code = api.CodeProviderUnavailable + } + return e +} + +// ShellQuote joins args into a POSIX shell command line. +func ShellQuote(args []string) string { + out := make([]string, len(args)) + for i, a := range args { + if a != "" && strings.IndexFunc(a, func(r rune) bool { + return !(r == '-' || r == '_' || r == '.' || r == '/' || r == '=' || r == ':' || r == ',' || r == '+' || + (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9')) + }) < 0 { + out[i] = a + continue + } + out[i] = "'" + strings.ReplaceAll(a, "'", `'\''`) + "'" + } + return strings.Join(out, " ") +} + +// CommandLine renders an ExecRequest as one shell command, honouring cwd. +func CommandLine(req api.ExecRequest) string { + cmd := req.Command + if cmd == "" { + cmd = ShellQuote(req.Args) + } + if req.Cwd != "" { + cmd = "cd " + ShellQuote([]string{req.Cwd}) + " && " + cmd + } + return cmd +} + +// Poll calls check until it reports done, ctx ends or timeout passes. +func Poll(ctx context.Context, every, timeout time.Duration, check func() (bool, error)) error { + deadline := time.Now().Add(timeout) + for { + done, err := check() + if err != nil || done { + return err + } + if time.Now().After(deadline) { + return adapter.Unavailable(fmt.Errorf("timed out after %s", timeout)) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(every): + } + } +} diff --git a/internal/audit/audit.go b/internal/audit/audit.go new file mode 100644 index 0000000..24d018d --- /dev/null +++ b/internal/audit/audit.go @@ -0,0 +1,71 @@ +// Package audit seals audit events into a hash chain. +// +// Each event's hash covers its content and the previous event's hash, so any +// edit, deletion or reordering of stored events is detectable by Verify. This +// is tamper-evident, not tamper-proof: an attacker with write access to the +// database can rewrite the whole chain. Exporting events (or the latest hash) +// to write-once storage outside the database closes that gap. +package audit + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + + "github.com/different-ai/differently/pkg/api" +) + +// Genesis is the PrevHash of the first event. +const Genesis = "0000000000000000000000000000000000000000000000000000000000000000" + +// Seal sets ev.PrevHash and ev.Hash given the previous event's hash. +func Seal(prev string, ev api.AuditEvent) api.AuditEvent { + if prev == "" { + prev = Genesis + } + ev.PrevHash = prev + ev.Hash = "" + ev.Hash = digest(ev) + return ev +} + +func digest(ev api.AuditEvent) string { + ev.Hash = "" + // encoding/json sorts map keys, which makes the encoding canonical for + // this struct. + b, _ := json.Marshal(ev) + sum := sha256.Sum256(append([]byte(ev.PrevHash), b...)) + return hex.EncodeToString(sum[:]) +} + +// VerifyResult reports the outcome of a chain check. +type VerifyResult struct { + OK bool `json:"ok"` + Events int `json:"events"` + BrokenAt int64 `json:"brokenAt,omitempty"` + Problem string `json:"problem,omitempty"` + HeadHash string `json:"headHash,omitempty"` +} + +// Verify checks a contiguous, ascending run of events. The first event's +// PrevHash is trusted as the anchor, so a partial range can be verified. +func Verify(events []api.AuditEvent) VerifyResult { + res := VerifyResult{OK: true, Events: len(events)} + for i, ev := range events { + if i > 0 { + prev := events[i-1] + if ev.Seq != prev.Seq+1 { + return VerifyResult{Events: len(events), BrokenAt: ev.Seq, Problem: fmt.Sprintf("sequence gap after %d", prev.Seq)} + } + if ev.PrevHash != prev.Hash { + return VerifyResult{Events: len(events), BrokenAt: ev.Seq, Problem: "previous hash mismatch"} + } + } + if digest(ev) != ev.Hash { + return VerifyResult{Events: len(events), BrokenAt: ev.Seq, Problem: "event content does not match its hash"} + } + res.HeadHash = ev.Hash + } + return res +} diff --git a/internal/audit/audit_test.go b/internal/audit/audit_test.go new file mode 100644 index 0000000..a6cc801 --- /dev/null +++ b/internal/audit/audit_test.go @@ -0,0 +1,37 @@ +package audit + +import ( + "testing" + "time" + + "github.com/different-ai/differently/pkg/api" +) + +func chain(n int) []api.AuditEvent { + var out []api.AuditEvent + prev := "" + for i := 1; i <= n; i++ { + ev := Seal(prev, api.AuditEvent{Seq: int64(i), Time: time.Unix(int64(i), 0).UTC(), Actor: "key_a", Action: "sandbox.create", Resource: "sbx_1", Details: map[string]string{"b": "2", "a": "1"}}) + out = append(out, ev) + prev = ev.Hash + } + return out +} + +func TestVerifyDetectsTampering(t *testing.T) { + events := chain(5) + if res := Verify(events); !res.OK || res.HeadHash != events[4].Hash { + t.Fatalf("intact chain failed: %+v", res) + } + + edited := chain(5) + edited[2].Actor = "someone-else" + if res := Verify(edited); res.OK || res.BrokenAt != 3 { + t.Fatalf("edit not detected: %+v", res) + } + + deleted := append(chain(5)[:2], chain(5)[3:]...) + if res := Verify(deleted); res.OK { + t.Fatalf("deletion not detected: %+v", res) + } +} diff --git a/internal/auditlog/auditlog.go b/internal/auditlog/auditlog.go new file mode 100644 index 0000000..470f3ac --- /dev/null +++ b/internal/auditlog/auditlog.go @@ -0,0 +1,66 @@ +// Package auditlog records audit events to the store's hash chain and, +// optionally, mirrors them as JSON lines to a file for shipping to +// write-once storage or a SIEM. +package auditlog + +import ( + "context" + "encoding/json" + "log/slog" + "os" + "sync" + "time" + + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/api" +) + +// Recorder appends audit events. +type Recorder struct { + Store store.Store + Log *slog.Logger + mu sync.Mutex + file *os.File +} + +// New returns a recorder; path may be empty. +func New(st store.Store, log *slog.Logger, path string) (*Recorder, error) { + r := &Recorder{Store: st, Log: log} + if path != "" { + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return nil, err + } + r.file = f + } + return r, nil +} + +// Record stores ev. Audit failures are logged, never silently dropped. +func (r *Recorder) Record(ctx context.Context, ev api.AuditEvent) { + if ev.Time.IsZero() { + ev.Time = time.Now().UTC() + } + if ev.Outcome == "" { + ev.Outcome = "success" + } + sealed, err := r.Store.AppendAudit(context.WithoutCancel(ctx), ev) + if err != nil { + r.Log.Error("audit append failed", "action", ev.Action, "resource", ev.Resource, "err", err) + return + } + if r.file != nil { + r.mu.Lock() + b, _ := json.Marshal(sealed) + _, _ = r.file.Write(append(b, '\n')) + r.mu.Unlock() + } +} + +// Close closes the mirror file. +func (r *Recorder) Close() error { + if r.file != nil { + return r.file.Close() + } + return nil +} diff --git a/internal/auth/auth.go b/internal/auth/auth.go new file mode 100644 index 0000000..5c297ff --- /dev/null +++ b/internal/auth/auth.go @@ -0,0 +1,167 @@ +// Package auth implements API keys, scopes and tenant authorization. +// +// Tokens look like dly__. Only a SHA-256 hash of the +// high-entropy secret is stored; the token is shown once at creation. A key +// belongs to the operator (no organization), an organization, or a single +// project, and carries scopes. Keys can only mint keys with a subset of their +// own scopes inside their own tenancy, so no key can escalate. +package auth + +import ( + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "slices" + "strings" + "time" + + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/api" +) + +// TokenPrefix starts every Differently API token. +const TokenPrefix = "dly_" + +// Scopes. +const ( + ScopeAdmin = "admin" // operator: every organization and admin API + ScopeOrgAdmin = "org:admin" // everything inside one organization + ScopeSandboxesRead = "sandboxes:read" // list and inspect sandboxes + ScopeSandboxesWrite = "sandboxes:write" // create, delete, lifecycle, snapshots + ScopeSandboxesExec = "sandboxes:exec" // run commands and transfer files + ScopeProjectsWrite = "projects:write" // create and delete projects + ScopeKeysWrite = "keys:write" // create and revoke keys + ScopeUsageRead = "usage:read" // usage, quotas + ScopeAuditRead = "audit:read" // audit events + ScopePoliciesWrite = "policies:write" // org/project placement policy +) + +// AllScopes lists every grantable scope. +var AllScopes = []string{ScopeAdmin, ScopeOrgAdmin, ScopeSandboxesRead, ScopeSandboxesWrite, ScopeSandboxesExec, ScopeProjectsWrite, ScopeKeysWrite, ScopeUsageRead, ScopeAuditRead, ScopePoliciesWrite} + +// Principal is an authenticated caller. +type Principal struct { + KeyID string + OrgID string // empty for operator keys + ProjectID string // set for project-scoped keys + Scopes []string +} + +// Operator reports whether p is an installation operator. +func (p Principal) Operator() bool { return slices.Contains(p.Scopes, ScopeAdmin) } + +// Has reports whether p holds scope (admin and org:admin imply tenant scopes). +func (p Principal) Has(scope string) bool { + if slices.Contains(p.Scopes, ScopeAdmin) || slices.Contains(p.Scopes, scope) { + return true + } + return scope != ScopeAdmin && slices.Contains(p.Scopes, ScopeOrgAdmin) +} + +// Actor is the audit identity. +func (p Principal) Actor() string { return "key:" + p.KeyID } + +// CanAccess reports whether p may act on a resource owned by org/project. +func (p Principal) CanAccess(orgID, projectID string) bool { + switch { + case p.Operator(): + return true + case p.OrgID != orgID: + return false + case p.ProjectID != "" && p.ProjectID != projectID: + return false + } + return true +} + +// Require returns a permission error unless p holds scope. +func (p Principal) Require(scope string) error { + if p.Has(scope) { + return nil + } + return api.Errorf(api.CodePermissionDenied, "this key lacks the %s scope", scope).With("scope", scope) +} + +// Hash returns the stored hash of a secret. +func Hash(secret string) string { + sum := sha256.Sum256([]byte(secret)) + return hex.EncodeToString(sum[:]) +} + +// ParseToken splits a token into key ID and secret. +func ParseToken(token string) (keyID, secret string, ok bool) { + if !strings.HasPrefix(token, TokenPrefix) { + return "", "", false + } + rest := strings.TrimPrefix(token, TokenPrefix) + i := strings.LastIndex(rest, "_") + if i <= 0 || i == len(rest)-1 { + return "", "", false + } + keyID, secret = rest[:i], rest[i+1:] + if len(secret) < 24 { + return "", "", false + } + return keyID, secret, true +} + +// NewToken creates a key ID and token for a new key. +func NewToken() (keyID, token, secret string) { + keyID = strings.ReplaceAll(id.New("key"), "_", "") + secret = id.Secret(40) + return keyID, TokenPrefix + keyID + "_" + secret, secret +} + +// Verify checks token against a stored key. +func Verify(k store.KeyRecord, secret string, now time.Time) error { + if subtle.ConstantTimeCompare([]byte(Hash(secret)), []byte(k.Hash)) != 1 { + return api.Errorf(api.CodeUnauthenticated, "invalid API key") + } + if k.Revoked { + return api.Errorf(api.CodeUnauthenticated, "API key revoked") + } + if k.ExpiresAt != nil && now.After(*k.ExpiresAt) { + return api.Errorf(api.CodeUnauthenticated, "API key expired") + } + return nil +} + +// CheckGrant validates that creator may mint a key for req. It returns the +// normalized org/project for the new key. +func CheckGrant(creator Principal, req api.CreateAPIKeyRequest) (orgID, projectID string, err error) { + if err := creator.Require(ScopeKeysWrite); err != nil { + return "", "", err + } + if len(req.Scopes) == 0 { + return "", "", api.Errorf(api.CodeInvalidRequest, "scopes are required") + } + for _, s := range req.Scopes { + if !slices.Contains(AllScopes, s) { + return "", "", api.Errorf(api.CodeInvalidRequest, "unknown scope %q", s) + } + if !creator.Has(s) || (s == ScopeOrgAdmin && !creator.Operator() && !slices.Contains(creator.Scopes, ScopeOrgAdmin)) { + return "", "", api.Errorf(api.CodePermissionDenied, "cannot grant scope %s you do not hold", s) + } + } + orgID, projectID = req.OrgID, req.ProjectID + if !creator.Operator() { + if orgID != "" && orgID != creator.OrgID { + return "", "", api.Errorf(api.CodePermissionDenied, "cannot create keys for another organization") + } + orgID = creator.OrgID + if creator.ProjectID != "" { + if projectID != "" && projectID != creator.ProjectID { + return "", "", api.Errorf(api.CodePermissionDenied, "cannot create keys for another project") + } + projectID = creator.ProjectID + } + } + if slices.Contains(req.Scopes, ScopeAdmin) && (orgID != "" || projectID != "") { + return "", "", api.Errorf(api.CodeInvalidRequest, "the admin scope is only valid on operator keys") + } + if orgID == "" && !slices.Contains(req.Scopes, ScopeAdmin) { + return "", "", api.Errorf(api.CodeInvalidRequest, "orgId is required for tenant keys") + } + return orgID, projectID, nil +} diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go new file mode 100644 index 0000000..4ab4ac4 --- /dev/null +++ b/internal/auth/auth_test.go @@ -0,0 +1,69 @@ +package auth + +import ( + "testing" + "time" + + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/api" +) + +func TestTokenRoundTrip(t *testing.T) { + keyID, token, secret := NewToken() + gotID, gotSecret, ok := ParseToken(token) + if !ok || gotID != keyID || gotSecret != secret { + t.Fatalf("parse %q -> %q %q %v", token, gotID, gotSecret, ok) + } + rec := store.KeyRecord{APIKey: api.APIKey{ID: keyID}, Hash: Hash(secret)} + if err := Verify(rec, secret, time.Now()); err != nil { + t.Fatal(err) + } + if err := Verify(rec, secret+"x", time.Now()); !api.IsCode(err, api.CodeUnauthenticated) { + t.Fatalf("wrong secret accepted: %v", err) + } + past := time.Now().Add(-time.Hour) + rec.ExpiresAt = &past + if err := Verify(rec, secret, time.Now()); err == nil { + t.Fatal("expired key accepted") + } + for _, bad := range []string{"", "dly_", "dly_abc_short", "sk_abc_" + secret} { + if _, _, ok := ParseToken(bad); ok { + t.Fatalf("parsed bad token %q", bad) + } + } +} + +// Scenario H: a project key cannot reach other tenants or escalate. +func TestProjectKeyIsContained(t *testing.T) { + p := Principal{KeyID: "k1", OrgID: "org1", ProjectID: "proj1", Scopes: []string{ScopeSandboxesRead, ScopeSandboxesWrite, ScopeKeysWrite}} + if !p.CanAccess("org1", "proj1") || p.CanAccess("org1", "proj2") || p.CanAccess("org2", "proj1") { + t.Fatal("project containment broken") + } + if p.Has(ScopeAdmin) || p.Has(ScopeAuditRead) { + t.Fatal("project key has scopes it was not granted") + } + if _, _, err := CheckGrant(p, api.CreateAPIKeyRequest{Scopes: []string{ScopeSandboxesExec}}); !api.IsCode(err, api.CodePermissionDenied) { + t.Fatalf("escalation allowed: %v", err) + } + if _, _, err := CheckGrant(p, api.CreateAPIKeyRequest{Scopes: []string{ScopeSandboxesRead}, ProjectID: "proj2"}); !api.IsCode(err, api.CodePermissionDenied) { + t.Fatalf("cross-project grant allowed: %v", err) + } + org, proj, err := CheckGrant(p, api.CreateAPIKeyRequest{Scopes: []string{ScopeSandboxesRead}}) + if err != nil || org != "org1" || proj != "proj1" { + t.Fatalf("narrow grant: %s %s %v", org, proj, err) + } +} + +func TestOrgAdminAndOperator(t *testing.T) { + orgAdmin := Principal{OrgID: "org1", Scopes: []string{ScopeOrgAdmin}} + if !orgAdmin.Has(ScopeSandboxesExec) || orgAdmin.Has(ScopeAdmin) || orgAdmin.CanAccess("org2", "") { + t.Fatal("org admin bounds wrong") + } + if _, _, err := CheckGrant(orgAdmin, api.CreateAPIKeyRequest{Scopes: []string{ScopeAdmin}}); err == nil { + t.Fatal("org admin minted operator key") + } + op := Principal{Scopes: []string{ScopeAdmin}} + if _, _, err := CheckGrant(op, api.CreateAPIKeyRequest{Scopes: []string{ScopeSandboxesRead}}); !api.IsCode(err, api.CodeInvalidRequest) { + t.Fatalf("tenant key without org accepted: %v", err) + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..532e0b1 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,372 @@ +// Package config loads and validates the server configuration. +// +// Secrets never appear in the file: the database DSN, bootstrap token, agent +// token and provider credentials are referenced by environment variable or +// file path and resolved at runtime, so the config can live in a ConfigMap or +// in Git. +package config + +import ( + "errors" + "fmt" + "os" + "slices" + "strings" + + "sigs.k8s.io/yaml" + + "github.com/different-ai/differently/pkg/api" +) + +// Config is the server configuration file. +type Config struct { + Listen string `json:"listen"` + PublicURL string `json:"publicURL,omitempty"` + Dev bool `json:"dev,omitempty"` + DataDir string `json:"dataDir,omitempty"` + Store StoreConfig `json:"store"` + Auth AuthConfig `json:"auth"` + Defaults Defaults `json:"defaults"` + Classifications []api.Classification `json:"classifications"` + Profiles []api.Profile `json:"profiles"` + Templates []api.Template `json:"templates,omitempty"` + Pools []PoolConfig `json:"pools"` + Policy api.PlacementPolicy `json:"policy"` + Scheduler SchedulerConfig `json:"scheduler"` + RateLimit RateLimitConfig `json:"rateLimit"` + Audit AuditConfig `json:"audit"` + Dashboard DashboardConfig `json:"dashboard"` +} + +// StoreConfig selects persistence. +type StoreConfig struct { + Driver string `json:"driver"` // memory | postgres + DSNEnv string `json:"dsnEnv,omitempty"` +} + +// AuthConfig names the environment variables holding bootstrap secrets. +type AuthConfig struct { + BootstrapTokenEnv string `json:"bootstrapTokenEnv"` + AgentTokenEnv string `json:"agentTokenEnv"` +} + +// Defaults apply when an organization, project or request omits a value. +type Defaults struct { + OrgQuota api.Quota `json:"orgQuota"` + ProjectQuota api.Quota `json:"projectQuota"` + Classification string `json:"classification"` + Profile string `json:"profile"` + Template string `json:"template,omitempty"` + Resources api.Resources `json:"resources"` + TTLSeconds int `json:"ttlSeconds"` +} + +// CredentialRef points at a secret without containing it. +type CredentialRef struct { + Env string `json:"env,omitempty"` + File string `json:"file,omitempty"` +} + +// String renders the reference for display. +func (c CredentialRef) String() string { + switch { + case c.Env != "": + return "env:" + c.Env + case c.File != "": + return "file:" + c.File + } + return "" +} + +// ParseCredentialRef parses "env:NAME" or "file:/path". +func ParseCredentialRef(s string) CredentialRef { + switch { + case strings.HasPrefix(s, "env:"): + return CredentialRef{Env: strings.TrimPrefix(s, "env:")} + case strings.HasPrefix(s, "file:"): + return CredentialRef{File: strings.TrimPrefix(s, "file:")} + } + return CredentialRef{} +} + +// Resolve reads the secret. A missing reference resolves to "". +func (c CredentialRef) Resolve() (string, error) { + switch { + case c.Env != "": + v := os.Getenv(c.Env) + if v == "" { + return "", fmt.Errorf("environment variable %s is empty", c.Env) + } + return v, nil + case c.File != "": + b, err := os.ReadFile(c.File) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil + } + return "", nil +} + +// PoolConfig declares a pool in the config file. +type PoolConfig struct { + Name string `json:"name"` + Adapter string `json:"adapter"` + Trust string `json:"trust,omitempty"` + Region string `json:"region,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + MaxConcurrency int `json:"maxConcurrency,omitempty"` + CostModel api.CostModel `json:"costModel,omitempty"` + Config map[string]string `json:"config,omitempty"` + Credentials CredentialRef `json:"credentials,omitempty"` +} + +// ToPool converts to the API representation. +func (p PoolConfig) ToPool() api.Pool { + enabled := p.Enabled == nil || *p.Enabled + return api.Pool{ + Name: p.Name, Adapter: p.Adapter, Trust: p.Trust, Region: p.Region, Labels: p.Labels, + Enabled: enabled, MaxConcurrency: p.MaxConcurrency, CostModel: p.CostModel, Config: p.Config, + CredentialRef: p.Credentials.String(), Source: "config", + } +} + +// SchedulerConfig tunes the dispatch and reconcile loops. +type SchedulerConfig struct { + IntervalMs int `json:"intervalMs"` + ProvisionTimeoutSeconds int `json:"provisionTimeoutSeconds"` + MaxAttempts int `json:"maxAttempts"` + ReconcileIntervalSecs int `json:"reconcileIntervalSeconds"` + AgentTimeoutSeconds int `json:"agentTimeoutSeconds"` +} + +// RateLimitConfig is a per-key token bucket. +type RateLimitConfig struct { + RequestsPerSecond float64 `json:"requestsPerSecond"` + Burst int `json:"burst"` +} + +// AuditConfig optionally mirrors audit events to a JSON-lines file. +type AuditConfig struct { + File string `json:"file,omitempty"` + // RecordCommands stores exec command lines in audit events. Commands can + // contain secrets, so the default records only the program name. + RecordCommands bool `json:"recordCommands,omitempty"` +} + +// DashboardConfig toggles the built-in web dashboard. +type DashboardConfig struct { + Enabled *bool `json:"enabled,omitempty"` +} + +// On reports whether the dashboard is served. +func (d DashboardConfig) On() bool { return d.Enabled == nil || *d.Enabled } + +// Adapter kinds that always run on third-party infrastructure. +var externalKinds = []string{"daytona", "freestyle"} + +// KnownAdapters lists adapter kinds the server can build. +var KnownAdapters = []string{"firecracker", "devhost", "fake", "remote", "daytona", "freestyle"} + +// Default returns a configuration with every default applied. +func Default() Config { + c := Config{} + c.applyDefaults() + return c +} + +// DevDefault is the zero-setup local configuration used by --dev. +func DevDefault(dataDir string) Config { + c := Config{ + Dev: true, + DataDir: dataDir, + Store: StoreConfig{Driver: "memory"}, + Pools: []PoolConfig{{ + Name: "local", Adapter: "devhost", Trust: api.TrustPrivate, Region: "local", + MaxConcurrency: 20, Config: map[string]string{"root": dataDir + "/devhost"}, + }}, + } + c.applyDefaults() + return c +} + +func (c *Config) applyDefaults() { + if c.Listen == "" { + c.Listen = ":8080" + } + if c.DataDir == "" { + c.DataDir = "/var/lib/differently" + } + if c.Store.Driver == "" { + c.Store.Driver = "memory" + } + if c.Store.Driver == "postgres" && c.Store.DSNEnv == "" { + c.Store.DSNEnv = "DIFFERENTLY_DATABASE_URL" + } + if c.Auth.BootstrapTokenEnv == "" { + c.Auth.BootstrapTokenEnv = "DIFFERENTLY_BOOTSTRAP_TOKEN" + } + if c.Auth.AgentTokenEnv == "" { + c.Auth.AgentTokenEnv = "DIFFERENTLY_AGENT_TOKEN" + } + if len(c.Classifications) == 0 { + c.Classifications = []api.Classification{ + {Name: "private-only", Description: "Must never leave private pools.", External: api.ExternalNever}, + {Name: "internal", Description: "May overflow to approved external pools.", External: api.ExternalOverflow}, + {Name: "public", Description: "No residency restriction beyond policy.", External: api.ExternalAllowed}, + } + } + if c.Defaults.Classification == "" { + c.Defaults.Classification = "internal" + } + if len(c.Profiles) == 0 { + c.Profiles = []api.Profile{ + {Name: "small", Resources: api.Resources{CPU: 1, MemoryMiB: 1024, DiskGiB: 5}}, + {Name: "medium", Resources: api.Resources{CPU: 2, MemoryMiB: 4096, DiskGiB: 10}}, + {Name: "large", Resources: api.Resources{CPU: 4, MemoryMiB: 8192, DiskGiB: 20}}, + } + } + if c.Defaults.Profile == "" { + c.Defaults.Profile = "small" + } + if c.Defaults.TTLSeconds == 0 { + c.Defaults.TTLSeconds = 3600 + } + if c.Policy.Scope == "" { + c.Policy.Scope = "global" + } + if c.Policy.External == "" { + c.Policy.External = api.ExternalNever + } + if c.Policy.Strategy == "" { + c.Policy.Strategy = api.StrategyPrivateFirst + } + for i := range c.Pools { + p := &c.Pools[i] + if slices.Contains(externalKinds, p.Adapter) { + p.Trust = api.TrustExternal + } + if p.Trust == "" { + p.Trust = api.TrustPrivate + } + } + s := &c.Scheduler + if s.IntervalMs == 0 { + s.IntervalMs = 250 + } + if s.ProvisionTimeoutSeconds == 0 { + s.ProvisionTimeoutSeconds = 120 + } + if s.MaxAttempts == 0 { + s.MaxAttempts = 3 + } + if s.ReconcileIntervalSecs == 0 { + s.ReconcileIntervalSecs = 30 + } + if s.AgentTimeoutSeconds == 0 { + s.AgentTimeoutSeconds = 30 + } + if c.RateLimit.RequestsPerSecond == 0 { + c.RateLimit.RequestsPerSecond = 50 + } + if c.RateLimit.Burst == 0 { + c.RateLimit.Burst = 100 + } +} + +// Load reads, defaults and validates a YAML or JSON config file. +func Load(path string) (Config, error) { + b, err := os.ReadFile(path) + if err != nil { + return Config{}, err + } + return Parse(b) +} + +// Parse defaults and validates raw YAML or JSON. +func Parse(b []byte) (Config, error) { + var c Config + if err := yaml.UnmarshalStrict(b, &c); err != nil { + return Config{}, fmt.Errorf("parse config: %w", err) + } + c.applyDefaults() + return c, c.Validate() +} + +// Validate reports every configuration problem at once. +func (c Config) Validate() error { + var errs []error + bad := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) } + if c.Store.Driver != "memory" && c.Store.Driver != "postgres" { + bad("store.driver must be memory or postgres, got %q", c.Store.Driver) + } + classes := map[string]bool{} + for _, cl := range c.Classifications { + classes[cl.Name] = true + if _, ok := map[string]bool{api.ExternalNever: true, api.ExternalOverflow: true, api.ExternalAllowed: true}[cl.External]; !ok { + bad("classification %s: external must be never, overflow or allowed", cl.Name) + } + } + if !classes[c.Defaults.Classification] { + bad("defaults.classification %q is not a defined classification", c.Defaults.Classification) + } + profiles := map[string]bool{} + for _, p := range c.Profiles { + profiles[p.Name] = true + if p.Resources.CPU <= 0 || p.Resources.MemoryMiB <= 0 { + bad("profile %s needs positive cpu and memoryMiB", p.Name) + } + } + if !profiles[c.Defaults.Profile] { + bad("defaults.profile %q is not a defined profile", c.Defaults.Profile) + } + names := map[string]bool{} + for _, p := range c.Pools { + switch { + case p.Name == "": + bad("every pool needs a name") + case names[p.Name]: + bad("duplicate pool %s", p.Name) + } + names[p.Name] = true + if !slices.Contains(KnownAdapters, p.Adapter) { + bad("pool %s: unknown adapter %q (known: %s)", p.Name, p.Adapter, strings.Join(KnownAdapters, ", ")) + } + if p.Trust != api.TrustPrivate && p.Trust != api.TrustExternal { + bad("pool %s: trust must be private or external", p.Name) + } + if p.Adapter == "devhost" && !c.Dev { + bad("pool %s: the devhost adapter has no isolation and is only allowed with dev: true", p.Name) + } + if p.Adapter == "remote" && p.Config["endpoint"] == "" { + bad("pool %s: remote adapter needs config.endpoint", p.Name) + } + } + for _, ep := range c.Policy.ExternalPools { + if !names[ep] { + bad("policy.externalPools names unknown pool %s", ep) + } + } + return errors.Join(errs...) +} + +// Classification returns a classification by name. +func (c Config) Classification(name string) (api.Classification, bool) { + for _, cl := range c.Classifications { + if cl.Name == name { + return cl, true + } + } + return api.Classification{}, false +} + +// Profile returns a profile by name. +func (c Config) Profile(name string) (api.Profile, bool) { + for _, p := range c.Profiles { + if p.Name == name { + return p, true + } + } + return api.Profile{}, false +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..29d9d79 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,64 @@ +package config + +import ( + "strings" + "testing" + + "github.com/different-ai/differently/pkg/api" +) + +func TestParseAppliesSafeDefaults(t *testing.T) { + c, err := Parse([]byte(` +pools: + - name: fc + adapter: firecracker + - name: burst + adapter: daytona + trust: private # ignored: providers are always external + credentials: {env: DAYTONA_API_KEY} +`)) + if err != nil { + t.Fatal(err) + } + if c.Policy.External != api.ExternalNever { + t.Fatalf("external must default to never, got %q", c.Policy.External) + } + if c.Pools[1].Trust != api.TrustExternal { + t.Fatal("daytona pool was allowed to claim private trust") + } + if got := c.Pools[1].ToPool().CredentialRef; got != "env:DAYTONA_API_KEY" { + t.Fatalf("credential ref %q", got) + } +} + +func TestValidateReportsProblems(t *testing.T) { + _, err := Parse([]byte(` +store: {driver: sqlite} +pools: + - {name: a, adapter: devhost} + - {name: a, adapter: kata} +policy: {externalPools: [ghost]} +`)) + if err == nil { + t.Fatal("expected errors") + } + for _, want := range []string{"store.driver", "only allowed with dev", "duplicate pool", "unknown adapter", "unknown pool ghost"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("missing %q in %v", want, err) + } + } + if _, err := Parse([]byte("unknownField: 1")); err == nil { + t.Fatal("unknown fields must be rejected") + } +} + +func TestCredentialRef(t *testing.T) { + t.Setenv("X_TOKEN", "secret") + v, err := ParseCredentialRef("env:X_TOKEN").Resolve() + if err != nil || v != "secret" { + t.Fatalf("%q %v", v, err) + } + if _, err := ParseCredentialRef("env:MISSING_TOKEN_XYZ").Resolve(); err == nil { + t.Fatal("empty env should error") + } +} diff --git a/internal/dashboard/dashboard.go b/internal/dashboard/dashboard.go new file mode 100644 index 0000000..1b462d6 --- /dev/null +++ b/internal/dashboard/dashboard.go @@ -0,0 +1,25 @@ +// Package dashboard serves the built-in operator and tenant web console. It +// is a static single-page app that calls the public API with the viewer's own +// API key, so it can do nothing the key cannot. +package dashboard + +import ( + "embed" + "io/fs" + "net/http" +) + +//go:embed static +var static embed.FS + +// Handler serves the dashboard assets. +func Handler() http.Handler { + sub, _ := fs.Sub(static, "static") + fs := http.FileServerFS(sub) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Referrer-Policy", "no-referrer") + fs.ServeHTTP(w, r) + }) +} diff --git a/internal/dashboard/static/app.js b/internal/dashboard/static/app.js new file mode 100644 index 0000000..5933356 --- /dev/null +++ b/internal/dashboard/static/app.js @@ -0,0 +1,110 @@ +// Differently console: a thin client over the public API. Every action uses +// the signed-in key, so the console can do nothing the key cannot. +"use strict"; +const $ = (s) => document.querySelector(s); +const esc = (v) => String(v ?? "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[c]); +let me = null; + +function key() { try { return sessionStorage.getItem("dly-key") || ""; } catch { return ""; } } +async function call(method, path, body) { + const res = await fetch(path, { method, headers: { Authorization: "Bearer " + key(), "Content-Type": "application/json" }, body: body ? JSON.stringify(body) : undefined }); + if (res.status === 204) return null; + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error((data.error && `${data.error.code}: ${data.error.message}`) || res.statusText); + return data; +} +function table(cols, rows) { + if (!rows.length) return "

Nothing here yet.

"; + return `
${cols.map((c) => ``).join("")}${rows.map((r) => `${r.map((v) => ``).join("")}`).join("")}
${esc(c)}
${v}
`; +} +const status = (s) => `${esc(s)}`; + +const views = { + async overview() { + const [sbx, caps, quota] = await Promise.all([call("GET", "/v1/sandboxes?limit=500"), call("GET", "/v1/capabilities"), me.orgId ? call("GET", "/v1/quotas").catch(() => null) : null]); + const by = {}; sbx.items.forEach((s) => (by[s.status] = (by[s.status] || 0) + 1)); + const cards = ["running", "queued", "provisioning", "stopped", "failed"].map((s) => `
${s}
${by[s] || 0}
`).join(""); + const queued = sbx.items.filter((s) => s.status === "queued"); + return `

Overview

${cards}
+

Pools

${table(["Pool", "Adapter", "Trust", "Region", "Health", "Capabilities"], caps.items.map((p) => [esc(p.pool), esc(p.adapter), esc(p.trust), esc(p.region), status(p.health), `${esc((p.capabilities || []).join(" "))}`]))} +

Queue

${table(["Sandbox", "Classification", "Waiting because"], queued.map((s) => [`${esc(s.id)}`, esc(s.classification), esc(s.statusReason)]))} + ${quota ? `

Quota

${esc(JSON.stringify(quota, null, 2))}
` : ""}`; + }, + async sandboxes() { + const sbx = await call("GET", "/v1/sandboxes?limit=200"); + return `

Sandboxes

+
+ ${table(["ID", "Status", "Pool", "Trust", "Size", "Class", "Created", ""], sbx.items.map((s) => [ + `${esc(s.id)}`, status(s.status), esc(s.pool), esc(s.trust), `${s.resources.cpu} vCPU / ${s.resources.memoryMiB} MiB`, esc(s.classification), + esc(new Date(s.createdAt).toLocaleString()), s.status === "terminated" || s.status === "failed" ? "" : ``]))}`; + }, + async usage() { + const u = await call("GET", "/v1/usage?groupBy=pool"); + return `

Usage

${esc(u.note)}

${table(["Pool", "Sandboxes", "Run hours", "vCPU hours", "Est. cost"], u.lines.map((l) => [esc(l.key), l.sandboxes, (l.runSeconds / 3600).toFixed(2), (l.cpuSeconds / 3600).toFixed(2), l.estimatedCost.toFixed(4)]))}`; + }, + async audit() { + const ev = await call("GET", "/v1/audit-events?limit=200"); + const verify = me.operator ? await call("GET", "/v1/admin/audit/verify") : null; + return `

Audit

${verify ? `

Hash chain: ${verify.ok ? "intact" : `broken at ${verify.brokenAt}: ${esc(verify.problem)}`} (${verify.events} events)

` : ""} + ${table(["#", "Time", "Actor", "Action", "Resource", "Details"], ev.items.slice().reverse().map((e) => [e.seq, esc(new Date(e.time).toLocaleString()), esc(e.actor), esc(e.action), `${esc(e.resource)}`, `${esc(JSON.stringify(e.details || {}))}`]))}`; + }, + async pools() { + const [pools, agents] = await Promise.all([call("GET", "/v1/admin/pools"), call("GET", "/v1/admin/agents")]); + return `

Pools

${table(["Pool", "Adapter", "Trust", "Enabled", "Draining", "Active", "Free CPU", "Health", ""], pools.items.map((p) => [esc(p.name), esc(p.adapter), esc(p.trust), p.enabled, p.draining, p.active, + p.capacity && p.capacity.bounded ? `${p.capacity.freeCPU}/${p.capacity.totalCPU}` : "provider", status(p.health && p.health.status), ``]))} +

Node agents

${table(["Agent", "Pool", "Node", "Free CPU", "Free MiB", "VMs", "Last seen", "Healthy"], agents.items.map((a) => [esc(a.id), esc(a.pool), esc(a.node), a.freeCPU, a.freeMemoryMiB, a.sandboxes, esc(new Date(a.lastSeen).toLocaleString()), a.healthy ? "yes" : "no"]))}`; + }, + async tenants() { + const orgs = await call("GET", "/v1/admin/organizations"); + return `

Tenants

+ ${table(["ID", "Name", "Suspended", "Quota"], orgs.items.map((o) => [`${esc(o.id)}`, esc(o.name), o.suspended ? "yes" : "no", `${esc(JSON.stringify(o.quota))}`]))}`; + }, + async trust() { + const t = await call("GET", "/v1/admin/trust-report"); + return `

Trust report

Telemetry: ${esc(t.telemetry)}. Global external placement: ${esc(t.globalPolicy.external)}. Audit chain: ${t.auditChain.ok ? "intact" : "broken"}.

+

Pools and trust boundaries

${table(["Pool", "Adapter", "Trust", "Endpoint", "Credential", "Approved for external work"], t.pools.map((p) => [esc(p.name), esc(p.adapter), esc(p.trust), `${esc(p.endpoint)}`, `${esc(p.credentialRef)}`, p.trust === "external" ? (p.approvedForExternalPlacement ? "yes" : "no") : "—"]))} +

Recent external placements

${table(["Sandbox", "Org", "Pool", "Class", "Created by", "Why"], (t.recentExternalPlacements || []).map((p) => [`${esc(p.sandboxId)}`, esc(p.orgId), esc(p.pool), esc(p.classification), esc(p.createdBy), esc(p.reason)]))} +

What Differently cannot prove

    ${t.limitations.map((l) => `
  • ${esc(l)}
    • `).join("")}
`; + }, +}; + +async function render() { + const name = (location.hash || "#overview").slice(1); + const view = views[name] || views.overview; + try { $("#view").innerHTML = await view(); } catch (e) { $("#view").innerHTML = `

${esc(e.message)}

`; } +} + +document.addEventListener("click", async (e) => { + const del = e.target.dataset && e.target.dataset.del; + const drain = e.target.dataset && e.target.dataset.drain; + try { + if (del && confirm(`Delete ${del}?`)) { await call("DELETE", "/v1/sandboxes/" + encodeURIComponent(del)); render(); } + if (drain) { await call("POST", `/v1/admin/pools/${encodeURIComponent(drain)}/${e.target.dataset.on === "true" ? "undrain" : "drain"}`); render(); } + } catch (err) { alert(err.message); } +}); +document.addEventListener("submit", async (e) => { + e.preventDefault(); + try { + if (e.target.id === "exec-form") { + const out = $("#exec-out"); out.hidden = false; out.textContent = "running…"; + const r = await call("POST", `/v1/sandboxes/${encodeURIComponent($("#exec-id").value)}/exec`, { command: $("#exec-cmd").value }); + out.textContent = `exit ${r.exitCode}\n${r.stdout}${r.stderr}`; + } + if (e.target.id === "org-form") { await call("POST", "/v1/admin/organizations", { name: $("#org-name").value, defaultProject: "default" }); render(); } + if (e.target.id === "login-form") { + try { sessionStorage.setItem("dly-key", $("#key").value.trim()); } catch {} + boot(); + } + } catch (err) { alert(err.message); } +}); +$("#logout").addEventListener("click", () => { try { sessionStorage.removeItem("dly-key"); } catch {} location.reload(); }); +window.addEventListener("hashchange", render); + +async function boot() { + if (!key()) { $("#login").hidden = false; return; } + try { me = await call("GET", "/v1/me"); } catch (e) { $("#login").hidden = false; $("#login-error").textContent = e.message; return; } + $("#login").hidden = true; $("#nav").hidden = false; + document.querySelectorAll(".admin").forEach((a) => (a.hidden = !me.operator)); + render(); +} +boot(); diff --git a/internal/dashboard/static/index.html b/internal/dashboard/static/index.html new file mode 100644 index 0000000..9d8c3df --- /dev/null +++ b/internal/dashboard/static/index.html @@ -0,0 +1,29 @@ + + + + + +Differently Console + + + +
+ Differentlyyour infrastructure, your rules, one API + +
+
+ +
+
+ + + diff --git a/internal/dashboard/static/style.css b/internal/dashboard/static/style.css new file mode 100644 index 0000000..21198e8 --- /dev/null +++ b/internal/dashboard/static/style.css @@ -0,0 +1,23 @@ +:root { --bg: #fafaf9; --fg: #1c1917; --muted: #78716c; --line: #e7e5e4; --card: #fff; --accent: #0f766e; --bad: #b91c1c; --warn: #b45309; } +@media (prefers-color-scheme: dark) { :root { --bg: #0c0a09; --fg: #f5f5f4; --muted: #a8a29e; --line: #292524; --card: #1c1917; --accent: #2dd4bf; --bad: #f87171; --warn: #fbbf24; } } +* { box-sizing: border-box; } +body { margin: 0; font: 14px/1.5 ui-sans-serif, system-ui, sans-serif; background: var(--bg); color: var(--fg); } +header { display: flex; flex-wrap: wrap; gap: 12px; align-items: center; padding: 12px 16px; border-bottom: 1px solid var(--line); } +.tag { color: var(--muted); font-size: 12px; } +nav { display: flex; flex-wrap: wrap; gap: 12px; margin-left: auto; } +nav a { color: var(--fg); text-decoration: none; } nav a:hover { color: var(--accent); } +main { padding: 16px; max-width: 1200px; margin: 0 auto; } +h1 { font-size: 20px; } h2 { font-size: 16px; margin-top: 24px; } +table { width: 100%; border-collapse: collapse; background: var(--card); border: 1px solid var(--line); } +th, td { text-align: left; padding: 6px 8px; border-bottom: 1px solid var(--line); vertical-align: top; } +th { color: var(--muted); font-weight: 500; font-size: 12px; } +.scroll { overflow-x: auto; } +.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); gap: 12px; } +.card { background: var(--card); border: 1px solid var(--line); border-radius: 8px; padding: 12px; } +.card .n { font-size: 24px; font-weight: 600; } +.ok { color: var(--accent); } .bad { color: var(--bad); } .warn { color: var(--warn); } +code, pre { font: 12px ui-monospace, monospace; } pre { background: var(--card); border: 1px solid var(--line); padding: 8px; overflow-x: auto; white-space: pre-wrap; } +input, button, select { font: inherit; padding: 6px 10px; border: 1px solid var(--line); border-radius: 6px; background: var(--card); color: var(--fg); } +button { cursor: pointer; } button.link { border: 0; background: none; color: var(--muted); padding: 0; } +form { display: flex; gap: 8px; flex-wrap: wrap; } #key { min-width: 280px; flex: 1; } +.error { color: var(--bad); } diff --git a/internal/guest/client.go b/internal/guest/client.go new file mode 100644 index 0000000..4baf4dc --- /dev/null +++ b/internal/guest/client.go @@ -0,0 +1,154 @@ +package guest + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/different-ai/differently/pkg/api" +) + +// Client talks to a guest agent. Dial opens a fresh connection to the guest, +// for example through a Firecracker vsock socket. +type Client struct { + Dial func(ctx context.Context) (net.Conn, error) + http *http.Client +} + +// NewClient returns a client using dial for every connection. +func NewClient(dial func(ctx context.Context) (net.Conn, error)) *Client { + c := &Client{Dial: dial} + c.http = &http.Client{Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return c.Dial(ctx) }, + DisableKeepAlives: true, + }} + return c +} + +// Health returns nil when the guest agent answers. +func (c *Client) Health(ctx context.Context) error { + resp, err := c.send(ctx, http.MethodGet, "/healthz", nil) + if err != nil { + return err + } + return resp.Body.Close() +} + +// WaitReady polls Health until it succeeds or timeout passes. +func (c *Client) WaitReady(ctx context.Context, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + var last error + for time.Now().Before(deadline) { + hctx, cancel := context.WithTimeout(ctx, 2*time.Second) + last = c.Health(hctx) + cancel() + if last == nil { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } + return fmt.Errorf("guest agent not ready after %s: %w", timeout, last) +} + +// Exec runs a command synchronously. +func (c *Client) Exec(ctx context.Context, req api.ExecRequest) (api.ExecResult, error) { + var out api.ExecResult + body, _ := json.Marshal(req) + resp, err := c.send(ctx, http.MethodPost, "/v1/exec", bytes.NewReader(body)) + if err != nil { + return out, err + } + defer resp.Body.Close() + // The guest is untrusted: never decode more than two full output buffers. + return out, json.NewDecoder(io.LimitReader(resp.Body, 2*MaxOutputBytes+1<<20)).Decode(&out) +} + +// ExecStream runs a command and forwards output events. +func (c *Client) ExecStream(ctx context.Context, req api.ExecRequest, emit func(api.ExecEvent) error) (int, error) { + body, _ := json.Marshal(req) + resp, err := c.send(ctx, http.MethodPost, "/v1/exec?stream=1", bytes.NewReader(body)) + if err != nil { + return -1, err + } + defer resp.Body.Close() + sc := bufio.NewScanner(resp.Body) + sc.Buffer(make([]byte, 64<<10), 8<<20) + for sc.Scan() { + var ev api.ExecEvent + if err := json.Unmarshal(sc.Bytes(), &ev); err != nil { + return -1, err + } + switch { + case ev.Error != "": + return -1, errors.New(ev.Error) + case ev.ExitCode != nil: + return *ev.ExitCode, nil + } + if err := emit(ev); err != nil { + return -1, err + } + } + return -1, errors.New("guest stream ended without exit code") +} + +// ReadFile downloads a file. +func (c *Client) ReadFile(ctx context.Context, p string) ([]byte, error) { + resp, err := c.send(ctx, http.MethodGet, "/v1/files/content?path="+url.QueryEscape(p), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, MaxFileBytes)) +} + +// WriteFile uploads a file. +func (c *Client) WriteFile(ctx context.Context, p string, data []byte) error { + resp, err := c.send(ctx, http.MethodPut, "/v1/files/content?path="+url.QueryEscape(p), bytes.NewReader(data)) + if err != nil { + return err + } + return resp.Body.Close() +} + +// ListFiles lists a directory. +func (c *Client) ListFiles(ctx context.Context, p string) ([]api.FileInfo, error) { + resp, err := c.send(ctx, http.MethodGet, "/v1/files?path="+url.QueryEscape(p), nil) + if err != nil { + return nil, err + } + defer resp.Body.Close() + var out []api.FileInfo + return out, json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&out) +} + +func (c *Client) send(ctx context.Context, method, path string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, "http://guest"+path, body) + if err != nil { + return nil, err + } + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 { + defer resp.Body.Close() + var env api.ErrorEnvelope + if json.NewDecoder(resp.Body).Decode(&env) == nil && env.Error != nil { + return nil, env.Error + } + return nil, fmt.Errorf("guest %s %s: HTTP %d", method, path, resp.StatusCode) + } + return resp, nil +} diff --git a/internal/guest/guest.go b/internal/guest/guest.go new file mode 100644 index 0000000..006b7c7 --- /dev/null +++ b/internal/guest/guest.go @@ -0,0 +1,369 @@ +// Package guest runs commands and file operations for a sandbox. +// +// The same Runner backs two places: the in-VM guest agent (Root "/"), which +// the node agent reaches over Firecracker vsock, and the devhost adapter +// (Root is a host directory, no isolation) used for local development. +package guest + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "io/fs" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "time" + + "github.com/different-ai/differently/pkg/api" +) + +const ( + // MaxOutputBytes caps each captured stream of a synchronous exec. + MaxOutputBytes = 10 << 20 + // DefaultTimeout applies when an exec request sets none. + DefaultTimeout = 10 * time.Minute + // MaxTimeout caps any exec request. + MaxTimeout = 2 * time.Hour + // MaxFileBytes bounds file reads and writes. + MaxFileBytes = 100 << 20 +) + +// Runner executes commands and file operations below Root. +type Runner struct { + // Root is the directory every path resolves under. "/" inside a VM. + Root string + // DefaultCwd is used when a request has no cwd. Relative to Root. + DefaultCwd string + // BaseEnv is applied to every command before request env. + BaseEnv map[string]string +} + +// Resolve maps a sandbox path to a host path, refusing escapes from Root. +func (r *Runner) Resolve(p string) (string, error) { + if p == "" { + p = "/" + } + clean := filepath.Clean("/" + p) + full := filepath.Join(r.Root, clean) + root := filepath.Clean(r.Root) + if full != root && !strings.HasPrefix(full, root+string(filepath.Separator)) && root != string(filepath.Separator) { + return "", api.Errorf(api.CodeInvalidRequest, "path %q escapes the sandbox root", p) + } + return full, nil +} + +func (r *Runner) command(ctx context.Context, req api.ExecRequest) (*exec.Cmd, context.CancelFunc, error) { + if req.Command == "" && len(req.Args) == 0 { + return nil, nil, api.Errorf(api.CodeInvalidRequest, "command or args is required") + } + timeout := DefaultTimeout + if req.TimeoutSeconds > 0 { + timeout = time.Duration(req.TimeoutSeconds) * time.Second + } + timeout = min(timeout, MaxTimeout) + ctx, cancel := context.WithTimeout(ctx, timeout) + var cmd *exec.Cmd + if len(req.Args) > 0 { + cmd = exec.CommandContext(ctx, req.Args[0], req.Args[1:]...) + } else { + cmd = exec.CommandContext(ctx, "/bin/sh", "-c", req.Command) + } + cwd := req.Cwd + if cwd == "" { + cwd = r.DefaultCwd + } + dir, err := r.Resolve(cwd) + if err != nil { + cancel() + return nil, nil, err + } + if st, err := os.Stat(dir); err != nil || !st.IsDir() { + dir = r.Root + } + cmd.Dir = dir + env := map[string]string{ + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOME": r.Root, + "LANG": "C.UTF-8", + } + if p := os.Getenv("PATH"); p != "" && r.Root != "/" { + env["PATH"] = p + } + for k, v := range r.BaseEnv { + env[k] = v + } + for k, v := range req.Env { + env[k] = v + } + cmd.Env = make([]string, 0, len(env)) + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + sort.Strings(cmd.Env) + if req.Stdin != "" { + cmd.Stdin = strings.NewReader(req.Stdin) + } + configureProcessGroup(cmd) + cmd.WaitDelay = 5 * time.Second + return cmd, cancel, nil +} + +// Exec runs a command to completion and captures its output. +func (r *Runner) Exec(ctx context.Context, req api.ExecRequest) (api.ExecResult, error) { + cmd, cancel, err := r.command(ctx, req) + if err != nil { + return api.ExecResult{}, err + } + defer cancel() + stdout, stderr := &capped{max: MaxOutputBytes}, &capped{max: MaxOutputBytes} + cmd.Stdout, cmd.Stderr = stdout, stderr + start := time.Now() + code, err := exitCode(cmd.Run()) + if err != nil { + return api.ExecResult{}, err + } + return api.ExecResult{ + ExitCode: code, + Stdout: stdout.String(), + Stderr: stderr.String(), + DurationMs: time.Since(start).Milliseconds(), + Truncated: stdout.truncated || stderr.truncated, + }, nil +} + +// ExecStream runs a command and emits output chunks as they arrive. +func (r *Runner) ExecStream(ctx context.Context, req api.ExecRequest, emit func(api.ExecEvent) error) (int, error) { + cmd, cancel, err := r.command(ctx, req) + if err != nil { + return -1, err + } + defer cancel() + var mu sync.Mutex + var emitErr error + send := func(stream string) io.Writer { + return writerFunc(func(p []byte) (int, error) { + mu.Lock() + defer mu.Unlock() + if emitErr == nil { + emitErr = emit(api.ExecEvent{Stream: stream, Data: string(p)}) + } + return len(p), nil + }) + } + cmd.Stdout, cmd.Stderr = send("stdout"), send("stderr") + code, err := exitCode(cmd.Run()) + if err != nil { + return -1, err + } + return code, emitErr +} + +func exitCode(err error) (int, error) { + if err == nil { + return 0, nil + } + var ee *exec.ExitError + if errors.As(err, &ee) { + return ee.ExitCode(), nil + } + if errors.Is(err, exec.ErrNotFound) || errors.Is(err, fs.ErrNotExist) { + return 127, nil + } + return -1, err +} + +// ReadFile returns a file's contents. +func (r *Runner) ReadFile(p string) ([]byte, error) { + full, err := r.Resolve(p) + if err != nil { + return nil, err + } + f, err := os.Open(full) + if err != nil { + return nil, fileErr(p, err) + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, MaxFileBytes+1)) + if err != nil { + return nil, err + } + if len(data) > MaxFileBytes { + return nil, api.Errorf(api.CodeInvalidRequest, "file larger than %d bytes", MaxFileBytes) + } + return data, nil +} + +// WriteFile writes a file, creating parent directories. +func (r *Runner) WriteFile(p string, data []byte) error { + full, err := r.Resolve(p) + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { + return fileErr(p, err) + } + return fileErr(p, os.WriteFile(full, data, 0o644)) +} + +// ListFiles lists a directory. +func (r *Runner) ListFiles(p string) ([]api.FileInfo, error) { + full, err := r.Resolve(p) + if err != nil { + return nil, err + } + entries, err := os.ReadDir(full) + if err != nil { + return nil, fileErr(p, err) + } + base := filepath.Clean("/" + p) + out := make([]api.FileInfo, 0, len(entries)) + for _, e := range entries { + info, err := e.Info() + if err != nil { + continue + } + out = append(out, api.FileInfo{ + Path: filepath.Join(base, e.Name()), + Name: e.Name(), + Size: info.Size(), + Mode: info.Mode().String(), + IsDir: e.IsDir(), + ModTime: info.ModTime().UTC(), + }) + } + return out, nil +} + +func fileErr(p string, err error) error { + switch { + case err == nil: + return nil + case errors.Is(err, fs.ErrNotExist): + return api.Errorf(api.CodeNotFound, "%s does not exist", p) + case errors.Is(err, fs.ErrPermission): + return api.Errorf(api.CodePermissionDenied, "%s: permission denied", p) + } + return api.Errorf(api.CodeInvalidRequest, "%s: %v", p, err) +} + +type capped struct { + buf bytes.Buffer + max int + truncated bool +} + +func (c *capped) Write(p []byte) (int, error) { + room := c.max - c.buf.Len() + if room <= 0 { + c.truncated = c.truncated || len(p) > 0 + return len(p), nil + } + if len(p) > room { + c.buf.Write(p[:room]) + c.truncated = true + return len(p), nil + } + return c.buf.Write(p) +} + +func (c *capped) String() string { return c.buf.String() } + +type writerFunc func([]byte) (int, error) + +func (f writerFunc) Write(p []byte) (int, error) { return f(p) } + +// Handler serves the guest agent API used by the node agent: +// +// GET /healthz +// POST /v1/exec[?stream=1] body api.ExecRequest +// GET /v1/files?path= list a directory +// GET /v1/files/content?path= +// PUT /v1/files/content?path= +func Handler(r *Runner) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) + }) + mux.HandleFunc("POST /v1/exec", func(w http.ResponseWriter, req *http.Request) { + var body api.ExecRequest + if err := json.NewDecoder(io.LimitReader(req.Body, 4<<20)).Decode(&body); err != nil { + writeErr(w, api.Errorf(api.CodeInvalidRequest, "invalid JSON: %v", err)) + return + } + if req.URL.Query().Get("stream") == "" { + res, err := r.Exec(req.Context(), body) + if err != nil { + writeErr(w, err) + return + } + writeJSON(w, http.StatusOK, res) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + flusher, _ := w.(http.Flusher) + enc := json.NewEncoder(w) + emit := func(ev api.ExecEvent) error { + err := enc.Encode(ev) + if flusher != nil { + flusher.Flush() + } + return err + } + code, err := r.ExecStream(req.Context(), body, emit) + if err != nil { + _ = emit(api.ExecEvent{Error: err.Error()}) + return + } + _ = emit(api.ExecEvent{ExitCode: &code}) + }) + mux.HandleFunc("GET /v1/files", func(w http.ResponseWriter, req *http.Request) { + items, err := r.ListFiles(req.URL.Query().Get("path")) + if err != nil { + writeErr(w, err) + return + } + writeJSON(w, http.StatusOK, items) + }) + mux.HandleFunc("GET /v1/files/content", func(w http.ResponseWriter, req *http.Request) { + data, err := r.ReadFile(req.URL.Query().Get("path")) + if err != nil { + writeErr(w, err) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(data) + }) + mux.HandleFunc("PUT /v1/files/content", func(w http.ResponseWriter, req *http.Request) { + data, err := io.ReadAll(io.LimitReader(req.Body, MaxFileBytes+1)) + if err == nil && len(data) > MaxFileBytes { + err = api.Errorf(api.CodeInvalidRequest, "file larger than %d bytes", MaxFileBytes) + } + if err == nil { + err = r.WriteFile(req.URL.Query().Get("path"), data) + } + if err != nil { + writeErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) + }) + return mux +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeErr(w http.ResponseWriter, err error) { + e := api.AsError(err) + writeJSON(w, e.HTTPStatus(), api.ErrorEnvelope{Error: e}) +} diff --git a/internal/guest/guest_test.go b/internal/guest/guest_test.go new file mode 100644 index 0000000..1d2e642 --- /dev/null +++ b/internal/guest/guest_test.go @@ -0,0 +1,85 @@ +package guest + +import ( + "context" + "net" + "net/http/httptest" + "strings" + "testing" + + "github.com/different-ai/differently/pkg/api" +) + +func TestRunnerExecAndFiles(t *testing.T) { + r := &Runner{Root: t.TempDir(), BaseEnv: map[string]string{"GREETING": "hi"}} + ctx := context.Background() + + res, err := r.Exec(ctx, api.ExecRequest{Command: `echo "$GREETING $WHO"; echo oops >&2; exit 3`, Env: map[string]string{"WHO": "there"}}) + if err != nil { + t.Fatal(err) + } + if res.ExitCode != 3 || strings.TrimSpace(res.Stdout) != "hi there" || strings.TrimSpace(res.Stderr) != "oops" { + t.Fatalf("unexpected result %+v", res) + } + + if err := r.WriteFile("/work/a.txt", []byte("hello")); err != nil { + t.Fatal(err) + } + data, err := r.ReadFile("work/a.txt") + if err != nil || string(data) != "hello" { + t.Fatalf("read back %q, %v", data, err) + } + items, err := r.ListFiles("/work") + if err != nil || len(items) != 1 || items[0].Path != "/work/a.txt" { + t.Fatalf("list %+v, %v", items, err) + } + if _, err := r.ReadFile("/missing"); !api.IsCode(err, api.CodeNotFound) { + t.Fatalf("want not_found, got %v", err) + } + // Paths cannot climb out of the root. + full, err := r.Resolve("../../etc/passwd") + if err != nil || !strings.HasPrefix(full, r.Root) { + t.Fatalf("resolve escaped root: %s %v", full, err) + } +} + +func TestRunnerTimeoutKillsProcessGroup(t *testing.T) { + r := &Runner{Root: t.TempDir()} + res, err := r.Exec(context.Background(), api.ExecRequest{Command: "sleep 30 & sleep 30", TimeoutSeconds: 1}) + if err != nil { + t.Fatal(err) + } + if res.ExitCode == 0 || res.DurationMs > 10_000 { + t.Fatalf("timeout not enforced: %+v", res) + } +} + +func TestClientOverHandler(t *testing.T) { + srv := httptest.NewServer(Handler(&Runner{Root: t.TempDir()})) + defer srv.Close() + addr := strings.TrimPrefix(srv.URL, "http://") + c := NewClient(func(ctx context.Context) (net.Conn, error) { + return (&net.Dialer{}).DialContext(ctx, "tcp", addr) + }) + ctx := context.Background() + if err := c.Health(ctx); err != nil { + t.Fatal(err) + } + if err := c.WriteFile(ctx, "/x/y", []byte("data")); err != nil { + t.Fatal(err) + } + if got, err := c.ReadFile(ctx, "/x/y"); err != nil || string(got) != "data" { + t.Fatalf("got %q %v", got, err) + } + var chunks []string + code, err := c.ExecStream(ctx, api.ExecRequest{Command: "printf a; printf b >&2"}, func(ev api.ExecEvent) error { + chunks = append(chunks, ev.Stream+":"+ev.Data) + return nil + }) + if err != nil || code != 0 || len(chunks) != 2 { + t.Fatalf("stream code=%d chunks=%v err=%v", code, chunks, err) + } + if _, err := c.ReadFile(ctx, "/nope"); !api.IsCode(err, api.CodeNotFound) { + t.Fatalf("want not_found over the wire, got %v", err) + } +} diff --git a/internal/guest/proc_other.go b/internal/guest/proc_other.go new file mode 100644 index 0000000..7908eee --- /dev/null +++ b/internal/guest/proc_other.go @@ -0,0 +1,7 @@ +//go:build !unix + +package guest + +import "os/exec" + +func configureProcessGroup(*exec.Cmd) {} diff --git a/internal/guest/proc_unix.go b/internal/guest/proc_unix.go new file mode 100644 index 0000000..49b62ca --- /dev/null +++ b/internal/guest/proc_unix.go @@ -0,0 +1,20 @@ +//go:build unix + +package guest + +import ( + "os/exec" + "syscall" +) + +// configureProcessGroup runs the command in its own process group so a +// timeout or cancellation kills every child, not just the shell. +func configureProcessGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.Cancel = func() error { + if cmd.Process == nil { + return nil + } + return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } +} diff --git a/internal/id/id.go b/internal/id/id.go new file mode 100644 index 0000000..2196df6 --- /dev/null +++ b/internal/id/id.go @@ -0,0 +1,73 @@ +// Package id generates sortable, prefixed resource identifiers. +// +// IDs are a type prefix plus a 26-character ULID-style body (48-bit +// millisecond timestamp, 80 random bits, Crockford base32), so they sort by +// creation time and are safe in URLs, labels and file names. +package id + +import ( + "crypto/rand" + "encoding/binary" + "strings" + "sync" + "time" +) + +const alphabet = "0123456789abcdefghjkmnpqrstvwxyz" + +var ( + mu sync.Mutex + lastMs uint64 + lastRand [10]byte +) + +// New returns prefix_. +func New(prefix string) string { + mu.Lock() + ms := uint64(time.Now().UnixMilli()) + var r [10]byte + if ms == lastMs { + // Monotonic within a millisecond: increment the previous random part. + r = lastRand + for i := len(r) - 1; i >= 0; i-- { + r[i]++ + if r[i] != 0 { + break + } + } + } else { + _, _ = rand.Read(r[:]) + } + lastMs, lastRand = ms, r + mu.Unlock() + + var b [16]byte + binary.BigEndian.PutUint16(b[0:2], uint16(ms>>32)) + binary.BigEndian.PutUint32(b[2:6], uint32(ms)) + copy(b[6:], r[:]) + return prefix + "_" + encode(b) +} + +// Secret returns n random bytes encoded in the same alphabet. +func Secret(n int) string { + buf := make([]byte, n) + _, _ = rand.Read(buf) + var sb strings.Builder + for _, c := range buf { + sb.WriteByte(alphabet[c&31]) + } + return sb.String() +} + +func encode(b [16]byte) string { + // 128 bits -> 26 base32 characters (first char carries 3 bits). + hi := binary.BigEndian.Uint64(b[0:8]) + lo := binary.BigEndian.Uint64(b[8:16]) + out := make([]byte, 26) + for i := 25; i >= 0; i-- { + out[i] = alphabet[lo&31] + lo = lo>>5 | (hi&31)<<59 + hi >>= 5 + } + return string(out) +} diff --git a/internal/id/id_test.go b/internal/id/id_test.go new file mode 100644 index 0000000..f070a11 --- /dev/null +++ b/internal/id/id_test.go @@ -0,0 +1,24 @@ +package id + +import ( + "strings" + "testing" +) + +func TestNewIsSortableAndUnique(t *testing.T) { + seen := map[string]bool{} + prev := "" + for i := 0; i < 5000; i++ { + v := New("sbx") + if !strings.HasPrefix(v, "sbx_") || len(v) != 30 { + t.Fatalf("bad id %q", v) + } + if seen[v] { + t.Fatalf("duplicate %q", v) + } + if v <= prev { + t.Fatalf("not monotonic: %q after %q", v, prev) + } + seen[v], prev = true, v + } +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..24f4f73 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,48 @@ +// Package metrics exposes Prometheus metrics for the control plane. +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +// Registry holds every Differently collector. +var Registry = prometheus.NewRegistry() + +var ( + // Transitions counts sandbox state changes. + Transitions = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "differently_sandbox_transitions_total", Help: "Sandbox state transitions.", + }, []string{"to", "pool"}) + // Placements counts placement outcomes. + Placements = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "differently_placement_decisions_total", Help: "Placement decisions by outcome.", + }, []string{"outcome", "pool"}) + // ProvisionSeconds measures time from activation to running. + ProvisionSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "differently_provision_seconds", Help: "Time to provision a sandbox.", + Buckets: []float64{.05, .1, .25, .5, 1, 2, 5, 10, 30, 60, 120}, + }, []string{"pool"}) + // QueueSeconds measures time spent queued before activation. + QueueSeconds = prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: "differently_queue_wait_seconds", Help: "Time sandboxes wait in the queue.", + Buckets: prometheus.ExponentialBuckets(0.01, 3, 12), + }) + // Requests counts API requests. + Requests = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "differently_api_requests_total", Help: "API requests by route and status.", + }, []string{"route", "status"}) + // RequestSeconds measures API latency. + RequestSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "differently_api_request_seconds", Help: "API request latency.", Buckets: prometheus.DefBuckets, + }, []string{"route"}) + // Reconciled counts reconciler corrections. + Reconciled = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "differently_reconcile_actions_total", Help: "Corrections made by the reconciler.", + }, []string{"action", "pool"}) +) + +func init() { + Registry.MustRegister(collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + Registry.MustRegister(Transitions, Placements, ProvisionSeconds, QueueSeconds, Requests, RequestSeconds, Reconciled) +} diff --git a/internal/policy/policy.go b/internal/policy/policy.go new file mode 100644 index 0000000..a8d263f --- /dev/null +++ b/internal/policy/policy.go @@ -0,0 +1,334 @@ +// Package policy decides where a sandbox may run and where it should run. +// +// Placement is filter-then-rank. Hard constraints are evaluated first and can +// only be narrowed by more specific scopes: classification, external mode, +// allowed/denied pools, regions, required capabilities, resource limits and +// cost ceilings. Only pools that pass every constraint are ranked by strategy. +// Security and residency therefore always win over cost or speed, and an +// external provider receives work only when every level permits it and it is +// explicitly approved — registering a provider authorizes nothing. +package policy + +import ( + "fmt" + "slices" + "sort" + "strings" + "time" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// PoolView is the scheduler's current knowledge of one pool. +type PoolView struct { + Pool api.Pool + Capabilities adapter.Capabilities + Capacity adapter.Capacity + Healthy bool + HealthDetail string + Active int +} + +// Input is everything placement needs. +type Input struct { + Sandbox api.Sandbox + Policies []api.PlacementPolicy // global, org, project — any may be absent + Classification api.Classification + Pools []PoolView + ExcludedPools []string // pools that already failed this sandbox + PinnedPool string // e.g. the pool holding a snapshot to restore + Now time.Time +} + +// Effective is the merged policy for one request. +type Effective struct { + External string + AllowedPools []string // nil means any + DeniedPools []string + ExternalPools []string // approved external pools; empty means none + Regions []string // nil means any + Strategy string + MaxCostPerHour float64 + MaxQueue time.Duration + Sources []string +} + +var externalRank = map[string]int{api.ExternalNever: 0, api.ExternalOverflow: 1, api.ExternalAllowed: 2} + +// ValidExternal reports whether s is a known external mode. +func ValidExternal(s string) bool { _, ok := externalRank[s]; return ok } + +// ValidStrategy reports whether s is a known strategy. +func ValidStrategy(s string) bool { + return s == api.StrategyPrivateFirst || s == api.StrategyCost || s == api.StrategySpread +} + +func stricter(a, b string) string { + if b == "" { + return a + } + if a == "" || externalRank[b] < externalRank[a] { + return b + } + return a +} + +func intersect(acc []string, next []string) []string { + if len(next) == 0 { + return acc + } + if acc == nil { + return slices.Clone(next) + } + var out []string + for _, v := range acc { + if slices.Contains(next, v) { + out = append(out, v) + } + } + if out == nil { + out = []string{} // an empty intersection allows nothing + } + return out +} + +// Merge combines policies, classification and the request preference. +func Merge(in Input) Effective { + eff := Effective{Strategy: api.StrategyPrivateFirst} + // Fail closed: with no global policy, nothing may leave private pools. + globalExternal := api.ExternalNever + var externalPools []string + for _, p := range in.Policies { + eff.Sources = append(eff.Sources, p.Scope) + if p.Scope == "global" { + if p.External != "" { + globalExternal = p.External + } + } else { + eff.External = stricter(eff.External, p.External) + } + eff.AllowedPools = intersect(eff.AllowedPools, p.AllowedPools) + eff.Regions = intersect(eff.Regions, p.AllowedRegions) + eff.DeniedPools = append(eff.DeniedPools, p.DeniedPools...) + externalPools = intersect(externalPools, p.ExternalPools) + if p.Strategy != "" { + eff.Strategy = p.Strategy // policies arrive least to most specific + } + if p.MaxCostPerHour > 0 && (eff.MaxCostPerHour == 0 || p.MaxCostPerHour < eff.MaxCostPerHour) { + eff.MaxCostPerHour = p.MaxCostPerHour + } + if q := time.Duration(p.MaxQueueSeconds) * time.Second; q > 0 && (eff.MaxQueue == 0 || q < eff.MaxQueue) { + eff.MaxQueue = q + } + } + eff.External = stricter(globalExternal, eff.External) + classExternal := in.Classification.External + if classExternal == "" { + classExternal = api.ExternalNever + } + eff.External = stricter(eff.External, classExternal) + if pl := in.Sandbox.Placement; pl != nil { + eff.External = stricter(eff.External, pl.External) + eff.AllowedPools = intersect(eff.AllowedPools, pl.AllowedPools) + eff.Regions = intersect(eff.Regions, pl.Regions) + if pl.Strategy != "" { + eff.Strategy = pl.Strategy + } + } + eff.ExternalPools = externalPools + return eff +} + +// Result is a placement decision plus the ranked pools to try in order. +type Result struct { + Decision *api.PlacementDecision + Ranked []string + // Err is set when no pool can ever satisfy the request. + Err *api.Error +} + +// Decide evaluates every pool and ranks the placeable ones. +func Decide(in Input) Result { + eff := Merge(in) + sb := in.Sandbox + dec := &api.PlacementDecision{Strategy: eff.Strategy, External: eff.External, Policies: eff.Sources, EvaluatedAt: in.Now} + if in.Classification.Name != "" { + dec.Policies = append(dec.Policies, "classification:"+in.Classification.Name) + } + + type cand struct { + view api.CandidateEvaluation + pv PoolView + } + var cands []cand + capabilityOnly := 0 + for _, pv := range in.Pools { + p := pv.Pool + ev := api.CandidateEvaluation{Pool: p.Name, Trust: p.Trust, Eligible: true} + reject := func(format string, args ...any) { + ev.Eligible = false + ev.Reasons = append(ev.Reasons, fmt.Sprintf(format, args...)) + } + if !p.Enabled { + reject("pool disabled") + } + if p.Draining { + reject("pool draining") + } + if slices.Contains(in.ExcludedPools, p.Name) { + reject("an earlier provisioning attempt on this pool failed") + } + if in.PinnedPool != "" && p.Name != in.PinnedPool { + reject("restore requires pool %s", in.PinnedPool) + } + if slices.Contains(eff.DeniedPools, p.Name) { + reject("denied by policy") + } + if eff.AllowedPools != nil && !slices.Contains(eff.AllowedPools, p.Name) { + reject("not in allowed pools") + } + if p.Trust != api.TrustPrivate { + switch { + case eff.External == api.ExternalNever: + reject("external placement not permitted (effective mode %q)", eff.External) + case !slices.Contains(eff.ExternalPools, p.Name): + reject("external pool not approved by policy") + } + } + if eff.Regions != nil && !slices.Contains(eff.Regions, p.Region) { + reject("region %q not allowed", p.Region) + } + missing := pv.Capabilities.Missing(sb.Capabilities) + policyOK := ev.Eligible + if len(missing) > 0 { + reject("missing capabilities: %s", strings.Join(missing, ", ")) + } + if ok, why := pv.Capabilities.Accepts(sb.Resources, sb.Template); !ok { + reject("%s", why) + } + cost := p.CostModel.HourlyCost(sb.Resources) + ev.EstimatedCost = cost + if eff.MaxCostPerHour > 0 && cost > eff.MaxCostPerHour { + reject("estimated cost %.4f/h exceeds ceiling %.4f/h", cost, eff.MaxCostPerHour) + } + if policyOK && !ev.Eligible && len(missing) > 0 && len(ev.Reasons) == 1 { + capabilityOnly++ + } + if ev.Eligible { + ev.HasCapacity = true + switch { + case !pv.Healthy: + ev.HasCapacity = false + ev.Reasons = append(ev.Reasons, "unhealthy: "+pv.HealthDetail) + case p.MaxConcurrency > 0 && pv.Active >= p.MaxConcurrency: + ev.HasCapacity = false + ev.Reasons = append(ev.Reasons, fmt.Sprintf("at concurrency limit %d", p.MaxConcurrency)) + case !pv.Capacity.Fits(sb.Resources): + ev.HasCapacity = false + ev.Reasons = append(ev.Reasons, "insufficient free capacity") + } + } + cands = append(cands, cand{view: ev, pv: pv}) + } + + privateReady := false + for _, c := range cands { + if c.view.Eligible && c.view.HasCapacity && c.view.Trust == api.TrustPrivate { + privateReady = true + } + } + var ready []cand + for i := range cands { + c := &cands[i] + if !c.view.Eligible || !c.view.HasCapacity { + continue + } + if c.view.Trust != api.TrustPrivate && eff.External == api.ExternalOverflow && privateReady { + c.view.Reasons = append(c.view.Reasons, "held for overflow: a private pool can take this now") + continue + } + ready = append(ready, *c) + } + + freeRatio := func(pv PoolView) float64 { + if !pv.Capacity.Bounded || pv.Capacity.TotalCPU == 0 { + if pv.Pool.MaxConcurrency > 0 { + return 1 - float64(pv.Active)/float64(pv.Pool.MaxConcurrency) + } + return 1 + } + return float64(pv.Capacity.FreeCPU) / float64(pv.Capacity.TotalCPU) + } + sort.SliceStable(ready, func(i, j int) bool { + a, b := ready[i], ready[j] + ap, bp := a.view.Trust == api.TrustPrivate, b.view.Trust == api.TrustPrivate + switch eff.Strategy { + case api.StrategyCost: + if a.view.EstimatedCost != b.view.EstimatedCost { + return a.view.EstimatedCost < b.view.EstimatedCost + } + if ap != bp { + return ap + } + case api.StrategySpread: + if fa, fb := freeRatio(a.pv), freeRatio(b.pv); fa != fb { + return fa > fb + } + if ap != bp { + return ap + } + default: // private-first + if ap != bp { + return ap + } + if a.view.EstimatedCost != b.view.EstimatedCost { + return a.view.EstimatedCost < b.view.EstimatedCost + } + if fa, fb := freeRatio(a.pv), freeRatio(b.pv); fa != fb { + return fa > fb + } + } + return a.view.Pool < b.view.Pool + }) + + res := Result{Decision: dec} + rank := map[string]int{} + for i, c := range ready { + res.Ranked = append(res.Ranked, c.view.Pool) + rank[c.view.Pool] = i + 1 + } + eligible := 0 + for _, c := range cands { + c.view.Rank = rank[c.view.Pool] + if c.view.Eligible { + eligible++ + } + dec.Candidates = append(dec.Candidates, c.view) + } + + switch { + case len(res.Ranked) > 0: + dec.Outcome, dec.Pool = "placed", res.Ranked[0] + dec.Reason = fmt.Sprintf("selected %s by %s strategy", res.Ranked[0], eff.Strategy) + case eligible > 0: + dec.Outcome = "waiting" + dec.Reason = "eligible pools exist but none can take the sandbox now; it stays queued" + default: + dec.Outcome = "rejected" + if capabilityOnly > 0 { + dec.Reason = "no permitted pool supports the requested capabilities" + res.Err = api.Errorf(api.CodeUnsupportedCapability, "%s", dec.Reason).With("capabilities", strings.Join(sb.Capabilities, ",")) + } else { + dec.Reason = "no pool satisfies placement policy for this sandbox" + res.Err = api.Errorf(api.CodeNoEligiblePool, "%s", dec.Reason).With("classification", sb.Classification) + } + } + return res +} + +// QueueExpired reports whether a waiting sandbox exceeded its queue budget. +func QueueExpired(in Input) bool { + eff := Merge(in) + return eff.MaxQueue > 0 && in.Now.Sub(in.Sandbox.CreatedAt) > eff.MaxQueue +} diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go new file mode 100644 index 0000000..f427bdb --- /dev/null +++ b/internal/policy/policy_test.go @@ -0,0 +1,147 @@ +package policy + +import ( + "testing" + "time" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +var ( + vmFeatures = []string{adapter.FeatureExec, adapter.FeatureFiles, adapter.FeatureSnapshotDisk, adapter.FeatureMicroVM} + extFeatures = []string{adapter.FeatureExec, adapter.FeatureFiles, adapter.FeaturePorts} +) + +func pool(name, trust string, free int, features []string, cpuHour float64) PoolView { + if trust == api.TrustExternal { + // External providers report unbounded capacity; the pool cap governs. + return PoolView{ + Pool: api.Pool{Name: name, Trust: trust, Enabled: true, Region: "us-east", MaxConcurrency: 10, CostModel: api.CostModel{CPUHour: cpuHour}}, + Capabilities: adapter.Capabilities{Features: features}, + Healthy: true, + } + } + return PoolView{ + Pool: api.Pool{Name: name, Trust: trust, Enabled: true, Region: "us-east", CostModel: api.CostModel{CPUHour: cpuHour}}, + Capabilities: adapter.Capabilities{Features: features}, + Capacity: adapter.Capacity{Bounded: true, TotalCPU: 16, FreeCPU: free, TotalMemoryMiB: 65536, FreeMemoryMiB: free * 4096}, + Healthy: true, + } +} + +func input(class api.Classification, policies []api.PlacementPolicy, pools ...PoolView) Input { + return Input{ + Sandbox: api.Sandbox{ID: "sbx_1", Classification: class.Name, Resources: api.Resources{CPU: 2, MemoryMiB: 2048}}, + Policies: policies, + Classification: class, + Pools: pools, + Now: time.Now(), + } +} + +var ( + internal = api.Classification{Name: "internal", External: api.ExternalAllowed} + privateOnly = api.Classification{Name: "private-only", External: api.ExternalNever} + overflow = []api.PlacementPolicy{{Scope: "global", External: api.ExternalOverflow, ExternalPools: []string{"daytona"}}} +) + +func TestPrivateFirstPrefersPrivateCapacity(t *testing.T) { + res := Decide(input(internal, overflow, pool("fc", api.TrustPrivate, 8, vmFeatures, 0.05), pool("daytona", api.TrustExternal, 0, extFeatures, 0.01))) + if res.Decision.Pool != "fc" || len(res.Ranked) != 1 { + t.Fatalf("want fc only, got %+v", res.Ranked) + } +} + +// Scenario D: explicitly authorized overflow goes external when private is full. +func TestOverflowWhenPrivateFull(t *testing.T) { + res := Decide(input(internal, overflow, pool("fc", api.TrustPrivate, 0, vmFeatures, 0.05), pool("daytona", api.TrustExternal, 0, extFeatures, 0.01))) + if res.Decision.Pool != "daytona" { + t.Fatalf("want overflow to daytona, got %+v", res.Decision) + } +} + +// Registering a provider authorizes nothing: without approval it is never used. +func TestUnapprovedExternalPoolIsNeverUsed(t *testing.T) { + policies := []api.PlacementPolicy{{Scope: "global", External: api.ExternalAllowed}} + res := Decide(input(internal, policies, pool("fc", api.TrustPrivate, 0, vmFeatures, 0.05), pool("daytona", api.TrustExternal, 0, extFeatures, 0.01))) + if res.Decision.Outcome != "waiting" || len(res.Ranked) != 0 { + t.Fatalf("unapproved external pool was ranked: %+v", res.Decision) + } +} + +// Scenario E: private-only work stays queued even when only external has room. +func TestPrivateOnlyNeverLeaves(t *testing.T) { + permissive := []api.PlacementPolicy{{Scope: "global", External: api.ExternalAllowed, ExternalPools: []string{"daytona"}}} + res := Decide(input(privateOnly, permissive, pool("fc", api.TrustPrivate, 0, vmFeatures, 0.05), pool("daytona", api.TrustExternal, 0, extFeatures, 0.01))) + if res.Decision.Outcome != "waiting" || len(res.Ranked) != 0 { + t.Fatalf("private-only escaped: %+v", res.Decision) + } + // With no private pool at all it is rejected rather than routed outside. + res = Decide(input(privateOnly, permissive, pool("daytona", api.TrustExternal, 0, extFeatures, 0.01))) + if res.Err == nil || res.Err.Code != api.CodeNoEligiblePool { + t.Fatalf("want no_eligible_pool, got %+v", res.Err) + } +} + +// A request or a narrower scope can tighten but never widen external access. +func TestNarrowerScopesOnlyTighten(t *testing.T) { + policies := append(overflow, api.PlacementPolicy{Scope: "project:p1", External: api.ExternalAllowed}) + in := input(internal, policies, pool("fc", api.TrustPrivate, 8, vmFeatures, 0.05), pool("daytona", api.TrustExternal, 0, extFeatures, 0.01)) + if eff := Merge(in); eff.External != api.ExternalOverflow { + t.Fatalf("project widened external to %q", eff.External) + } + in.Sandbox.Placement = &api.Placement{External: api.ExternalNever} + in.Pools[0].Capacity.FreeCPU = 0 + if res := Decide(in); len(res.Ranked) != 0 { + t.Fatalf("request opt-out ignored: %v", res.Ranked) + } +} + +// Scenario I: a capability mismatch is a precise error, or another capable +// authorized backend is chosen. +func TestCapabilityMismatch(t *testing.T) { + in := input(internal, nil, pool("fc", api.TrustPrivate, 8, vmFeatures, 0.05)) + in.Sandbox.Capabilities = []string{adapter.FeaturePorts} + res := Decide(in) + if res.Err == nil || res.Err.Code != api.CodeUnsupportedCapability { + t.Fatalf("want unsupported_capability, got %+v", res.Err) + } + in = input(internal, overflow, pool("fc", api.TrustPrivate, 8, vmFeatures, 0.05), pool("daytona", api.TrustExternal, 0, extFeatures, 0.01)) + in.Sandbox.Capabilities = []string{adapter.FeaturePorts} + if res := Decide(in); res.Decision.Pool != "daytona" { + t.Fatalf("want capable authorized backend daytona, got %+v", res.Decision) + } +} + +func TestCostStrategyAndCeiling(t *testing.T) { + policies := []api.PlacementPolicy{{Scope: "global", Strategy: api.StrategyCost, MaxCostPerHour: 0.15}} + res := Decide(input(internal, policies, + pool("expensive", api.TrustPrivate, 8, vmFeatures, 0.10), // 0.20/h, over ceiling + pool("cheap", api.TrustPrivate, 8, vmFeatures, 0.02), + pool("mid", api.TrustPrivate, 8, vmFeatures, 0.05))) + if len(res.Ranked) != 2 || res.Ranked[0] != "cheap" || res.Ranked[1] != "mid" { + t.Fatalf("ranked %v", res.Ranked) + } +} + +func TestDrainingAndAllowedPools(t *testing.T) { + a, b := pool("a", api.TrustPrivate, 8, vmFeatures, 0), pool("b", api.TrustPrivate, 8, vmFeatures, 0) + a.Pool.Draining = true + res := Decide(input(internal, []api.PlacementPolicy{{Scope: "org:o1", AllowedPools: []string{"a", "b"}}, {Scope: "project:p1", AllowedPools: []string{"a"}}}, a, b)) + if res.Err == nil { + t.Fatalf("project narrowed to a (draining) yet got %v", res.Ranked) + } +} + +func TestDecisionExplainsEveryPool(t *testing.T) { + res := Decide(input(privateOnly, overflow, pool("fc", api.TrustPrivate, 8, vmFeatures, 0), pool("daytona", api.TrustExternal, 8, extFeatures, 0))) + if len(res.Decision.Candidates) != 2 { + t.Fatalf("candidates %+v", res.Decision.Candidates) + } + for _, c := range res.Decision.Candidates { + if c.Pool == "daytona" && (c.Eligible || len(c.Reasons) == 0) { + t.Fatalf("daytona should be ineligible with a reason: %+v", c) + } + } +} diff --git a/internal/pools/pools.go b/internal/pools/pools.go new file mode 100644 index 0000000..2594fd8 --- /dev/null +++ b/internal/pools/pools.go @@ -0,0 +1,274 @@ +// Package pools builds execution adapters for registered pools and keeps a +// cached view of their health, capabilities and capacity for placement. +package pools + +import ( + "context" + "fmt" + "log/slog" + "reflect" + "sync" + "time" + + "github.com/different-ai/differently/internal/policy" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// Deps are shared resources adapter factories may need. +type Deps struct { + Store store.Store + AgentToken string + AgentTimeout time.Duration + Dev bool + DataDir string + Logger *slog.Logger + // Resolve returns the secret behind a pool's credential reference. + Resolve func(ref string) (string, error) +} + +// Factory builds an adapter for a pool. +type Factory func(p api.Pool, deps Deps) (adapter.ExecutionAdapter, error) + +var ( + factoriesMu sync.RWMutex + factories = map[string]Factory{} +) + +// Register makes an adapter kind available. Built-in adapters register from +// the server's wiring; plugins can register at init. +func Register(kind string, f Factory) { + factoriesMu.Lock() + defer factoriesMu.Unlock() + factories[kind] = f +} + +// Kinds lists registered adapter kinds. +func Kinds() []string { + factoriesMu.RLock() + defer factoriesMu.RUnlock() + var out []string + for k := range factories { + out = append(out, k) + } + return out +} + +type entry struct { + pool api.Pool + adapter adapter.ExecutionAdapter + caps adapter.Capabilities + capacity adapter.Capacity + healthy bool + detail string + checkedAt time.Time + buildErr error +} + +// Manager owns one adapter per pool. +type Manager struct { + deps Deps + mu sync.RWMutex + entries map[string]*entry +} + +// NewManager returns an empty manager. +func NewManager(deps Deps) *Manager { + if deps.Logger == nil { + deps.Logger = slog.Default() + } + return &Manager{deps: deps, entries: map[string]*entry{}} +} + +// Set installs a pre-built adapter (tests, embedding). +func (m *Manager) Set(p api.Pool, a adapter.ExecutionAdapter) { + m.mu.Lock() + defer m.mu.Unlock() + m.entries[p.Name] = &entry{pool: p, adapter: a} +} + +// Sync builds adapters for new or changed pools and drops removed ones. +func (m *Manager) Sync(pools []api.Pool) { + m.mu.Lock() + defer m.mu.Unlock() + seen := map[string]bool{} + for _, p := range pools { + seen[p.Name] = true + old, ok := m.entries[p.Name] + if ok && old.adapter != nil && sameBuild(old.pool, p) { + old.pool = p + continue + } + if ok && old.adapter != nil && old.buildErr == nil && factoryFor(p.Adapter) == nil { + old.pool = p // pre-built via Set + continue + } + e := &entry{pool: p} + if f := factoryFor(p.Adapter); f == nil { + e.buildErr = fmt.Errorf("adapter kind %q is not available in this build", p.Adapter) + } else if a, err := f(p, m.deps); err != nil { + e.buildErr = err + } else { + e.adapter = a + } + if e.buildErr != nil { + e.detail = e.buildErr.Error() + m.deps.Logger.Warn("pool adapter unavailable", "pool", p.Name, "err", e.buildErr) + } + m.entries[p.Name] = e + } + for name := range m.entries { + if !seen[name] { + delete(m.entries, name) + } + } +} + +func sameBuild(a, b api.Pool) bool { + return a.Adapter == b.Adapter && a.CredentialRef == b.CredentialRef && reflect.DeepEqual(a.Config, b.Config) +} + +func factoryFor(kind string) Factory { + factoriesMu.RLock() + defer factoriesMu.RUnlock() + return factories[kind] +} + +// Adapter returns the adapter for a pool. +func (m *Manager) Adapter(name string) (adapter.ExecutionAdapter, error) { + m.mu.RLock() + defer m.mu.RUnlock() + e, ok := m.entries[name] + switch { + case !ok: + return nil, api.Errorf(api.CodeNotFound, "pool %s is not registered", name) + case e.adapter == nil: + return nil, api.Errorf(api.CodeProviderUnavailable, "pool %s: %v", name, e.buildErr) + } + return e.adapter, nil +} + +// Refresh probes every pool's health, capabilities and capacity. +func (m *Manager) Refresh(ctx context.Context) { + m.mu.RLock() + list := make([]*entry, 0, len(m.entries)) + for _, e := range m.entries { + list = append(list, e) + } + m.mu.RUnlock() + var wg sync.WaitGroup + for _, e := range list { + if e.adapter == nil { + continue + } + wg.Add(1) + go func(e *entry) { + defer wg.Done() + pctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + healthy, detail := true, "" + caps, err := e.adapter.Capabilities(pctx) + if err == nil { + caps.Features = adapter.EffectiveFeatures(e.adapter, caps.Features) + err = e.adapter.Health(pctx) + } + var capacity adapter.Capacity + if err == nil { + capacity, err = e.adapter.Capacity(pctx) + } + if err != nil { + healthy, detail = false, err.Error() + } + m.mu.Lock() + if err == nil { + e.caps, e.capacity = caps, capacity + } + e.healthy, e.detail, e.checkedAt = healthy, detail, time.Now().UTC() + m.mu.Unlock() + }(e) + } + wg.Wait() +} + +// Supports reports whether a pool advertises feature (declared by the +// adapter and backed by an implemented interface). +func (m *Manager) Supports(ctx context.Context, name, feature string) bool { + m.mu.RLock() + e, ok := m.entries[name] + m.mu.RUnlock() + if !ok || e.adapter == nil { + return false + } + caps := e.caps + if caps.Features == nil { + c, err := e.adapter.Capabilities(ctx) + if err != nil { + return false + } + c.Features = adapter.EffectiveFeatures(e.adapter, c.Features) + caps = c + } + return caps.Has(feature) +} + +// MarkUnhealthy records a provisioning failure until the next good probe. +func (m *Manager) MarkUnhealthy(name string, err error) { + m.mu.Lock() + defer m.mu.Unlock() + if e, ok := m.entries[name]; ok { + e.healthy, e.detail = false, err.Error() + } +} + +// Views returns placement inputs for every pool. +func (m *Manager) Views(active map[string]int) []policy.PoolView { + m.mu.RLock() + defer m.mu.RUnlock() + out := make([]policy.PoolView, 0, len(m.entries)) + for _, e := range m.entries { + out = append(out, policy.PoolView{ + Pool: e.pool, Capabilities: e.caps, Capacity: e.capacity, + Healthy: e.healthy && e.adapter != nil, HealthDetail: e.detail, Active: active[e.pool.Name], + }) + } + return out +} + +// Decorate fills runtime fields (health, capacity, capabilities) on a pool. +func (m *Manager) Decorate(p *api.Pool, active int) { + m.mu.RLock() + defer m.mu.RUnlock() + p.Active = active + e, ok := m.entries[p.Name] + if !ok { + return + } + status := "unknown" + switch { + case e.adapter == nil: + status = "unavailable" + case !e.checkedAt.IsZero() && e.healthy: + status = "healthy" + case !e.checkedAt.IsZero(): + status = "unavailable" + } + p.Health = &api.PoolHealth{Status: status, Message: e.detail, CheckedAt: e.checkedAt} + p.Capabilities = e.caps.Features + c := e.capacity + p.Capacity = &api.PoolCapacity{Bounded: c.Bounded, TotalCPU: c.TotalCPU, FreeCPU: c.FreeCPU, TotalMemoryMiB: c.TotalMemoryMiB, FreeMemoryMiB: c.FreeMemoryMiB, Nodes: c.Nodes} +} + +// Endpoints lists the external endpoints configured adapters reach, for the +// trust report. +func (m *Manager) Endpoints() map[string]string { + m.mu.RLock() + defer m.mu.RUnlock() + out := map[string]string{} + for name, e := range m.entries { + if ep, ok := e.adapter.(interface{ Endpoint() string }); ok { + out[name] = ep.Endpoint() + } + } + return out +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go new file mode 100644 index 0000000..9b945aa --- /dev/null +++ b/internal/scheduler/scheduler.go @@ -0,0 +1,603 @@ +// Package scheduler moves sandboxes through their lifecycle. +// +// One leader runs the loops (the postgres store elects it with an advisory +// lock); API replicas only write intent to the store. Each tick: +// +// 1. dispatch: queued sandboxes, in fair order, are placed by the policy +// engine and atomically activated on a pool (re-checking tenant quota and +// pool concurrency); +// 2. provision: activated sandboxes are created through their pool adapter, +// using the sandbox ID as idempotency key so retries never double-create; +// 3. terminate: sandboxes marked terminating are deleted on the backend; +// 4. expire: sandboxes past their TTL are marked terminating. +// +// The reconciler separately compares each pool's instances with the store, +// deleting orphans, failing sandboxes whose instance vanished, and recovering +// provisioning that was interrupted by a crash. +package scheduler + +import ( + "context" + "log/slog" + "slices" + "strings" + "sync" + "time" + + "github.com/different-ai/differently/internal/auditlog" + "github.com/different-ai/differently/internal/config" + "github.com/different-ai/differently/internal/metrics" + "github.com/different-ai/differently/internal/policy" + "github.com/different-ai/differently/internal/pools" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// RetryDelay is the wait before a sandbox is reconsidered for placement. +const RetryDelay = 2 * time.Second + +// Scheduler runs dispatch, provisioning, termination and reconciliation. +type Scheduler struct { + Store store.Store + Pools *pools.Manager + Config config.Config + Installation string + Audit *auditlog.Recorder + Log *slog.Logger + Now func() time.Time + + wake chan struct{} + mu sync.Mutex + inflight map[string]bool + wg sync.WaitGroup +} + +// New returns a scheduler. +func New(st store.Store, pm *pools.Manager, cfg config.Config, installation string, rec *auditlog.Recorder, log *slog.Logger) *Scheduler { + return &Scheduler{ + Store: st, Pools: pm, Config: cfg, Installation: installation, Audit: rec, Log: log, + Now: func() time.Time { return time.Now().UTC() }, wake: make(chan struct{}, 1), inflight: map[string]bool{}, + } +} + +// Wake asks the loop to run a tick soon. +func (s *Scheduler) Wake() { + select { + case s.wake <- struct{}{}: + default: + } +} + +// Run loops until ctx ends. +func (s *Scheduler) Run(ctx context.Context) { + tick := time.NewTicker(time.Duration(s.Config.Scheduler.IntervalMs) * time.Millisecond) + refresh := time.NewTicker(5 * time.Second) + reconcile := time.NewTicker(time.Duration(s.Config.Scheduler.ReconcileIntervalSecs) * time.Second) + defer tick.Stop() + defer refresh.Stop() + defer reconcile.Stop() + s.Pools.Refresh(ctx) + for { + select { + case <-ctx.Done(): + s.wg.Wait() + return + case <-refresh.C: + s.Pools.Refresh(ctx) + continue + case <-reconcile.C: + if s.leader(ctx) { + s.Reconcile(ctx) + } + continue + case <-tick.C: + case <-s.wake: + } + if s.leader(ctx) { + s.Tick(ctx) + } + } +} + +func (s *Scheduler) leader(ctx context.Context) bool { + ok, err := s.Store.TryLeader(ctx) + if err != nil { + s.Log.Warn("leader check failed", "err", err) + } + return ok +} + +// Tick runs one dispatch/provision/terminate/expire pass. +func (s *Scheduler) Tick(ctx context.Context) { + s.dispatch(ctx) + s.startProvisioning(ctx) + s.terminate(ctx) + s.expire(ctx) +} + +// Drain waits for in-flight provisioning and termination. +func (s *Scheduler) Drain() { s.wg.Wait() } + +func (s *Scheduler) policies(ctx context.Context, orgID, projectID string) []api.PlacementPolicy { + global := s.Config.Policy + if p, ok, _ := s.Store.GetPolicy(ctx, "global"); ok { + global = p + } + out := []api.PlacementPolicy{global} + for _, scope := range []string{"org:" + orgID, "project:" + projectID} { + if p, ok, _ := s.Store.GetPolicy(ctx, scope); ok { + out = append(out, p) + } + } + return out +} + +// PlacementInput assembles the policy input for a sandbox. +func (s *Scheduler) PlacementInput(ctx context.Context, rec store.SandboxRecord, active map[string]int) policy.Input { + class, _ := s.Config.Classification(rec.Classification) + in := policy.Input{ + Sandbox: rec.Sandbox, Policies: s.policies(ctx, rec.OrgID, rec.ProjectID), Classification: class, + Pools: s.Pools.Views(active), ExcludedPools: rec.ExcludedPools, Now: s.Now(), + } + if rec.FromSnapshot != "" { + if snap, err := s.Store.GetSnapshot(ctx, rec.FromSnapshot); err == nil { + in.PinnedPool = snap.Pool + } + } + return in +} + +func (s *Scheduler) dispatch(ctx context.Context) { + now := s.Now() + cands, err := s.Store.QueuedCandidates(ctx, now, 100) + if err != nil || len(cands) == 0 { + return + } + active, err := s.Store.PoolActive(ctx) + if err != nil { + return + } + orgs := map[string]api.Organization{} + for _, rec := range cands { + org, ok := orgs[rec.OrgID] + if !ok { + if org, err = s.Store.GetOrg(ctx, rec.OrgID); err != nil { + continue + } + orgs[rec.OrgID] = org + } + if org.Suspended { + s.wait(ctx, rec.ID, "organization suspended", nil) + continue + } + project, err := s.Store.GetProject(ctx, rec.ProjectID) + if err != nil { + continue + } + in := s.PlacementInput(ctx, rec, active) + res := policy.Decide(in) + if (res.Err != nil || len(res.Ranked) == 0) && len(in.ExcludedPools) > 0 { + // Every other pool is out: retry the pools that failed before. + in.ExcludedPools = nil + res = policy.Decide(in) + } + if res.Err != nil { + metrics.Placements.WithLabelValues("rejected", "").Inc() + s.fail(ctx, rec.ID, res.Err.Message, res.Decision) + continue + } + if len(res.Ranked) == 0 { + if policy.QueueExpired(in) { + s.fail(ctx, rec.ID, "queue timeout: no eligible pool had capacity in time", res.Decision) + continue + } + metrics.Placements.WithLabelValues("waiting", "").Inc() + s.wait(ctx, rec.ID, "waiting for capacity: "+summarize(res.Decision), res.Decision) + continue + } + s.activate(ctx, rec, res, org, project, active) + } +} + +func (s *Scheduler) activate(ctx context.Context, rec store.SandboxRecord, res policy.Result, org api.Organization, project api.Project, active map[string]int) { + for _, name := range res.Ranked { + pool, err := s.Store.GetPool(ctx, name) + if err != nil { + continue + } + dec := *res.Decision + dec.Pool, dec.Outcome = name, "placed" + if name != res.Decision.Pool { + dec.Reason += "; " + res.Decision.Pool + " filled concurrently, used next ranked pool " + name + } + out, err := s.Store.ActivateSandbox(ctx, rec.ID, store.Activation{ + Pool: name, Adapter: pool.Adapter, Trust: pool.Trust, Region: pool.Region, MaxConcurrency: pool.MaxConcurrency, + OrgQuota: org.Quota, ProjectQuota: project.Quota, Decision: &dec, Now: s.Now(), + }) + switch { + case err == nil: + active[name]++ + metrics.Placements.WithLabelValues("placed", name).Inc() + metrics.QueueSeconds.Observe(s.Now().Sub(rec.CreatedAt).Seconds()) + s.Audit.Record(ctx, api.AuditEvent{ + Actor: "scheduler", OrgID: out.OrgID, ProjectID: out.ProjectID, Action: "sandbox.placed", Resource: out.ID, + Details: map[string]string{"pool": name, "trust": pool.Trust, "strategy": dec.Strategy, "external": dec.External, "reason": dec.Reason}, + }) + return + case api.IsCode(err, api.CodeQuotaExceeded): + s.wait(ctx, rec.ID, "waiting for tenant quota: "+err.Error(), res.Decision) + return + case api.IsCode(err, adapter.CodeInsufficientCapacity): + continue + default: + s.Log.Warn("activation failed", "sandbox", rec.ID, "pool", name, "err", err) + return + } + } + s.wait(ctx, rec.ID, "waiting for capacity: ranked pools filled concurrently", res.Decision) +} + +func summarize(d *api.PlacementDecision) string { + var parts []string + for _, c := range d.Candidates { + if c.Eligible && len(c.Reasons) > 0 { + parts = append(parts, c.Pool+" ("+strings.Join(c.Reasons, "; ")+")") + } + } + return strings.Join(parts, ", ") +} + +func (s *Scheduler) wait(ctx context.Context, sid, reason string, dec *api.PlacementDecision) { + _, _ = s.Store.UpdateSandbox(ctx, sid, func(r *store.SandboxRecord) error { + if r.Status != api.StatusQueued { + return nil + } + r.StatusReason = reason + if dec != nil { + r.Decision = dec + } + r.NextAttemptAt = s.Now().Add(RetryDelay) + return nil + }) +} + +func (s *Scheduler) fail(ctx context.Context, sid, reason string, dec *api.PlacementDecision) { + rec, err := s.Store.UpdateSandbox(ctx, sid, func(r *store.SandboxRecord) error { + now := s.Now() + stopClock(r, now) + r.Status, r.StatusReason, r.EndedAt, r.UpdatedAt = api.StatusFailed, reason, &now, now + if dec != nil { + r.Decision = dec + } + return nil + }) + if err == nil { + metrics.Transitions.WithLabelValues(string(api.StatusFailed), rec.Pool).Inc() + s.Audit.Record(ctx, api.AuditEvent{Actor: "scheduler", OrgID: rec.OrgID, ProjectID: rec.ProjectID, Action: "sandbox.failed", Resource: sid, Outcome: "failure", Details: map[string]string{"reason": reason}}) + } +} + +func (s *Scheduler) claim(sid string) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.inflight[sid] { + return false + } + s.inflight[sid] = true + return true +} + +func (s *Scheduler) release(sid string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.inflight, sid) +} + +func (s *Scheduler) startProvisioning(ctx context.Context) { + recs, err := s.Store.ListSandboxes(ctx, store.SandboxFilter{Statuses: []api.SandboxStatus{api.StatusProvisioning}}) + if err != nil { + return + } + for _, rec := range recs { + if !s.claim(rec.ID) { + continue + } + s.wg.Add(1) + go func(rec store.SandboxRecord) { + defer s.wg.Done() + defer s.release(rec.ID) + s.provision(context.WithoutCancel(ctx), rec) + }(rec) + } +} + +func (s *Scheduler) provision(ctx context.Context, rec store.SandboxRecord) { + a, err := s.Pools.Adapter(rec.Pool) + if err != nil { + s.retry(ctx, rec, err) + return + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(s.Config.Scheduler.ProvisionTimeoutSeconds)*time.Second) + defer cancel() + labels := map[string]string{adapter.LabelOrg: rec.OrgID, adapter.LabelProject: rec.ProjectID} + for k, v := range rec.Labels { + labels[k] = v + } + start := s.Now() + inst, err := a.Create(ctx, adapter.CreateRequest{ + ID: rec.ID, Installation: s.Installation, Template: rec.Template, Resources: rec.Resources, + Env: rec.Env, Labels: labels, TTLSeconds: rec.TTLSeconds, FromSnapshot: rec.SnapshotBackendRef, + }) + if err != nil { + s.retry(ctx, rec, err) + return + } + metrics.ProvisionSeconds.WithLabelValues(rec.Pool).Observe(s.Now().Sub(start).Seconds()) + out, err := s.Store.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { + now := s.Now() + r.BackendRef = inst.Ref + if inst.Region != "" { + r.Region = inst.Region + } + if r.Status != api.StatusProvisioning { + return nil // deleted while provisioning: the terminator now has the ref + } + r.Status, r.StatusReason, r.UpdatedAt = api.StatusRunning, "", now + if r.StartedAt == nil { + r.StartedAt = &now + } + r.RunningSince = &now + if r.TTLSeconds > 0 { + exp := now.Add(time.Duration(r.TTLSeconds) * time.Second) + r.ExpiresAt = &exp + } + return nil + }) + if err == nil && out.Status == api.StatusRunning { + metrics.Transitions.WithLabelValues(string(api.StatusRunning), rec.Pool).Inc() + s.Audit.Record(ctx, api.AuditEvent{Actor: "scheduler", OrgID: out.OrgID, ProjectID: out.ProjectID, Action: "sandbox.running", Resource: out.ID, Details: map[string]string{"pool": out.Pool, "backendRef": inst.Ref}}) + } +} + +// retry requeues after a provisioning failure or fails the sandbox once +// attempts are exhausted. Transient capacity errors do not count against +// pool health. +func (s *Scheduler) retry(ctx context.Context, rec store.SandboxRecord, cause error) { + capacity := api.IsCode(cause, adapter.CodeInsufficientCapacity) + if api.IsCode(cause, api.CodeInvalidRequest) { + // The backend rejected the request itself; retrying cannot help and + // the pool is not unhealthy. + s.fail(ctx, rec.ID, "backend rejected the sandbox: "+cause.Error(), nil) + return + } + if !capacity { + s.Pools.MarkUnhealthy(rec.Pool, cause) + s.Log.Warn("provisioning failed", "sandbox", rec.ID, "pool", rec.Pool, "attempt", rec.Attempts, "err", cause) + } + if !capacity && rec.Attempts >= s.Config.Scheduler.MaxAttempts { + s.fail(ctx, rec.ID, "provisioning failed after "+itoa(rec.Attempts)+" attempts: "+cause.Error(), nil) + return + } + _, _ = s.Store.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { + if r.Status != api.StatusProvisioning { + return nil + } + now := s.Now() + if !capacity && !slices.Contains(r.ExcludedPools, r.Pool) { + r.ExcludedPools = append(r.ExcludedPools, r.Pool) + } + r.Status, r.Pool, r.Adapter, r.Trust, r.Region = api.StatusQueued, "", "", "", "" + r.StatusReason = "retrying after provisioning error: " + cause.Error() + backoff := RetryDelay * time.Duration(1< 0 { + r.Attempts-- // capacity misses are not failed attempts + } + } + r.NextAttemptAt, r.UpdatedAt = now.Add(backoff), now + return nil + }) + s.Audit.Record(ctx, api.AuditEvent{Actor: "scheduler", OrgID: rec.OrgID, ProjectID: rec.ProjectID, Action: "sandbox.provision_retry", Resource: rec.ID, Outcome: "failure", Details: map[string]string{"pool": rec.Pool, "error": cause.Error()}}) +} + +func (s *Scheduler) terminate(ctx context.Context) { + recs, err := s.Store.ListSandboxes(ctx, store.SandboxFilter{Statuses: []api.SandboxStatus{api.StatusTerminating}}) + if err != nil { + return + } + for _, rec := range recs { + if !s.claim(rec.ID) { + continue // provisioning still running: it will record the ref first + } + s.wg.Add(1) + go func(rec store.SandboxRecord) { + defer s.wg.Done() + defer s.release(rec.ID) + s.destroy(context.WithoutCancel(ctx), rec) + }(rec) + } +} + +func (s *Scheduler) destroy(ctx context.Context, rec store.SandboxRecord) { + if rec.BackendRef != "" { + a, err := s.Pools.Adapter(rec.Pool) + if err == nil { + dctx, cancel := context.WithTimeout(ctx, 60*time.Second) + err = a.Delete(dctx, rec.BackendRef) + cancel() + } + if err != nil { + s.Log.Warn("terminate failed; will retry", "sandbox", rec.ID, "pool", rec.Pool, "err", err) + return + } + } + out, err := s.Store.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { + now := s.Now() + stopClock(r, now) + r.Status, r.EndedAt, r.UpdatedAt = api.StatusTerminated, &now, now + return nil + }) + if err == nil { + s.nudge(ctx) + metrics.Transitions.WithLabelValues(string(api.StatusTerminated), out.Pool).Inc() + s.Audit.Record(ctx, api.AuditEvent{Actor: "scheduler", OrgID: out.OrgID, ProjectID: out.ProjectID, Action: "sandbox.terminated", Resource: out.ID, Details: map[string]string{"pool": out.Pool, "reason": out.StatusReason}}) + } +} + +func (s *Scheduler) expire(ctx context.Context) { + now := s.Now() + recs, err := s.Store.ListSandboxes(ctx, store.SandboxFilter{Statuses: []api.SandboxStatus{api.StatusRunning, api.StatusPaused, api.StatusStopped}}) + if err != nil { + return + } + for _, rec := range recs { + if rec.ExpiresAt == nil || rec.ExpiresAt.After(now) { + continue + } + _, _ = s.Store.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { + r.Status, r.StatusReason, r.UpdatedAt = api.StatusTerminating, "ttl expired", now + return nil + }) + s.Audit.Record(ctx, api.AuditEvent{Actor: "scheduler", OrgID: rec.OrgID, ProjectID: rec.ProjectID, Action: "sandbox.expired", Resource: rec.ID}) + } +} + +// nudge makes waiting sandboxes eligible for the next tick after quota or +// capacity was released, instead of waiting out their retry delay. +func (s *Scheduler) nudge(ctx context.Context) { + now := s.Now() + waiting, err := s.Store.ListSandboxes(ctx, store.SandboxFilter{Statuses: []api.SandboxStatus{api.StatusQueued}, Limit: 200}) + if err != nil { + return + } + for _, rec := range waiting { + if rec.NextAttemptAt.After(now) && rec.Attempts == 0 { + _, _ = s.Store.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { + if r.Status == api.StatusQueued { + r.NextAttemptAt = now + } + return nil + }) + } + } + s.Wake() +} + +// Reconcile corrects drift between the store and every healthy pool. +func (s *Scheduler) Reconcile(ctx context.Context) { + poolsList, err := s.Store.ListPools(ctx) + if err != nil { + return + } + for _, p := range poolsList { + a, err := s.Pools.Adapter(p.Name) + if err != nil { + continue + } + lctx, cancel := context.WithTimeout(ctx, 30*time.Second) + instances, err := a.List(lctx, s.Installation) + cancel() + if err != nil { + // Never act on a partial view: a failed list is not "no instances". + s.Log.Warn("reconcile list failed", "pool", p.Name, "err", err) + continue + } + s.reconcilePool(ctx, p.Name, a, instances) + } +} + +func (s *Scheduler) reconcilePool(ctx context.Context, pool string, a adapter.ExecutionAdapter, instances []adapter.Instance) { + byRef := map[string]adapter.Instance{} + bySandbox := map[string]adapter.Instance{} + for _, in := range instances { + byRef[in.Ref] = in + bySandbox[in.SandboxID] = in + } + // Orphans: backend instances whose sandbox is gone, terminal or elsewhere. + for _, in := range instances { + rec, err := s.Store.GetSandbox(ctx, in.SandboxID) + orphan := api.IsCode(err, api.CodeNotFound) || (err == nil && (rec.Status.Terminal() || (rec.Pool != pool && rec.Status != api.StatusQueued))) + if !orphan { + continue + } + if err := a.Delete(ctx, in.Ref); err != nil { + s.Log.Warn("orphan delete failed", "pool", pool, "ref", in.Ref, "err", err) + continue + } + metrics.Reconciled.WithLabelValues("orphan_deleted", pool).Inc() + s.Audit.Record(ctx, api.AuditEvent{Actor: "reconciler", OrgID: rec.OrgID, ProjectID: rec.ProjectID, Action: "reconcile.orphan_deleted", Resource: in.SandboxID, Details: map[string]string{"pool": pool, "ref": in.Ref}}) + } + recs, err := s.Store.ListSandboxes(ctx, store.SandboxFilter{Pool: pool, Statuses: []api.SandboxStatus{api.StatusProvisioning, api.StatusRunning, api.StatusPaused, api.StatusStopped}}) + if err != nil { + return + } + now := s.Now() + timeout := time.Duration(s.Config.Scheduler.ProvisionTimeoutSeconds) * time.Second + for _, rec := range recs { + switch rec.Status { + case api.StatusProvisioning: + if !s.claim(rec.ID) { + continue // provisioning is in flight in this process + } + stale := rec.ProvisionStartedAt == nil || now.Sub(*rec.ProvisionStartedAt) > timeout + if in, ok := bySandbox[rec.ID]; ok && stale { + // Created before a crash: adopt it. + _, _ = s.Store.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { + r.BackendRef, r.Status, r.RunningSince, r.UpdatedAt = in.Ref, api.StatusRunning, &now, now + if r.StartedAt == nil { + r.StartedAt = &now + } + if r.TTLSeconds > 0 && r.ExpiresAt == nil { + exp := now.Add(time.Duration(r.TTLSeconds) * time.Second) + r.ExpiresAt = &exp + } + return nil + }) + metrics.Reconciled.WithLabelValues("adopted", pool).Inc() + } else if stale { + s.release(rec.ID) + s.retry(ctx, rec, adapter.Unavailable(errString("provisioning interrupted"))) + continue + } + s.release(rec.ID) + default: + if rec.BackendRef == "" { + continue + } + if _, ok := byRef[rec.BackendRef]; ok { + continue + } + s.fail(ctx, rec.ID, "backend instance disappeared", nil) + metrics.Reconciled.WithLabelValues("lost", pool).Inc() + } + } +} + +// stopClock folds the current running interval into RunSeconds. +func stopClock(r *store.SandboxRecord, now time.Time) { + if r.RunningSince != nil { + r.RunSeconds += now.Sub(*r.RunningSince).Seconds() + r.RunningSince = nil + } +} + +// StopClock is exported for lifecycle handlers. +func StopClock(r *store.SandboxRecord, now time.Time) { stopClock(r, now) } + +type errString string + +func (e errString) Error() string { return string(e) } + +func itoa(n int) string { + if n == 0 { + return "0" + } + var b []byte + for n > 0 { + b = append([]byte{byte('0' + n%10)}, b...) + n /= 10 + } + return string(b) +} diff --git a/internal/server/admin.go b/internal/server/admin.go new file mode 100644 index 0000000..a70b5af --- /dev/null +++ b/internal/server/admin.go @@ -0,0 +1,495 @@ +package server + +import ( + "net/http" + "slices" + "strings" + "time" + + "github.com/different-ai/differently/internal/audit" + "github.com/different-ai/differently/internal/auth" + "github.com/different-ai/differently/internal/config" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/internal/version" + "github.com/different-ai/differently/pkg/api" +) + +func (s *Server) adminRoutes(authed func(string, handlerFunc)) { + admin := func(pattern string, h handlerFunc) { + authed(pattern, func(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if !p.Operator() { + return api.Errorf(api.CodePermissionDenied, "operator (admin) key required") + } + return h(w, r, p) + }) + } + admin("GET /v1/admin/organizations", s.listOrgs) + admin("POST /v1/admin/organizations", s.createOrg) + admin("GET /v1/admin/organizations/{id}", s.getOrg) + admin("PATCH /v1/admin/organizations/{id}", s.updateOrg) + admin("POST /v1/admin/organizations/{id}/suspend", s.suspendOrg(true)) + admin("POST /v1/admin/organizations/{id}/resume", s.suspendOrg(false)) + admin("PATCH /v1/admin/projects/{id}", s.updateProjectQuota) + admin("GET /v1/admin/pools", s.listPools) + admin("POST /v1/admin/pools", s.createPool) + admin("GET /v1/admin/pools/{name}", s.getPool) + admin("PATCH /v1/admin/pools/{name}", s.updatePool) + admin("DELETE /v1/admin/pools/{name}", s.deletePool) + admin("POST /v1/admin/pools/{name}/drain", s.drainPool(true)) + admin("POST /v1/admin/pools/{name}/undrain", s.drainPool(false)) + admin("GET /v1/admin/agents", s.listAgents) + admin("POST /v1/admin/sandboxes/{id}/terminate", s.forceTerminate) + admin("GET /v1/admin/trust-report", s.trustReport) + admin("GET /v1/admin/audit/verify", s.verifyAudit) + admin("POST /v1/admin/reconcile", func(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + s.Scheduler.Reconcile(r.Context()) + s.audit(r, p, "admin.reconcile", "installation", "", "", nil) + writeJSON(w, 200, map[string]bool{"ok": true}) + return nil + }) +} + +func (s *Server) listOrgs(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + list, err := s.Store.ListOrgs(r.Context()) + if err != nil { + return err + } + writeJSON(w, 200, api.List[api.Organization]{Items: append([]api.Organization{}, list...)}) + return nil +} + +func (s *Server) createOrg(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req struct { + Name string `json:"name"` + Quota *api.Quota `json:"quota,omitempty"` + DefaultProject string `json:"defaultProject,omitempty"` + } + if err := decode(r, &req); err != nil { + return err + } + if strings.TrimSpace(req.Name) == "" { + return api.Errorf(api.CodeInvalidRequest, "name is required") + } + org := api.Organization{ID: id.New("org"), Name: req.Name, Quota: s.Config.Defaults.OrgQuota, CreatedAt: s.now()} + if req.Quota != nil { + org.Quota = *req.Quota + } + if err := s.Store.CreateOrg(r.Context(), org); err != nil { + return err + } + s.audit(r, p, "org.create", org.ID, org.ID, "", map[string]string{"name": org.Name}) + resp := map[string]any{"organization": org} + if req.DefaultProject != "" { + proj := api.Project{ID: id.New("prj"), OrgID: org.ID, Name: req.DefaultProject, Quota: s.Config.Defaults.ProjectQuota, CreatedAt: s.now()} + if err := s.Store.CreateProject(r.Context(), proj); err != nil { + return err + } + s.audit(r, p, "project.create", proj.ID, org.ID, proj.ID, map[string]string{"name": proj.Name}) + resp["project"] = proj + } + writeJSON(w, http.StatusCreated, resp) + return nil +} + +func (s *Server) getOrg(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + org, err := s.Store.GetOrg(r.Context(), r.PathValue("id")) + if err != nil { + return err + } + writeJSON(w, 200, org) + return nil +} + +func (s *Server) updateOrg(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req struct { + Name string `json:"name,omitempty"` + Quota *api.Quota `json:"quota,omitempty"` + } + if err := decode(r, &req); err != nil { + return err + } + org, err := s.Store.UpdateOrg(r.Context(), r.PathValue("id"), func(o *api.Organization) error { + if req.Name != "" { + o.Name = req.Name + } + if req.Quota != nil { + o.Quota = *req.Quota + } + return nil + }) + if err != nil { + return err + } + s.audit(r, p, "org.update", org.ID, org.ID, "", nil) + writeJSON(w, 200, org) + return nil +} + +// suspendOrg blocks new work for a tenant. With terminate=true in the body, +// its running sandboxes are terminated too. +func (s *Server) suspendOrg(suspend bool) handlerFunc { + return func(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req struct { + Terminate bool `json:"terminate,omitempty"` + Reason string `json:"reason,omitempty"` + } + if err := decode(r, &req); err != nil { + return err + } + org, err := s.Store.UpdateOrg(r.Context(), r.PathValue("id"), func(o *api.Organization) error { o.Suspended = suspend; return nil }) + if err != nil { + return err + } + terminated := 0 + if suspend && req.Terminate { + recs, _ := s.Store.ListSandboxes(r.Context(), store.SandboxFilter{OrgID: org.ID}) + for _, rec := range recs { + if !rec.Status.Terminal() { + if _, err := s.markTerminating(r.Context(), rec.ID, "organization suspended: "+req.Reason); err == nil { + terminated++ + } + } + } + } + action := "org.resume" + if suspend { + action = "org.suspend" + } + s.audit(r, p, action, org.ID, org.ID, "", map[string]string{"reason": req.Reason, "terminated": itoa64(int64(terminated))}) + writeJSON(w, 200, map[string]any{"organization": org, "terminated": terminated}) + return nil + } +} + +func (s *Server) updateProjectQuota(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req struct { + Quota api.Quota `json:"quota"` + } + if err := decode(r, &req); err != nil { + return err + } + proj, err := s.Store.UpdateProject(r.Context(), r.PathValue("id"), func(pr *api.Project) error { pr.Quota = req.Quota; return nil }) + if err != nil { + return err + } + s.audit(r, p, "project.quota_update", proj.ID, proj.OrgID, proj.ID, nil) + writeJSON(w, 200, proj) + return nil +} + +func (s *Server) decoratedPools(r *http.Request) ([]api.Pool, error) { + list, err := s.Store.ListPools(r.Context()) + if err != nil { + return nil, err + } + active, _ := s.Store.PoolActive(r.Context()) + for i := range list { + s.Pools.Decorate(&list[i], active[list[i].Name]) + } + return list, nil +} + +func (s *Server) listPools(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + list, err := s.decoratedPools(r) + if err != nil { + return err + } + writeJSON(w, 200, api.List[api.Pool]{Items: append([]api.Pool{}, list...)}) + return nil +} + +func (s *Server) getPool(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + pool, err := s.Store.GetPool(r.Context(), r.PathValue("name")) + if err != nil { + return err + } + active, _ := s.Store.PoolActive(r.Context()) + s.Pools.Decorate(&pool, active[pool.Name]) + writeJSON(w, 200, pool) + return nil +} + +// createPool registers a pool at runtime. Credentials are references to +// environment variables or mounted files on the server, never values, so +// secrets stay in the operator's secret store and out of the database. +func (s *Server) createPool(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var pc config.PoolConfig + if err := decode(r, &pc); err != nil { + return err + } + if _, err := s.Store.GetPool(r.Context(), pc.Name); err == nil { + return api.Errorf(api.CodeConflict, "pool %s exists", pc.Name) + } + cfg := s.Config + cfg.Pools = []config.PoolConfig{pc} + cfg.Policy.ExternalPools = nil + parsed, err := validatePool(cfg) + if err != nil { + return err + } + pool := parsed.ToPool() + pool.Source = "api" + if err := s.Store.UpsertPool(r.Context(), pool); err != nil { + return err + } + if err := s.SyncPools(r.Context()); err != nil { + return err + } + s.audit(r, p, "pool.create", pool.Name, "", "", map[string]string{"adapter": pool.Adapter, "trust": pool.Trust, "credentialRef": pool.CredentialRef}) + writeJSON(w, http.StatusCreated, pool) + return nil +} + +func validatePool(cfg config.Config) (config.PoolConfig, error) { + b, _ := jsonMarshal(cfg) + parsed, err := config.Parse(b) + if err != nil { + return config.PoolConfig{}, api.Errorf(api.CodeInvalidRequest, "%v", err) + } + return parsed.Pools[0], nil +} + +func (s *Server) updatePool(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req struct { + Enabled *bool `json:"enabled,omitempty"` + MaxConcurrency *int `json:"maxConcurrency,omitempty"` + CostModel *api.CostModel `json:"costModel,omitempty"` + Region *string `json:"region,omitempty"` + } + if err := decode(r, &req); err != nil { + return err + } + pool, err := s.Store.GetPool(r.Context(), r.PathValue("name")) + if err != nil { + return err + } + if req.Enabled != nil { + pool.Enabled = *req.Enabled + } + if req.MaxConcurrency != nil { + pool.MaxConcurrency = *req.MaxConcurrency + } + if req.CostModel != nil { + pool.CostModel = *req.CostModel + } + if req.Region != nil { + pool.Region = *req.Region + } + if err := s.Store.UpsertPool(r.Context(), pool); err != nil { + return err + } + _ = s.SyncPools(r.Context()) + s.audit(r, p, "pool.update", pool.Name, "", "", nil) + writeJSON(w, 200, pool) + return nil +} + +func (s *Server) deletePool(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + name := r.PathValue("name") + if slices.ContainsFunc(s.Config.Pools, func(pc config.PoolConfig) bool { return pc.Name == name }) { + return api.Errorf(api.CodeConflict, "pool %s is declared in the config file; remove it there", name) + } + if err := s.Store.DeletePool(r.Context(), name); err != nil { + return err + } + _ = s.SyncPools(r.Context()) + s.audit(r, p, "pool.delete", name, "", "", nil) + w.WriteHeader(http.StatusNoContent) + return nil +} + +// drainPool stops new placements on a pool; running sandboxes continue. +func (s *Server) drainPool(drain bool) handlerFunc { + return func(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + pool, err := s.Store.GetPool(r.Context(), r.PathValue("name")) + if err != nil { + return err + } + pool.Draining = drain + if err := s.Store.UpsertPool(r.Context(), pool); err != nil { + return err + } + _ = s.SyncPools(r.Context()) + action := "pool.undrain" + if drain { + action = "pool.drain" + } + s.audit(r, p, action, pool.Name, "", "", nil) + writeJSON(w, 200, pool) + return nil + } +} + +func (s *Server) listAgents(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + list, err := s.Store.ListAgents(r.Context(), r.URL.Query().Get("pool")) + if err != nil { + return err + } + cutoff := s.now().Add(-time.Duration(s.Config.Scheduler.AgentTimeoutSeconds) * time.Second) + for i := range list { + list[i].Healthy = list[i].LastSeen.After(cutoff) + } + writeJSON(w, 200, api.List[api.Agent]{Items: append([]api.Agent{}, list...)}) + return nil +} + +func (s *Server) forceTerminate(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req struct { + Reason string `json:"reason"` + } + if err := decode(r, &req); err != nil { + return err + } + rec, err := s.Store.GetSandbox(r.Context(), r.PathValue("id")) + if err != nil { + return err + } + out, err := s.markTerminating(r.Context(), rec.ID, "terminated by operator: "+req.Reason) + if err != nil { + return err + } + s.audit(r, p, "admin.sandbox_terminate", rec.ID, rec.OrgID, rec.ProjectID, map[string]string{"reason": req.Reason}) + writeJSON(w, http.StatusAccepted, out.Sandbox) + return nil +} + +func (s *Server) verifyAudit(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + events, err := s.Store.ListAudit(r.Context(), store.AuditFilter{}) + if err != nil { + return err + } + writeJSON(w, 200, audit.Verify(events)) + return nil +} + +// TrustReport is what an operator inspects to see where workloads can go, +// where they went and what Differently cannot prove. +type TrustReport struct { + GeneratedAt time.Time `json:"generatedAt"` + Installation string `json:"installation"` + Version string `json:"version"` + Telemetry string `json:"telemetry"` + ControlPlane map[string]string `json:"controlPlane"` + Pools []TrustPool `json:"pools"` + GlobalPolicy api.PlacementPolicy `json:"globalPolicy"` + ScopedPolicies []api.PlacementPolicy `json:"scopedPolicies"` + Classifications []api.Classification `json:"classifications"` + Placements []api.UsageLine `json:"placementsByPool"` + ExternalRecent []TrustPlacement `json:"recentExternalPlacements"` + Agents []api.Agent `json:"agents"` + AdminActions []api.AuditEvent `json:"recentAdminActions"` + AuditChain audit.VerifyResult `json:"auditChain"` + Limitations []string `json:"limitations"` +} + +// TrustPool summarizes a pool's trust boundary. +type TrustPool struct { + Name string `json:"name"` + Adapter string `json:"adapter"` + Trust string `json:"trust"` + Enabled bool `json:"enabled"` + Draining bool `json:"draining"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + CredentialRef string `json:"credentialRef,omitempty"` + ApprovedExternal bool `json:"approvedForExternalPlacement"` + Health string `json:"health"` +} + +// TrustPlacement is one sandbox placed outside private pools. +type TrustPlacement struct { + SandboxID string `json:"sandboxId"` + OrgID string `json:"orgId"` + Pool string `json:"pool"` + Classification string `json:"classification"` + CreatedBy string `json:"createdBy"` + PlacedAt time.Time `json:"placedAt"` + Reason string `json:"reason"` +} + +// Limitations are the residual risks Differently does not control. +var Limitations = []string{ + "Host administrators (root on KVM nodes, Kubernetes cluster-admins) can read guest memory and disks; Differently does not protect workloads from the infrastructure operator.", + "Cloud providers hosting your nodes can access their hardware; private pools are only as private as the infrastructure under them.", + "External providers (trust=external pools) run the workload and can see its code, data and environment variables.", + "The audit chain is tamper-evident, not tamper-proof: anyone with database write access can rewrite it. Export events (audit.file) to write-once storage to anchor it.", + "Actions taken directly on backends (kubectl, provider consoles, SSH to nodes) bypass Differently and are not in its audit log.", + "Egress from Firecracker sandboxes is controlled only by node networking you configure; Differently does not inspect traffic.", + "Exec commands are audited by program name unless audit.recordCommands is enabled; command arguments and outputs are not retained.", + "Usage costs are estimates from configured cost models, not reconciled provider invoices.", + "Firecracker microVMs do not run under the jailer yet; they rely on Firecracker's built-in seccomp filters and the agent's container isolation.", + "The control plane reaches node agents over plain HTTP authenticated by one shared agent token; a compromised node can impersonate other agents until mTLS lands.", + "Sandbox environment variables are stored unencrypted in the control-plane database; protect it with storage encryption and access controls.", + "Audit logging is fail-open: if an audit append fails the action still proceeds and the failure is only logged.", + "Sandbox egress rules cover IPv4 only.", +} + +func (s *Server) trustReport(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + ctx := r.Context() + rep := TrustReport{ + GeneratedAt: s.now(), Installation: s.Installation, Version: version.Version, + Telemetry: "none: Differently has no telemetry or phone-home code paths", + ControlPlane: map[string]string{"store": s.Config.Store.Driver, "publicURL": s.Config.PublicURL, "auditMirror": s.Config.Audit.File, "dev": itoa64(map[bool]int64{true: 1}[s.Config.Dev])}, + Classifications: s.Config.Classifications, Limitations: Limitations, + } + rep.GlobalPolicy = s.Config.Policy + policies, _ := s.Store.ListPolicies(ctx) + for _, pol := range policies { + if pol.Scope == "global" { + rep.GlobalPolicy = pol + } else { + rep.ScopedPolicies = append(rep.ScopedPolicies, pol) + } + } + pools, err := s.decoratedPools(r) + if err != nil { + return err + } + endpoints := s.Pools.Endpoints() + for _, pl := range pools { + rep.Pools = append(rep.Pools, TrustPool{ + Name: pl.Name, Adapter: pl.Adapter, Trust: pl.Trust, Enabled: pl.Enabled, Draining: pl.Draining, Region: pl.Region, + Endpoint: endpoints[pl.Name], CredentialRef: pl.CredentialRef, Health: pl.Health.Status, + ApprovedExternal: pl.Trust == api.TrustExternal && rep.GlobalPolicy.External != api.ExternalNever && slices.Contains(rep.GlobalPolicy.ExternalPools, pl.Name), + }) + } + recs, _ := s.Store.ListSandboxes(ctx, store.SandboxFilter{}) + byPool := map[string]*api.UsageLine{} + for _, rec := range recs { + if rec.Pool == "" { + continue + } + l, ok := byPool[rec.Pool] + if !ok { + l = &api.UsageLine{Key: rec.Pool} + byPool[rec.Pool] = l + } + l.Sandboxes++ + l.RunSeconds += rec.RunSeconds + if rec.Trust == api.TrustExternal && len(rep.ExternalRecent) < 50 { + placed := rec.CreatedAt + reason := "" + if rec.Decision != nil { + placed, reason = rec.Decision.EvaluatedAt, rec.Decision.Reason + } + rep.ExternalRecent = append(rep.ExternalRecent, TrustPlacement{SandboxID: rec.ID, OrgID: rec.OrgID, Pool: rec.Pool, Classification: rec.Classification, CreatedBy: rec.CreatedBy, PlacedAt: placed, Reason: reason}) + } + } + for _, l := range byPool { + rep.Placements = append(rep.Placements, *l) + } + slices.SortFunc(rep.Placements, func(a, b api.UsageLine) int { return strings.Compare(a.Key, b.Key) }) + rep.Agents, _ = s.Store.ListAgents(ctx, "") + events, _ := s.Store.ListAudit(ctx, store.AuditFilter{}) + rep.AuditChain = audit.Verify(events) + for i := len(events) - 1; i >= 0 && len(rep.AdminActions) < 50; i-- { + ev := events[i] + if strings.HasPrefix(ev.Action, "admin.") || strings.HasPrefix(ev.Action, "pool.") || strings.HasPrefix(ev.Action, "policy.") || + strings.HasPrefix(ev.Action, "org.") || ev.Action == "key.create" || ev.Action == "key.revoke" || ev.Action == "key.rotate" { + rep.AdminActions = append(rep.AdminActions, ev) + } + } + writeJSON(w, 200, rep) + return nil +} diff --git a/internal/server/routes_test.go b/internal/server/routes_test.go new file mode 100644 index 0000000..0092ab5 --- /dev/null +++ b/internal/server/routes_test.go @@ -0,0 +1,40 @@ +package server_test + +import ( + "os" + "regexp" + "strings" + "testing" + + "sigs.k8s.io/yaml" +) + +// TestOpenAPICoversRoutes keeps api/openapi.yaml in step with the router. +func TestOpenAPICoversRoutes(t *testing.T) { + spec, err := os.ReadFile("../../api/openapi.yaml") + if err != nil { + t.Fatal(err) + } + var doc map[string]any + if err := yaml.Unmarshal(spec, &doc); err != nil || doc["openapi"] != "3.1.0" { + t.Fatalf("api/openapi.yaml does not parse as OpenAPI 3.1: %v", err) + } + var src strings.Builder + for _, f := range []string{"server.go", "sandboxes.go", "tenancy.go", "admin.go"} { + b, _ := os.ReadFile(f) + src.Write(b) + } + routes := regexp.MustCompile(`"(GET|POST|PUT|PATCH|DELETE) (/[^"{]*(?:\{[a-z]+\}[^"{]*)*)"`).FindAllStringSubmatch(src.String(), -1) + if len(routes) < 50 { + t.Fatalf("found only %d routes; regexp drifted", len(routes)) + } + for _, r := range routes { + path := strings.TrimSuffix(r[2], "{$}") + if path == "/" || strings.HasPrefix(path, "/dashboard") || path == "/openapi.yaml" { + continue + } + if !strings.Contains(string(spec), "\n "+path+":") { + t.Errorf("route %s %s is missing from api/openapi.yaml", r[1], path) + } + } +} diff --git a/internal/server/sandboxes.go b/internal/server/sandboxes.go new file mode 100644 index 0000000..5404725 --- /dev/null +++ b/internal/server/sandboxes.go @@ -0,0 +1,880 @@ +package server + +import ( + "context" + "fmt" + "io" + "net/http" + "slices" + "strings" + "time" + + "github.com/different-ai/differently/internal/auth" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/policy" + "github.com/different-ai/differently/internal/scheduler" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// KnownFeatures are the capability names a request may ask for. +var KnownFeatures = []string{ + adapter.FeatureExec, adapter.FeatureExecStream, adapter.FeatureFiles, adapter.FeatureStopStart, adapter.FeaturePauseResume, + adapter.FeatureSnapshotDisk, adapter.FeatureSnapshotMem, adapter.FeatureFork, adapter.FeaturePersistDisk, adapter.FeaturePorts, + adapter.FeatureSSH, adapter.FeaturePTY, adapter.FeatureEgressControl, adapter.FeatureGPU, adapter.FeatureArchAMD64, + adapter.FeatureArchARM64, adapter.FeatureMicroVM, adapter.FeatureContainer, +} + +func (s *Server) sandboxRoutes(authed func(string, handlerFunc)) { + authed("POST /v1/sandboxes", s.createSandbox) + authed("GET /v1/sandboxes", s.listSandboxes) + authed("GET /v1/sandboxes/{id}", s.getSandbox) + authed("PATCH /v1/sandboxes/{id}", s.updateSandbox) + authed("DELETE /v1/sandboxes/{id}", s.deleteSandbox) + authed("GET /v1/sandboxes/{id}/wait", s.waitSandbox) + authed("GET /v1/sandboxes/{id}/placement", s.explainPlacement) + authed("POST /v1/sandboxes/{id}/exec", s.exec) + authed("GET /v1/sandboxes/{id}/files", s.listFiles) + authed("GET /v1/sandboxes/{id}/files/content", s.readFile) + authed("PUT /v1/sandboxes/{id}/files/content", s.writeFile) + authed("POST /v1/sandboxes/{id}/stop", s.lifecycle("stop")) + authed("POST /v1/sandboxes/{id}/start", s.lifecycle("start")) + authed("POST /v1/sandboxes/{id}/pause", s.lifecycle("pause")) + authed("POST /v1/sandboxes/{id}/resume", s.lifecycle("resume")) + authed("POST /v1/sandboxes/{id}/snapshots", s.createSnapshot) + authed("POST /v1/sandboxes/{id}/fork", s.forkSandbox) + authed("POST /v1/sandboxes/{id}/ports", s.exposePort) + authed("GET /v1/snapshots", s.listSnapshots) + authed("GET /v1/snapshots/{id}", s.getSnapshot) + authed("DELETE /v1/snapshots/{id}", s.deleteSnapshot) +} + +// resolveProject finds the project a request targets and checks access. +func (s *Server) resolveProject(ctx context.Context, p auth.Principal, projectID string) (api.Organization, api.Project, error) { + if p.ProjectID != "" { + if projectID != "" && projectID != p.ProjectID { + return api.Organization{}, api.Project{}, api.Errorf(api.CodePermissionDenied, "this key is limited to project %s", p.ProjectID) + } + projectID = p.ProjectID + } + if projectID == "" { + return api.Organization{}, api.Project{}, api.Errorf(api.CodeInvalidRequest, "projectId is required for organization and operator keys") + } + proj, err := s.Store.GetProject(ctx, projectID) + if err != nil || !p.CanAccess(proj.OrgID, proj.ID) { + return api.Organization{}, api.Project{}, api.Errorf(api.CodeNotFound, "project %s not found", projectID) + } + org, err := s.Store.GetOrg(ctx, proj.OrgID) + if err != nil { + return api.Organization{}, api.Project{}, err + } + if org.Suspended { + return org, proj, api.Errorf(api.CodeSuspended, "organization %s is suspended", org.ID) + } + return org, proj, nil +} + +// loadSandbox fetches a sandbox the principal may see. Sandboxes in other +// tenants are reported as not found, never as forbidden. +func (s *Server) loadSandbox(r *http.Request, p auth.Principal) (store.SandboxRecord, error) { + rec, err := s.Store.GetSandbox(r.Context(), r.PathValue("id")) + if err != nil || !p.CanAccess(rec.OrgID, rec.ProjectID) { + return rec, api.Errorf(api.CodeNotFound, "sandbox %s not found", r.PathValue("id")) + } + return rec, nil +} + +func (s *Server) createSandbox(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + var req api.CreateSandboxRequest + if err := decode(r, &req); err != nil { + return err + } + rec, err := s.buildSandbox(r.Context(), p, req) + if err != nil { + return err + } + rec.IdempotencyKey = r.Header.Get("Idempotency-Key") + return s.admit(w, r, p, rec) +} + +func (s *Server) admit(w http.ResponseWriter, r *http.Request, p auth.Principal, rec store.SandboxRecord) error { + ctx := r.Context() + org, err := s.Store.GetOrg(ctx, rec.OrgID) + if err != nil { + return err + } + proj, err := s.Store.GetProject(ctx, rec.ProjectID) + if err != nil { + return err + } + // Fail fast with a precise error when no pool could ever take this. + active, _ := s.Store.PoolActive(ctx) + if res := policy.Decide(s.Scheduler.PlacementInput(ctx, rec, active)); res.Err != nil { + return res.Err + } + out, existed, err := s.Store.AdmitSandbox(ctx, rec, store.Admission{OrgQuota: org.Quota, ProjectQuota: proj.Quota}) + if err != nil { + return err + } + if existed { + writeJSON(w, http.StatusOK, out.Sandbox) + return nil + } + s.audit(r, p, "sandbox.create", out.ID, out.OrgID, out.ProjectID, map[string]string{ + "classification": out.Classification, "cpu": fmt.Sprint(out.Resources.CPU), "memoryMiB": fmt.Sprint(out.Resources.MemoryMiB), "template": out.Template, + }) + s.Scheduler.Wake() + writeJSON(w, http.StatusAccepted, out.Sandbox) + return nil +} + +func (s *Server) buildSandbox(ctx context.Context, p auth.Principal, req api.CreateSandboxRequest) (store.SandboxRecord, error) { + org, proj, err := s.resolveProject(ctx, p, req.ProjectID) + if err != nil { + return store.SandboxRecord{}, err + } + profileName := req.Profile + if profileName == "" && req.Resources == nil { + profileName = s.Config.Defaults.Profile + } + resources := s.Config.Defaults.Resources + if def, ok := s.Config.Profile(s.Config.Defaults.Profile); ok && resources == (api.Resources{}) { + resources = def.Resources // explicit partial resources start from the default profile + } + template := s.Config.Defaults.Template + caps := slices.Clone(req.Capabilities) + if profileName != "" { + prof, ok := s.Config.Profile(profileName) + if !ok { + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "unknown profile %q", profileName) + } + resources = prof.Resources + if prof.Template != "" { + template = prof.Template + } + for _, c := range prof.Capabilities { + if !slices.Contains(caps, c) { + caps = append(caps, c) + } + } + } + if req.Resources != nil { + if req.Resources.CPU > 0 { + resources.CPU = req.Resources.CPU + } + if req.Resources.MemoryMiB > 0 { + resources.MemoryMiB = req.Resources.MemoryMiB + } + if req.Resources.DiskGiB > 0 { + resources.DiskGiB = req.Resources.DiskGiB + } + } + if req.Template != "" { + template = req.Template + } + switch { + case resources.CPU <= 0 || resources.CPU > 256: + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "resources.cpu must be between 1 and 256") + case resources.MemoryMiB < 128 || resources.MemoryMiB > 1<<20: + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "resources.memoryMiB must be between 128 and 1048576") + case resources.DiskGiB < 0 || resources.DiskGiB > 16384: + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "resources.diskGiB must be between 0 and 16384") + } + for _, c := range caps { + if !slices.Contains(KnownFeatures, c) { + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "unknown capability %q", c) + } + } + class := req.Classification + if class == "" { + class = s.Config.Defaults.Classification + } + if _, ok := s.Config.Classification(class); !ok { + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "unknown classification %q", class) + } + if pl := req.Placement; pl != nil { + if pl.Strategy != "" && !policy.ValidStrategy(pl.Strategy) { + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "unknown placement strategy %q", pl.Strategy) + } + if pl.External != "" && !policy.ValidExternal(pl.External) { + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "placement.external must be never, overflow or allowed") + } + } + ttl := req.TTLSeconds + if ttl == 0 { + ttl = s.Config.Defaults.TTLSeconds + } + for k := range req.Labels { + if strings.HasPrefix(k, "differently.dev/") { + return store.SandboxRecord{}, api.Errorf(api.CodeInvalidRequest, "label prefix differently.dev/ is reserved") + } + } + now := s.now() + rec := store.SandboxRecord{ + Sandbox: api.Sandbox{ + ID: id.New("sbx"), OrgID: org.ID, ProjectID: proj.ID, Status: api.StatusQueued, Template: template, Profile: profileName, + Resources: resources, Capabilities: caps, Classification: class, Placement: req.Placement, Labels: req.Labels, + Priority: req.Priority, TTLSeconds: ttl, CreatedBy: p.Actor(), CreatedAt: now, UpdatedAt: now, FromSnapshot: req.FromSnapshot, + }, + Env: req.Env, NextAttemptAt: now, + } + for k := range req.Env { + rec.EnvKeys = append(rec.EnvKeys, k) + } + slices.Sort(rec.EnvKeys) + if req.FromSnapshot != "" { + snap, err := s.Store.GetSnapshot(ctx, req.FromSnapshot) + if err != nil || snap.OrgID != org.ID || !p.CanAccess(snap.OrgID, snap.ProjectID) { + return rec, api.Errorf(api.CodeNotFound, "snapshot %s not found", req.FromSnapshot) + } + rec.SnapshotBackendRef = snap.BackendRef + } + return rec, nil +} + +func (s *Server) listSandboxes(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesRead); err != nil { + return err + } + q := r.URL.Query() + f := store.SandboxFilter{OrgID: p.OrgID, ProjectID: p.ProjectID, Cursor: q.Get("cursor"), Limit: min(max(queryInt(r, "limit", 50), 1), 500), Pool: q.Get("pool")} + if p.Operator() && q.Get("orgId") != "" { + f.OrgID = q.Get("orgId") + } + if v := q.Get("projectId"); v != "" && (p.ProjectID == "" || v == p.ProjectID) { + f.ProjectID = v + } + if v := q.Get("status"); v != "" { + for _, st := range strings.Split(v, ",") { + f.Statuses = append(f.Statuses, api.SandboxStatus(st)) + } + } + for _, l := range q["label"] { + if k, v, ok := strings.Cut(l, "="); ok { + if f.Label == nil { + f.Label = map[string]string{} + } + f.Label[k] = v + } + } + recs, err := s.Store.ListSandboxes(r.Context(), f) + if err != nil { + return err + } + out := api.List[api.Sandbox]{Items: make([]api.Sandbox, 0, len(recs))} + for _, rec := range recs { + out.Items = append(out.Items, rec.Sandbox) + } + if len(recs) == f.Limit { + out.NextCursor = recs[len(recs)-1].ID + } + writeJSON(w, 200, out) + return nil +} + +func (s *Server) getSandbox(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesRead); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + writeJSON(w, 200, rec.Sandbox) + return nil +} + +func (s *Server) updateSandbox(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + var req api.UpdateSandboxRequest + if err := decode(r, &req); err != nil { + return err + } + org, _ := s.Store.GetOrg(r.Context(), rec.OrgID) + proj, _ := s.Store.GetProject(r.Context(), rec.ProjectID) + if req.TTLSeconds != nil { + for _, q := range []api.Quota{org.Quota, proj.Quota} { + if q.MaxTTLSeconds > 0 && *req.TTLSeconds > q.MaxTTLSeconds { + return api.Errorf(api.CodeQuotaExceeded, "ttl exceeds limit %ds", q.MaxTTLSeconds).With("dimension", "ttlSeconds") + } + } + } + out, err := s.Store.UpdateSandbox(r.Context(), rec.ID, func(x *store.SandboxRecord) error { + if x.Status.Terminal() { + return api.Errorf(api.CodeInvalidState, "sandbox is %s", x.Status) + } + for k, v := range req.Labels { + if strings.HasPrefix(k, "differently.dev/") { + return api.Errorf(api.CodeInvalidRequest, "label prefix differently.dev/ is reserved") + } + if x.Labels == nil { + x.Labels = map[string]string{} + } + if v == "" { + delete(x.Labels, k) + } else { + x.Labels[k] = v + } + } + if req.TTLSeconds != nil { + x.TTLSeconds = *req.TTLSeconds + if x.StartedAt != nil { + exp := s.now().Add(time.Duration(*req.TTLSeconds) * time.Second) + x.ExpiresAt = &exp + } + } + x.UpdatedAt = s.now() + return nil + }) + if err != nil { + return err + } + s.audit(r, p, "sandbox.update", out.ID, out.OrgID, out.ProjectID, nil) + writeJSON(w, 200, out.Sandbox) + return nil +} + +func (s *Server) deleteSandbox(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + out, err := s.markTerminating(r.Context(), rec.ID, "deleted by "+p.Actor()) + if err != nil { + return err + } + s.audit(r, p, "sandbox.delete", out.ID, out.OrgID, out.ProjectID, nil) + writeJSON(w, http.StatusAccepted, out.Sandbox) + return nil +} + +// markTerminating requests deletion. Queued sandboxes have no backend +// instance and terminate immediately; others are torn down by the scheduler. +func (s *Server) markTerminating(ctx context.Context, sid, reason string) (store.SandboxRecord, error) { + out, err := s.Store.UpdateSandbox(ctx, sid, func(x *store.SandboxRecord) error { + now := s.now() + switch { + case x.Status.Terminal(): + return nil + case x.Status == api.StatusQueued: + x.Status, x.EndedAt = api.StatusTerminated, &now + default: + x.Status = api.StatusTerminating + } + x.StatusReason, x.UpdatedAt = reason, now + return nil + }) + s.Scheduler.Wake() + return out, err +} + +func (s *Server) waitSandbox(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesRead); err != nil { + return err + } + want := api.SandboxStatus(r.URL.Query().Get("status")) + if want == "" { + want = api.StatusRunning + } + timeout := time.Duration(min(max(queryInt(r, "timeoutSeconds", 60), 1), 300)) * time.Second + deadline := time.Now().Add(timeout) + for { + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + if rec.Status == want || rec.Status.Terminal() || time.Now().After(deadline) { + writeJSON(w, 200, rec.Sandbox) + return nil + } + select { + case <-r.Context().Done(): + return r.Context().Err() + case <-time.After(100 * time.Millisecond): + } + } +} + +func (s *Server) explainPlacement(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesRead); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + out := map[string]any{"sandboxId": rec.ID, "status": rec.Status, "pool": rec.Pool, "trust": rec.Trust, "decision": rec.Decision} + if rec.Status == api.StatusQueued { + active, _ := s.Store.PoolActive(r.Context()) + out["current"] = policy.Decide(s.Scheduler.PlacementInput(r.Context(), rec, active)).Decision + } + writeJSON(w, 200, out) + return nil +} + +// running returns the sandbox's adapter after checking it is running. +func (s *Server) running(r *http.Request, p auth.Principal, scope string) (store.SandboxRecord, adapter.ExecutionAdapter, error) { + if err := p.Require(scope); err != nil { + return store.SandboxRecord{}, nil, err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return rec, nil, err + } + if rec.Status != api.StatusRunning { + return rec, nil, api.Errorf(api.CodeInvalidState, "sandbox %s is %s, not running", rec.ID, rec.Status).With("status", string(rec.Status)) + } + a, err := s.Pools.Adapter(rec.Pool) + return rec, a, err +} + +func unsupported(rec store.SandboxRecord, op string) error { + return api.Errorf(api.CodeUnsupportedOperation, "the %s backend of pool %s does not support %s", rec.Adapter, rec.Pool, op). + With("operation", op).With("pool", rec.Pool).With("adapter", rec.Adapter) +} + +func (s *Server) exec(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + rec, a, err := s.running(r, p, auth.ScopeSandboxesExec) + if err != nil { + return err + } + var req api.ExecRequest + if err := decode(r, &req); err != nil { + return err + } + if req.Command == "" && len(req.Args) == 0 { + return api.Errorf(api.CodeInvalidRequest, "command or args is required") + } + program := req.Command + if len(req.Args) > 0 { + program = req.Args[0] + } + detail := map[string]string{"program": strings.Fields(program + " ")[0]} + if s.Config.Audit.RecordCommands { + detail["command"] = truncate(req.Command+strings.Join(req.Args, " "), 512) + } + s.audit(r, p, "sandbox.exec", rec.ID, rec.OrgID, rec.ProjectID, detail) + if r.URL.Query().Get("stream") != "" || strings.Contains(r.Header.Get("Accept"), "text/event-stream") { + return s.streamExec(w, r, rec, a, req) + } + res, err := a.Exec(r.Context(), rec.BackendRef, req) + if err != nil { + return err + } + writeJSON(w, 200, res) + return nil +} + +// streamExec sends server-sent events: stdout/stderr chunks, then exit. +func (s *Server) streamExec(w http.ResponseWriter, r *http.Request, rec store.SandboxRecord, a adapter.ExecutionAdapter, req api.ExecRequest) error { + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("X-Accel-Buffering", "no") + w.WriteHeader(200) + flusher, _ := w.(http.Flusher) + send := func(event string, ev api.ExecEvent) error { + b, _ := jsonMarshal(ev) + if _, err := fmt.Fprintf(w, "event: %s\ndata: %s\n\n", event, b); err != nil { + return err + } + if flusher != nil { + flusher.Flush() + } + return nil + } + var code int + var err error + if st, ok := a.(adapter.Streamer); ok { + code, err = st.ExecStream(r.Context(), rec.BackendRef, req, func(ev api.ExecEvent) error { return send(ev.Stream, ev) }) + } else { + var res api.ExecResult + res, err = a.Exec(r.Context(), rec.BackendRef, req) + if err == nil { + if res.Stdout != "" { + _ = send("stdout", api.ExecEvent{Stream: "stdout", Data: res.Stdout}) + } + if res.Stderr != "" { + _ = send("stderr", api.ExecEvent{Stream: "stderr", Data: res.Stderr}) + } + code = res.ExitCode + } + } + if err != nil { + return send("error", api.ExecEvent{Error: err.Error()}) + } + return send("exit", api.ExecEvent{ExitCode: &code}) +} + +func (s *Server) listFiles(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + rec, a, err := s.running(r, p, auth.ScopeSandboxesExec) + if err != nil { + return err + } + fs, ok := a.(adapter.FileSystem) + if !ok || !s.Pools.Supports(r.Context(), rec.Pool, adapter.FeatureFiles) { + return unsupported(rec, adapter.FeatureFiles) + } + items, err := fs.ListFiles(r.Context(), rec.BackendRef, r.URL.Query().Get("path")) + if err != nil { + return err + } + if items == nil { + items = []api.FileInfo{} + } + writeJSON(w, 200, api.List[api.FileInfo]{Items: items}) + return nil +} + +func (s *Server) readFile(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + rec, a, err := s.running(r, p, auth.ScopeSandboxesExec) + if err != nil { + return err + } + fs, ok := a.(adapter.FileSystem) + if !ok || !s.Pools.Supports(r.Context(), rec.Pool, adapter.FeatureFiles) { + return unsupported(rec, adapter.FeatureFiles) + } + path := r.URL.Query().Get("path") + data, err := fs.ReadFile(r.Context(), rec.BackendRef, path) + if err != nil { + return err + } + s.audit(r, p, "sandbox.file_read", rec.ID, rec.OrgID, rec.ProjectID, map[string]string{"path": truncate(path, 256)}) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(data) + return nil +} + +func (s *Server) writeFile(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + rec, a, err := s.running(r, p, auth.ScopeSandboxesExec) + if err != nil { + return err + } + fs, ok := a.(adapter.FileSystem) + if !ok || !s.Pools.Supports(r.Context(), rec.Pool, adapter.FeatureFiles) { + return unsupported(rec, adapter.FeatureFiles) + } + path := r.URL.Query().Get("path") + if path == "" { + return api.Errorf(api.CodeInvalidRequest, "path is required") + } + const maxUpload = 100 << 20 + data, err := io.ReadAll(io.LimitReader(r.Body, maxUpload+1)) + if err != nil { + return err + } + if len(data) > maxUpload { + return api.Errorf(api.CodeInvalidRequest, "upload larger than %d bytes", maxUpload) + } + if err := fs.WriteFile(r.Context(), rec.BackendRef, path, data); err != nil { + return err + } + s.audit(r, p, "sandbox.file_write", rec.ID, rec.OrgID, rec.ProjectID, map[string]string{"path": truncate(path, 256), "bytes": fmt.Sprint(len(data))}) + w.WriteHeader(http.StatusNoContent) + return nil +} + +// lifecycle handles stop/start/pause/resume. +func (s *Server) lifecycle(action string) handlerFunc { + type step struct { + from, to api.SandboxStatus + feature string + } + steps := map[string]step{ + "stop": {api.StatusRunning, api.StatusStopped, adapter.FeatureStopStart}, + "start": {api.StatusStopped, api.StatusRunning, adapter.FeatureStopStart}, + "pause": {api.StatusRunning, api.StatusPaused, adapter.FeaturePauseResume}, + "resume": {api.StatusPaused, api.StatusRunning, adapter.FeaturePauseResume}, + } + st := steps[action] + return func(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + if rec.Status == st.to { + writeJSON(w, 200, rec.Sandbox) // idempotent + return nil + } + if rec.Status != st.from { + return api.Errorf(api.CodeInvalidState, "cannot %s a %s sandbox", action, rec.Status).With("status", string(rec.Status)) + } + a, err := s.Pools.Adapter(rec.Pool) + if err != nil { + return err + } + if !s.Pools.Supports(r.Context(), rec.Pool, st.feature) { + return unsupported(rec, st.feature) + } + if action == "start" { + // Starting re-reserves CPU and memory: check quota atomically + // first, and release the reservation if the backend fails. + org, _ := s.Store.GetOrg(r.Context(), rec.OrgID) + proj, _ := s.Store.GetProject(r.Context(), rec.ProjectID) + // The reservation marks it running (not provisioning, which the + // scheduler would pick up) until the backend confirms. + if _, err := s.Store.ReserveCompute(r.Context(), rec.ID, api.StatusStopped, api.StatusRunning, org.Quota, proj.Quota, s.now()); err != nil { + return err + } + st.from = api.StatusRunning + } + switch action { + case "stop", "start": + l, ok := a.(adapter.Lifecycle) + if !ok { + return unsupported(rec, st.feature) + } + if action == "stop" { + err = l.Stop(r.Context(), rec.BackendRef) + } else { + err = l.Start(r.Context(), rec.BackendRef) + } + default: + pz, ok := a.(adapter.Pauser) + if !ok { + return unsupported(rec, st.feature) + } + if action == "pause" { + err = pz.Pause(r.Context(), rec.BackendRef) + } else { + err = pz.Resume(r.Context(), rec.BackendRef) + } + } + if err != nil { + if action == "start" { + _, _ = s.Store.UpdateSandbox(r.Context(), rec.ID, func(x *store.SandboxRecord) error { + if x.Status == api.StatusRunning { + x.Status = api.StatusStopped + } + return nil + }) + } + return err + } + out, err := s.Store.UpdateSandbox(r.Context(), rec.ID, func(x *store.SandboxRecord) error { + if x.Status != st.from { + return api.Errorf(api.CodeConflict, "sandbox changed to %s concurrently", x.Status) + } + now := s.now() + if st.to == api.StatusRunning { + x.RunningSince = &now + } else { + scheduler.StopClock(x, now) + } + x.Status, x.UpdatedAt = st.to, now + return nil + }) + if err != nil { + return err + } + s.audit(r, p, "sandbox."+action, out.ID, out.OrgID, out.ProjectID, nil) + writeJSON(w, 200, out.Sandbox) + return nil + } +} + +func (s *Server) snapshot(r *http.Request, p auth.Principal, rec store.SandboxRecord, kind string) (store.SnapshotRecord, error) { + if kind == "" { + kind = "disk" + } + if kind != "disk" && kind != "memory" { + return store.SnapshotRecord{}, api.Errorf(api.CodeInvalidRequest, "kind must be disk or memory") + } + if rec.Status != api.StatusRunning && rec.Status != api.StatusStopped && rec.Status != api.StatusPaused { + return store.SnapshotRecord{}, api.Errorf(api.CodeInvalidState, "cannot snapshot a %s sandbox", rec.Status) + } + a, err := s.Pools.Adapter(rec.Pool) + if err != nil { + return store.SnapshotRecord{}, err + } + sn, ok := a.(adapter.Snapshotter) + feature := adapter.FeatureSnapshotDisk + if kind == "memory" { + feature = adapter.FeatureSnapshotMem + } + if !ok || !s.Pools.Supports(r.Context(), rec.Pool, feature) { + return store.SnapshotRecord{}, unsupported(rec, feature) + } + org, _ := s.Store.GetOrg(r.Context(), rec.OrgID) + if usage, _ := s.Store.Usage(r.Context(), rec.OrgID, ""); org.Quota.MaxSnapshots > 0 && usage.Snapshots >= org.Quota.MaxSnapshots { + return store.SnapshotRecord{}, api.Errorf(api.CodeQuotaExceeded, "organization snapshot limit %d reached", org.Quota.MaxSnapshots).With("dimension", "snapshots") + } + snap, err := sn.Snapshot(r.Context(), rec.BackendRef, kind) + if err != nil { + return store.SnapshotRecord{}, err + } + out := store.SnapshotRecord{ + Snapshot: api.Snapshot{ + ID: id.New("snp"), SandboxID: rec.ID, OrgID: rec.OrgID, ProjectID: rec.ProjectID, Pool: rec.Pool, + Kind: snap.Kind, Status: "ready", SizeBytes: snap.SizeBytes, CreatedAt: s.now(), CreatedBy: p.Actor(), + }, + BackendRef: snap.Ref, + } + if err := s.Store.CreateSnapshot(r.Context(), out); err != nil { + return out, err + } + s.audit(r, p, "snapshot.create", out.ID, out.OrgID, out.ProjectID, map[string]string{"sandbox": rec.ID, "kind": out.Kind, "pool": rec.Pool}) + return out, nil +} + +func (s *Server) createSnapshot(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + var req api.CreateSnapshotRequest + if err := decode(r, &req); err != nil { + return err + } + snap, err := s.snapshot(r, p, rec, req.Kind) + if err != nil { + return err + } + writeJSON(w, http.StatusCreated, snap.Snapshot) + return nil +} + +// forkSandbox snapshots a sandbox and starts a new one from it on the same +// pool. Snapshots are not portable across backends. +func (s *Server) forkSandbox(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + rec, err := s.loadSandbox(r, p) + if err != nil { + return err + } + kind := "disk" + if s.Pools.Supports(r.Context(), rec.Pool, adapter.FeatureSnapshotMem) && rec.Status == api.StatusRunning { + kind = "memory" + } + snap, err := s.snapshot(r, p, rec, kind) + if err != nil { + return err + } + labels := map[string]string{"forked-from": rec.ID} + for k, v := range rec.Labels { + labels[k] = v + } + child, err := s.buildSandbox(r.Context(), p, api.CreateSandboxRequest{ + ProjectID: rec.ProjectID, Template: rec.Template, Resources: &rec.Resources, Capabilities: rec.Capabilities, + Classification: rec.Classification, Placement: rec.Placement, TTLSeconds: rec.TTLSeconds, Env: rec.Env, Labels: labels, FromSnapshot: snap.ID, + }) + if err != nil { + return err + } + child.Profile = rec.Profile + return s.admit(w, r, p, child) +} + +func (s *Server) exposePort(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + rec, a, err := s.running(r, p, auth.ScopeSandboxesWrite) + if err != nil { + return err + } + var req struct { + Port int `json:"port"` + } + if err := decode(r, &req); err != nil { + return err + } + if req.Port < 1 || req.Port > 65535 { + return api.Errorf(api.CodeInvalidRequest, "port must be 1-65535") + } + pe, ok := a.(adapter.PortExposer) + if !ok || !s.Pools.Supports(r.Context(), rec.Pool, adapter.FeaturePorts) { + return unsupported(rec, adapter.FeaturePorts) + } + u, err := pe.ExposePort(r.Context(), rec.BackendRef, req.Port) + if err != nil { + return err + } + s.audit(r, p, "sandbox.port_expose", rec.ID, rec.OrgID, rec.ProjectID, map[string]string{"port": fmt.Sprint(req.Port), "pool": rec.Pool}) + writeJSON(w, 200, api.PortURL{Port: req.Port, URL: u}) + return nil +} + +func (s *Server) listSnapshots(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesRead); err != nil { + return err + } + recs, err := s.Store.ListSnapshots(r.Context(), p.OrgID, p.ProjectID) + if err != nil { + return err + } + out := api.List[api.Snapshot]{Items: []api.Snapshot{}} + for _, sn := range recs { + out.Items = append(out.Items, sn.Snapshot) + } + writeJSON(w, 200, out) + return nil +} + +func (s *Server) loadSnapshot(r *http.Request, p auth.Principal) (store.SnapshotRecord, error) { + sn, err := s.Store.GetSnapshot(r.Context(), r.PathValue("id")) + if err != nil || !p.CanAccess(sn.OrgID, sn.ProjectID) { + return sn, api.Errorf(api.CodeNotFound, "snapshot %s not found", r.PathValue("id")) + } + return sn, nil +} + +func (s *Server) getSnapshot(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesRead); err != nil { + return err + } + sn, err := s.loadSnapshot(r, p) + if err != nil { + return err + } + writeJSON(w, 200, sn.Snapshot) + return nil +} + +func (s *Server) deleteSnapshot(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeSandboxesWrite); err != nil { + return err + } + sn, err := s.loadSnapshot(r, p) + if err != nil { + return err + } + if a, err := s.Pools.Adapter(sn.Pool); err == nil { + if snr, ok := a.(adapter.Snapshotter); ok { + if err := snr.DeleteSnapshot(r.Context(), sn.BackendRef); err != nil { + return err + } + } + } + if err := s.Store.DeleteSnapshot(r.Context(), sn.ID); err != nil { + return err + } + s.audit(r, p, "snapshot.delete", sn.ID, sn.OrgID, sn.ProjectID, nil) + w.WriteHeader(http.StatusNoContent) + return nil +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..80b79aa --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,374 @@ +// Package server implements the Differently HTTP API, the built-in dashboard +// and the node-agent registration endpoint. +package server + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus/promhttp" + + apispec "github.com/different-ai/differently/api" + "github.com/different-ai/differently/internal/auditlog" + "github.com/different-ai/differently/internal/auth" + "github.com/different-ai/differently/internal/config" + "github.com/different-ai/differently/internal/dashboard" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/metrics" + "github.com/different-ai/differently/internal/pools" + "github.com/different-ai/differently/internal/scheduler" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/internal/version" + "github.com/different-ai/differently/pkg/api" +) + +// Server is the API server. +type Server struct { + Config config.Config + Store store.Store + Pools *pools.Manager + Scheduler *scheduler.Scheduler + Audit *auditlog.Recorder + Log *slog.Logger + Installation string + AgentToken string + + limiter *limiter + now func() time.Time +} + +// New wires a server. +func New(cfg config.Config, st store.Store, pm *pools.Manager, sch *scheduler.Scheduler, rec *auditlog.Recorder, log *slog.Logger, installation, agentToken string) *Server { + return &Server{ + Config: cfg, Store: st, Pools: pm, Scheduler: sch, Audit: rec, Log: log, Installation: installation, AgentToken: agentToken, + limiter: newLimiter(cfg.RateLimit.RequestsPerSecond, cfg.RateLimit.Burst), + now: func() time.Time { return time.Now().UTC() }, + } +} + +// Bootstrap creates the operator key from the bootstrap token and loads +// config-declared pools. In dev mode it also creates a default organization +// and project and writes a local operator token file. +func (s *Server) Bootstrap(ctx context.Context) error { + token := os.Getenv(s.Config.Auth.BootstrapTokenEnv) + if token == "" && s.Config.Dev { + var err error + if token, err = devToken(s.Config.DataDir); err != nil { + return err + } + } + if token == "" { + s.Log.Warn("no bootstrap token configured; set " + s.Config.Auth.BootstrapTokenEnv + " to create the first operator key") + } + if token != "" { + keyID, secret, ok := auth.ParseToken(token) + if !ok { + return fmt.Errorf("%s is not a valid Differently token (want dly__)", s.Config.Auth.BootstrapTokenEnv) + } + existing, err := s.Store.GetKey(ctx, keyID) + switch { + case api.IsCode(err, api.CodeNotFound): + err = s.Store.CreateKey(ctx, store.KeyRecord{ + APIKey: api.APIKey{ID: keyID, Name: "bootstrap operator", Scopes: []string{auth.ScopeAdmin}, CreatedBy: "bootstrap", CreatedAt: s.now()}, + Hash: auth.Hash(secret), + }) + if err == nil { + s.Audit.Record(ctx, api.AuditEvent{Actor: "bootstrap", Action: "key.create", Resource: keyID, Details: map[string]string{"scopes": auth.ScopeAdmin}}) + } + case err == nil && existing.Hash != auth.Hash(secret): + // A revoked bootstrap key stays revoked even if the token changes. + _, err = s.Store.UpdateKey(ctx, keyID, func(k *store.KeyRecord) error { k.Hash = auth.Hash(secret); return nil }) + s.Audit.Record(ctx, api.AuditEvent{Actor: "bootstrap", Action: "key.rotate", Resource: keyID, Details: map[string]string{"revoked": itoa64(map[bool]int64{true: 1}[existing.Revoked])}}) + } + if err != nil { + return err + } + } + for _, pc := range s.Config.Pools { + p := pc.ToPool() + if old, err := s.Store.GetPool(ctx, p.Name); err == nil && old.Source == "config" { + p.Draining = old.Draining // drain state is operational, keep it across restarts + } + if err := s.Store.UpsertPool(ctx, p); err != nil { + return err + } + } + if s.Config.Dev { + if _, err := s.Store.GetOrg(ctx, "org_default"); api.IsCode(err, api.CodeNotFound) { + _ = s.Store.CreateOrg(ctx, api.Organization{ID: "org_default", Name: "Default", Quota: s.Config.Defaults.OrgQuota, CreatedAt: s.now()}) + _ = s.Store.CreateProject(ctx, api.Project{ID: "prj_default", OrgID: "org_default", Name: "default", Quota: s.Config.Defaults.ProjectQuota, CreatedAt: s.now()}) + } + } + return s.SyncPools(ctx) +} + +// SyncPools rebuilds adapters from the stored pool registry. +func (s *Server) SyncPools(ctx context.Context) error { + list, err := s.Store.ListPools(ctx) + if err != nil { + return err + } + s.Pools.Sync(list) + return nil +} + +func devToken(dir string) (string, error) { + path := filepath.Join(dir, "token") + if b, err := os.ReadFile(path); err == nil { + return strings.TrimSpace(string(b)), nil + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err + } + token := auth.TokenPrefix + "devoperator_" + id.Secret(40) + return token, os.WriteFile(path, []byte(token+"\n"), 0o600) +} + +// handlerFunc is an authenticated endpoint. +type handlerFunc func(w http.ResponseWriter, r *http.Request, p auth.Principal) error + +// Handler returns the HTTP handler. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + public := func(pattern string, h http.HandlerFunc) { mux.Handle(pattern, s.observe(pattern, h)) } + authed := func(pattern string, h handlerFunc) { mux.Handle(pattern, s.observe(pattern, s.authenticate(h))) } + + public("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { writeJSON(w, 200, map[string]string{"status": "ok"}) }) + public("GET /readyz", s.ready) + public("GET /version", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 200, api.VersionInfo{Version: version.Version, Commit: version.Commit, APIVersion: "v1", Installation: s.Installation}) + }) + mux.Handle("GET /metrics", promhttp.HandlerFor(metrics.Registry, promhttp.HandlerOpts{})) + public("GET /openapi.yaml", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + _, _ = w.Write(apispec.OpenAPI) + }) + if s.Config.Dashboard.On() { + mux.Handle("GET /dashboard/", http.StripPrefix("/dashboard/", dashboard.Handler())) + mux.Handle("GET /{$}", http.RedirectHandler("/dashboard/", http.StatusFound)) + } + public("POST /internal/v1/agents/heartbeat", s.agentHeartbeat) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + writeErr(w, r, api.Errorf(api.CodeNotFound, "no route for %s %s", r.Method, r.URL.Path)) + }) + + authed("GET /v1/me", s.me) + s.sandboxRoutes(authed) + s.tenancyRoutes(authed) + s.adminRoutes(authed) + return s.recover(mux) +} + +func (s *Server) ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + if _, err := s.Store.Installation(ctx); err != nil { + writeErr(w, r, api.Errorf(api.CodeProviderUnavailable, "store unavailable: %v", err)) + return + } + writeJSON(w, 200, map[string]string{"status": "ready"}) +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(code int) { + if w.status == 0 { + w.status = code + } + w.ResponseWriter.WriteHeader(code) +} + +func (w *statusWriter) Write(b []byte) (int, error) { + if w.status == 0 { + w.status = 200 + } + return w.ResponseWriter.Write(b) +} + +func (w *statusWriter) Flush() { + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +type ctxKey int + +const requestIDKey ctxKey = 1 + +func requestID(r *http.Request) string { + v, _ := r.Context().Value(requestIDKey).(string) + return v +} + +// observe adds a request ID, structured access logs and metrics. +func (s *Server) observe(route string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rid := r.Header.Get("X-Request-Id") + if rid == "" || len(rid) > 64 { + rid = id.New("req") + } + w.Header().Set("X-Request-Id", rid) + r = r.WithContext(context.WithValue(r.Context(), requestIDKey, rid)) + sw := &statusWriter{ResponseWriter: w} + start := time.Now() + next.ServeHTTP(sw, r) + if sw.status == 0 { + sw.status = 200 + } + metrics.Requests.WithLabelValues(route, strconv.Itoa(sw.status)).Inc() + metrics.RequestSeconds.WithLabelValues(route).Observe(time.Since(start).Seconds()) + if route != "GET /healthz" && route != "GET /readyz" { + s.Log.Info("request", "method", r.Method, "path", r.URL.Path, "status", sw.status, "ms", time.Since(start).Milliseconds(), "requestId", rid) + } + }) +} + +func (s *Server) recover(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if v := recover(); v != nil { + s.Log.Error("panic serving request", "path", r.URL.Path, "panic", v) + writeErr(w, r, api.Errorf(api.CodeInternal, "internal error")) + } + }() + next.ServeHTTP(w, r) + }) +} + +// authenticate resolves the bearer token to a principal and rate-limits it. +func (s *Server) authenticate(h handlerFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if token == "" { + token = r.Header.Get("X-Api-Key") + } + keyID, secret, ok := auth.ParseToken(token) + if !ok { + writeErr(w, r, api.Errorf(api.CodeUnauthenticated, "missing or malformed API key; send Authorization: Bearer dly_...")) + return + } + k, err := s.Store.GetKey(r.Context(), keyID) + if err == nil { + err = auth.Verify(k, secret, s.now()) + } else if api.IsCode(err, api.CodeNotFound) { + err = api.Errorf(api.CodeUnauthenticated, "invalid API key") + } + if err != nil { + writeErr(w, r, err) + return + } + if !s.limiter.allow(keyID) { + w.Header().Set("Retry-After", "1") + writeErr(w, r, api.Errorf(api.CodeRateLimited, "rate limit exceeded for this key")) + return + } + if k.LastUsedAt == nil || s.now().Sub(*k.LastUsedAt) > time.Minute { + now := s.now() + _, _ = s.Store.UpdateKey(r.Context(), keyID, func(rec *store.KeyRecord) error { rec.LastUsedAt = &now; return nil }) + } + p := auth.Principal{KeyID: k.ID, OrgID: k.OrgID, ProjectID: k.ProjectID, Scopes: k.Scopes} + if p.OrgID != "" { + if org, err := s.Store.GetOrg(r.Context(), p.OrgID); err == nil && org.Suspended && r.Method != http.MethodGet { + writeErr(w, r, api.Errorf(api.CodeSuspended, "organization %s is suspended", org.ID)) + return + } + } + if err := h(w, r, p); err != nil { + writeErr(w, r, err) + } + }) +} + +func (s *Server) me(w http.ResponseWriter, _ *http.Request, p auth.Principal) error { + writeJSON(w, 200, map[string]any{"keyId": p.KeyID, "orgId": p.OrgID, "projectId": p.ProjectID, "scopes": p.Scopes, "operator": p.Operator()}) + return nil +} + +// audit records an event attributed to p. +func (s *Server) audit(r *http.Request, p auth.Principal, action, resource, orgID, projectID string, details map[string]string) { + s.Audit.Record(r.Context(), api.AuditEvent{Actor: p.Actor(), OrgID: orgID, ProjectID: projectID, Action: action, Resource: resource, Details: details, RequestID: requestID(r)}) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeErr(w http.ResponseWriter, r *http.Request, err error) { + e := api.AsError(err) + if e.Code == "insufficient_capacity" { + e = &api.Error{Code: api.CodeProviderUnavailable, Message: e.Message, Details: e.Details} + } + out := *e + out.RequestID = requestID(r) + writeJSON(w, out.HTTPStatus(), api.ErrorEnvelope{Error: &out}) +} + +func decode(r *http.Request, v any) error { + dec := json.NewDecoder(io.LimitReader(r.Body, 4<<20)) + dec.DisallowUnknownFields() + if err := dec.Decode(v); err != nil && !errors.Is(err, io.EOF) { + return api.Errorf(api.CodeInvalidRequest, "invalid JSON body: %v", err) + } + return nil +} + +func queryInt(r *http.Request, key string, def int) int { + if v, err := strconv.Atoi(r.URL.Query().Get(key)); err == nil { + return v + } + return def +} + +// limiter is a per-key token bucket. Limits are per API replica. +type limiter struct { + rate float64 + burst float64 + mu sync.Mutex + buckets map[string]*bucket +} + +type bucket struct { + tokens float64 + last time.Time +} + +func newLimiter(rate float64, burst int) *limiter { + return &limiter{rate: rate, burst: float64(burst), buckets: map[string]*bucket{}} +} + +func (l *limiter) allow(key string) bool { + if l.rate <= 0 { + return true + } + l.mu.Lock() + defer l.mu.Unlock() + now := time.Now() + b, ok := l.buckets[key] + if !ok { + b = &bucket{tokens: l.burst, last: now} + l.buckets[key] = b + } + b.tokens = min(l.burst, b.tokens+now.Sub(b.last).Seconds()*l.rate) + b.last = now + if b.tokens < 1 { + return false + } + b.tokens-- + return true +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..91dee87 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,467 @@ +package server_test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/different-ai/differently/internal/adapters/fake" + "github.com/different-ai/differently/internal/auditlog" + "github.com/different-ai/differently/internal/config" + "github.com/different-ai/differently/internal/pools" + "github.com/different-ai/differently/internal/scheduler" + "github.com/different-ai/differently/internal/server" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +const operatorToken = "dly_bootstrap_0123456789abcdefghjkmnpqrstvwxyz" + +type harness struct { + t *testing.T + srv *httptest.Server + sched *scheduler.Scheduler + store *store.Memory + fc *fake.Adapter + ext *fake.Adapter +} + +// newHarness runs the real server, scheduler and memory store with two fake +// pools: "fc" (private, 4 CPU) and "burst" (external, unbounded). +func newHarness(t *testing.T, policy api.PlacementPolicy) *harness { + t.Helper() + t.Setenv("DIFFERENTLY_BOOTSTRAP_TOKEN", operatorToken) + cfg := config.Default() + cfg.Policy = policy + cfg.Policy.Scope = "global" + cfg.RateLimit.RequestsPerSecond = 0 + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + st := store.NewMemory() + rec, _ := auditlog.New(st, log, "") + pm := pools.NewManager(pools.Deps{Store: st, Logger: log}) + h := &harness{t: t, store: st, + fc: fake.New(fake.Options{Trust: api.TrustPrivate, TotalCPU: 4, TotalMemoryMiB: 16384}), + ext: fake.New(fake.Options{Trust: api.TrustExternal, Features: []string{adapter.FeatureExec, adapter.FeatureFiles, adapter.FeaturePorts}}), + } + ctx := context.Background() + for _, p := range []struct { + pool api.Pool + a adapter.ExecutionAdapter + }{ + {api.Pool{Name: "fc", Adapter: "fake", Trust: api.TrustPrivate, Enabled: true, Region: "dc1", Source: "config", CostModel: api.CostModel{CPUHour: 0.04}}, h.fc}, + {api.Pool{Name: "burst", Adapter: "fake", Trust: api.TrustExternal, Enabled: true, Region: "us", MaxConcurrency: 10, Source: "config", CostModel: api.CostModel{CPUHour: 0.02}}, h.ext}, + } { + _ = st.UpsertPool(ctx, p.pool) + pm.Set(p.pool, p.a) + } + inst, _ := st.Installation(ctx) + h.sched = scheduler.New(st, pm, cfg, inst, rec, log) + srv := server.New(cfg, st, pm, h.sched, rec, log, inst, "agent-token") + if err := srv.Bootstrap(ctx); err != nil { + t.Fatal(err) + } + pm.Refresh(ctx) + h.srv = httptest.NewServer(srv.Handler()) + t.Cleanup(h.srv.Close) + return h +} + +// tick runs scheduler passes until provisioning settles. +func (h *harness) tick() { + for i := 0; i < 3; i++ { + h.sched.Tick(context.Background()) + h.sched.Drain() + } +} + +func (h *harness) call(method, path, token string, body any, out any) int { + h.t.Helper() + var r io.Reader + if body != nil { + if b, ok := body.([]byte); ok { + r = bytes.NewReader(b) + } else { + b, _ := json.Marshal(body) + r = bytes.NewReader(b) + } + } + req, _ := http.NewRequest(method, h.srv.URL+path, r) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := http.DefaultClient.Do(req) + if err != nil { + h.t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if out != nil { + if s, ok := out.(*string); ok { + *s = string(raw) + } else if err := json.Unmarshal(raw, out); err != nil && len(raw) > 0 { + h.t.Fatalf("%s %s: decode %q: %v", method, path, raw, err) + } + } + return resp.StatusCode +} + +func (h *harness) must(want int, method, path, token string, body, out any) { + h.t.Helper() + var raw string + if got := h.call(method, path, token, body, &raw); got != want { + h.t.Fatalf("%s %s: status %d, want %d: %s", method, path, got, want, raw) + } + if out != nil { + if err := json.Unmarshal([]byte(raw), out); err != nil { + h.t.Fatalf("decode %s: %v", raw, err) + } + } +} + +// tenant creates an organization, project and project key. +func (h *harness) tenant(name string, quota api.Quota) (orgID, projectID, token string) { + var created struct { + Organization api.Organization `json:"organization"` + Project api.Project `json:"project"` + } + h.must(201, "POST", "/v1/admin/organizations", operatorToken, map[string]any{"name": name, "quota": quota, "defaultProject": "default"}, &created) + var key api.CreateAPIKeyResponse + h.must(201, "POST", "/v1/api-keys", operatorToken, api.CreateAPIKeyRequest{ + Name: "ci", OrgID: created.Organization.ID, ProjectID: created.Project.ID, + Scopes: []string{"sandboxes:read", "sandboxes:write", "sandboxes:exec", "keys:write", "usage:read"}, + }, &key) + return created.Organization.ID, created.Project.ID, key.Token +} + +func (h *harness) create(token string, req api.CreateSandboxRequest) api.Sandbox { + var sb api.Sandbox + h.must(202, "POST", "/v1/sandboxes", token, req, &sb) + return sb +} + +func (h *harness) get(token, id string) api.Sandbox { + var sb api.Sandbox + h.must(200, "GET", "/v1/sandboxes/"+id, token, nil, &sb) + return sb +} + +var overflowPolicy = api.PlacementPolicy{External: api.ExternalOverflow, ExternalPools: []string{"burst"}} + +// Scenario A: install, admin, key, sandbox lifecycle with no hosted service. +func TestOwnCloudQuickstart(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + _, _, token := h.tenant("acme", api.Quota{}) + sb := h.create(token, api.CreateSandboxRequest{Profile: "small", Env: map[string]string{"SECRET": "x"}}) + if sb.Status != api.StatusQueued || len(sb.EnvKeys) != 1 { + t.Fatalf("created %+v", sb) + } + h.tick() + sb = h.get(token, sb.ID) + if sb.Status != api.StatusRunning || sb.Pool != "fc" || sb.Trust != api.TrustPrivate { + t.Fatalf("not running privately: %+v", sb) + } + var res api.ExecResult + h.must(200, "POST", "/v1/sandboxes/"+sb.ID+"/exec", token, api.ExecRequest{Command: "node --version"}, &res) + if !strings.Contains(res.Stdout, "node --version") { + t.Fatalf("exec %+v", res) + } + h.must(204, "PUT", "/v1/sandboxes/"+sb.ID+"/files/content?path=/a.txt", token, []byte("hello"), nil) + var content string + h.call("GET", "/v1/sandboxes/"+sb.ID+"/files/content?path=/a.txt", token, nil, &content) + if content != "hello" { + t.Fatalf("file %q", content) + } + var stream string + h.call("POST", "/v1/sandboxes/"+sb.ID+"/exec?stream=1", token, api.ExecRequest{Command: "echo hi"}, &stream) + if !strings.Contains(stream, "event: stdout") || !strings.Contains(stream, "event: exit") { + t.Fatalf("stream %q", stream) + } + partial := h.create(token, api.CreateSandboxRequest{Resources: &api.Resources{CPU: 2}}) + if partial.Resources.CPU != 2 || partial.Resources.MemoryMiB != 1024 { + t.Fatalf("partial resources should start from the default profile: %+v", partial.Resources) + } + h.must(202, "DELETE", "/v1/sandboxes/"+partial.ID, token, nil, nil) + h.must(202, "DELETE", "/v1/sandboxes/"+sb.ID, token, nil, nil) + h.tick() + if got := h.get(token, sb.ID); got.Status != api.StatusTerminated || h.fc.Count() != 0 { + t.Fatalf("not cleaned up: %s, %d instances", got.Status, h.fc.Count()) + } + var verify struct { + OK bool `json:"ok"` + Events int `json:"events"` + } + h.must(200, "GET", "/v1/admin/audit/verify", operatorToken, nil, &verify) + if !verify.OK || verify.Events < 6 { + t.Fatalf("audit chain %+v", verify) + } +} + +// Scenarios D and E: overflow only for eligible work; private-only waits. +func TestHybridOverflowAndPrivateOnly(t *testing.T) { + h := newHarness(t, overflowPolicy) + _, _, token := h.tenant("acme", api.Quota{}) + // Fill the private pool (4 CPU). + first := h.create(token, api.CreateSandboxRequest{Resources: &api.Resources{CPU: 4, MemoryMiB: 4096}}) + h.tick() + if got := h.get(token, first.ID); got.Pool != "fc" { + t.Fatalf("first sandbox should be private: %+v", got) + } + h.sched.Pools.Refresh(context.Background()) + + secret := h.create(token, api.CreateSandboxRequest{Classification: "private-only", Resources: &api.Resources{CPU: 2, MemoryMiB: 2048}}) + general := h.create(token, api.CreateSandboxRequest{Classification: "internal", Resources: &api.Resources{CPU: 2, MemoryMiB: 2048}}) + h.tick() + if got := h.get(token, secret.ID); got.Status != api.StatusQueued || !strings.Contains(got.StatusReason, "waiting") { + t.Fatalf("private-only must stay queued, got %+v", got) + } + if got := h.get(token, general.ID); got.Pool != "burst" || got.Trust != api.TrustExternal { + t.Fatalf("eligible work should overflow: %+v", got) + } + // Opting out per request keeps work private. + optOut := h.create(token, api.CreateSandboxRequest{Placement: &api.Placement{External: api.ExternalNever}}) + h.tick() + if got := h.get(token, optOut.ID); got.Status != api.StatusQueued { + t.Fatalf("opt-out escaped: %+v", got) + } + // Scenario L: the operator can see why. + var explain map[string]any + h.must(200, "GET", "/v1/sandboxes/"+general.ID+"/placement", token, nil, &explain) + if explain["trust"] != "external" || explain["decision"] == nil { + t.Fatalf("placement explanation %v", explain) + } + var report server.TrustReport + h.must(200, "GET", "/v1/admin/trust-report", operatorToken, nil, &report) + if len(report.ExternalRecent) != 1 || report.ExternalRecent[0].SandboxID != general.ID || len(report.Limitations) == 0 || !report.AuditChain.OK { + t.Fatalf("trust report %+v", report) + } +} + +// Scenario B: with external disabled nothing reaches the external pool. +func TestFullyPrivateDeployment(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{External: api.ExternalNever}) + _, _, token := h.tenant("acme", api.Quota{}) + for i := 0; i < 4; i++ { + h.create(token, api.CreateSandboxRequest{Classification: "public", Resources: &api.Resources{CPU: 2, MemoryMiB: 1024}}) + } + h.tick() + if h.ext.Count() != 0 { + t.Fatalf("%d sandboxes reached the external provider", h.ext.Count()) + } +} + +// Scenario C: per-tenant concurrency, fair queueing and cleanup. +func TestMultiTenantQuotas(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + _, _, a := h.tenant("a", api.Quota{MaxActiveSandboxes: 1, MaxQueued: 2}) + _, _, b := h.tenant("b", api.Quota{MaxActiveSandboxes: 2}) + a1 := h.create(a, api.CreateSandboxRequest{}) + h.tick() + a2 := h.create(a, api.CreateSandboxRequest{}) + h.create(a, api.CreateSandboxRequest{}) + var e api.ErrorEnvelope + if code := h.call("POST", "/v1/sandboxes", a, api.CreateSandboxRequest{}, &e); code != 429 || e.Error.Code != api.CodeQuotaExceeded { + t.Fatalf("queue limit not enforced: %d %+v", code, e.Error) + } + b1 := h.create(b, api.CreateSandboxRequest{}) + h.tick() + if h.get(a, a1.ID).Status != api.StatusRunning || h.get(a, a2.ID).Status != api.StatusQueued || h.get(b, b1.ID).Status != api.StatusRunning { + t.Fatal("concurrency limit or fairness broken") + } + if code := h.call("POST", "/v1/sandboxes", a, api.CreateSandboxRequest{Resources: &api.Resources{CPU: 64, MemoryMiB: 1024}}, &e); code != 422 { + t.Fatalf("oversized request should be rejected up front, got %d %+v", code, e.Error) + } + // Stopping releases CPU; starting must re-check quota (a1 holds the only slot). + h.must(200, "POST", "/v1/sandboxes/"+a1.ID+"/stop", a, nil, nil) + h.must(200, "POST", "/v1/sandboxes/"+a1.ID+"/start", a, nil, nil) + h.must(202, "DELETE", "/v1/sandboxes/"+a1.ID, a, nil, nil) + h.tick() + h.tick() + if got := h.get(a, a2.ID).Status; got != api.StatusRunning { + t.Fatalf("queued sandbox did not start after release: %s", got) + } + var q api.QuotaReport + h.must(200, "GET", "/v1/quotas", a, nil, &q) + if q.OrgUsage.ActiveSandboxes != 1 || q.OrgUsage.Queued != 1 { + t.Fatalf("usage %+v", q.OrgUsage) + } +} + +// Scenario H: a project key reaches only its own resources. +func TestScopedKeyIsolation(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + _, _, a := h.tenant("a", api.Quota{}) + _, projB, b := h.tenant("b", api.Quota{}) + sbB := h.create(b, api.CreateSandboxRequest{}) + if code := h.call("GET", "/v1/sandboxes/"+sbB.ID, a, nil, nil); code != 404 { + t.Fatalf("cross-tenant read returned %d", code) + } + if code := h.call("POST", "/v1/sandboxes", a, api.CreateSandboxRequest{ProjectID: projB}, nil); code != 403 { + t.Fatalf("cross-project create returned %d", code) + } + if code := h.call("POST", "/v1/api-keys", a, api.CreateAPIKeyRequest{Name: "x", Scopes: []string{"admin"}}, nil); code != 403 { + t.Fatalf("escalation returned %d", code) + } + if code := h.call("GET", "/v1/admin/pools", a, nil, nil); code != 403 { + t.Fatalf("admin API returned %d", code) + } + orgA := "" + var me map[string]any + h.must(200, "GET", "/v1/me", a, nil, &me) + orgA, _ = me["orgId"].(string) + var adminA api.CreateAPIKeyResponse + h.must(201, "POST", "/v1/api-keys", operatorToken, api.CreateAPIKeyRequest{Name: "a-admin", OrgID: orgA, Scopes: []string{"org:admin"}}, &adminA) + h.must(200, "PUT", "/v1/policies/project:"+projB, operatorToken, api.PlacementPolicy{External: api.ExternalNever}, nil) + if code := h.call("PUT", "/v1/policies/project:"+projB, adminA.Token, api.PlacementPolicy{}, nil); code != 404 { + t.Fatalf("cross-tenant policy update returned %d", code) + } + if code := h.call("DELETE", "/v1/policies/project:"+projB, adminA.Token, nil, nil); code != 404 { + t.Fatalf("cross-tenant policy delete returned %d", code) + } + var pools map[string]any + h.must(200, "GET", "/v1/capabilities", a, nil, &pools) + if strings.Contains(mustJSON(pools), "credential") { + t.Fatal("tenant view leaks credential references") + } + if code := h.call("GET", "/v1/sandboxes", "dly_nope_0123456789abcdefghjkmnpqrstvwxyz", nil, nil); code != 401 { + t.Fatalf("bad key returned %d", code) + } +} + +// Scenario I: unsupported capabilities fail precisely or route elsewhere. +func TestCapabilityMismatch(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + _, _, token := h.tenant("a", api.Quota{}) + var e api.ErrorEnvelope + if code := h.call("POST", "/v1/sandboxes", token, api.CreateSandboxRequest{Capabilities: []string{"gpu"}}, &e); code != 422 || e.Error.Code != api.CodeUnsupportedCapability { + t.Fatalf("want unsupported_capability, got %d %+v", code, e.Error) + } + h2 := newHarness(t, overflowPolicy) + _, _, token2 := h2.tenant("a", api.Quota{}) + sb := h2.create(token2, api.CreateSandboxRequest{Capabilities: []string{"ports.expose"}}) + h2.tick() + if got := h2.get(token2, sb.ID); got.Pool != "fc" { + // The fake private pool supports ports too, so private-first keeps it home. + t.Fatalf("placed on %s", got.Pool) + } + // Lifecycle operations the backend lacks return unsupported_operation. + plain := h2.create(token2, api.CreateSandboxRequest{Placement: &api.Placement{AllowedPools: []string{"burst"}}}) + h2.tick() + e = api.ErrorEnvelope{} + if code := h2.call("POST", "/v1/sandboxes/"+plain.ID+"/pause", token2, nil, &e); code != 501 || e.Error.Code != api.CodeUnsupportedOperation { + t.Fatalf("want unsupported_operation, got %d %+v", code, e.Error) + } +} + +// Scenario F: an operator policy change moves new work without app changes. +func TestProviderSubstitution(t *testing.T) { + h := newHarness(t, overflowPolicy) + _, _, token := h.tenant("a", api.Quota{}) + before := h.create(token, api.CreateSandboxRequest{}) + h.must(200, "PUT", "/v1/policies/global", operatorToken, api.PlacementPolicy{External: api.ExternalAllowed, ExternalPools: []string{"burst"}, DeniedPools: []string{"fc"}}, nil) + after := h.create(token, api.CreateSandboxRequest{}) + h.tick() + if h.get(token, before.ID).Pool != "fc" && h.get(token, before.ID).Pool != "burst" { + t.Fatal("first sandbox not placed") + } + if got := h.get(token, after.ID); got.Pool != "burst" { + t.Fatalf("policy change ignored: %+v", got) + } +} + +// Scenario J: provider failures retry safely, orphans and lost instances are +// reconciled, and quota stays correct. +func TestFailureRecoveryAndReconcile(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + orgID, _, token := h.tenant("a", api.Quota{MaxActiveSandboxes: 5}) + h.fc.FailCreates = 1 + sb := h.create(token, api.CreateSandboxRequest{}) + h.tick() + if got := h.get(token, sb.ID); got.Status != api.StatusQueued || got.Attempts != 1 { + t.Fatalf("failed attempt should requeue: %+v", got) + } + // Retry after backoff; the pool recovers on the next health probe. + _, _ = h.store.UpdateSandbox(context.Background(), sb.ID, func(r *store.SandboxRecord) error { r.NextAttemptAt = time.Now(); return nil }) + h.sched.Pools.Refresh(context.Background()) + h.tick() + if got := h.get(token, sb.ID); got.Status != api.StatusRunning || h.fc.Count() != 1 { + t.Fatalf("retry did not converge to exactly one instance: %+v (%d)", got, h.fc.Count()) + } + inst, _ := h.store.Installation(context.Background()) + h.fc.Inject("sbx_ghost", inst) + running, _ := h.store.GetSandbox(context.Background(), sb.ID) + h.fc.Forget(running.BackendRef) + h.sched.Reconcile(context.Background()) + if h.fc.Count() != 0 { + t.Fatal("orphan was not deleted") + } + if got := h.get(token, sb.ID); got.Status != api.StatusFailed { + t.Fatalf("lost instance not detected: %+v", got) + } + u, _ := h.store.Usage(context.Background(), orgID, "") + if u.ActiveSandboxes != 0 || u.CPU != 0 { + t.Fatalf("quota leaked: %+v", u) + } +} + +func TestIdempotentCreateAndSnapshots(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + _, _, token := h.tenant("a", api.Quota{}) + body, _ := json.Marshal(api.CreateSandboxRequest{}) + send := func() (int, api.Sandbox) { + req, _ := http.NewRequest("POST", h.srv.URL+"/v1/sandboxes", bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Idempotency-Key", "job-42") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var sb api.Sandbox + _ = json.NewDecoder(resp.Body).Decode(&sb) + return resp.StatusCode, sb + } + c1, s1 := send() + c2, s2 := send() + if c1 != 202 || c2 != 200 || s1.ID != s2.ID { + t.Fatalf("replay: %d %s / %d %s", c1, s1.ID, c2, s2.ID) + } + h.tick() + h.must(204, "PUT", "/v1/sandboxes/"+s1.ID+"/files/content?path=/state", token, []byte("v1"), nil) + var child api.Sandbox + h.must(202, "POST", "/v1/sandboxes/"+s1.ID+"/fork", token, nil, &child) + h.tick() + var data string + h.call("GET", "/v1/sandboxes/"+child.ID+"/files/content?path=/state", token, nil, &data) + if child.Labels["forked-from"] != s1.ID || data != "v1" { + t.Fatalf("fork %+v data %q", child, data) + } + var snaps api.List[api.Snapshot] + h.must(200, "GET", "/v1/snapshots", token, nil, &snaps) + if len(snaps.Items) != 1 || snaps.Items[0].Pool != "fc" { + t.Fatalf("snapshots %+v", snaps) + } +} + +func TestSuspendOrganization(t *testing.T) { + h := newHarness(t, api.PlacementPolicy{}) + orgID, _, token := h.tenant("a", api.Quota{}) + sb := h.create(token, api.CreateSandboxRequest{}) + h.tick() + h.must(200, "POST", "/v1/admin/organizations/"+orgID+"/suspend", operatorToken, map[string]any{"terminate": true, "reason": "abuse"}, nil) + h.tick() + if got := h.get(token, sb.ID); got.Status != api.StatusTerminated { + t.Fatalf("suspended tenant's sandbox still %s", got.Status) + } + var e api.ErrorEnvelope + if code := h.call("POST", "/v1/sandboxes", token, api.CreateSandboxRequest{}, &e); code != 403 || e.Error.Code != api.CodeSuspended { + t.Fatalf("suspended org created work: %d %+v", code, e.Error) + } +} + +func mustJSON(v any) string { b, _ := json.Marshal(v); return string(b) } diff --git a/internal/server/tenancy.go b/internal/server/tenancy.go new file mode 100644 index 0000000..ab1f3ab --- /dev/null +++ b/internal/server/tenancy.go @@ -0,0 +1,504 @@ +package server + +import ( + "crypto/subtle" + "encoding/json" + "net/http" + "slices" + "sort" + "strings" + "time" + + "github.com/different-ai/differently/internal/auth" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/policy" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/api" +) + +func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +func (s *Server) tenancyRoutes(authed func(string, handlerFunc)) { + authed("GET /v1/projects", s.listProjects) + authed("POST /v1/projects", s.createProject) + authed("GET /v1/projects/{id}", s.getProject) + authed("DELETE /v1/projects/{id}", s.deleteProject) + authed("GET /v1/api-keys", s.listKeys) + authed("POST /v1/api-keys", s.createKey) + authed("DELETE /v1/api-keys/{id}", s.revokeKey) + authed("GET /v1/quotas", s.quotas) + authed("GET /v1/usage", s.usage) + authed("GET /v1/audit-events", s.listAudit) + authed("GET /v1/policies", s.listPolicies) + authed("PUT /v1/policies/{scope}", s.putPolicy) + authed("DELETE /v1/policies/{scope}", s.deletePolicy) + authed("GET /v1/capabilities", s.capabilities) + authed("GET /v1/profiles", func(w http.ResponseWriter, _ *http.Request, _ auth.Principal) error { + writeJSON(w, 200, api.List[api.Profile]{Items: s.Config.Profiles}) + return nil + }) + authed("GET /v1/templates", func(w http.ResponseWriter, _ *http.Request, _ auth.Principal) error { + writeJSON(w, 200, api.List[api.Template]{Items: append([]api.Template{}, s.Config.Templates...)}) + return nil + }) + authed("GET /v1/classifications", func(w http.ResponseWriter, _ *http.Request, _ auth.Principal) error { + writeJSON(w, 200, api.List[api.Classification]{Items: s.Config.Classifications}) + return nil + }) +} + +// orgFor resolves the organization a tenant call targets. +func orgFor(r *http.Request, p auth.Principal) (string, error) { + if !p.Operator() { + return p.OrgID, nil + } + if v := r.URL.Query().Get("orgId"); v != "" { + return v, nil + } + return "", api.Errorf(api.CodeInvalidRequest, "operator keys must pass orgId") +} + +func (s *Server) listProjects(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + orgID := p.OrgID + if p.Operator() { + orgID = r.URL.Query().Get("orgId") + } + list, err := s.Store.ListProjects(r.Context(), orgID) + if err != nil { + return err + } + out := api.List[api.Project]{Items: []api.Project{}} + for _, pr := range list { + if p.CanAccess(pr.OrgID, pr.ID) { + out.Items = append(out.Items, pr) + } + } + writeJSON(w, 200, out) + return nil +} + +func (s *Server) createProject(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeProjectsWrite); err != nil { + return err + } + if p.ProjectID != "" { + return api.Errorf(api.CodePermissionDenied, "project keys cannot create projects") + } + var req struct { + Name string `json:"name"` + OrgID string `json:"orgId,omitempty"` + Quota *api.Quota `json:"quota,omitempty"` + } + if err := decode(r, &req); err != nil { + return err + } + if strings.TrimSpace(req.Name) == "" { + return api.Errorf(api.CodeInvalidRequest, "name is required") + } + orgID := p.OrgID + if p.Operator() { + orgID = req.OrgID + } + if _, err := s.Store.GetOrg(r.Context(), orgID); err != nil { + return api.Errorf(api.CodeNotFound, "organization %q not found", orgID) + } + proj := api.Project{ID: id.New("prj"), OrgID: orgID, Name: req.Name, Quota: s.Config.Defaults.ProjectQuota, CreatedAt: s.now()} + if req.Quota != nil { + if !p.Operator() && !slices.Contains(p.Scopes, auth.ScopeOrgAdmin) { + return api.Errorf(api.CodePermissionDenied, "setting a project quota requires org:admin") + } + proj.Quota = *req.Quota + } + if err := s.Store.CreateProject(r.Context(), proj); err != nil { + return err + } + s.audit(r, p, "project.create", proj.ID, proj.OrgID, proj.ID, map[string]string{"name": proj.Name}) + writeJSON(w, http.StatusCreated, proj) + return nil +} + +func (s *Server) getProject(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + proj, err := s.Store.GetProject(r.Context(), r.PathValue("id")) + if err != nil || !p.CanAccess(proj.OrgID, proj.ID) { + return api.Errorf(api.CodeNotFound, "project %s not found", r.PathValue("id")) + } + writeJSON(w, 200, proj) + return nil +} + +func (s *Server) deleteProject(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeProjectsWrite); err != nil { + return err + } + proj, err := s.Store.GetProject(r.Context(), r.PathValue("id")) + if err != nil || !p.CanAccess(proj.OrgID, proj.ID) || p.ProjectID != "" { + return api.Errorf(api.CodeNotFound, "project %s not found", r.PathValue("id")) + } + if err := s.Store.DeleteProject(r.Context(), proj.ID); err != nil { + return err + } + s.audit(r, p, "project.delete", proj.ID, proj.OrgID, proj.ID, nil) + w.WriteHeader(http.StatusNoContent) + return nil +} + +func (s *Server) listKeys(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeKeysWrite); err != nil { + return err + } + orgID := p.OrgID + if p.Operator() { + orgID = r.URL.Query().Get("orgId") + } + list, err := s.Store.ListKeys(r.Context(), orgID, p.ProjectID) + if err != nil { + return err + } + out := api.List[api.APIKey]{Items: []api.APIKey{}} + for _, k := range list { + if p.Operator() || k.OrgID != "" { + out.Items = append(out.Items, k.APIKey) + } + } + writeJSON(w, 200, out) + return nil +} + +func (s *Server) createKey(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + var req api.CreateAPIKeyRequest + if err := decode(r, &req); err != nil { + return err + } + orgID, projectID, err := auth.CheckGrant(p, req) + if err != nil { + return err + } + if orgID != "" { + if _, err := s.Store.GetOrg(r.Context(), orgID); err != nil { + return api.Errorf(api.CodeNotFound, "organization %s not found", orgID) + } + } + if projectID != "" { + proj, err := s.Store.GetProject(r.Context(), projectID) + if err != nil || proj.OrgID != orgID { + return api.Errorf(api.CodeNotFound, "project %s not found", projectID) + } + } + keyID, token, secret := auth.NewToken() + k := store.KeyRecord{ + APIKey: api.APIKey{ID: keyID, Name: req.Name, OrgID: orgID, ProjectID: projectID, Scopes: req.Scopes, CreatedBy: p.Actor(), CreatedAt: s.now()}, + Hash: auth.Hash(secret), + } + if req.TTLHours > 0 { + exp := s.now().Add(time.Duration(req.TTLHours) * time.Hour) + k.ExpiresAt = &exp + } + if err := s.Store.CreateKey(r.Context(), k); err != nil { + return err + } + s.audit(r, p, "key.create", keyID, orgID, projectID, map[string]string{"scopes": strings.Join(req.Scopes, ","), "name": req.Name}) + writeJSON(w, http.StatusCreated, api.CreateAPIKeyResponse{APIKey: k.APIKey, Token: token}) + return nil +} + +func (s *Server) revokeKey(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeKeysWrite); err != nil { + return err + } + k, err := s.Store.GetKey(r.Context(), r.PathValue("id")) + if err != nil || (!p.Operator() && (k.OrgID != p.OrgID || (p.ProjectID != "" && k.ProjectID != p.ProjectID))) { + return api.Errorf(api.CodeNotFound, "api key %s not found", r.PathValue("id")) + } + if _, err := s.Store.UpdateKey(r.Context(), k.ID, func(x *store.KeyRecord) error { x.Revoked = true; return nil }); err != nil { + return err + } + s.audit(r, p, "key.revoke", k.ID, k.OrgID, k.ProjectID, nil) + w.WriteHeader(http.StatusNoContent) + return nil +} + +func (s *Server) quotas(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeUsageRead); err != nil { + return err + } + orgID, err := orgFor(r, p) + if err != nil { + return err + } + org, err := s.Store.GetOrg(r.Context(), orgID) + if err != nil { + return err + } + ou, err := s.Store.Usage(r.Context(), orgID, "") + if err != nil { + return err + } + rep := api.QuotaReport{OrgID: orgID, OrgQuota: org.Quota, OrgUsage: ou} + projectID := r.URL.Query().Get("projectId") + if p.ProjectID != "" { + projectID = p.ProjectID + } + if projectID != "" { + proj, err := s.Store.GetProject(r.Context(), projectID) + if err != nil || proj.OrgID != orgID { + return api.Errorf(api.CodeNotFound, "project %s not found", projectID) + } + pu, _ := s.Store.Usage(r.Context(), orgID, projectID) + rep.ProjectID, rep.ProjectQuota, rep.ProjectUsage = projectID, &proj.Quota, &pu + } + writeJSON(w, 200, rep) + return nil +} + +// usage aggregates estimated usage from sandbox records. Run time is the +// accumulated running time of sandboxes that overlap the window; costs use +// each pool's configured cost model. Reconciled provider invoices are not +// imported yet, so every figure is an estimate. +func (s *Server) usage(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeUsageRead); err != nil { + return err + } + now := s.now() + from, to := now.Add(-30*24*time.Hour), now + if v, err := time.Parse(time.RFC3339, r.URL.Query().Get("from")); err == nil { + from = v + } + if v, err := time.Parse(time.RFC3339, r.URL.Query().Get("to")); err == nil { + to = v + } + groupBy := r.URL.Query().Get("groupBy") + if groupBy == "" { + groupBy = "pool" + } + if !slices.Contains([]string{"pool", "project", "org", "trust"}, groupBy) { + return api.Errorf(api.CodeInvalidRequest, "groupBy must be pool, project, org or trust") + } + f := store.SandboxFilter{OrgID: p.OrgID, ProjectID: p.ProjectID} + if p.Operator() { + f.OrgID = r.URL.Query().Get("orgId") + } + recs, err := s.Store.ListSandboxes(r.Context(), f) + if err != nil { + return err + } + poolList, _ := s.Store.ListPools(r.Context()) + costs := map[string]api.CostModel{} + for _, pl := range poolList { + costs[pl.Name] = pl.CostModel + } + lines := map[string]*api.UsageLine{} + total := api.UsageLine{Key: "total"} + for _, rec := range recs { + if rec.CreatedAt.After(to) || (rec.EndedAt != nil && rec.EndedAt.Before(from)) { + continue + } + key := map[string]string{"pool": rec.Pool, "project": rec.ProjectID, "org": rec.OrgID, "trust": rec.Trust}[groupBy] + if key == "" { + key = "(unplaced)" + } + l, ok := lines[key] + if !ok { + l = &api.UsageLine{Key: key} + lines[key] = l + } + run := rec.RunSeconds + if rec.RunningSince != nil { + run += now.Sub(*rec.RunningSince).Seconds() + } + add := api.UsageLine{ + Sandboxes: 1, RunSeconds: run, QueueSeconds: rec.QueueSeconds, + CPUSeconds: run * float64(rec.Resources.CPU), MemoryGiBSeconds: run * float64(rec.Resources.MemoryMiB) / 1024, + EstimatedCost: run / 3600 * costs[rec.Pool].HourlyCost(rec.Resources), + } + for _, t := range []*api.UsageLine{l, &total} { + t.Sandboxes += add.Sandboxes + t.RunSeconds += add.RunSeconds + t.QueueSeconds += add.QueueSeconds + t.CPUSeconds += add.CPUSeconds + t.MemoryGiBSeconds += add.MemoryGiBSeconds + t.EstimatedCost += add.EstimatedCost + } + } + rep := api.UsageReport{From: from, To: to, GroupBy: groupBy, Total: total, Note: "estimated from configured pool cost models; provider invoices are not reconciled"} + for _, l := range lines { + rep.Lines = append(rep.Lines, *l) + } + sort.Slice(rep.Lines, func(i, j int) bool { return rep.Lines[i].Key < rep.Lines[j].Key }) + writeJSON(w, 200, rep) + return nil +} + +func (s *Server) listAudit(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + if err := p.Require(auth.ScopeAuditRead); err != nil { + return err + } + f := store.AuditFilter{OrgID: p.OrgID, Resource: r.URL.Query().Get("resource"), Action: r.URL.Query().Get("action"), AfterSeq: int64(queryInt(r, "afterSeq", 0)), Limit: min(max(queryInt(r, "limit", 100), 1), 1000)} + if p.Operator() { + f.OrgID = r.URL.Query().Get("orgId") + } + events, err := s.Store.ListAudit(r.Context(), f) + if err != nil { + return err + } + if events == nil { + events = []api.AuditEvent{} + } + out := api.List[api.AuditEvent]{Items: events} + if len(events) == f.Limit { + out.NextCursor = itoa64(events[len(events)-1].Seq) + } + writeJSON(w, 200, out) + return nil +} + +func itoa64(n int64) string { b, _ := json.Marshal(n); return string(b) } + +// policyScopeAllowed reports whether p may change scope. +func policyScopeAllowed(p auth.Principal, scope string) bool { + if p.Operator() { + return scope == "global" || strings.HasPrefix(scope, "org:") || strings.HasPrefix(scope, "project:") + } + if !p.Has(auth.ScopePoliciesWrite) { + return false + } + if scope == "org:"+p.OrgID { + return p.ProjectID == "" + } + return strings.HasPrefix(scope, "project:") && (p.ProjectID == "" || scope == "project:"+p.ProjectID) +} + +func (s *Server) listPolicies(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + list, err := s.Store.ListPolicies(r.Context()) + if err != nil { + return err + } + global := s.Config.Policy + out := api.List[api.PlacementPolicy]{Items: []api.PlacementPolicy{}} + hasGlobal := false + for _, pol := range list { + hasGlobal = hasGlobal || pol.Scope == "global" + visible := p.Operator() || pol.Scope == "global" || pol.Scope == "org:"+p.OrgID + if strings.HasPrefix(pol.Scope, "project:") && !p.Operator() { + proj, err := s.Store.GetProject(r.Context(), strings.TrimPrefix(pol.Scope, "project:")) + visible = err == nil && p.CanAccess(proj.OrgID, proj.ID) + } + if visible { + out.Items = append(out.Items, pol) + } + } + if !hasGlobal { + out.Items = append([]api.PlacementPolicy{global}, out.Items...) + } + writeJSON(w, 200, out) + return nil +} + +func (s *Server) putPolicy(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + scope := r.PathValue("scope") + if !policyScopeAllowed(p, scope) { + return api.Errorf(api.CodePermissionDenied, "this key cannot change policy %s", scope) + } + if strings.HasPrefix(scope, "project:") && !p.Operator() { + proj, err := s.Store.GetProject(r.Context(), strings.TrimPrefix(scope, "project:")) + if err != nil || !p.CanAccess(proj.OrgID, proj.ID) { + return api.Errorf(api.CodeNotFound, "project not found") + } + } + var pol api.PlacementPolicy + if err := decode(r, &pol); err != nil { + return err + } + pol.Scope, pol.UpdatedAt, pol.UpdatedBy = scope, s.now(), p.Actor() + if pol.External != "" && !policy.ValidExternal(pol.External) { + return api.Errorf(api.CodeInvalidRequest, "external must be never, overflow or allowed") + } + if pol.Strategy != "" && !policy.ValidStrategy(pol.Strategy) { + return api.Errorf(api.CodeInvalidRequest, "unknown strategy %q", pol.Strategy) + } + if err := s.Store.PutPolicy(r.Context(), pol); err != nil { + return err + } + b, _ := json.Marshal(pol) + s.audit(r, p, "policy.update", scope, p.OrgID, p.ProjectID, map[string]string{"policy": truncate(string(b), 1024)}) + s.Scheduler.Wake() + writeJSON(w, 200, pol) + return nil +} + +func (s *Server) deletePolicy(w http.ResponseWriter, r *http.Request, p auth.Principal) error { + scope := r.PathValue("scope") + if !policyScopeAllowed(p, scope) { + return api.Errorf(api.CodePermissionDenied, "this key cannot change policy %s", scope) + } + if strings.HasPrefix(scope, "project:") && !p.Operator() { + proj, err := s.Store.GetProject(r.Context(), strings.TrimPrefix(scope, "project:")) + if err != nil || !p.CanAccess(proj.OrgID, proj.ID) { + return api.Errorf(api.CodeNotFound, "project not found") + } + } + if err := s.Store.DeletePolicy(r.Context(), scope); err != nil { + return err + } + s.audit(r, p, "policy.delete", scope, p.OrgID, p.ProjectID, nil) + w.WriteHeader(http.StatusNoContent) + return nil +} + +// capabilities lists pools as tenants see them: no config or credentials. +func (s *Server) capabilities(w http.ResponseWriter, r *http.Request, _ auth.Principal) error { + list, err := s.Store.ListPools(r.Context()) + if err != nil { + return err + } + active, _ := s.Store.PoolActive(r.Context()) + type view struct { + Pool string `json:"pool"` + Adapter string `json:"adapter"` + Trust string `json:"trust"` + Region string `json:"region,omitempty"` + Enabled bool `json:"enabled"` + Health string `json:"health"` + Capabilities []string `json:"capabilities"` + } + out := api.List[view]{Items: []view{}} + for _, pl := range list { + s.Pools.Decorate(&pl, active[pl.Name]) + out.Items = append(out.Items, view{Pool: pl.Name, Adapter: pl.Adapter, Trust: pl.Trust, Region: pl.Region, Enabled: pl.Enabled && !pl.Draining, Health: pl.Health.Status, Capabilities: pl.Capabilities}) + } + writeJSON(w, 200, out) + return nil +} + +// agentHeartbeat registers or refreshes a node agent. +func (s *Server) agentHeartbeat(w http.ResponseWriter, r *http.Request) { + token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if s.AgentToken == "" || subtle.ConstantTimeCompare([]byte(token), []byte(s.AgentToken)) != 1 { + writeErr(w, r, api.Errorf(api.CodeUnauthenticated, "invalid agent token")) + return + } + var a api.Agent + if err := decode(r, &a); err != nil { + writeErr(w, r, err) + return + } + if a.ID == "" || a.Pool == "" || a.Endpoint == "" || strings.Contains(a.ID, ":") { + writeErr(w, r, api.Errorf(api.CodeInvalidRequest, "id (without ':'), pool and endpoint are required")) + return + } + pool, err := s.Store.GetPool(r.Context(), a.Pool) + if err != nil || pool.Adapter != "firecracker" { + writeErr(w, r, api.Errorf(api.CodeNotFound, "no firecracker pool named %s is registered", a.Pool)) + return + } + known, _ := s.Store.ListAgents(r.Context(), a.Pool) + isNew := !slices.ContainsFunc(known, func(k api.Agent) bool { return k.ID == a.ID && k.Endpoint == a.Endpoint }) + a.LastSeen, a.Healthy = s.now(), true + if err := s.Store.UpsertAgent(r.Context(), a); err != nil { + writeErr(w, r, err) + return + } + if isNew { + s.Audit.Record(r.Context(), api.AuditEvent{Actor: "agent:" + a.ID, Action: "agent.register", Resource: a.ID, Details: map[string]string{"pool": a.Pool, "node": a.Node, "endpoint": a.Endpoint, "version": a.Version}, RequestID: requestID(r)}) + } + writeJSON(w, 200, map[string]any{"ok": true, "installation": s.Installation}) +} diff --git a/internal/store/memory.go b/internal/store/memory.go new file mode 100644 index 0000000..77e5feb --- /dev/null +++ b/internal/store/memory.go @@ -0,0 +1,576 @@ +package store + +import ( + "context" + "encoding/json" + "sort" + "sync" + "time" + + "github.com/different-ai/differently/internal/audit" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/pkg/api" +) + +// Memory is an in-process Store for development and tests. State is lost on +// restart; use postgres for anything durable. +type Memory struct { + mu sync.Mutex + installation string + orgs map[string]api.Organization + projects map[string]api.Project + keys map[string]KeyRecord + pools map[string]api.Pool + policies map[string]api.PlacementPolicy + sandboxes map[string]SandboxRecord + snapshots map[string]SnapshotRecord + agents map[string]api.Agent + audit []api.AuditEvent +} + +// NewMemory returns an empty memory store. +func NewMemory() *Memory { + return &Memory{ + orgs: map[string]api.Organization{}, + projects: map[string]api.Project{}, + keys: map[string]KeyRecord{}, + pools: map[string]api.Pool{}, + policies: map[string]api.PlacementPolicy{}, + sandboxes: map[string]SandboxRecord{}, + snapshots: map[string]SnapshotRecord{}, + agents: map[string]api.Agent{}, + } +} + +// clone deep-copies v so callers never alias stored state. +func clone[T any](v T) T { + b, err := json.Marshal(v) + if err != nil { + panic(err) + } + var out T + if err := json.Unmarshal(b, &out); err != nil { + panic(err) + } + return out +} + +func (m *Memory) Installation(context.Context) (string, error) { + m.mu.Lock() + defer m.mu.Unlock() + if m.installation == "" { + m.installation = id.New("inst") + } + return m.installation, nil +} + +func (m *Memory) CreateOrg(_ context.Context, o api.Organization) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.orgs[o.ID]; ok { + return api.Errorf(api.CodeConflict, "organization %s exists", o.ID) + } + m.orgs[o.ID] = clone(o) + return nil +} + +func (m *Memory) GetOrg(_ context.Context, id string) (api.Organization, error) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.orgs[id] + if !ok { + return o, NotFound("organization", id) + } + return clone(o), nil +} + +func (m *Memory) ListOrgs(context.Context) ([]api.Organization, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]api.Organization, 0, len(m.orgs)) + for _, o := range m.orgs { + out = append(out, clone(o)) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (m *Memory) UpdateOrg(_ context.Context, id string, fn func(*api.Organization) error) (api.Organization, error) { + m.mu.Lock() + defer m.mu.Unlock() + o, ok := m.orgs[id] + if !ok { + return o, NotFound("organization", id) + } + o = clone(o) + if err := fn(&o); err != nil { + return o, err + } + m.orgs[id] = clone(o) + return o, nil +} + +func (m *Memory) CreateProject(_ context.Context, p api.Project) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.projects[p.ID]; ok { + return api.Errorf(api.CodeConflict, "project %s exists", p.ID) + } + m.projects[p.ID] = clone(p) + return nil +} + +func (m *Memory) GetProject(_ context.Context, id string) (api.Project, error) { + m.mu.Lock() + defer m.mu.Unlock() + p, ok := m.projects[id] + if !ok { + return p, NotFound("project", id) + } + return clone(p), nil +} + +func (m *Memory) ListProjects(_ context.Context, orgID string) ([]api.Project, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []api.Project + for _, p := range m.projects { + if orgID == "" || p.OrgID == orgID { + out = append(out, clone(p)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (m *Memory) UpdateProject(_ context.Context, id string, fn func(*api.Project) error) (api.Project, error) { + m.mu.Lock() + defer m.mu.Unlock() + p, ok := m.projects[id] + if !ok { + return p, NotFound("project", id) + } + p = clone(p) + if err := fn(&p); err != nil { + return p, err + } + m.projects[id] = clone(p) + return p, nil +} + +func (m *Memory) DeleteProject(_ context.Context, id string) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.projects[id]; !ok { + return NotFound("project", id) + } + for _, s := range m.sandboxes { + if s.ProjectID == id && !s.Status.Terminal() { + return api.Errorf(api.CodeConflict, "project %s still has sandbox %s", id, s.ID) + } + } + delete(m.projects, id) + return nil +} + +func (m *Memory) CreateKey(_ context.Context, k KeyRecord) error { + m.mu.Lock() + defer m.mu.Unlock() + if _, ok := m.keys[k.ID]; ok { + return api.Errorf(api.CodeConflict, "key %s exists", k.ID) + } + m.keys[k.ID] = clone(k) + return nil +} + +func (m *Memory) GetKey(_ context.Context, id string) (KeyRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + k, ok := m.keys[id] + if !ok { + return k, NotFound("api key", id) + } + return clone(k), nil +} + +func (m *Memory) ListKeys(_ context.Context, orgID, projectID string) ([]KeyRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []KeyRecord + for _, k := range m.keys { + if (orgID == "" || k.OrgID == orgID) && (projectID == "" || k.ProjectID == projectID) { + out = append(out, clone(k)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (m *Memory) UpdateKey(_ context.Context, id string, fn func(*KeyRecord) error) (KeyRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + k, ok := m.keys[id] + if !ok { + return k, NotFound("api key", id) + } + k = clone(k) + if err := fn(&k); err != nil { + return k, err + } + m.keys[id] = clone(k) + return k, nil +} + +func (m *Memory) UpsertPool(_ context.Context, p api.Pool) error { + m.mu.Lock() + defer m.mu.Unlock() + p.Health, p.Capacity, p.Active = nil, nil, 0 + m.pools[p.Name] = clone(p) + return nil +} + +func (m *Memory) GetPool(_ context.Context, name string) (api.Pool, error) { + m.mu.Lock() + defer m.mu.Unlock() + p, ok := m.pools[name] + if !ok { + return p, NotFound("pool", name) + } + return clone(p), nil +} + +func (m *Memory) ListPools(context.Context) ([]api.Pool, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]api.Pool, 0, len(m.pools)) + for _, p := range m.pools { + out = append(out, clone(p)) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +func (m *Memory) DeletePool(_ context.Context, name string) error { + m.mu.Lock() + defer m.mu.Unlock() + for _, s := range m.sandboxes { + if s.Pool == name && !s.Status.Terminal() { + return api.Errorf(api.CodeConflict, "pool %s still hosts sandbox %s; drain it first", name, s.ID) + } + } + delete(m.pools, name) + return nil +} + +func (m *Memory) PutPolicy(_ context.Context, p api.PlacementPolicy) error { + m.mu.Lock() + defer m.mu.Unlock() + m.policies[p.Scope] = clone(p) + return nil +} + +func (m *Memory) GetPolicy(_ context.Context, scope string) (api.PlacementPolicy, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + p, ok := m.policies[scope] + return clone(p), ok, nil +} + +func (m *Memory) ListPolicies(context.Context) ([]api.PlacementPolicy, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := make([]api.PlacementPolicy, 0, len(m.policies)) + for _, p := range m.policies { + out = append(out, clone(p)) + } + sort.Slice(out, func(i, j int) bool { return out[i].Scope < out[j].Scope }) + return out, nil +} + +func (m *Memory) DeletePolicy(_ context.Context, scope string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.policies, scope) + return nil +} + +func (m *Memory) tally(orgID, projectID string) api.QuotaUsage { + var recs []SandboxRecord + for _, s := range m.sandboxes { + if s.OrgID == orgID && (projectID == "" || s.ProjectID == projectID) { + recs = append(recs, s) + } + } + snaps := 0 + for _, s := range m.snapshots { + if s.OrgID == orgID && (projectID == "" || s.ProjectID == projectID) { + snaps++ + } + } + return Tally(recs, snaps) +} + +func (m *Memory) AdmitSandbox(_ context.Context, rec SandboxRecord, adm Admission) (SandboxRecord, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if rec.IdempotencyKey != "" { + for _, s := range m.sandboxes { + if s.ProjectID == rec.ProjectID && s.IdempotencyKey == rec.IdempotencyKey { + return clone(s), true, nil + } + } + } + if err := CheckAdmission("organization", m.tally(rec.OrgID, ""), adm.OrgQuota, rec.Resources, rec.TTLSeconds); err != nil { + return rec, false, err + } + if err := CheckAdmission("project", m.tally(rec.OrgID, rec.ProjectID), adm.ProjectQuota, rec.Resources, rec.TTLSeconds); err != nil { + return rec, false, err + } + rec.Status = api.StatusQueued + m.sandboxes[rec.ID] = clone(rec) + return clone(rec), false, nil +} + +func (m *Memory) GetSandbox(_ context.Context, id string) (SandboxRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.sandboxes[id] + if !ok { + return s, NotFound("sandbox", id) + } + return clone(s), nil +} + +func (m *Memory) ListSandboxes(_ context.Context, f SandboxFilter) ([]SandboxRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []SandboxRecord + for _, s := range m.sandboxes { + if f.Matches(s) && (f.Cursor == "" || s.ID < f.Cursor) { + out = append(out, s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID > out[j].ID }) + if f.Limit > 0 && len(out) > f.Limit { + out = out[:f.Limit] + } + return clone(out), nil +} + +func (m *Memory) UpdateSandbox(_ context.Context, id string, fn func(*SandboxRecord) error) (SandboxRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.sandboxes[id] + if !ok { + return s, NotFound("sandbox", id) + } + s = clone(s) + if err := fn(&s); err != nil { + return s, err + } + m.sandboxes[id] = clone(s) + return s, nil +} + +func (m *Memory) QueuedCandidates(_ context.Context, now time.Time, limit int) ([]SandboxRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + var queued []SandboxRecord + activeCPU := map[string]int{} + for _, s := range m.sandboxes { + switch { + case s.Status == api.StatusQueued && !s.NextAttemptAt.After(now): + queued = append(queued, s) + case s.Status.ReservesCompute() && s.Status != api.StatusQueued: + activeCPU[s.OrgID] += s.Resources.CPU + } + } + out := FairOrder(queued, activeCPU) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return clone(out), nil +} + +func (m *Memory) ActivateSandbox(_ context.Context, sid string, act Activation) (SandboxRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.sandboxes[sid] + if !ok { + return s, NotFound("sandbox", sid) + } + if s.Status != api.StatusQueued { + return s, api.Errorf(api.CodeInvalidState, "sandbox %s is %s, not queued", sid, s.Status) + } + if err := CheckActivation("organization", m.tally(s.OrgID, ""), act.OrgQuota, s.Resources); err != nil { + return s, err + } + if err := CheckActivation("project", m.tally(s.OrgID, s.ProjectID), act.ProjectQuota, s.Resources); err != nil { + return s, err + } + if act.MaxConcurrency > 0 { + n := 0 + for _, o := range m.sandboxes { + if o.Pool == act.Pool && HoldsPoolSlot(o.Status) { + n++ + } + } + if n >= act.MaxConcurrency { + return s, &api.Error{Code: "insufficient_capacity", Message: "pool " + act.Pool + " is at its concurrency limit"} + } + } + s = clone(s) + applyActivation(&s, act) + m.sandboxes[sid] = clone(s) + return s, nil +} + +func applyActivation(s *SandboxRecord, act Activation) { + now := act.Now + s.Status = api.StatusProvisioning + s.StatusReason = "" + s.Pool, s.Adapter, s.Trust, s.Region = act.Pool, act.Adapter, act.Trust, act.Region + s.Decision = act.Decision + s.Attempts++ + s.QueueSeconds = now.Sub(s.CreatedAt).Seconds() - s.RunSeconds + s.ProvisionStartedAt = &now + s.UpdatedAt = now +} + +func (m *Memory) ReserveCompute(_ context.Context, sid string, from, to api.SandboxStatus, oq, pq api.Quota, now time.Time) (SandboxRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.sandboxes[sid] + if !ok { + return s, NotFound("sandbox", sid) + } + if s.Status != from { + return s, api.Errorf(api.CodeInvalidState, "sandbox %s is %s, not %s", sid, s.Status, from) + } + if err := CheckCompute("organization", m.tally(s.OrgID, ""), oq, s.Resources); err != nil { + return s, err + } + if err := CheckCompute("project", m.tally(s.OrgID, s.ProjectID), pq, s.Resources); err != nil { + return s, err + } + s = clone(s) + s.Status, s.UpdatedAt = to, now + m.sandboxes[sid] = clone(s) + return s, nil +} + +func (m *Memory) Usage(_ context.Context, orgID, projectID string) (api.QuotaUsage, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.tally(orgID, projectID), nil +} + +func (m *Memory) PoolActive(context.Context) (map[string]int, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string]int{} + for _, s := range m.sandboxes { + if s.Pool != "" && HoldsPoolSlot(s.Status) { + out[s.Pool]++ + } + } + return out, nil +} + +func (m *Memory) CreateSnapshot(_ context.Context, s SnapshotRecord) error { + m.mu.Lock() + defer m.mu.Unlock() + m.snapshots[s.ID] = clone(s) + return nil +} + +func (m *Memory) GetSnapshot(_ context.Context, id string) (SnapshotRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.snapshots[id] + if !ok { + return s, NotFound("snapshot", id) + } + return clone(s), nil +} + +func (m *Memory) ListSnapshots(_ context.Context, orgID, projectID string) ([]SnapshotRecord, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []SnapshotRecord + for _, s := range m.snapshots { + if (orgID == "" || s.OrgID == orgID) && (projectID == "" || s.ProjectID == projectID) { + out = append(out, clone(s)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID > out[j].ID }) + return out, nil +} + +func (m *Memory) DeleteSnapshot(_ context.Context, id string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.snapshots, id) + return nil +} + +func (m *Memory) AppendAudit(_ context.Context, ev api.AuditEvent) (api.AuditEvent, error) { + m.mu.Lock() + defer m.mu.Unlock() + prev := "" + if n := len(m.audit); n > 0 { + prev = m.audit[n-1].Hash + } + ev.Seq = int64(len(m.audit) + 1) + if ev.Time.IsZero() { + ev.Time = time.Now().UTC() + } + ev = audit.Seal(prev, clone(ev)) + m.audit = append(m.audit, ev) + return ev, nil +} + +func (m *Memory) ListAudit(_ context.Context, f AuditFilter) ([]api.AuditEvent, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []api.AuditEvent + for _, ev := range m.audit { + if ev.Seq <= f.AfterSeq || + (f.OrgID != "" && ev.OrgID != f.OrgID) || + (f.Resource != "" && ev.Resource != f.Resource) || + (f.Action != "" && ev.Action != f.Action) { + continue + } + out = append(out, clone(ev)) + if f.Limit > 0 && len(out) >= f.Limit { + break + } + } + return out, nil +} + +func (m *Memory) UpsertAgent(_ context.Context, a api.Agent) error { + m.mu.Lock() + defer m.mu.Unlock() + m.agents[a.ID] = clone(a) + return nil +} + +func (m *Memory) ListAgents(_ context.Context, pool string) ([]api.Agent, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []api.Agent + for _, a := range m.agents { + if pool == "" || a.Pool == pool { + out = append(out, clone(a)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (m *Memory) TryLeader(context.Context) (bool, error) { return true, nil } + +func (m *Memory) Close() error { return nil } + +var _ Store = (*Memory)(nil) diff --git a/internal/store/memory_test.go b/internal/store/memory_test.go new file mode 100644 index 0000000..60573fd --- /dev/null +++ b/internal/store/memory_test.go @@ -0,0 +1,12 @@ +package store_test + +import ( + "testing" + + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/internal/store/storetest" +) + +func TestMemoryConformance(t *testing.T) { + storetest.Run(t, func(*testing.T) store.Store { return store.NewMemory() }) +} diff --git a/internal/store/postgres/migrations/0001_init.sql b/internal/store/postgres/migrations/0001_init.sql new file mode 100644 index 0000000..941c2ca --- /dev/null +++ b/internal/store/postgres/migrations/0001_init.sql @@ -0,0 +1,75 @@ +-- Differently control-plane schema. Resources are stored as JSONB documents +-- with the columns needed for locking, filtering and ordering promoted. + +CREATE TABLE IF NOT EXISTS meta ( + key text PRIMARY KEY, + value text NOT NULL +); + +CREATE TABLE IF NOT EXISTS organizations ( + id text PRIMARY KEY, + doc jsonb NOT NULL +); + +CREATE TABLE IF NOT EXISTS projects ( + id text PRIMARY KEY, + org_id text NOT NULL REFERENCES organizations (id), + doc jsonb NOT NULL +); +CREATE INDEX IF NOT EXISTS projects_org ON projects (org_id); + +CREATE TABLE IF NOT EXISTS api_keys ( + id text PRIMARY KEY, + org_id text NOT NULL DEFAULT '', + project_id text NOT NULL DEFAULT '', + doc jsonb NOT NULL +); +CREATE INDEX IF NOT EXISTS api_keys_org ON api_keys (org_id, project_id); + +CREATE TABLE IF NOT EXISTS pools ( + name text PRIMARY KEY, + doc jsonb NOT NULL +); + +CREATE TABLE IF NOT EXISTS policies ( + scope text PRIMARY KEY, + doc jsonb NOT NULL +); + +CREATE TABLE IF NOT EXISTS sandboxes ( + id text PRIMARY KEY, + org_id text NOT NULL, + project_id text NOT NULL, + pool text NOT NULL DEFAULT '', + status text NOT NULL, + idempotency_key text NOT NULL DEFAULT '', + next_attempt_at timestamptz NOT NULL DEFAULT now(), + doc jsonb NOT NULL +); +CREATE INDEX IF NOT EXISTS sandboxes_queue ON sandboxes (status, next_attempt_at) WHERE status = 'queued'; +CREATE INDEX IF NOT EXISTS sandboxes_org ON sandboxes (org_id, project_id, status); +CREATE INDEX IF NOT EXISTS sandboxes_pool ON sandboxes (pool, status); +CREATE UNIQUE INDEX IF NOT EXISTS sandboxes_idempotency ON sandboxes (project_id, idempotency_key) WHERE idempotency_key <> ''; + +CREATE TABLE IF NOT EXISTS snapshots ( + id text PRIMARY KEY, + org_id text NOT NULL, + project_id text NOT NULL, + doc jsonb NOT NULL +); +CREATE INDEX IF NOT EXISTS snapshots_org ON snapshots (org_id, project_id); + +CREATE TABLE IF NOT EXISTS audit_events ( + seq bigint PRIMARY KEY, + org_id text NOT NULL DEFAULT '', + resource text NOT NULL DEFAULT '', + action text NOT NULL, + doc jsonb NOT NULL +); +CREATE INDEX IF NOT EXISTS audit_events_org ON audit_events (org_id, seq); + +CREATE TABLE IF NOT EXISTS agents ( + id text PRIMARY KEY, + pool text NOT NULL, + doc jsonb NOT NULL +); diff --git a/internal/store/postgres/postgres.go b/internal/store/postgres/postgres.go new file mode 100644 index 0000000..5c05baf --- /dev/null +++ b/internal/store/postgres/postgres.go @@ -0,0 +1,709 @@ +// Package postgres is the production Store. Quota-sensitive operations run +// in transactions that lock the organization row (and the pool row for +// concurrency caps) in a fixed order, so concurrent replicas cannot +// double-allocate. Scheduler leadership is a session advisory lock. +package postgres + +import ( + "context" + "embed" + "encoding/json" + "errors" + "fmt" + "io/fs" + "sort" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/different-ai/differently/internal/audit" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/api" +) + +//go:embed migrations/*.sql +var migrations embed.FS + +const ( + lockMigrate int64 = 0x646c795f6d6967 // "dly_mig" + lockLeader int64 = 0x646c795f6c6472 // "dly_ldr" + lockAudit int64 = 0x646c795f617564 // "dly_aud" +) + +// Store implements store.Store on PostgreSQL. +type Store struct { + pool *pgxpool.Pool + + leaderMu sync.Mutex + leaderConn *pgxpool.Conn +} + +// Open connects to dsn. +func Open(ctx context.Context, dsn string) (*Store, error) { + p, err := pgxpool.New(ctx, dsn) + if err != nil { + return nil, err + } + if err := p.Ping(ctx); err != nil { + p.Close() + return nil, err + } + return &Store{pool: p}, nil +} + +// Migrate applies pending migrations under an advisory lock. +func (s *Store) Migrate(ctx context.Context) error { + conn, err := s.pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", lockMigrate); err != nil { + return err + } + defer conn.Exec(context.WithoutCancel(ctx), "SELECT pg_advisory_unlock($1)", lockMigrate) + if _, err := conn.Exec(ctx, "CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"); err != nil { + return err + } + names, _ := fs.Glob(migrations, "migrations/*.sql") + sort.Strings(names) + for _, name := range names { + var done bool + if err := conn.QueryRow(ctx, "SELECT EXISTS (SELECT 1 FROM schema_migrations WHERE version = $1)", name).Scan(&done); err != nil { + return err + } + if done { + continue + } + sql, _ := migrations.ReadFile(name) + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(sql)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("%s: %w", name, err) + } + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version) VALUES ($1)", name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} + +// Close releases leadership and the pool. +func (s *Store) Close() error { + s.leaderMu.Lock() + if s.leaderConn != nil { + s.leaderConn.Release() + s.leaderConn = nil + } + s.leaderMu.Unlock() + s.pool.Close() + return nil +} + +type querier interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +func (s *Store) tx(ctx context.Context, fn func(pgx.Tx) error) error { + return pgx.BeginFunc(ctx, s.pool, fn) +} + +func getDoc[T any](ctx context.Context, q querier, kind, id, sql string, args ...any) (T, error) { + var out T + var raw []byte + err := q.QueryRow(ctx, sql, args...).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return out, store.NotFound(kind, id) + } + if err != nil { + return out, err + } + return out, json.Unmarshal(raw, &out) +} + +func listDocs[T any](ctx context.Context, q querier, sql string, args ...any) ([]T, error) { + rows, err := q.Query(ctx, sql, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []T + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, err + } + var v T + if err := json.Unmarshal(raw, &v); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +func conflictOnDup(err error, msg string) error { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return api.Errorf(api.CodeConflict, "%s", msg) + } + return err +} + +func (s *Store) Installation(ctx context.Context) (string, error) { + if _, err := s.pool.Exec(ctx, "INSERT INTO meta (key, value) VALUES ('installation', $1) ON CONFLICT DO NOTHING", id.New("inst")); err != nil { + return "", err + } + var v string + return v, s.pool.QueryRow(ctx, "SELECT value FROM meta WHERE key = 'installation'").Scan(&v) +} + +func (s *Store) CreateOrg(ctx context.Context, o api.Organization) error { + _, err := s.pool.Exec(ctx, "INSERT INTO organizations (id, doc) VALUES ($1, $2)", o.ID, o) + return conflictOnDup(err, "organization "+o.ID+" exists") +} + +func (s *Store) GetOrg(ctx context.Context, oid string) (api.Organization, error) { + return getDoc[api.Organization](ctx, s.pool, "organization", oid, "SELECT doc FROM organizations WHERE id = $1", oid) +} + +func (s *Store) ListOrgs(ctx context.Context) ([]api.Organization, error) { + return listDocs[api.Organization](ctx, s.pool, "SELECT doc FROM organizations ORDER BY id") +} + +func (s *Store) UpdateOrg(ctx context.Context, oid string, fn func(*api.Organization) error) (api.Organization, error) { + var out api.Organization + err := s.tx(ctx, func(tx pgx.Tx) error { + o, err := getDoc[api.Organization](ctx, tx, "organization", oid, "SELECT doc FROM organizations WHERE id = $1 FOR UPDATE", oid) + if err != nil { + return err + } + if err := fn(&o); err != nil { + return err + } + out = o + _, err = tx.Exec(ctx, "UPDATE organizations SET doc = $2 WHERE id = $1", oid, o) + return err + }) + return out, err +} + +func (s *Store) CreateProject(ctx context.Context, p api.Project) error { + _, err := s.pool.Exec(ctx, "INSERT INTO projects (id, org_id, doc) VALUES ($1, $2, $3)", p.ID, p.OrgID, p) + return conflictOnDup(err, "project "+p.ID+" exists") +} + +func (s *Store) GetProject(ctx context.Context, pid string) (api.Project, error) { + return getDoc[api.Project](ctx, s.pool, "project", pid, "SELECT doc FROM projects WHERE id = $1", pid) +} + +func (s *Store) ListProjects(ctx context.Context, orgID string) ([]api.Project, error) { + return listDocs[api.Project](ctx, s.pool, "SELECT doc FROM projects WHERE $1 = '' OR org_id = $1 ORDER BY id", orgID) +} + +func (s *Store) UpdateProject(ctx context.Context, pid string, fn func(*api.Project) error) (api.Project, error) { + var out api.Project + err := s.tx(ctx, func(tx pgx.Tx) error { + p, err := getDoc[api.Project](ctx, tx, "project", pid, "SELECT doc FROM projects WHERE id = $1 FOR UPDATE", pid) + if err != nil { + return err + } + if err := fn(&p); err != nil { + return err + } + out = p + _, err = tx.Exec(ctx, "UPDATE projects SET doc = $2 WHERE id = $1", pid, p) + return err + }) + return out, err +} + +func (s *Store) DeleteProject(ctx context.Context, pid string) error { + return s.tx(ctx, func(tx pgx.Tx) error { + p, err := getDoc[api.Project](ctx, tx, "project", pid, "SELECT doc FROM projects WHERE id = $1", pid) + if err != nil { + return err + } + // Lock the organization so no admission races the delete. + if _, err := tx.Exec(ctx, "SELECT 1 FROM organizations WHERE id = $1 FOR UPDATE", p.OrgID); err != nil { + return err + } + var busy string + err = tx.QueryRow(ctx, "SELECT id FROM sandboxes WHERE project_id = $1 AND status NOT IN ('terminated','failed') LIMIT 1", pid).Scan(&busy) + if err == nil { + return api.Errorf(api.CodeConflict, "project %s still has sandbox %s", pid, busy) + } + if !errors.Is(err, pgx.ErrNoRows) { + return err + } + _, err = tx.Exec(ctx, "DELETE FROM projects WHERE id = $1", pid) + return err + }) +} + +func (s *Store) CreateKey(ctx context.Context, k store.KeyRecord) error { + _, err := s.pool.Exec(ctx, "INSERT INTO api_keys (id, org_id, project_id, doc) VALUES ($1, $2, $3, $4)", k.ID, k.OrgID, k.ProjectID, k) + return conflictOnDup(err, "key "+k.ID+" exists") +} + +func (s *Store) GetKey(ctx context.Context, kid string) (store.KeyRecord, error) { + return getDoc[store.KeyRecord](ctx, s.pool, "api key", kid, "SELECT doc FROM api_keys WHERE id = $1", kid) +} + +func (s *Store) ListKeys(ctx context.Context, orgID, projectID string) ([]store.KeyRecord, error) { + return listDocs[store.KeyRecord](ctx, s.pool, "SELECT doc FROM api_keys WHERE ($1 = '' OR org_id = $1) AND ($2 = '' OR project_id = $2) ORDER BY id", orgID, projectID) +} + +func (s *Store) UpdateKey(ctx context.Context, kid string, fn func(*store.KeyRecord) error) (store.KeyRecord, error) { + var out store.KeyRecord + err := s.tx(ctx, func(tx pgx.Tx) error { + k, err := getDoc[store.KeyRecord](ctx, tx, "api key", kid, "SELECT doc FROM api_keys WHERE id = $1 FOR UPDATE", kid) + if err != nil { + return err + } + if err := fn(&k); err != nil { + return err + } + out = k + _, err = tx.Exec(ctx, "UPDATE api_keys SET doc = $2 WHERE id = $1", kid, k) + return err + }) + return out, err +} + +func (s *Store) UpsertPool(ctx context.Context, p api.Pool) error { + p.Health, p.Capacity, p.Active = nil, nil, 0 + _, err := s.pool.Exec(ctx, "INSERT INTO pools (name, doc) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET doc = EXCLUDED.doc", p.Name, p) + return err +} + +func (s *Store) GetPool(ctx context.Context, name string) (api.Pool, error) { + return getDoc[api.Pool](ctx, s.pool, "pool", name, "SELECT doc FROM pools WHERE name = $1", name) +} + +func (s *Store) ListPools(ctx context.Context) ([]api.Pool, error) { + return listDocs[api.Pool](ctx, s.pool, "SELECT doc FROM pools ORDER BY name") +} + +func (s *Store) DeletePool(ctx context.Context, name string) error { + return s.tx(ctx, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, "SELECT 1 FROM pools WHERE name = $1 FOR UPDATE", name); err != nil { + return err + } + var busy string + err := tx.QueryRow(ctx, "SELECT id FROM sandboxes WHERE pool = $1 AND status NOT IN ('terminated','failed','queued') LIMIT 1", name).Scan(&busy) + if err == nil { + return api.Errorf(api.CodeConflict, "pool %s still hosts sandbox %s; drain it first", name, busy) + } + if !errors.Is(err, pgx.ErrNoRows) { + return err + } + _, err = tx.Exec(ctx, "DELETE FROM pools WHERE name = $1", name) + return err + }) +} + +func (s *Store) PutPolicy(ctx context.Context, p api.PlacementPolicy) error { + _, err := s.pool.Exec(ctx, "INSERT INTO policies (scope, doc) VALUES ($1, $2) ON CONFLICT (scope) DO UPDATE SET doc = EXCLUDED.doc", p.Scope, p) + return err +} + +func (s *Store) GetPolicy(ctx context.Context, scope string) (api.PlacementPolicy, bool, error) { + p, err := getDoc[api.PlacementPolicy](ctx, s.pool, "policy", scope, "SELECT doc FROM policies WHERE scope = $1", scope) + if api.IsCode(err, api.CodeNotFound) { + return p, false, nil + } + return p, err == nil, err +} + +func (s *Store) ListPolicies(ctx context.Context) ([]api.PlacementPolicy, error) { + return listDocs[api.PlacementPolicy](ctx, s.pool, "SELECT doc FROM policies ORDER BY scope") +} + +func (s *Store) DeletePolicy(ctx context.Context, scope string) error { + _, err := s.pool.Exec(ctx, "DELETE FROM policies WHERE scope = $1", scope) + return err +} + +func tally(ctx context.Context, q querier, orgID, projectID string) (api.QuotaUsage, error) { + recs, err := listDocs[store.SandboxRecord](ctx, q, + "SELECT doc FROM sandboxes WHERE org_id = $1 AND ($2 = '' OR project_id = $2) AND status NOT IN ('terminated','failed')", orgID, projectID) + if err != nil { + return api.QuotaUsage{}, err + } + var snaps int + if err := q.QueryRow(ctx, "SELECT count(*) FROM snapshots WHERE org_id = $1 AND ($2 = '' OR project_id = $2)", orgID, projectID).Scan(&snaps); err != nil { + return api.QuotaUsage{}, err + } + return store.Tally(recs, snaps), nil +} + +func lockOrg(ctx context.Context, tx pgx.Tx, orgID string) error { + var x int + err := tx.QueryRow(ctx, "SELECT 1 FROM organizations WHERE id = $1 FOR UPDATE", orgID).Scan(&x) + if errors.Is(err, pgx.ErrNoRows) { + return store.NotFound("organization", orgID) + } + return err +} + +func (s *Store) AdmitSandbox(ctx context.Context, rec store.SandboxRecord, adm store.Admission) (store.SandboxRecord, bool, error) { + var out store.SandboxRecord + existed := false + err := s.tx(ctx, func(tx pgx.Tx) error { + if err := lockOrg(ctx, tx, rec.OrgID); err != nil { + return err + } + if rec.IdempotencyKey != "" { + prev, err := getDoc[store.SandboxRecord](ctx, tx, "sandbox", "", "SELECT doc FROM sandboxes WHERE project_id = $1 AND idempotency_key = $2", rec.ProjectID, rec.IdempotencyKey) + if err == nil { + out, existed = prev, true + return nil + } + if !api.IsCode(err, api.CodeNotFound) { + return err + } + } + ou, err := tally(ctx, tx, rec.OrgID, "") + if err != nil { + return err + } + if err := store.CheckAdmission("organization", ou, adm.OrgQuota, rec.Resources, rec.TTLSeconds); err != nil { + return err + } + pu, err := tally(ctx, tx, rec.OrgID, rec.ProjectID) + if err != nil { + return err + } + if err := store.CheckAdmission("project", pu, adm.ProjectQuota, rec.Resources, rec.TTLSeconds); err != nil { + return err + } + rec.Status = api.StatusQueued + out = rec + return insertSandbox(ctx, tx, rec) + }) + return out, existed, err +} + +func insertSandbox(ctx context.Context, q querier, r store.SandboxRecord) error { + _, err := q.Exec(ctx, `INSERT INTO sandboxes (id, org_id, project_id, pool, status, idempotency_key, next_attempt_at, doc) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`, r.ID, r.OrgID, r.ProjectID, r.Pool, string(r.Status), r.IdempotencyKey, r.NextAttemptAt, r) + return err +} + +func saveSandbox(ctx context.Context, q querier, r store.SandboxRecord) error { + _, err := q.Exec(ctx, "UPDATE sandboxes SET pool = $2, status = $3, next_attempt_at = $4, doc = $5 WHERE id = $1", + r.ID, r.Pool, string(r.Status), r.NextAttemptAt, r) + return err +} + +func (s *Store) GetSandbox(ctx context.Context, sid string) (store.SandboxRecord, error) { + return getDoc[store.SandboxRecord](ctx, s.pool, "sandbox", sid, "SELECT doc FROM sandboxes WHERE id = $1", sid) +} + +func (s *Store) ListSandboxes(ctx context.Context, f store.SandboxFilter) ([]store.SandboxRecord, error) { + where := []string{"TRUE"} + var args []any + add := func(clause string, v any) { + args = append(args, v) + where = append(where, fmt.Sprintf(clause, len(args))) + } + if f.OrgID != "" { + add("org_id = $%d", f.OrgID) + } + if f.ProjectID != "" { + add("project_id = $%d", f.ProjectID) + } + if f.Pool != "" { + add("pool = $%d", f.Pool) + } + if len(f.Statuses) > 0 { + st := make([]string, len(f.Statuses)) + for i, v := range f.Statuses { + st[i] = string(v) + } + add("status = ANY($%d)", st) + } + if len(f.Label) > 0 { + b, _ := json.Marshal(f.Label) + add("doc->'labels' @> $%d::jsonb", string(b)) + } + if f.Cursor != "" { + add("id < $%d", f.Cursor) + } + sql := "SELECT doc FROM sandboxes WHERE " + strings.Join(where, " AND ") + " ORDER BY id DESC" + if f.Limit > 0 { + sql += fmt.Sprintf(" LIMIT %d", f.Limit) + } + return listDocs[store.SandboxRecord](ctx, s.pool, sql, args...) +} + +func (s *Store) UpdateSandbox(ctx context.Context, sid string, fn func(*store.SandboxRecord) error) (store.SandboxRecord, error) { + var out store.SandboxRecord + err := s.tx(ctx, func(tx pgx.Tx) error { + r, err := getDoc[store.SandboxRecord](ctx, tx, "sandbox", sid, "SELECT doc FROM sandboxes WHERE id = $1 FOR UPDATE", sid) + if err != nil { + return err + } + if err := fn(&r); err != nil { + return err + } + out = r + return saveSandbox(ctx, tx, r) + }) + return out, err +} + +func (s *Store) QueuedCandidates(ctx context.Context, now time.Time, limit int) ([]store.SandboxRecord, error) { + queued, err := listDocs[store.SandboxRecord](ctx, s.pool, + "SELECT doc FROM sandboxes WHERE status = 'queued' AND next_attempt_at <= $1 ORDER BY id LIMIT 1000", now) + if err != nil || len(queued) == 0 { + return nil, err + } + rows, err := s.pool.Query(ctx, "SELECT org_id, coalesce(sum((doc->'resources'->>'cpu')::int), 0) FROM sandboxes WHERE status IN ('provisioning','running','paused','terminating') GROUP BY org_id") + if err != nil { + return nil, err + } + active := map[string]int{} + for rows.Next() { + var org string + var cpu int + if err := rows.Scan(&org, &cpu); err != nil { + rows.Close() + return nil, err + } + active[org] = cpu + } + rows.Close() + out := store.FairOrder(queued, active) + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return out, nil +} + +func (s *Store) ActivateSandbox(ctx context.Context, sid string, act store.Activation) (store.SandboxRecord, error) { + var out store.SandboxRecord + err := s.tx(ctx, func(tx pgx.Tx) error { + var orgID string + if err := tx.QueryRow(ctx, "SELECT org_id FROM sandboxes WHERE id = $1", sid).Scan(&orgID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return store.NotFound("sandbox", sid) + } + return err + } + // Lock order everywhere: organization, then pool, then sandbox. + if err := lockOrg(ctx, tx, orgID); err != nil { + return err + } + if act.MaxConcurrency > 0 { + if _, err := tx.Exec(ctx, "SELECT 1 FROM pools WHERE name = $1 FOR UPDATE", act.Pool); err != nil { + return err + } + } + r, err := getDoc[store.SandboxRecord](ctx, tx, "sandbox", sid, "SELECT doc FROM sandboxes WHERE id = $1 FOR UPDATE", sid) + if err != nil { + return err + } + if r.Status != api.StatusQueued { + return api.Errorf(api.CodeInvalidState, "sandbox %s is %s, not queued", sid, r.Status) + } + ou, err := tally(ctx, tx, r.OrgID, "") + if err != nil { + return err + } + if err := store.CheckActivation("organization", ou, act.OrgQuota, r.Resources); err != nil { + return err + } + pu, err := tally(ctx, tx, r.OrgID, r.ProjectID) + if err != nil { + return err + } + if err := store.CheckActivation("project", pu, act.ProjectQuota, r.Resources); err != nil { + return err + } + if act.MaxConcurrency > 0 { + var n int + if err := tx.QueryRow(ctx, "SELECT count(*) FROM sandboxes WHERE pool = $1 AND status IN ('provisioning','running','paused','stopped','terminating')", act.Pool).Scan(&n); err != nil { + return err + } + if n >= act.MaxConcurrency { + return &api.Error{Code: "insufficient_capacity", Message: "pool " + act.Pool + " is at its concurrency limit"} + } + } + now := act.Now + r.Status, r.StatusReason = api.StatusProvisioning, "" + r.Pool, r.Adapter, r.Trust, r.Region = act.Pool, act.Adapter, act.Trust, act.Region + r.Decision = act.Decision + r.Attempts++ + r.QueueSeconds = now.Sub(r.CreatedAt).Seconds() - r.RunSeconds + r.ProvisionStartedAt, r.UpdatedAt = &now, now + out = r + return saveSandbox(ctx, tx, r) + }) + return out, err +} + +func (s *Store) ReserveCompute(ctx context.Context, sid string, from, to api.SandboxStatus, oq, pq api.Quota, now time.Time) (store.SandboxRecord, error) { + var out store.SandboxRecord + err := s.tx(ctx, func(tx pgx.Tx) error { + var orgID string + if err := tx.QueryRow(ctx, "SELECT org_id FROM sandboxes WHERE id = $1", sid).Scan(&orgID); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return store.NotFound("sandbox", sid) + } + return err + } + if err := lockOrg(ctx, tx, orgID); err != nil { + return err + } + r, err := getDoc[store.SandboxRecord](ctx, tx, "sandbox", sid, "SELECT doc FROM sandboxes WHERE id = $1 FOR UPDATE", sid) + if err != nil { + return err + } + if r.Status != from { + return api.Errorf(api.CodeInvalidState, "sandbox %s is %s, not %s", sid, r.Status, from) + } + ou, err := tally(ctx, tx, r.OrgID, "") + if err != nil { + return err + } + if err := store.CheckCompute("organization", ou, oq, r.Resources); err != nil { + return err + } + pu, err := tally(ctx, tx, r.OrgID, r.ProjectID) + if err != nil { + return err + } + if err := store.CheckCompute("project", pu, pq, r.Resources); err != nil { + return err + } + r.Status, r.UpdatedAt = to, now + out = r + return saveSandbox(ctx, tx, r) + }) + return out, err +} + +func (s *Store) Usage(ctx context.Context, orgID, projectID string) (api.QuotaUsage, error) { + return tally(ctx, s.pool, orgID, projectID) +} + +func (s *Store) PoolActive(ctx context.Context) (map[string]int, error) { + rows, err := s.pool.Query(ctx, "SELECT pool, count(*) FROM sandboxes WHERE pool <> '' AND status IN ('provisioning','running','paused','stopped','terminating') GROUP BY pool") + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]int{} + for rows.Next() { + var name string + var n int + if err := rows.Scan(&name, &n); err != nil { + return nil, err + } + out[name] = n + } + return out, rows.Err() +} + +func (s *Store) CreateSnapshot(ctx context.Context, sn store.SnapshotRecord) error { + _, err := s.pool.Exec(ctx, "INSERT INTO snapshots (id, org_id, project_id, doc) VALUES ($1, $2, $3, $4)", sn.ID, sn.OrgID, sn.ProjectID, sn) + return err +} + +func (s *Store) GetSnapshot(ctx context.Context, sid string) (store.SnapshotRecord, error) { + return getDoc[store.SnapshotRecord](ctx, s.pool, "snapshot", sid, "SELECT doc FROM snapshots WHERE id = $1", sid) +} + +func (s *Store) ListSnapshots(ctx context.Context, orgID, projectID string) ([]store.SnapshotRecord, error) { + return listDocs[store.SnapshotRecord](ctx, s.pool, "SELECT doc FROM snapshots WHERE ($1 = '' OR org_id = $1) AND ($2 = '' OR project_id = $2) ORDER BY id DESC", orgID, projectID) +} + +func (s *Store) DeleteSnapshot(ctx context.Context, sid string) error { + _, err := s.pool.Exec(ctx, "DELETE FROM snapshots WHERE id = $1", sid) + return err +} + +func (s *Store) AppendAudit(ctx context.Context, ev api.AuditEvent) (api.AuditEvent, error) { + err := s.tx(ctx, func(tx pgx.Tx) error { + if _, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock($1)", lockAudit); err != nil { + return err + } + var seq int64 + var prev string + err := tx.QueryRow(ctx, "SELECT seq, doc->>'hash' FROM audit_events ORDER BY seq DESC LIMIT 1").Scan(&seq, &prev) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + ev.Seq = seq + 1 + if ev.Time.IsZero() { + ev.Time = time.Now().UTC() + } + ev.Time = ev.Time.UTC().Truncate(time.Microsecond) // match timestamp precision on reload + ev = audit.Seal(prev, ev) + _, err = tx.Exec(ctx, "INSERT INTO audit_events (seq, org_id, resource, action, doc) VALUES ($1, $2, $3, $4, $5)", ev.Seq, ev.OrgID, ev.Resource, ev.Action, ev) + return err + }) + return ev, err +} + +func (s *Store) ListAudit(ctx context.Context, f store.AuditFilter) ([]api.AuditEvent, error) { + limit := f.Limit + if limit <= 0 { + limit = 1 << 30 + } + return listDocs[api.AuditEvent](ctx, s.pool, `SELECT doc FROM audit_events + WHERE seq > $1 AND ($2 = '' OR org_id = $2) AND ($3 = '' OR resource = $3) AND ($4 = '' OR action = $4) + ORDER BY seq LIMIT $5`, f.AfterSeq, f.OrgID, f.Resource, f.Action, limit) +} + +func (s *Store) UpsertAgent(ctx context.Context, a api.Agent) error { + _, err := s.pool.Exec(ctx, "INSERT INTO agents (id, pool, doc) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET pool = EXCLUDED.pool, doc = EXCLUDED.doc", a.ID, a.Pool, a) + return err +} + +func (s *Store) ListAgents(ctx context.Context, pool string) ([]api.Agent, error) { + return listDocs[api.Agent](ctx, s.pool, "SELECT doc FROM agents WHERE $1 = '' OR pool = $1 ORDER BY id", pool) +} + +// TryLeader holds a session advisory lock on a dedicated connection. If the +// connection drops, Postgres releases the lock and another replica takes over. +func (s *Store) TryLeader(ctx context.Context) (bool, error) { + s.leaderMu.Lock() + defer s.leaderMu.Unlock() + if s.leaderConn != nil { + if err := s.leaderConn.Ping(ctx); err == nil { + return true, nil + } + s.leaderConn.Release() + s.leaderConn = nil + } + conn, err := s.pool.Acquire(ctx) + if err != nil { + return false, err + } + var ok bool + if err := conn.QueryRow(ctx, "SELECT pg_try_advisory_lock($1)", lockLeader).Scan(&ok); err != nil || !ok { + conn.Release() + return false, err + } + s.leaderConn = conn + return true, nil +} + +var _ store.Store = (*Store)(nil) diff --git a/internal/store/postgres/postgres_test.go b/internal/store/postgres/postgres_test.go new file mode 100644 index 0000000..ef7527c --- /dev/null +++ b/internal/store/postgres/postgres_test.go @@ -0,0 +1,34 @@ +package postgres + +import ( + "context" + "os" + "testing" + + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/internal/store/storetest" +) + +// TestConformance runs the shared store suite against a real database when +// DIFFERENTLY_TEST_POSTGRES_URL is set (each subtest gets a fresh schema). +func TestConformance(t *testing.T) { + dsn := os.Getenv("DIFFERENTLY_TEST_POSTGRES_URL") + if dsn == "" { + t.Skip("set DIFFERENTLY_TEST_POSTGRES_URL to run the postgres conformance suite") + } + storetest.Run(t, func(t *testing.T) store.Store { + ctx := context.Background() + s, err := Open(ctx, dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + for _, tbl := range []string{"sandboxes", "snapshots", "audit_events", "agents", "api_keys", "projects", "organizations", "pools", "policies", "meta", "schema_migrations"} { + _, _ = s.pool.Exec(ctx, "DROP TABLE IF EXISTS "+tbl+" CASCADE") + } + if err := s.Migrate(ctx); err != nil { + t.Fatal(err) + } + return s + }) +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 0000000..3bf5a3e --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,300 @@ +// Package store persists control-plane state. +// +// Two implementations share this contract: memory (development and tests) and +// postgres (production, HA). Quota-sensitive transitions — admitting a request +// and activating a queued sandbox onto a pool — are single atomic store +// operations so concurrent requests cannot double-allocate or bypass quota. +// Quota usage is always derived from sandbox state rather than kept in +// separate counters, so a crash cannot leave usage out of sync. +package store + +import ( + "context" + "slices" + "sort" + "time" + + "github.com/different-ai/differently/pkg/api" +) + +// SandboxRecord is a sandbox plus fields that never leave the control plane. +type SandboxRecord struct { + api.Sandbox + BackendRef string `json:"backendRef,omitempty"` + Env map[string]string `json:"env,omitempty"` + IdempotencyKey string `json:"idempotencyKey,omitempty"` + NextAttemptAt time.Time `json:"nextAttemptAt"` + RunningSince *time.Time `json:"runningSince,omitempty"` + RunSeconds float64 `json:"runSeconds"` + QueueSeconds float64 `json:"queueSeconds"` + ExcludedPools []string `json:"excludedPools,omitempty"` + SnapshotBackendRef string `json:"snapshotBackendRef,omitempty"` + ProvisionStartedAt *time.Time `json:"provisionStartedAt,omitempty"` +} + +// KeyRecord is an API key plus its secret hash. +type KeyRecord struct { + api.APIKey + Hash string `json:"hash"` +} + +// SnapshotRecord is a snapshot plus its backend reference. +type SnapshotRecord struct { + api.Snapshot + BackendRef string `json:"backendRef"` +} + +// SandboxFilter selects sandboxes for listing. +type SandboxFilter struct { + OrgID string + ProjectID string + Pool string + Statuses []api.SandboxStatus + Label map[string]string + Cursor string // return IDs strictly older than this + Limit int +} + +// Matches reports whether rec passes the filter (ignoring cursor and limit). +func (f SandboxFilter) Matches(rec SandboxRecord) bool { + if f.OrgID != "" && rec.OrgID != f.OrgID { + return false + } + if f.ProjectID != "" && rec.ProjectID != f.ProjectID { + return false + } + if f.Pool != "" && rec.Pool != f.Pool { + return false + } + if len(f.Statuses) > 0 && !slices.Contains(f.Statuses, rec.Status) { + return false + } + for k, v := range f.Label { + if rec.Labels[k] != v { + return false + } + } + return true +} + +// AuditFilter selects audit events. +type AuditFilter struct { + OrgID string + Resource string + Action string + AfterSeq int64 + Limit int +} + +// Admission is the input to AdmitSandbox. +type Admission struct { + OrgQuota api.Quota + ProjectQuota api.Quota +} + +// Activation is the input to ActivateSandbox: placing a queued sandbox on a +// pool after re-checking tenant quota and pool concurrency atomically. +type Activation struct { + Pool string + Adapter string + Trust string + Region string + MaxConcurrency int + OrgQuota api.Quota + ProjectQuota api.Quota + Decision *api.PlacementDecision + Now time.Time +} + +// Store is the persistence contract. +type Store interface { + // Installation returns the stable installation ID, creating it once. + Installation(ctx context.Context) (string, error) + + CreateOrg(ctx context.Context, org api.Organization) error + GetOrg(ctx context.Context, id string) (api.Organization, error) + ListOrgs(ctx context.Context) ([]api.Organization, error) + UpdateOrg(ctx context.Context, id string, fn func(*api.Organization) error) (api.Organization, error) + + CreateProject(ctx context.Context, p api.Project) error + GetProject(ctx context.Context, id string) (api.Project, error) + ListProjects(ctx context.Context, orgID string) ([]api.Project, error) + UpdateProject(ctx context.Context, id string, fn func(*api.Project) error) (api.Project, error) + // DeleteProject fails with conflict while non-terminal sandboxes exist. + DeleteProject(ctx context.Context, id string) error + + CreateKey(ctx context.Context, k KeyRecord) error + GetKey(ctx context.Context, id string) (KeyRecord, error) + ListKeys(ctx context.Context, orgID, projectID string) ([]KeyRecord, error) + UpdateKey(ctx context.Context, id string, fn func(*KeyRecord) error) (KeyRecord, error) + + UpsertPool(ctx context.Context, p api.Pool) error + GetPool(ctx context.Context, name string) (api.Pool, error) + ListPools(ctx context.Context) ([]api.Pool, error) + DeletePool(ctx context.Context, name string) error + + PutPolicy(ctx context.Context, p api.PlacementPolicy) error + GetPolicy(ctx context.Context, scope string) (api.PlacementPolicy, bool, error) + ListPolicies(ctx context.Context) ([]api.PlacementPolicy, error) + DeletePolicy(ctx context.Context, scope string) error + + // AdmitSandbox atomically checks admission limits and inserts rec as + // queued. A repeated IdempotencyKey within a project returns the existing + // record and existed=true. + AdmitSandbox(ctx context.Context, rec SandboxRecord, adm Admission) (out SandboxRecord, existed bool, err error) + GetSandbox(ctx context.Context, id string) (SandboxRecord, error) + ListSandboxes(ctx context.Context, f SandboxFilter) ([]SandboxRecord, error) + UpdateSandbox(ctx context.Context, id string, fn func(*SandboxRecord) error) (SandboxRecord, error) + // QueuedCandidates returns runnable queued sandboxes in fair order. + QueuedCandidates(ctx context.Context, now time.Time, limit int) ([]SandboxRecord, error) + // ActivateSandbox moves a queued sandbox to provisioning on a pool. + ActivateSandbox(ctx context.Context, id string, act Activation) (SandboxRecord, error) + // ReserveCompute atomically moves a sandbox from one status to another + // when the move re-reserves CPU and memory (for example stopped to + // running), re-checking tenant compute quota under the same lock as + // ActivateSandbox. + ReserveCompute(ctx context.Context, id string, from, to api.SandboxStatus, orgQuota, projectQuota api.Quota, now time.Time) (SandboxRecord, error) + // Usage tallies quota usage for an organization or project. + Usage(ctx context.Context, orgID, projectID string) (api.QuotaUsage, error) + // PoolActive counts sandboxes holding a slot on each pool. + PoolActive(ctx context.Context) (map[string]int, error) + + CreateSnapshot(ctx context.Context, s SnapshotRecord) error + GetSnapshot(ctx context.Context, id string) (SnapshotRecord, error) + ListSnapshots(ctx context.Context, orgID, projectID string) ([]SnapshotRecord, error) + DeleteSnapshot(ctx context.Context, id string) error + + // AppendAudit seals ev into the chain and stores it. + AppendAudit(ctx context.Context, ev api.AuditEvent) (api.AuditEvent, error) + ListAudit(ctx context.Context, f AuditFilter) ([]api.AuditEvent, error) + + UpsertAgent(ctx context.Context, a api.Agent) error + ListAgents(ctx context.Context, pool string) ([]api.Agent, error) + + // TryLeader reports whether this process holds scheduler leadership. + TryLeader(ctx context.Context) (bool, error) + Close() error +} + +// HoldsPoolSlot reports whether a sandbox occupies pool concurrency. +func HoldsPoolSlot(s api.SandboxStatus) bool { + switch s { + case api.StatusProvisioning, api.StatusRunning, api.StatusPaused, api.StatusStopped, api.StatusTerminating: + return true + } + return false +} + +// Tally computes quota usage from records. +func Tally(recs []SandboxRecord, snapshots int) api.QuotaUsage { + var u api.QuotaUsage + for _, r := range recs { + switch { + case r.Status.Terminal(): + continue + case r.Status == api.StatusQueued: + u.Queued++ + continue + } + u.ActiveSandboxes++ + u.DiskGiB += r.Resources.DiskGiB + if r.Status.ReservesCompute() { + u.CPU += r.Resources.CPU + u.MemoryMiB += r.Resources.MemoryMiB + } + } + u.Snapshots = snapshots + return u +} + +// CheckActivation returns a quota_exceeded error if activating r would push +// usage over q. +func CheckActivation(scope string, u api.QuotaUsage, q api.Quota, r api.Resources) error { + over := func(limit, used, add int) bool { return limit > 0 && used+add > limit } + switch { + case over(q.MaxActiveSandboxes, u.ActiveSandboxes, 1): + return api.Errorf(api.CodeQuotaExceeded, "%s active sandbox limit %d reached", scope, q.MaxActiveSandboxes).With("dimension", "activeSandboxes") + case over(q.MaxCPU, u.CPU, r.CPU): + return api.Errorf(api.CodeQuotaExceeded, "%s CPU limit %d reached", scope, q.MaxCPU).With("dimension", "cpu") + case over(q.MaxMemoryMiB, u.MemoryMiB, r.MemoryMiB): + return api.Errorf(api.CodeQuotaExceeded, "%s memory limit %d MiB reached", scope, q.MaxMemoryMiB).With("dimension", "memoryMiB") + case over(q.MaxDiskGiB, u.DiskGiB, r.DiskGiB): + return api.Errorf(api.CodeQuotaExceeded, "%s disk limit %d GiB reached", scope, q.MaxDiskGiB).With("dimension", "diskGiB") + } + return nil +} + +// CheckCompute returns a quota_exceeded error if re-reserving r's CPU and +// memory would push usage over q. The sandbox already counts as active. +func CheckCompute(scope string, u api.QuotaUsage, q api.Quota, r api.Resources) error { + switch { + case q.MaxCPU > 0 && u.CPU+r.CPU > q.MaxCPU: + return api.Errorf(api.CodeQuotaExceeded, "%s CPU limit %d reached", scope, q.MaxCPU).With("dimension", "cpu") + case q.MaxMemoryMiB > 0 && u.MemoryMiB+r.MemoryMiB > q.MaxMemoryMiB: + return api.Errorf(api.CodeQuotaExceeded, "%s memory limit %d MiB reached", scope, q.MaxMemoryMiB).With("dimension", "memoryMiB") + } + return nil +} + +// CheckAdmission rejects requests that can never fit or would overflow the +// queue. Concurrency limits are enforced at activation, so a tenant at its +// limit queues instead of failing. +func CheckAdmission(scope string, u api.QuotaUsage, q api.Quota, r api.Resources, ttl int) error { + switch { + case q.MaxCPU > 0 && r.CPU > q.MaxCPU: + return api.Errorf(api.CodeQuotaExceeded, "request CPU %d exceeds %s limit %d", r.CPU, scope, q.MaxCPU).With("dimension", "cpu") + case q.MaxMemoryMiB > 0 && r.MemoryMiB > q.MaxMemoryMiB: + return api.Errorf(api.CodeQuotaExceeded, "request memory exceeds %s limit %d MiB", scope, q.MaxMemoryMiB).With("dimension", "memoryMiB") + case q.MaxDiskGiB > 0 && r.DiskGiB > q.MaxDiskGiB: + return api.Errorf(api.CodeQuotaExceeded, "request disk exceeds %s limit %d GiB", scope, q.MaxDiskGiB).With("dimension", "diskGiB") + case q.MaxTTLSeconds > 0 && ttl > q.MaxTTLSeconds: + return api.Errorf(api.CodeQuotaExceeded, "ttl %ds exceeds %s limit %ds", ttl, scope, q.MaxTTLSeconds).With("dimension", "ttlSeconds") + case q.MaxQueued > 0 && u.Queued+1 > q.MaxQueued: + return api.Errorf(api.CodeQuotaExceeded, "%s queue limit %d reached", scope, q.MaxQueued).With("dimension", "queued") + } + return nil +} + +// FairOrder orders queued sandboxes for dispatch. Organizations with less +// active CPU go first and organizations are interleaved round-robin, so one +// tenant's burst cannot starve others; within an organization higher priority +// then older requests go first. +func FairOrder(queued []SandboxRecord, activeCPUByOrg map[string]int) []SandboxRecord { + byOrg := map[string][]SandboxRecord{} + var orgs []string + for _, r := range queued { + if _, ok := byOrg[r.OrgID]; !ok { + orgs = append(orgs, r.OrgID) + } + byOrg[r.OrgID] = append(byOrg[r.OrgID], r) + } + for _, list := range byOrg { + sort.SliceStable(list, func(i, j int) bool { + if list[i].Priority != list[j].Priority { + return list[i].Priority > list[j].Priority + } + return list[i].ID < list[j].ID + }) + } + sort.SliceStable(orgs, func(i, j int) bool { + a, b := activeCPUByOrg[orgs[i]], activeCPUByOrg[orgs[j]] + if a != b { + return a < b + } + return orgs[i] < orgs[j] + }) + out := make([]SandboxRecord, 0, len(queued)) + for i := 0; len(out) < len(queued); i++ { + for _, o := range orgs { + if i < len(byOrg[o]) { + out = append(out, byOrg[o][i]) + } + } + } + return out +} + +// NotFound builds a not_found error for a resource kind. +func NotFound(kind, id string) error { + return api.Errorf(api.CodeNotFound, "%s %s not found", kind, id) +} diff --git a/internal/store/storetest/storetest.go b/internal/store/storetest/storetest.go new file mode 100644 index 0000000..77866c4 --- /dev/null +++ b/internal/store/storetest/storetest.go @@ -0,0 +1,254 @@ +// Package storetest is a conformance suite every Store implementation runs. +package storetest + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/different-ai/differently/internal/audit" + "github.com/different-ai/differently/internal/id" + "github.com/different-ai/differently/internal/store" + "github.com/different-ai/differently/pkg/api" +) + +// Run executes the suite. newStore must return an empty store. +func Run(t *testing.T, newStore func(t *testing.T) store.Store) { + t.Run("ConcurrentAdmissionRespectsQueueLimit", func(t *testing.T) { concurrentAdmission(t, newStore(t)) }) + t.Run("ConcurrentActivationRespectsQuota", func(t *testing.T) { concurrentActivation(t, newStore(t)) }) + t.Run("PoolConcurrencyCap", func(t *testing.T) { poolCap(t, newStore(t)) }) + t.Run("Idempotency", func(t *testing.T) { idempotency(t, newStore(t)) }) + t.Run("FairOrderAcrossOrgs", func(t *testing.T) { fairOrder(t, newStore(t)) }) + t.Run("AuditChain", func(t *testing.T) { auditChain(t, newStore(t)) }) + t.Run("ProjectDeleteGuard", func(t *testing.T) { projectGuard(t, newStore(t)) }) + t.Run("ConcurrentStartRespectsCPUQuota", func(t *testing.T) { concurrentStart(t, newStore(t)) }) +} + +func seed(t *testing.T, s store.Store, org string) (api.Organization, api.Project) { + t.Helper() + ctx := context.Background() + o := api.Organization{ID: org, Name: org, CreatedAt: time.Now().UTC()} + if err := s.CreateOrg(ctx, o); err != nil { + t.Fatal(err) + } + p := api.Project{ID: org + "-proj", OrgID: org, Name: "default", CreatedAt: time.Now().UTC()} + if err := s.CreateProject(ctx, p); err != nil { + t.Fatal(err) + } + return o, p +} + +func sandbox(p api.Project, cpu int) store.SandboxRecord { + now := time.Now().UTC() + return store.SandboxRecord{Sandbox: api.Sandbox{ + ID: id.New("sbx"), OrgID: p.OrgID, ProjectID: p.ID, Resources: api.Resources{CPU: cpu, MemoryMiB: 512, DiskGiB: 1}, + Classification: "internal", CreatedAt: now, UpdatedAt: now, + }} +} + +func concurrentAdmission(t *testing.T, s store.Store) { + _, p := seed(t, s, "org-a") + var ok atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if _, _, err := s.AdmitSandbox(context.Background(), sandbox(p, 1), store.Admission{OrgQuota: api.Quota{MaxQueued: 10}}); err == nil { + ok.Add(1) + } else if !api.IsCode(err, api.CodeQuotaExceeded) { + t.Errorf("unexpected error %v", err) + } + }() + } + wg.Wait() + if ok.Load() != 10 { + t.Fatalf("admitted %d, want exactly 10", ok.Load()) + } +} + +func concurrentActivation(t *testing.T, s store.Store) { + ctx := context.Background() + _, p := seed(t, s, "org-b") + var ids []string + for i := 0; i < 20; i++ { + rec, _, err := s.AdmitSandbox(ctx, sandbox(p, 2), store.Admission{}) + if err != nil { + t.Fatal(err) + } + ids = append(ids, rec.ID) + } + quota := api.Quota{MaxActiveSandboxes: 5, MaxCPU: 6} + var ok atomic.Int32 + var wg sync.WaitGroup + for _, sid := range ids { + wg.Add(1) + go func(sid string) { + defer wg.Done() + _, err := s.ActivateSandbox(ctx, sid, store.Activation{Pool: "p1", OrgQuota: quota, Now: time.Now().UTC()}) + if err == nil { + ok.Add(1) + } + }(sid) + } + wg.Wait() + // CPU 6 / 2 per sandbox = 3, tighter than the active limit of 5. + if ok.Load() != 3 { + t.Fatalf("activated %d, want exactly 3", ok.Load()) + } + u, err := s.Usage(ctx, p.OrgID, "") + if err != nil { + t.Fatal(err) + } + if u.CPU != 6 || u.Queued != 17 || u.ActiveSandboxes != 3 { + t.Fatalf("usage %+v", u) + } + // Terminating releases quota with no counter to fix up. + var active string + recs, _ := s.ListSandboxes(ctx, store.SandboxFilter{Statuses: []api.SandboxStatus{api.StatusProvisioning}}) + active = recs[0].ID + if _, err := s.UpdateSandbox(ctx, active, func(r *store.SandboxRecord) error { r.Status = api.StatusTerminated; return nil }); err != nil { + t.Fatal(err) + } + if u, _ := s.Usage(ctx, p.OrgID, ""); u.CPU != 4 { + t.Fatalf("cpu after release %d", u.CPU) + } +} + +func poolCap(t *testing.T, s store.Store) { + ctx := context.Background() + _, p := seed(t, s, "org-c") + var ok atomic.Int32 + var wg sync.WaitGroup + for i := 0; i < 10; i++ { + rec, _, err := s.AdmitSandbox(ctx, sandbox(p, 1), store.Admission{}) + if err != nil { + t.Fatal(err) + } + wg.Add(1) + go func(sid string) { + defer wg.Done() + _, err := s.ActivateSandbox(ctx, sid, store.Activation{Pool: "small", MaxConcurrency: 2, Now: time.Now().UTC()}) + if err == nil { + ok.Add(1) + } else if !api.IsCode(err, "insufficient_capacity") { + t.Errorf("unexpected %v", err) + } + }(rec.ID) + } + wg.Wait() + if ok.Load() != 2 { + t.Fatalf("pool admitted %d, want 2", ok.Load()) + } + counts, _ := s.PoolActive(ctx) + if counts["small"] != 2 { + t.Fatalf("pool active %v", counts) + } +} + +func idempotency(t *testing.T, s store.Store) { + ctx := context.Background() + _, p := seed(t, s, "org-d") + a := sandbox(p, 1) + a.IdempotencyKey = "req-1" + first, existed, err := s.AdmitSandbox(ctx, a, store.Admission{}) + if err != nil || existed { + t.Fatal(err, existed) + } + b := sandbox(p, 1) + b.IdempotencyKey = "req-1" + second, existed, err := s.AdmitSandbox(ctx, b, store.Admission{}) + if err != nil || !existed || second.ID != first.ID { + t.Fatalf("idempotent replay returned %s existed=%v err=%v", second.ID, existed, err) + } +} + +func fairOrder(t *testing.T, s store.Store) { + ctx := context.Background() + _, big := seed(t, s, "org-big") + _, small := seed(t, s, "org-small") + for i := 0; i < 6; i++ { + if _, _, err := s.AdmitSandbox(ctx, sandbox(big, 1), store.Admission{}); err != nil { + t.Fatal(err) + } + } + if _, _, err := s.AdmitSandbox(ctx, sandbox(small, 1), store.Admission{}); err != nil { + t.Fatal(err) + } + got, err := s.QueuedCandidates(ctx, time.Now().UTC().Add(time.Second), 3) + if err != nil { + t.Fatal(err) + } + seenSmall := false + for _, r := range got { + seenSmall = seenSmall || r.OrgID == "org-small" + } + if !seenSmall { + t.Fatalf("small org starved behind big org's burst: %v", got) + } +} + +func auditChain(t *testing.T, s store.Store) { + ctx := context.Background() + for i := 0; i < 5; i++ { + if _, err := s.AppendAudit(ctx, api.AuditEvent{Actor: "key_x", Action: "sandbox.create", Resource: "sbx", Outcome: "success"}); err != nil { + t.Fatal(err) + } + } + events, err := s.ListAudit(ctx, store.AuditFilter{}) + if err != nil { + t.Fatal(err) + } + if res := audit.Verify(events); !res.OK || res.Events != 5 { + t.Fatalf("chain %+v", res) + } +} + +func concurrentStart(t *testing.T, s store.Store) { + ctx := context.Background() + _, p := seed(t, s, "org-f") + var ids []string + for i := 0; i < 6; i++ { + rec, _, err := s.AdmitSandbox(ctx, sandbox(p, 2), store.Admission{}) + if err != nil { + t.Fatal(err) + } + if _, err := s.ActivateSandbox(ctx, rec.ID, store.Activation{Pool: "p", Now: time.Now().UTC()}); err != nil { + t.Fatal(err) + } + if _, err := s.UpdateSandbox(ctx, rec.ID, func(r *store.SandboxRecord) error { r.Status = api.StatusStopped; return nil }); err != nil { + t.Fatal(err) + } + ids = append(ids, rec.ID) + } + var ok atomic.Int32 + var wg sync.WaitGroup + for _, sid := range ids { + wg.Add(1) + go func(sid string) { + defer wg.Done() + if _, err := s.ReserveCompute(ctx, sid, api.StatusStopped, api.StatusRunning, api.Quota{MaxCPU: 6}, api.Quota{}, time.Now().UTC()); err == nil { + ok.Add(1) + } else if !api.IsCode(err, api.CodeQuotaExceeded) { + t.Errorf("unexpected %v", err) + } + }(sid) + } + wg.Wait() + if ok.Load() != 3 { + t.Fatalf("started %d stopped sandboxes under a 6-CPU quota, want 3", ok.Load()) + } +} + +func projectGuard(t *testing.T, s store.Store) { + ctx := context.Background() + _, p := seed(t, s, "org-e") + if _, _, err := s.AdmitSandbox(ctx, sandbox(p, 1), store.Admission{}); err != nil { + t.Fatal(err) + } + if err := s.DeleteProject(ctx, p.ID); !api.IsCode(err, api.CodeConflict) { + t.Fatalf("want conflict, got %v", err) + } +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..adbf371 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,9 @@ +// Package version carries build metadata set with -ldflags. +package version + +var ( + // Version is the release version (set at build time). + Version = "0.1.0-dev" + // Commit is the source commit (set at build time). + Commit = "unknown" +) diff --git a/internal/wiring/wiring.go b/internal/wiring/wiring.go new file mode 100644 index 0000000..9d22051 --- /dev/null +++ b/internal/wiring/wiring.go @@ -0,0 +1,93 @@ +// Package wiring registers the built-in adapter kinds with the pool manager. +package wiring + +import ( + "fmt" + "path/filepath" + + "github.com/different-ai/differently/internal/adapters/daytona" + "github.com/different-ai/differently/internal/adapters/devhost" + "github.com/different-ai/differently/internal/adapters/fake" + "github.com/different-ai/differently/internal/adapters/firecracker" + "github.com/different-ai/differently/internal/adapters/freestyle" + "github.com/different-ai/differently/internal/config" + "github.com/different-ai/differently/internal/pools" + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/adapter/adapterhttp" + "github.com/different-ai/differently/pkg/api" +) + +// Register installs every built-in adapter factory. +func Register() { + pools.Register("firecracker", func(p api.Pool, d pools.Deps) (adapter.ExecutionAdapter, error) { + if d.AgentToken == "" { + return nil, fmt.Errorf("firecracker pools need the agent token (auth.agentTokenEnv)") + } + return firecracker.New(p.Name, d.Store, d.AgentToken, d.AgentTimeout), nil + }) + pools.Register("remote", func(p api.Pool, d pools.Deps) (adapter.ExecutionAdapter, error) { + token, err := resolve(p, d) + if err != nil { + return nil, err + } + return &remote{Client: adapterhttp.NewClient(p.Config["endpoint"], token), trust: p.Trust}, nil + }) + pools.Register("daytona", func(p api.Pool, d pools.Deps) (adapter.ExecutionAdapter, error) { + key, err := resolve(p, d) + if err != nil { + return nil, err + } + return daytona.New(daytona.FromConfig(p.Config, key)), nil + }) + pools.Register("freestyle", func(p api.Pool, d pools.Deps) (adapter.ExecutionAdapter, error) { + key, err := resolve(p, d) + if err != nil { + return nil, err + } + return freestyle.New(freestyle.FromConfig(p.Config, key)), nil + }) + pools.Register("devhost", func(p api.Pool, d pools.Deps) (adapter.ExecutionAdapter, error) { + if !d.Dev { + return nil, fmt.Errorf("devhost has no isolation and only runs in dev mode") + } + root := p.Config["root"] + if root == "" { + root = filepath.Join(d.DataDir, "devhost", p.Name) + } + return devhost.New(root) + }) + pools.Register("fake", func(p api.Pool, d pools.Deps) (adapter.ExecutionAdapter, error) { + if !d.Dev { + return nil, fmt.Errorf("the fake adapter only runs in dev mode") + } + return fake.New(fake.Options{Trust: p.Trust, Region: p.Region}), nil + }) +} + +func resolve(p api.Pool, d pools.Deps) (string, error) { + if p.CredentialRef == "" { + return "", nil + } + if d.Resolve != nil { + return d.Resolve(p.CredentialRef) + } + return config.ParseCredentialRef(p.CredentialRef).Resolve() +} + +// remote wraps the HTTP plugin client, discovering capabilities lazily. +type remote struct { + *adapterhttp.Client + trust string +} + +func (r *remote) Info() adapter.Info { + info := r.Client.Info() + if info.Kind == "" { + info = adapter.Info{Kind: "remote", Protocol: adapter.ProtocolVersion} + } + info.Trust = r.trust // the operator's declaration wins over the plugin's claim + return info +} + +// Endpoint is reported in the trust report. +func (r *remote) Endpoint() string { return r.Client.BaseURL } diff --git a/pkg/adapter/adapter.go b/pkg/adapter/adapter.go new file mode 100644 index 0000000..9ebe472 --- /dev/null +++ b/pkg/adapter/adapter.go @@ -0,0 +1,295 @@ +// Package adapter is the versioned contract between the Differently control +// plane and execution backends. +// +// An ExecutionAdapter creates and manages sandbox instances on one backend +// (direct Firecracker, an external provider API, a community plugin). The +// required interface is deliberately small; optional behaviour is expressed by +// implementing the optional interfaces below (FileSystem, Lifecycle, Pauser, +// Snapshotter, Streamer, PortExposer). The control plane advertises a feature +// only when the adapter both declares it and implements the interface, so a +// request for an unsupported operation fails with a precise error instead of +// silently degrading. +// +// Adapters run in-process (built in) or out-of-process behind the HTTP +// protocol in package adapterhttp. +package adapter + +import ( + "context" + "slices" + + "github.com/different-ai/differently/pkg/api" +) + +// ProtocolVersion is the adapter contract version. Breaking changes bump it. +const ProtocolVersion = "1" + +// Feature names advertised in Capabilities.Features and requested through +// CreateSandboxRequest.Capabilities. +const ( + FeatureExec = "exec" + FeatureExecStream = "exec.stream" + FeatureFiles = "files" + FeatureStopStart = "lifecycle.stop-start" + FeaturePauseResume = "lifecycle.pause-resume" + FeatureSnapshotDisk = "snapshots.disk" + FeatureSnapshotMem = "snapshots.memory" + FeatureFork = "fork" + FeaturePersistDisk = "persistence.disk" + FeaturePorts = "ports.expose" + FeatureSSH = "ssh" + FeaturePTY = "pty" + FeatureEgressControl = "network.egress-control" + FeatureGPU = "gpu" + FeatureArchAMD64 = "arch.amd64" + FeatureArchARM64 = "arch.arm64" + FeatureMicroVM = "isolation.microvm" + FeatureContainer = "isolation.container" + FeatureNoIsolation = "isolation.none" +) + +// Well-known labels adapters attach to backend resources so the reconciler can +// find what this installation owns. +const ( + LabelSandboxID = "differently.dev/sandbox-id" + LabelInstallation = "differently.dev/installation" + LabelOrg = "differently.dev/org" + LabelProject = "differently.dev/project" +) + +// Instance states reported by adapters. +const ( + StatePending = "pending" + StateRunning = "running" + StateStopped = "stopped" + StatePaused = "paused" + StateError = "error" +) + +// CodeInsufficientCapacity signals a transient lack of capacity. The scheduler +// requeues the sandbox without marking the pool unhealthy. +const CodeInsufficientCapacity = "insufficient_capacity" + +// Info identifies an adapter implementation. +type Info struct { + Kind string `json:"kind"` + Version string `json:"version"` + Trust string `json:"trust"` + Isolation string `json:"isolation"` + Description string `json:"description,omitempty"` + Protocol string `json:"protocol"` +} + +// Capabilities is what an adapter can do and the envelope it accepts. +type Capabilities struct { + Features []string `json:"features"` + MaxCPU int `json:"maxCPU,omitempty"` + MaxMemoryMiB int `json:"maxMemoryMiB,omitempty"` + MaxDiskGiB int `json:"maxDiskGiB,omitempty"` + Templates []string `json:"templates,omitempty"` // empty means any + Regions []string `json:"regions,omitempty"` +} + +// Has reports whether a feature is advertised. +func (c Capabilities) Has(feature string) bool { return slices.Contains(c.Features, feature) } + +// Missing returns the requested features that are not advertised. +func (c Capabilities) Missing(requested []string) []string { + var out []string + for _, f := range requested { + if !c.Has(f) { + out = append(out, f) + } + } + return out +} + +// Accepts reports whether a resource envelope and template fit the limits. +func (c Capabilities) Accepts(r api.Resources, template string) (bool, string) { + switch { + case c.MaxCPU > 0 && r.CPU > c.MaxCPU: + return false, "cpu above adapter maximum" + case c.MaxMemoryMiB > 0 && r.MemoryMiB > c.MaxMemoryMiB: + return false, "memory above adapter maximum" + case c.MaxDiskGiB > 0 && r.DiskGiB > c.MaxDiskGiB: + return false, "disk above adapter maximum" + case template != "" && len(c.Templates) > 0 && !slices.Contains(c.Templates, template): + return false, "template " + template + " not available" + } + return true, "" +} + +// Capacity is the free capacity an adapter currently reports. Unbounded +// adapters (most external providers) rely on the pool's concurrency cap. +type Capacity struct { + Bounded bool `json:"bounded"` + TotalCPU int `json:"totalCPU,omitempty"` + FreeCPU int `json:"freeCPU,omitempty"` + TotalMemoryMiB int `json:"totalMemoryMiB,omitempty"` + FreeMemoryMiB int `json:"freeMemoryMiB,omitempty"` + Nodes int `json:"nodes,omitempty"` + // LargestFreeCPU and LargestFreeMemoryMiB describe the roomiest single + // node, so fragmented capacity is not mistaken for room. + LargestFreeCPU int `json:"largestFreeCPU,omitempty"` + LargestFreeMemoryMiB int `json:"largestFreeMemoryMiB,omitempty"` +} + +// Fits reports whether one sandbox of r fits. +func (c Capacity) Fits(r api.Resources) bool { + if !c.Bounded { + return true + } + cpu, mem := c.FreeCPU, c.FreeMemoryMiB + if c.LargestFreeCPU > 0 || c.LargestFreeMemoryMiB > 0 { + cpu, mem = c.LargestFreeCPU, c.LargestFreeMemoryMiB + } + return r.CPU <= cpu && r.MemoryMiB <= mem +} + +// CreateRequest asks an adapter for a new instance. ID is the Differently +// sandbox ID and is the idempotency key: creating the same ID twice must +// return the existing instance, never a second one. +type CreateRequest struct { + ID string `json:"id"` + Installation string `json:"installation"` + Template string `json:"template,omitempty"` + Resources api.Resources `json:"resources"` + Env map[string]string `json:"env,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + TTLSeconds int `json:"ttlSeconds,omitempty"` + FromSnapshot string `json:"fromSnapshot,omitempty"` +} + +// Instance is a backend resource backing a sandbox. +type Instance struct { + Ref string `json:"ref"` + SandboxID string `json:"sandboxId"` + State string `json:"state"` + Region string `json:"region,omitempty"` + Message string `json:"message,omitempty"` +} + +// Snapshot is a backend snapshot reference. +type Snapshot struct { + Ref string `json:"ref"` + Kind string `json:"kind"` + SizeBytes int64 `json:"sizeBytes,omitempty"` +} + +// ExecutionAdapter is the required contract every backend implements. +type ExecutionAdapter interface { + Info() Info + Capabilities(ctx context.Context) (Capabilities, error) + Capacity(ctx context.Context) (Capacity, error) + Health(ctx context.Context) error + Create(ctx context.Context, req CreateRequest) (Instance, error) + Get(ctx context.Context, ref string) (Instance, error) + // List returns instances owned by the given installation. + List(ctx context.Context, installation string) ([]Instance, error) + // Delete is idempotent: deleting a missing instance succeeds. + Delete(ctx context.Context, ref string) error + Exec(ctx context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) +} + +// Streamer streams command output as it is produced. +type Streamer interface { + ExecStream(ctx context.Context, ref string, req api.ExecRequest, emit func(api.ExecEvent) error) (exitCode int, err error) +} + +// FileSystem reads and writes files inside an instance. +type FileSystem interface { + ReadFile(ctx context.Context, ref, path string) ([]byte, error) + WriteFile(ctx context.Context, ref, path string, data []byte) error + ListFiles(ctx context.Context, ref, path string) ([]api.FileInfo, error) +} + +// Lifecycle stops an instance while keeping its disk, and starts it again. +type Lifecycle interface { + Stop(ctx context.Context, ref string) error + Start(ctx context.Context, ref string) error +} + +// Pauser freezes an instance in memory and resumes it. +type Pauser interface { + Pause(ctx context.Context, ref string) error + Resume(ctx context.Context, ref string) error +} + +// Snapshotter captures instance state. Restoring happens through +// CreateRequest.FromSnapshot on the same adapter. +type Snapshotter interface { + Snapshot(ctx context.Context, ref, kind string) (Snapshot, error) + DeleteSnapshot(ctx context.Context, snapshotRef string) error +} + +// PortExposer returns a reachable URL for a service port. +type PortExposer interface { + ExposePort(ctx context.Context, ref string, port int) (string, error) +} + +// EffectiveFeatures returns declared features that the adapter actually +// implements. Declaring "files" without implementing FileSystem drops it. +func EffectiveFeatures(a ExecutionAdapter, declared []string) []string { + implements := func(f string) bool { + switch f { + case FeatureExecStream: + _, ok := a.(Streamer) + return ok + case FeatureFiles: + _, ok := a.(FileSystem) + return ok + case FeatureStopStart, FeaturePersistDisk: + _, ok := a.(Lifecycle) + return ok + case FeaturePauseResume: + _, ok := a.(Pauser) + return ok + case FeatureSnapshotDisk, FeatureSnapshotMem, FeatureFork: + _, ok := a.(Snapshotter) + return ok + case FeaturePorts: + _, ok := a.(PortExposer) + return ok + } + return true + } + out := make([]string, 0, len(declared)) + for _, f := range declared { + if implements(f) && !slices.Contains(out, f) { + out = append(out, f) + } + } + slices.Sort(out) + return out +} + +// NotFound is the error adapters return for a missing instance. +func NotFound(ref string) error { + return api.Errorf(api.CodeNotFound, "instance %s not found", ref) +} + +// Unsupported is the error for an operation a backend cannot perform. +func Unsupported(op string) error { + return api.Errorf(api.CodeUnsupportedOperation, "operation %s is not supported by this backend", op).With("operation", op) +} + +// Unavailable wraps a backend failure that should count against pool health. +func Unavailable(err error) error { + return api.Errorf(api.CodeProviderUnavailable, "backend unavailable: %v", err) +} + +// InsufficientCapacity is returned when the backend has no room right now. +func InsufficientCapacity(msg string) error { + return &api.Error{Code: CodeInsufficientCapacity, Message: msg} +} + +// OwnershipLabels returns the labels adapters should stamp on resources. +func OwnershipLabels(req CreateRequest) map[string]string { + out := map[string]string{} + for k, v := range req.Labels { + out[k] = v + } + out[LabelSandboxID] = req.ID + out[LabelInstallation] = req.Installation + return out +} diff --git a/pkg/adapter/adapterhttp/adapterhttp.go b/pkg/adapter/adapterhttp/adapterhttp.go new file mode 100644 index 0000000..73e2eed --- /dev/null +++ b/pkg/adapter/adapterhttp/adapterhttp.go @@ -0,0 +1,576 @@ +// Package adapterhttp serves an adapter.ExecutionAdapter over HTTP and +// provides a client that implements the same interface. +// +// This is the out-of-process plugin protocol: the Differently node agent speaks +// it, and community backends can implement it in any language. Routes live +// under /adapter/v1, bodies are JSON, errors use the api.ErrorEnvelope shape, +// and streamed exec output is newline-delimited JSON (one api.ExecEvent per +// line). An optional bearer token authenticates the control plane. +package adapterhttp + +import ( + "bufio" + "bytes" + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/different-ai/differently/pkg/adapter" + "github.com/different-ai/differently/pkg/api" +) + +// ProtocolHeader carries the protocol version on every response. +const ProtocolHeader = "Differently-Adapter-Protocol" + +// MaxFileBytes bounds file transfers through the protocol. +const MaxFileBytes = 100 << 20 + +// InfoResponse is returned by GET /adapter/v1/info. +type InfoResponse struct { + Info adapter.Info `json:"info"` + Capabilities adapter.Capabilities `json:"capabilities"` +} + +type snapshotRequest struct { + Kind string `json:"kind"` +} + +type portRequest struct { + Port int `json:"port"` +} + +type portResponse struct { + URL string `json:"url"` +} + +// Handler serves a. If token is non-empty every request must present it as a +// bearer token. +func Handler(a adapter.ExecutionAdapter, token string) http.Handler { + mux := http.NewServeMux() + h := &handler{a: a} + mux.HandleFunc("GET /adapter/v1/info", h.info) + mux.HandleFunc("GET /adapter/v1/capacity", h.capacity) + mux.HandleFunc("GET /adapter/v1/health", h.health) + mux.HandleFunc("POST /adapter/v1/instances", h.create) + mux.HandleFunc("GET /adapter/v1/instances", h.list) + mux.HandleFunc("GET /adapter/v1/instances/{ref}", h.get) + mux.HandleFunc("DELETE /adapter/v1/instances/{ref}", h.delete) + mux.HandleFunc("POST /adapter/v1/instances/{ref}/exec", h.exec) + mux.HandleFunc("GET /adapter/v1/instances/{ref}/files", h.listFiles) + mux.HandleFunc("GET /adapter/v1/instances/{ref}/files/content", h.readFile) + mux.HandleFunc("PUT /adapter/v1/instances/{ref}/files/content", h.writeFile) + mux.HandleFunc("POST /adapter/v1/instances/{ref}/{action}", h.action) + mux.HandleFunc("POST /adapter/v1/instances/{ref}/snapshots", h.snapshot) + mux.HandleFunc("DELETE /adapter/v1/snapshots/{ref}", h.deleteSnapshot) + mux.HandleFunc("POST /adapter/v1/instances/{ref}/ports", h.port) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(ProtocolHeader, adapter.ProtocolVersion) + if token != "" { + got := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if subtle.ConstantTimeCompare([]byte(got), []byte(token)) != 1 { + writeErr(w, api.Errorf(api.CodeUnauthenticated, "invalid adapter token")) + return + } + } + mux.ServeHTTP(w, r) + }) +} + +type handler struct{ a adapter.ExecutionAdapter } + +func (h *handler) info(w http.ResponseWriter, r *http.Request) { + caps, err := h.a.Capabilities(r.Context()) + if err != nil { + writeErr(w, err) + return + } + caps.Features = adapter.EffectiveFeatures(h.a, caps.Features) + info := h.a.Info() + info.Protocol = adapter.ProtocolVersion + writeJSON(w, http.StatusOK, InfoResponse{Info: info, Capabilities: caps}) +} + +func (h *handler) capacity(w http.ResponseWriter, r *http.Request) { + c, err := h.a.Capacity(r.Context()) + respond(w, c, err) +} + +func (h *handler) health(w http.ResponseWriter, r *http.Request) { + if err := h.a.Health(r.Context()); err != nil { + writeErr(w, adapter.Unavailable(err)) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (h *handler) create(w http.ResponseWriter, r *http.Request) { + var req adapter.CreateRequest + if !decode(w, r, &req) { + return + } + inst, err := h.a.Create(r.Context(), req) + respond(w, inst, err) +} + +func (h *handler) list(w http.ResponseWriter, r *http.Request) { + items, err := h.a.List(r.Context(), r.URL.Query().Get("installation")) + if items == nil { + items = []adapter.Instance{} + } + respond(w, items, err) +} + +func (h *handler) get(w http.ResponseWriter, r *http.Request) { + inst, err := h.a.Get(r.Context(), r.PathValue("ref")) + respond(w, inst, err) +} + +func (h *handler) delete(w http.ResponseWriter, r *http.Request) { + if err := h.a.Delete(r.Context(), r.PathValue("ref")); err != nil { + writeErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handler) exec(w http.ResponseWriter, r *http.Request) { + var req api.ExecRequest + if !decode(w, r, &req) { + return + } + ref := r.PathValue("ref") + if r.URL.Query().Get("stream") == "" { + res, err := h.a.Exec(r.Context(), ref, req) + respond(w, res, err) + return + } + s, ok := h.a.(adapter.Streamer) + if !ok { + writeErr(w, adapter.Unsupported("exec.stream")) + return + } + w.Header().Set("Content-Type", "application/x-ndjson") + w.WriteHeader(http.StatusOK) + flusher, _ := w.(http.Flusher) + enc := json.NewEncoder(w) + emit := func(ev api.ExecEvent) error { + if err := enc.Encode(ev); err != nil { + return err + } + if flusher != nil { + flusher.Flush() + } + return nil + } + code, err := s.ExecStream(r.Context(), ref, req, emit) + if err != nil { + _ = emit(api.ExecEvent{Error: err.Error()}) + return + } + _ = emit(api.ExecEvent{ExitCode: &code}) +} + +func (h *handler) fs(w http.ResponseWriter) (adapter.FileSystem, bool) { + fs, ok := h.a.(adapter.FileSystem) + if !ok { + writeErr(w, adapter.Unsupported("files")) + } + return fs, ok +} + +func (h *handler) listFiles(w http.ResponseWriter, r *http.Request) { + fs, ok := h.fs(w) + if !ok { + return + } + items, err := fs.ListFiles(r.Context(), r.PathValue("ref"), r.URL.Query().Get("path")) + if items == nil { + items = []api.FileInfo{} + } + respond(w, items, err) +} + +func (h *handler) readFile(w http.ResponseWriter, r *http.Request) { + fs, ok := h.fs(w) + if !ok { + return + } + data, err := fs.ReadFile(r.Context(), r.PathValue("ref"), r.URL.Query().Get("path")) + if err != nil { + writeErr(w, err) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(data) +} + +func (h *handler) writeFile(w http.ResponseWriter, r *http.Request) { + fs, ok := h.fs(w) + if !ok { + return + } + data, err := io.ReadAll(io.LimitReader(r.Body, MaxFileBytes+1)) + if err != nil { + writeErr(w, err) + return + } + if len(data) > MaxFileBytes { + writeErr(w, api.Errorf(api.CodeInvalidRequest, "file larger than %d bytes", MaxFileBytes)) + return + } + if err := fs.WriteFile(r.Context(), r.PathValue("ref"), r.URL.Query().Get("path"), data); err != nil { + writeErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handler) action(w http.ResponseWriter, r *http.Request) { + ref, action := r.PathValue("ref"), r.PathValue("action") + var err error + switch action { + case "stop", "start": + l, ok := h.a.(adapter.Lifecycle) + if !ok { + writeErr(w, adapter.Unsupported(action)) + return + } + if action == "stop" { + err = l.Stop(r.Context(), ref) + } else { + err = l.Start(r.Context(), ref) + } + case "pause", "resume": + p, ok := h.a.(adapter.Pauser) + if !ok { + writeErr(w, adapter.Unsupported(action)) + return + } + if action == "pause" { + err = p.Pause(r.Context(), ref) + } else { + err = p.Resume(r.Context(), ref) + } + default: + writeErr(w, api.Errorf(api.CodeNotFound, "unknown action %s", action)) + return + } + if err != nil { + writeErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handler) snapshot(w http.ResponseWriter, r *http.Request) { + s, ok := h.a.(adapter.Snapshotter) + if !ok { + writeErr(w, adapter.Unsupported("snapshot")) + return + } + var req snapshotRequest + if !decode(w, r, &req) { + return + } + snap, err := s.Snapshot(r.Context(), r.PathValue("ref"), req.Kind) + respond(w, snap, err) +} + +func (h *handler) deleteSnapshot(w http.ResponseWriter, r *http.Request) { + s, ok := h.a.(adapter.Snapshotter) + if !ok { + writeErr(w, adapter.Unsupported("snapshot")) + return + } + if err := s.DeleteSnapshot(r.Context(), r.PathValue("ref")); err != nil { + writeErr(w, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *handler) port(w http.ResponseWriter, r *http.Request) { + p, ok := h.a.(adapter.PortExposer) + if !ok { + writeErr(w, adapter.Unsupported("ports.expose")) + return + } + var req portRequest + if !decode(w, r, &req) { + return + } + u, err := p.ExposePort(r.Context(), r.PathValue("ref"), req.Port) + respond(w, portResponse{URL: u}, err) +} + +func decode(w http.ResponseWriter, r *http.Request, v any) bool { + if err := json.NewDecoder(io.LimitReader(r.Body, 4<<20)).Decode(v); err != nil && !errors.Is(err, io.EOF) { + writeErr(w, api.Errorf(api.CodeInvalidRequest, "invalid JSON body: %v", err)) + return false + } + return true +} + +func respond(w http.ResponseWriter, v any, err error) { + if err != nil { + writeErr(w, err) + return + } + writeJSON(w, http.StatusOK, v) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeErr(w http.ResponseWriter, err error) { + e := api.AsError(err) + status := e.HTTPStatus() + if e.Code == adapter.CodeInsufficientCapacity { + status = http.StatusServiceUnavailable + } + writeJSON(w, status, api.ErrorEnvelope{Error: e}) +} + +// Client implements adapter.ExecutionAdapter and every optional interface +// against a remote adapter. Optional operations the remote does not support +// return unsupported_operation errors. +type Client struct { + BaseURL string + Token string + HTTP *http.Client + + info InfoResponse +} + +// NewClient returns a client for baseURL. +func NewClient(baseURL, token string) *Client { + // No short client timeout: exec and transfers may run for hours and are + // bounded by the caller's context instead. + return &Client{BaseURL: strings.TrimRight(baseURL, "/"), Token: token, HTTP: &http.Client{Timeout: 2*time.Hour + 15*time.Minute}} +} + +// Discover fetches the remote info and capabilities. +func (c *Client) Discover(ctx context.Context) (InfoResponse, error) { + var out InfoResponse + err := c.do(ctx, http.MethodGet, "/adapter/v1/info", nil, &out) + if err == nil { + c.info = out + } + return out, err +} + +// Info returns the last discovered info. +func (c *Client) Info() adapter.Info { return c.info.Info } + +// Capabilities fetches capabilities from the remote. +func (c *Client) Capabilities(ctx context.Context) (adapter.Capabilities, error) { + out, err := c.Discover(ctx) + return out.Capabilities, err +} + +// Capacity implements adapter.ExecutionAdapter. +func (c *Client) Capacity(ctx context.Context) (adapter.Capacity, error) { + var out adapter.Capacity + return out, c.do(ctx, http.MethodGet, "/adapter/v1/capacity", nil, &out) +} + +// Health implements adapter.ExecutionAdapter. +func (c *Client) Health(ctx context.Context) error { + return c.do(ctx, http.MethodGet, "/adapter/v1/health", nil, nil) +} + +// Create implements adapter.ExecutionAdapter. +func (c *Client) Create(ctx context.Context, req adapter.CreateRequest) (adapter.Instance, error) { + var out adapter.Instance + return out, c.do(ctx, http.MethodPost, "/adapter/v1/instances", req, &out) +} + +// Get implements adapter.ExecutionAdapter. +func (c *Client) Get(ctx context.Context, ref string) (adapter.Instance, error) { + var out adapter.Instance + return out, c.do(ctx, http.MethodGet, inst(ref, ""), nil, &out) +} + +// List implements adapter.ExecutionAdapter. +func (c *Client) List(ctx context.Context, installation string) ([]adapter.Instance, error) { + var out []adapter.Instance + return out, c.do(ctx, http.MethodGet, "/adapter/v1/instances?installation="+url.QueryEscape(installation), nil, &out) +} + +// Delete implements adapter.ExecutionAdapter. +func (c *Client) Delete(ctx context.Context, ref string) error { + err := c.do(ctx, http.MethodDelete, inst(ref, ""), nil, nil) + if api.IsCode(err, api.CodeNotFound) { + return nil + } + return err +} + +// Exec implements adapter.ExecutionAdapter. +func (c *Client) Exec(ctx context.Context, ref string, req api.ExecRequest) (api.ExecResult, error) { + var out api.ExecResult + return out, c.do(ctx, http.MethodPost, inst(ref, "/exec"), req, &out) +} + +// ExecStream implements adapter.Streamer. +func (c *Client) ExecStream(ctx context.Context, ref string, req api.ExecRequest, emit func(api.ExecEvent) error) (int, error) { + body, _ := json.Marshal(req) + resp, err := c.send(ctx, http.MethodPost, inst(ref, "/exec?stream=1"), bytes.NewReader(body), "application/json") + if err != nil { + return -1, err + } + defer resp.Body.Close() + sc := bufio.NewScanner(resp.Body) + sc.Buffer(make([]byte, 64<<10), 8<<20) + for sc.Scan() { + var ev api.ExecEvent + if err := json.Unmarshal(sc.Bytes(), &ev); err != nil { + return -1, fmt.Errorf("decode stream: %w", err) + } + switch { + case ev.Error != "": + return -1, errors.New(ev.Error) + case ev.ExitCode != nil: + return *ev.ExitCode, nil + } + if err := emit(ev); err != nil { + return -1, err + } + } + if err := sc.Err(); err != nil { + return -1, err + } + return -1, errors.New("stream ended without exit code") +} + +// ReadFile implements adapter.FileSystem. +func (c *Client) ReadFile(ctx context.Context, ref, path string) ([]byte, error) { + resp, err := c.send(ctx, http.MethodGet, inst(ref, "/files/content?path="+url.QueryEscape(path)), nil, "") + if err != nil { + return nil, err + } + defer resp.Body.Close() + return io.ReadAll(io.LimitReader(resp.Body, MaxFileBytes)) +} + +// WriteFile implements adapter.FileSystem. +func (c *Client) WriteFile(ctx context.Context, ref, path string, data []byte) error { + resp, err := c.send(ctx, http.MethodPut, inst(ref, "/files/content?path="+url.QueryEscape(path)), bytes.NewReader(data), "application/octet-stream") + if err != nil { + return err + } + return resp.Body.Close() +} + +// ListFiles implements adapter.FileSystem. +func (c *Client) ListFiles(ctx context.Context, ref, path string) ([]api.FileInfo, error) { + var out []api.FileInfo + return out, c.do(ctx, http.MethodGet, inst(ref, "/files?path="+url.QueryEscape(path)), nil, &out) +} + +// Stop implements adapter.Lifecycle. +func (c *Client) Stop(ctx context.Context, ref string) error { + return c.do(ctx, http.MethodPost, inst(ref, "/stop"), nil, nil) +} + +// Start implements adapter.Lifecycle. +func (c *Client) Start(ctx context.Context, ref string) error { + return c.do(ctx, http.MethodPost, inst(ref, "/start"), nil, nil) +} + +// Pause implements adapter.Pauser. +func (c *Client) Pause(ctx context.Context, ref string) error { + return c.do(ctx, http.MethodPost, inst(ref, "/pause"), nil, nil) +} + +// Resume implements adapter.Pauser. +func (c *Client) Resume(ctx context.Context, ref string) error { + return c.do(ctx, http.MethodPost, inst(ref, "/resume"), nil, nil) +} + +// Snapshot implements adapter.Snapshotter. +func (c *Client) Snapshot(ctx context.Context, ref, kind string) (adapter.Snapshot, error) { + var out adapter.Snapshot + return out, c.do(ctx, http.MethodPost, inst(ref, "/snapshots"), snapshotRequest{Kind: kind}, &out) +} + +// DeleteSnapshot implements adapter.Snapshotter. +func (c *Client) DeleteSnapshot(ctx context.Context, ref string) error { + return c.do(ctx, http.MethodDelete, "/adapter/v1/snapshots/"+url.PathEscape(ref), nil, nil) +} + +// ExposePort implements adapter.PortExposer. +func (c *Client) ExposePort(ctx context.Context, ref string, port int) (string, error) { + var out portResponse + return out.URL, c.do(ctx, http.MethodPost, inst(ref, "/ports"), portRequest{Port: port}, &out) +} + +func inst(ref, suffix string) string { return "/adapter/v1/instances/" + url.PathEscape(ref) + suffix } + +func (c *Client) do(ctx context.Context, method, path string, in, out any) error { + var body io.Reader + ct := "" + if in != nil { + b, err := json.Marshal(in) + if err != nil { + return err + } + body, ct = bytes.NewReader(b), "application/json" + } + resp, err := c.send(ctx, method, path, body, ct) + if err != nil { + return err + } + defer resp.Body.Close() + if out == nil || resp.StatusCode == http.StatusNoContent { + return nil + } + return json.NewDecoder(resp.Body).Decode(out) +} + +func (c *Client) send(ctx context.Context, method, path string, body io.Reader, contentType string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, body) + if err != nil { + return nil, err + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + if c.Token != "" { + req.Header.Set("Authorization", "Bearer "+c.Token) + } + resp, err := c.HTTP.Do(req) + if err != nil { + return nil, adapter.Unavailable(err) + } + if resp.StatusCode >= 300 { + defer resp.Body.Close() + var env api.ErrorEnvelope + if json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&env) == nil && env.Error != nil { + return nil, env.Error + } + return nil, adapter.Unavailable(fmt.Errorf("%s %s: HTTP %s", method, path, strconv.Itoa(resp.StatusCode))) + } + return resp, nil +} + +var ( + _ adapter.ExecutionAdapter = (*Client)(nil) + _ adapter.Streamer = (*Client)(nil) + _ adapter.FileSystem = (*Client)(nil) + _ adapter.Lifecycle = (*Client)(nil) + _ adapter.Pauser = (*Client)(nil) + _ adapter.Snapshotter = (*Client)(nil) + _ adapter.PortExposer = (*Client)(nil) +) diff --git a/pkg/api/errors.go b/pkg/api/errors.go new file mode 100644 index 0000000..3788be2 --- /dev/null +++ b/pkg/api/errors.go @@ -0,0 +1,96 @@ +package api + +import ( + "errors" + "fmt" + "net/http" +) + +// Error codes. Clients should branch on Code, never on Message. +const ( + CodeInvalidRequest = "invalid_request" + CodeUnauthenticated = "unauthenticated" + CodePermissionDenied = "permission_denied" + CodeNotFound = "not_found" + CodeConflict = "conflict" + CodeQuotaExceeded = "quota_exceeded" + CodeNoEligiblePool = "no_eligible_pool" + CodeUnsupportedCapability = "unsupported_capability" + CodeUnsupportedOperation = "unsupported_operation" + CodeProviderUnavailable = "provider_unavailable" + CodeInvalidState = "invalid_state" + CodeRateLimited = "rate_limited" + CodeSuspended = "suspended" + CodeInternal = "internal" +) + +// Error is the structured error returned by every endpoint. +type Error struct { + Code string `json:"code"` + Message string `json:"message"` + Details map[string]string `json:"details,omitempty"` + RequestID string `json:"requestId,omitempty"` +} + +// ErrorEnvelope wraps Error on the wire: {"error": {...}}. +type ErrorEnvelope struct { + Error *Error `json:"error"` +} + +func (e *Error) Error() string { return e.Code + ": " + e.Message } + +// HTTPStatus maps an error code to its HTTP status. +func (e *Error) HTTPStatus() int { + switch e.Code { + case CodeInvalidRequest: + return http.StatusBadRequest + case CodeUnauthenticated: + return http.StatusUnauthorized + case CodePermissionDenied, CodeSuspended: + return http.StatusForbidden + case CodeNotFound: + return http.StatusNotFound + case CodeConflict, CodeInvalidState: + return http.StatusConflict + case CodeQuotaExceeded, CodeRateLimited: + return http.StatusTooManyRequests + case CodeNoEligiblePool, CodeUnsupportedCapability: + return http.StatusUnprocessableEntity + case CodeUnsupportedOperation: + return http.StatusNotImplemented + case CodeProviderUnavailable: + return http.StatusServiceUnavailable + } + return http.StatusInternalServerError +} + +// With returns a copy of e with an extra detail. +func (e *Error) With(key, value string) *Error { + c := *e + c.Details = map[string]string{} + for k, v := range e.Details { + c.Details[k] = v + } + c.Details[key] = value + return &c +} + +// Errorf builds an Error. +func Errorf(code, format string, args ...any) *Error { + return &Error{Code: code, Message: fmt.Sprintf(format, args...)} +} + +// AsError extracts an *Error from err, wrapping unknown errors as internal. +func AsError(err error) *Error { + var e *Error + if errors.As(err, &e) { + return e + } + return &Error{Code: CodeInternal, Message: err.Error()} +} + +// IsCode reports whether err carries the given code. +func IsCode(err error, code string) bool { + var e *Error + return errors.As(err, &e) && e.Code == code +} diff --git a/pkg/api/types.go b/pkg/api/types.go new file mode 100644 index 0000000..770f271 --- /dev/null +++ b/pkg/api/types.go @@ -0,0 +1,455 @@ +// Package api defines the public, vendor-neutral Differently API resources. +// +// These types are the wire contract for /v1. They deliberately contain no +// Kubernetes, Firecracker or provider-specific fields: backends are reached +// through pools and capabilities, never named in application code. +package api + +import "time" + +// SandboxStatus is the lifecycle state of a sandbox. +type SandboxStatus string + +const ( + StatusQueued SandboxStatus = "queued" + StatusProvisioning SandboxStatus = "provisioning" + StatusRunning SandboxStatus = "running" + StatusStopped SandboxStatus = "stopped" + StatusPaused SandboxStatus = "paused" + StatusTerminating SandboxStatus = "terminating" + StatusTerminated SandboxStatus = "terminated" + StatusFailed SandboxStatus = "failed" +) + +// Terminal reports whether no further transitions are possible. +func (s SandboxStatus) Terminal() bool { + return s == StatusTerminated || s == StatusFailed +} + +// ReservesCompute reports whether the sandbox counts against CPU and memory +// quota. Paused microVMs keep their memory resident, so they still reserve. +func (s SandboxStatus) ReservesCompute() bool { + switch s { + case StatusQueued, StatusProvisioning, StatusRunning, StatusPaused, StatusTerminating: + return true + } + return false +} + +// Workload trust boundaries for pools. +const ( + TrustPrivate = "private" // infrastructure the operator controls + TrustExternal = "external" // a third-party provider runs the workload +) + +// External placement modes, from most to least restrictive. +const ( + ExternalNever = "never" // never place outside private pools + ExternalOverflow = "overflow" // only when no private pool has capacity + ExternalAllowed = "allowed" // external pools rank alongside private ones +) + +// Placement strategies. +const ( + StrategyPrivateFirst = "private-first" + StrategyCost = "cost" + StrategySpread = "spread" +) + +// Resources is the compute envelope of a sandbox. +type Resources struct { + CPU int `json:"cpu"` + MemoryMiB int `json:"memoryMiB"` + DiskGiB int `json:"diskGiB"` +} + +// Placement is a request-level placement preference. A request can only +// narrow what operator, organization and project policies allow. +type Placement struct { + Strategy string `json:"strategy,omitempty"` + AllowedPools []string `json:"allowedPools,omitempty"` + Regions []string `json:"regions,omitempty"` + // External can lower the effective external mode (e.g. "never"), never raise it. + External string `json:"external,omitempty"` +} + +// CreateSandboxRequest is the body of POST /v1/sandboxes. +type CreateSandboxRequest struct { + ProjectID string `json:"projectId,omitempty"` + Template string `json:"template,omitempty"` + Profile string `json:"profile,omitempty"` + Resources *Resources `json:"resources,omitempty"` + Capabilities []string `json:"capabilities,omitempty"` + Classification string `json:"classification,omitempty"` + Placement *Placement `json:"placement,omitempty"` + TTLSeconds int `json:"ttlSeconds,omitempty"` + Env map[string]string `json:"env,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Priority int `json:"priority,omitempty"` + FromSnapshot string `json:"fromSnapshot,omitempty"` +} + +// UpdateSandboxRequest is the body of PATCH /v1/sandboxes/{id}. +type UpdateSandboxRequest struct { + Labels map[string]string `json:"labels,omitempty"` + TTLSeconds *int `json:"ttlSeconds,omitempty"` +} + +// Sandbox is an isolated compute environment. +type Sandbox struct { + ID string `json:"id"` + OrgID string `json:"orgId"` + ProjectID string `json:"projectId"` + Status SandboxStatus `json:"status"` + StatusReason string `json:"statusReason,omitempty"` + Template string `json:"template,omitempty"` + Profile string `json:"profile,omitempty"` + Resources Resources `json:"resources"` + Capabilities []string `json:"capabilities,omitempty"` + Classification string `json:"classification"` + Placement *Placement `json:"placement,omitempty"` + Pool string `json:"pool,omitempty"` + Adapter string `json:"adapter,omitempty"` + Trust string `json:"trust,omitempty"` + Region string `json:"region,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + EnvKeys []string `json:"envKeys,omitempty"` + Priority int `json:"priority"` + TTLSeconds int `json:"ttlSeconds,omitempty"` + ExpiresAt *time.Time `json:"expiresAt,omitempty"` + FromSnapshot string `json:"fromSnapshot,omitempty"` + Attempts int `json:"attempts"` + CreatedBy string `json:"createdBy"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `json:"updatedAt"` + StartedAt *time.Time `json:"startedAt,omitempty"` + EndedAt *time.Time `json:"endedAt,omitempty"` + Decision *PlacementDecision `json:"decision,omitempty"` +} + +// PlacementDecision explains where a sandbox was placed and why. +type PlacementDecision struct { + Pool string `json:"pool,omitempty"` + Strategy string `json:"strategy"` + External string `json:"external"` + Outcome string `json:"outcome"` // placed | waiting | rejected + Reason string `json:"reason"` + Policies []string `json:"policies"` + Candidates []CandidateEvaluation `json:"candidates"` + EvaluatedAt time.Time `json:"evaluatedAt"` +} + +// CandidateEvaluation is the verdict for one pool during placement. +type CandidateEvaluation struct { + Pool string `json:"pool"` + Trust string `json:"trust"` + Eligible bool `json:"eligible"` + HasCapacity bool `json:"hasCapacity"` + EstimatedCost float64 `json:"estimatedCostPerHour"` + Rank int `json:"rank,omitempty"` + Reasons []string `json:"reasons,omitempty"` +} + +// ExecRequest runs a command in a sandbox. Command is run by /bin/sh -c; +// Args, when set, is executed directly without a shell. +type ExecRequest struct { + Command string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Env map[string]string `json:"env,omitempty"` + Cwd string `json:"cwd,omitempty"` + Stdin string `json:"stdin,omitempty"` + TimeoutSeconds int `json:"timeoutSeconds,omitempty"` +} + +// ExecResult is the outcome of a completed command. +type ExecResult struct { + ExitCode int `json:"exitCode"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + DurationMs int64 `json:"durationMs"` + Truncated bool `json:"truncated,omitempty"` +} + +// ExecEvent is one server-sent event of a streamed command. +type ExecEvent struct { + Stream string `json:"stream,omitempty"` // stdout | stderr + Data string `json:"data,omitempty"` + ExitCode *int `json:"exitCode,omitempty"` + Error string `json:"error,omitempty"` +} + +// FileInfo describes a file inside a sandbox. +type FileInfo struct { + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + Mode string `json:"mode"` + IsDir bool `json:"isDir"` + ModTime time.Time `json:"modTime"` +} + +// Snapshot captures sandbox state for restore or fork. +type Snapshot struct { + ID string `json:"id"` + SandboxID string `json:"sandboxId"` + OrgID string `json:"orgId"` + ProjectID string `json:"projectId"` + Pool string `json:"pool"` + Kind string `json:"kind"` // disk | memory + Status string `json:"status"` + SizeBytes int64 `json:"sizeBytes,omitempty"` + CreatedAt time.Time `json:"createdAt"` + CreatedBy string `json:"createdBy"` +} + +// CreateSnapshotRequest is the body of POST /v1/sandboxes/{id}/snapshots. +type CreateSnapshotRequest struct { + Kind string `json:"kind,omitempty"` // disk (default) | memory +} + +// PortURL is a reachable address for a sandbox service port. +type PortURL struct { + Port int `json:"port"` + URL string `json:"url"` +} + +// Quota limits. Zero means unlimited at that level. +type Quota struct { + MaxActiveSandboxes int `json:"maxActiveSandboxes,omitempty"` + MaxQueued int `json:"maxQueued,omitempty"` + MaxCPU int `json:"maxCPU,omitempty"` + MaxMemoryMiB int `json:"maxMemoryMiB,omitempty"` + MaxDiskGiB int `json:"maxDiskGiB,omitempty"` + MaxTTLSeconds int `json:"maxTTLSeconds,omitempty"` + MaxSnapshots int `json:"maxSnapshots,omitempty"` + PoolBudgetsPerHour map[string]float64 `json:"poolBudgetsPerHour,omitempty"` +} + +// QuotaUsage is the current consumption counted against a quota. +type QuotaUsage struct { + ActiveSandboxes int `json:"activeSandboxes"` + Queued int `json:"queued"` + CPU int `json:"cpu"` + MemoryMiB int `json:"memoryMiB"` + DiskGiB int `json:"diskGiB"` + Snapshots int `json:"snapshots"` +} + +// QuotaReport pairs limits with usage for an organization and project. +type QuotaReport struct { + OrgID string `json:"orgId"` + ProjectID string `json:"projectId,omitempty"` + OrgQuota Quota `json:"orgQuota"` + OrgUsage QuotaUsage `json:"orgUsage"` + ProjectQuota *Quota `json:"projectQuota,omitempty"` + ProjectUsage *QuotaUsage `json:"projectUsage,omitempty"` +} + +// Organization is a tenant. +type Organization struct { + ID string `json:"id"` + Name string `json:"name"` + Suspended bool `json:"suspended"` + Quota Quota `json:"quota"` + CreatedAt time.Time `json:"createdAt"` +} + +// Project groups sandboxes and keys within an organization. +type Project struct { + ID string `json:"id"` + OrgID string `json:"orgId"` + Name string `json:"name"` + Quota Quota `json:"quota"` + CreatedAt time.Time `json:"createdAt"` +} + +// APIKey is a scoped credential. The secret is returned only once on creation. +type APIKey struct { + ID string `json:"id"` + Name string `json:"name"` + OrgID string `json:"orgId,omitempty"` + ProjectID string `json:"projectId,omitempty"` + Scopes []string `json:"scopes"` + CreatedBy string `json:"createdBy"` + CreatedAt time.Time `json:"createdAt"` + ExpiresAt *time.Time `json:"expiresAt,omitempty"` + LastUsedAt *time.Time `json:"lastUsedAt,omitempty"` + Revoked bool `json:"revoked"` +} + +// CreateAPIKeyRequest is the body of POST /v1/api-keys. +type CreateAPIKeyRequest struct { + Name string `json:"name"` + OrgID string `json:"orgId,omitempty"` + ProjectID string `json:"projectId,omitempty"` + Scopes []string `json:"scopes"` + TTLHours int `json:"ttlHours,omitempty"` +} + +// CreateAPIKeyResponse includes the one-time token. +type CreateAPIKeyResponse struct { + APIKey + Token string `json:"token"` +} + +// CostModel prices pool consumption for estimates and chargeback. +type CostModel struct { + Currency string `json:"currency,omitempty"` + CPUHour float64 `json:"cpuHour,omitempty"` + MemoryGiBHour float64 `json:"memoryGiBHour,omitempty"` + DiskGiBHour float64 `json:"diskGiBHour,omitempty"` + SandboxHour float64 `json:"sandboxHour,omitempty"` +} + +// HourlyCost estimates the hourly price of a resource envelope. +func (c CostModel) HourlyCost(r Resources) float64 { + return c.SandboxHour + float64(r.CPU)*c.CPUHour + + float64(r.MemoryMiB)/1024*c.MemoryGiBHour + float64(r.DiskGiB)*c.DiskGiBHour +} + +// Pool is a registered execution backend instance. +type Pool struct { + Name string `json:"name"` + Adapter string `json:"adapter"` + Trust string `json:"trust"` + Region string `json:"region,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Enabled bool `json:"enabled"` + Draining bool `json:"draining"` + MaxConcurrency int `json:"maxConcurrency,omitempty"` + CostModel CostModel `json:"costModel"` + Config map[string]string `json:"config,omitempty"` + CredentialRef string `json:"credentialRef,omitempty"` + Source string `json:"source"` // config | api + Capabilities []string `json:"capabilities,omitempty"` + Health *PoolHealth `json:"health,omitempty"` + Capacity *PoolCapacity `json:"capacity,omitempty"` + Active int `json:"active"` +} + +// PoolHealth is the last observed health of a pool. +type PoolHealth struct { + Status string `json:"status"` // healthy | degraded | unavailable | unknown + Message string `json:"message,omitempty"` + CheckedAt time.Time `json:"checkedAt"` +} + +// PoolCapacity is the last observed free capacity of a pool. +type PoolCapacity struct { + Bounded bool `json:"bounded"` + TotalCPU int `json:"totalCPU,omitempty"` + FreeCPU int `json:"freeCPU,omitempty"` + TotalMemoryMiB int `json:"totalMemoryMiB,omitempty"` + FreeMemoryMiB int `json:"freeMemoryMiB,omitempty"` + Nodes int `json:"nodes,omitempty"` +} + +// PlacementPolicy constrains where sandboxes in a scope may run. +// Scope is "global", "org:" or "project:". More specific scopes can +// only narrow security constraints; strategy prefers the most specific value. +type PlacementPolicy struct { + Scope string `json:"scope"` + AllowedPools []string `json:"allowedPools,omitempty"` + DeniedPools []string `json:"deniedPools,omitempty"` + AllowedRegions []string `json:"allowedRegions,omitempty"` + External string `json:"external,omitempty"` + ExternalPools []string `json:"externalPools,omitempty"` + Strategy string `json:"strategy,omitempty"` + MaxCostPerHour float64 `json:"maxCostPerHour,omitempty"` + MaxQueueSeconds int `json:"maxQueueSeconds,omitempty"` + UpdatedAt time.Time `json:"updatedAt"` + UpdatedBy string `json:"updatedBy,omitempty"` +} + +// Classification is an operator-defined data sensitivity level. +type Classification struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + // External is the most permissive external mode this class allows. + External string `json:"external"` +} + +// Profile is a named resource preset. +type Profile struct { + Name string `json:"name"` + Resources Resources `json:"resources"` + Template string `json:"template,omitempty"` + Capabilities []string `json:"capabilities,omitempty"` +} + +// Template is a bootable image known to one or more pools. +type Template struct { + Name string `json:"name"` + Description string `json:"description,omitempty"` + Pools []string `json:"pools,omitempty"` +} + +// AuditEvent is one entry of the tamper-evident audit chain. +type AuditEvent struct { + Seq int64 `json:"seq"` + Time time.Time `json:"time"` + Actor string `json:"actor"` + OrgID string `json:"orgId,omitempty"` + ProjectID string `json:"projectId,omitempty"` + Action string `json:"action"` + Resource string `json:"resource"` + Outcome string `json:"outcome"` + Details map[string]string `json:"details,omitempty"` + RequestID string `json:"requestId,omitempty"` + PrevHash string `json:"prevHash"` + Hash string `json:"hash"` +} + +// UsageLine is one aggregated usage row. +type UsageLine struct { + Key string `json:"key"` + Sandboxes int `json:"sandboxes"` + CPUSeconds float64 `json:"cpuSeconds"` + MemoryGiBSeconds float64 `json:"memoryGiBSeconds"` + RunSeconds float64 `json:"runSeconds"` + QueueSeconds float64 `json:"queueSeconds"` + EstimatedCost float64 `json:"estimatedCost"` +} + +// UsageReport aggregates usage over a time window. +type UsageReport struct { + From time.Time `json:"from"` + To time.Time `json:"to"` + GroupBy string `json:"groupBy"` + Lines []UsageLine `json:"lines"` + Total UsageLine `json:"total"` + Note string `json:"note"` +} + +// Agent is a registered node agent serving a private pool. +type Agent struct { + ID string `json:"id"` + Pool string `json:"pool"` + Node string `json:"node"` + Endpoint string `json:"endpoint"` + Version string `json:"version"` + Arch string `json:"arch"` + TotalCPU int `json:"totalCPU"` + FreeCPU int `json:"freeCPU"` + TotalMemoryMiB int `json:"totalMemoryMiB"` + FreeMemoryMiB int `json:"freeMemoryMiB"` + Sandboxes int `json:"sandboxes"` + Capabilities []string `json:"capabilities"` + Templates []string `json:"templates,omitempty"` + LastSeen time.Time `json:"lastSeen"` + Healthy bool `json:"healthy"` +} + +// List is a paginated collection. +type List[T any] struct { + Items []T `json:"items"` + NextCursor string `json:"nextCursor,omitempty"` +} + +// VersionInfo is returned by GET /version. +type VersionInfo struct { + Version string `json:"version"` + Commit string `json:"commit"` + APIVersion string `json:"apiVersion"` + Installation string `json:"installation"` +} From 483ad93c59c887166be23357fa798029cd8aea40 Mon Sep 17 00:00:00 2001 From: Jalil Date: Wed, 28 Oct 2026 13:04:09 -0700 Subject: [PATCH 02/76] feat: add the Firecracker node agent and in-guest agent differently-agent runs Firecracker microVMs on KVM nodes, isolates each VM on its own tap device and firewall chain, reaches the guest over vsock, and serves the adapter protocol to the control plane. It supports stop/start from the retained disk, pause/resume, crash-consistent disk snapshots and recovery after restarts. Signed-off-by: Jalil --- cmd/differently-agent/main.go | 224 ++++++++ cmd/differently-guest/main.go | 38 ++ cmd/differently-guest/vsock_linux.go | 11 + cmd/differently-guest/vsock_other.go | 12 + internal/agent/agent.go | 798 +++++++++++++++++++++++++++ internal/agent/agent_test.go | 280 ++++++++++ internal/agent/network.go | 73 +++ internal/firecracker/client.go | 152 +++++ internal/firecracker/client_test.go | 66 +++ 9 files changed, 1654 insertions(+) create mode 100644 cmd/differently-agent/main.go create mode 100644 cmd/differently-guest/main.go create mode 100644 cmd/differently-guest/vsock_linux.go create mode 100644 cmd/differently-guest/vsock_other.go create mode 100644 internal/agent/agent.go create mode 100644 internal/agent/agent_test.go create mode 100644 internal/agent/network.go create mode 100644 internal/firecracker/client.go create mode 100644 internal/firecracker/client_test.go diff --git a/cmd/differently-agent/main.go b/cmd/differently-agent/main.go new file mode 100644 index 0000000..6c5e036 --- /dev/null +++ b/cmd/differently-agent/main.go @@ -0,0 +1,224 @@ +// Command differently-agent runs on each KVM node. It manages Firecracker +// microVMs locally, serves them to the control plane over the adapter +// protocol and heartbeats its capacity. +// +// differently-agent --server URL --pool NAME [flags] serve +// differently-agent doctor check this node +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "os/exec" + "os/signal" + "runtime" + "strings" + "syscall" + "time" + + "github.com/different-ai/differently/internal/agent" + "github.com/different-ai/differently/internal/version" + "github.com/different-ai/differently/pkg/adapter/adapterhttp" +) + +type flags struct { + server, pool, listen, advertise, node, stateDir, imagesDir, kernel, firecracker, jailer, egress, dns string + reserveCPU, reserveMem int + overcommit float64 +} + +func main() { + cmd := "serve" + args := os.Args[1:] + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + cmd, args = args[0], args[1:] + } + host, _ := os.Hostname() + f := flags{} + fs := flag.NewFlagSet("differently-agent", flag.ExitOnError) + fs.StringVar(&f.server, "server", os.Getenv("DIFFERENTLY_SERVER"), "control plane URL") + fs.StringVar(&f.pool, "pool", os.Getenv("DIFFERENTLY_POOL"), "firecracker pool this node serves") + fs.StringVar(&f.listen, "listen", ":9090", "adapter API listen address") + fs.StringVar(&f.advertise, "advertise-url", os.Getenv("DIFFERENTLY_ADVERTISE_URL"), "URL the control plane uses to reach this agent") + fs.StringVar(&f.node, "node", envOr("NODE_NAME", host), "node name") + fs.StringVar(&f.stateDir, "state-dir", "/var/lib/differently/agent", "microVM state directory") + fs.StringVar(&f.imagesDir, "images-dir", "/var/lib/differently/images", "directory of