Sandbox templates: content-keyed records, derived-image strategies, template forks, GC

AGENTS.md

@@ -10,6 +10,7 @@ a service, not a library.
 It owns:
 
 - Host records and lifecycle state in Postgres
+- Template records, async template builds, and their provider images
 - Inline provisioning in `POST /hosts`
 - Provider VM creation and deletion (exe.dev, AWS, Hetzner, Exoscale,
   local Docker, Docker Sandboxes)
@@ -18,8 +19,9 @@ It owns:
 - An SSH gateway for hosts of gateway providers (`python -m gateway.server`)
 - Account-bound exe.dev HTTP proxy resources
 
-Periodic maintenance runs as cron jobs: `python -m hosts.janitor` reaps
-expired hosts, `python -m hosts.pool` tops up the warm pool.
+Periodic maintenance runs as cron jobs: `python -m janitor` reaps expired
+hosts and abandoned, failed, or unused templates, and `python -m hosts.pool`
+tops up the warm pool.
 
 No backwards compatibility is required unless a caller contract is explicitly
 documented in this repo.
@@ -47,8 +49,10 @@ src/
   hosts/             # Host API, models, schemas, service, janitor, pool, auth
   gateway/           # SSH gateway for gateway-provider hosts
   http_proxies/      # HTTP proxy API, schemas, service, deps
+  janitor/           # Cron entry point that runs the host and template reapers
   providers/         # VM provider ABC, capabilities, registry, adapters
   networking/        # Network provider framework and Tailscale adapter
+  templates/         # Template API, models, service, and janitor
   conftest.py        # Test env defaults and database reset fixture
 alembic/             # Database migrations
 api-tests/           # Playwright black-box API tests

alembic/env.py

@@ -6,6 +6,7 @@
 from core.database import Base
 from core.settings import get_settings
 from hosts import models  # noqa: F401
+from templates import models as template_models  # noqa: F401
 
 config = context.config
 

alembic/versions/0004_templates.py

@@ -0,0 +1,48 @@
+"""reusable provider templates
+
+Revision ID: 0004_templates
+Revises: 0003_host_public_key
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "0004_templates"
+down_revision: str | None = "0003_host_public_key"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "templates",
+        sa.Column("id", sa.Uuid(), nullable=False),
+        sa.Column("provider", sa.String(length=20), nullable=False),
+        sa.Column("base_image", sa.Text(), nullable=False),
+        sa.Column("requirements_hash", sa.String(length=64), nullable=False),
+        sa.Column("setup_script", sa.Text(), nullable=False),
+        sa.Column("label", sa.Text(), nullable=False),
+        sa.Column("handle", sa.Text(), nullable=False),
+        sa.Column("status", sa.String(length=32), nullable=False),
+        sa.Column("last_error", sa.Text(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        "ix_templates_provider_base_image_requirements_hash",
+        "templates",
+        ["provider", "base_image", "requirements_hash"],
+        unique=True,
+    )
+
+
+def downgrade() -> None:
+    op.drop_index(
+        "ix_templates_provider_base_image_requirements_hash",
+        table_name="templates",
+    )
+    op.drop_table("templates")

docs/api.md

@@ -13,6 +13,8 @@ Every endpoint except `GET /healthz` requires
 ## Endpoints
 
 - `POST /hosts` · `GET /hosts` · `GET /hosts/{id}` · `DELETE /hosts/{id}`
+- `POST /templates` · `GET /templates` · `GET /templates/{id}` ·
+  `DELETE /templates/{id}`
 - `POST /http-proxies` · `DELETE /http-proxies/{name}` ·
   `POST|DELETE /http-proxies/{name}/hosts/{host_id}`
 - `GET /doctor` — read-only dependency diagnostics

docs/architecture.md

@@ -29,6 +29,8 @@ that true:
 ```text
 hosts.api          HTTP request/response concerns only
 hosts.service      host lifecycle behavior (HostService)
+templates.api      template request/response concerns only
+templates.service  template build and delete behavior (TemplateService)
 providers/<name>   one package per VM provider
 networking/        network provider framework + Tailscale adapter
 core/              settings, database, exception base
@@ -71,14 +73,15 @@ the core settings knowing any provider exists.
 
 Not every provider supports every feature, and the host contract must
 not grow provider-shaped warts. Optional features are capability
-mix-ins: `HttpProxyCapability` declares the http-proxy surface and the
-exe provider implements it. `resolve_capability` narrows a specific
-provider instance to a capability — the default provider for
-account-bound operations, the host's own provider for host-bound ones
-— and raises the shared `CapabilityUnsupportedError` when that
-provider doesn't implement it, which the routes surface as a clear
-error. New provider-specific features should follow this pattern
-rather than widening `VMProvider` or the host schema.
+mix-ins: `HttpProxyCapability` declares the http-proxy surface, and
+`TemplateCapability` declares the template create and delete surface.
+`resolve_capability` narrows a specific provider instance
+to a capability — the default provider for account-bound operations,
+the host's own provider for host-bound ones — and raises the shared
+`CapabilityUnsupportedError` when that provider does not implement it,
+which the routes surface as a clear error. New provider-specific
+features must follow this pattern rather than widening `VMProvider`
+or the host schema.
 
 The review question that guards the whole design: *does this change
 leak a provider into the contract?*
@@ -96,6 +99,19 @@ successful key returns the original host instead of a duplicate.
 Caller `env` is stored for provisioning and never returned by the API;
 keys in `hosts.schemas.RESERVED_HOST_ENV_KEYS` are rejected.
 
+A template is a persistent provider image keyed by provider, base image,
+and setup-script hash. `POST /templates` creates a `building` record and
+returns `202 Accepted`. Callers poll until the template becomes
+`available` or `failed`. Templates outlive hosts. Each provider builds
+and deletes its own templates behind `TemplateCapability`.
+
+A host request can name an available template by its ID — the ID that
+the create returned. The template's handle becomes the host image. An
+explicit `image`
+wins over the template, and the template wins over the provider default.
+Host creation never builds a missing or unavailable template. It returns
+a client error, and the caller decides when to build.
+
 Every host is a renewable lease. A create without `expires_at` gets
 `now + LEASE_DEFAULT_TTL`, so a host whose owner disappears lapses and
 self-reaps instead of leaking VM cost; an explicit `expires_at: null`
@@ -106,13 +122,18 @@ hosts renew — unclaimed warm-pool members belong to pool maintenance
 and refuse with `409`.
 
 Two maintenance commands run as cron jobs from the same image:
-`hosts.janitor` reaps expired and orphaned hosts, `hosts.pool` keeps a
-warm pool of pre-provisioned hosts per provider (`POOL_SIZES`, with
-`POOL_SIZE` as the default provider's target) to hide provider cold
-starts. Pool members are warmed with the provider's default image and
-size, so a request that customizes its host — `image`, `env`,
-`instance_type`, or `disk_gb` — always provisions fresh instead of
-claiming a warm host.
+
+- `janitor` reaps expired and orphaned hosts, marks abandoned template
+  builds `failed`, and deletes failed or unused templates.
+- `hosts.pool` keeps a warm pool of pre-provisioned hosts per provider
+  (`POOL_SIZES`, with `POOL_SIZE` as the default provider's target) to
+  hide provider cold starts.
+
+When you edit a template setup script, the hash changes. The old
+template ages out after its last lease. Pool members
+are warmed with the provider's default image and size, so a request that
+customizes its host — `image`, `env`, `template`, `instance_type`, or
+`disk_gb` — always provisions fresh instead of claiming a warm host.
 
 ## Diagnostics
 

docs/deploy.md

@@ -20,15 +20,17 @@ docker run --rm -p 8780:8780 --env-file drukbox.env "$IMAGE"
 docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/alembic upgrade head
 
 # Maintenance (cron, e.g. every 10-15 min)
-docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/python -m hosts.janitor
+docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/python -m janitor
 docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/python -m hosts.pool
 ```
 
-The janitor reaps expired and orphaned hosts. The pool maintainer
+The janitor reaps expired and orphaned hosts, marks abandoned template
+builds failed, keeps failed builds for diagnosis, and deletes failed or
+unused templates. The pool maintainer
 pre-provisions warm hosts per provider and only does anything when at
 least one provider has a warm target (`POOL_SIZES` / `POOL_SIZE`).
-Schedule both under your cron infrastructure (k8s `CronJob`, systemd
-timer) from the same image and env file.
+Schedule both under your cron infrastructure (k8s `CronJob`,
+systemd timer) from the same image and env file.
 
 Use Postgres in production (`postgresql+psycopg://...`). SQLite
 (`sqlite+aiosqlite:///./drukbox.db`) is for single-process demos and
@@ -100,9 +102,8 @@ socket is host-root-equivalent. Do not expose a docker-backed drukbox to
 untrusted callers.
 
 Janitor and pool one-off containers using the Docker provider need the
-same socket mount and socket-GID supplemental group. `DOCKER_HOST` remains
-available when the daemon is remote or rootless instead of exposed through
-`/var/run/docker.sock`.
+same socket mount and socket-GID supplemental group. `DOCKER_HOST` remains available when the daemon is remote or
+rootless instead of exposed through `/var/run/docker.sock`.
 
 ## Local microVMs with Docker Sandboxes
 
@@ -294,6 +295,9 @@ Core, optional:
 | `SERVICE_LABEL` | `drukbox` | Label stamped onto provider resources (VM tags, SG tags). |
 | `UVICORN_HOST` | `0.0.0.0` | API bind address. Set `127.0.0.1` to restrict to loopback. |
 | `PROVISIONING_GRACE_SECONDS` | `600` | Safety TTL on in-flight hosts so the janitor reaps row + VM if the client disconnects mid-provision. Must exceed the worst-case provision duration. |
+| `TEMPLATE_BUILD_TIMEOUT_MINUTES` | `60` | Max age of an unfinished template build before the template janitor marks it failed. |
+| `TEMPLATE_FAILED_RETENTION_HOURS` | `24` | How long failed template records and diagnostics remain before the template janitor deletes them. |
+| `TEMPLATE_UNUSED_TTL_DAYS` | `14` | How long an available template remains after its last use, or creation when never used. |
 | `LEASE_DEFAULT_TTL` | `86400` | Lease TTL in seconds for hosts created without an explicit `expires_at`, and the extension applied by an empty `POST /hosts/{id}/renew`. An explicit `expires_at: null` at create time opts out of expiry entirely. |
 | `IDEMPOTENCY_KEY_TTL_HOURS` | `24` | Retention period for successful `Idempotency-Key` mappings. |
 | `POOL_SIZES` | `{}` | Warm hosts to keep ready per provider, as JSON (e.g. `{"exe": 2, "hetzner": 1}`). Overrides `POOL_SIZE` for the providers it names. |
@@ -319,6 +323,9 @@ exe.dev provider:
 | --- | --- | --- |
 | `EXE_API_TOKEN` | — (required) | Bearer token for the exe.dev exec API. |
 | `EXE_DEFAULT_IMAGE` | — (required) | Image used when the caller omits `image`. |
+| `EXE_TEMPLATE_REGISTRY` | — | Repository prefix for derived template images. A VM created from this registry gets `--registry-auth` so exe.dev can pull a private image. |
+| `EXE_REGISTRY_USERNAME` | — | Username for the derived-template image registry. |
+| `EXE_REGISTRY_PASSWORD` | — | Password or token for the derived-template image registry. |
 | `EXE_API_URL` | `https://exe.dev` | API base URL. |
 | `EXE_API_TIMEOUT` | `30.0` | Timeout for exe.dev API calls. |
 | `EXE_BOOTSTRAP_SSH_TIMEOUT_SECONDS` | `30.0` | ssh-keyscan retry budget for a fresh exe.dev sandbox. |

docs/security.md

@@ -76,9 +76,10 @@ covered in [Networking](networking.md). The security-relevant summary:
 
 ## Secrets and in-VM metadata
 
-Provider tokens (`EXE_API_TOKEN`, `HETZNER_API_TOKEN`, Tailscale OAuth)
-and AWS credentials are read from the environment / the AWS SDK default
-chain and never written to the database or returned by the API. Caller
+Provider tokens (`EXE_API_TOKEN`, `EXE_REGISTRY_PASSWORD`,
+`HETZNER_API_TOKEN`, Tailscale OAuth) and AWS credentials are read from
+the environment / the AWS SDK default chain and never written to the
+database or returned by the API. Caller
 `env` is write-only: it is delivered to the VM but never echoed in any
 response, and reserved keys (`TAILSCALE_AUTHKEY`) are rejected at the
 schema.

pyproject.toml

@@ -10,8 +10,10 @@ packages = [
     "src/gateway",
     "src/hosts",
     "src/http_proxies",
+    "src/janitor",
     "src/networking",
     "src/providers",
+    "src/templates",
 ]
 exclude = ["**/tests", "**/tests/**"]
 

src/api/app.py

@@ -13,6 +13,7 @@
 from http_proxies.api import router as http_proxies_router
 from networking.tailscale import Tailscale
 from providers.registry import iter_initialized_vm_providers
+from templates.api import router as templates_router
 
 _log_level = getattr(logging, os.environ.get("LOG_LEVEL", "INFO").upper(), logging.INFO)
 
@@ -71,4 +72,5 @@ async def healthz() -> dict[str, str]:
 
 app.include_router(hosts_router)
 app.include_router(http_proxies_router)
+app.include_router(templates_router)
 app.include_router(diagnostics_router)

src/conftest.py

@@ -41,6 +41,7 @@ def load_test_env() -> None:
 async def reset_database() -> AsyncGenerator[None]:
     from core.database import Base, engine
     from hosts import models  # noqa: F401
+    from templates import models as template_models  # noqa: F401
 
     async with engine.begin() as connection:
         await connection.run_sync(Base.metadata.drop_all)

src/core/settings.py

@@ -70,6 +70,24 @@ class Settings(BaseSettings):
         validation_alias="PROVISIONING_GRACE_SECONDS",
         description="Safety TTL on the host row while provisioning is in flight.",
     )
+    template_build_timeout_minutes: int = Field(
+        default=60,
+        gt=0,
+        validation_alias="TEMPLATE_BUILD_TIMEOUT_MINUTES",
+        description="Minutes before an unfinished template build is considered abandoned.",
+    )
+    template_failed_retention_hours: int = Field(
+        default=24,
+        gt=0,
+        validation_alias="TEMPLATE_FAILED_RETENTION_HOURS",
+        description="Hours to retain failed templates for diagnosis before deletion.",
+    )
+    template_unused_ttl_days: int = Field(
+        default=14,
+        gt=0,
+        validation_alias="TEMPLATE_UNUSED_TTL_DAYS",
+        description="Days to retain an available template after its last use.",
+    )
     lease_default_ttl: int = Field(
         default=86400,
         gt=0,

src/core/tests/test_settings.py

@@ -92,6 +92,9 @@ def test_tailscale_disabled_ignores_missing_credentials(monkeypatch: pytest.Monk
         "POOL_SIZE",
         "POOL_HOST_MAX_AGE_HOURS",
         "POOL_MAX_CREATES_PER_TICK",
+        "TEMPLATE_BUILD_TIMEOUT_MINUTES",
+        "TEMPLATE_FAILED_RETENTION_HOURS",
+        "TEMPLATE_UNUSED_TTL_DAYS",
     ],
 )
 def test_numeric_settings_reject_negative_values(monkeypatch: pytest.MonkeyPatch, key: str) -> None:
@@ -100,6 +103,22 @@ def test_numeric_settings_reject_negative_values(monkeypatch: pytest.MonkeyPatch
         _settings_with(monkeypatch, env)
 
 
+def test_template_maintenance_defaults(monkeypatch: pytest.MonkeyPatch) -> None:
+    settings = _settings_with(
+        monkeypatch,
+        {
+            **_base_env(),
+            "TEMPLATE_BUILD_TIMEOUT_MINUTES": None,
+            "TEMPLATE_FAILED_RETENTION_HOURS": None,
+            "TEMPLATE_UNUSED_TTL_DAYS": None,
+        },
+    )
+
+    assert settings.template_build_timeout_minutes == 60
+    assert settings.template_failed_retention_hours == 24
+    assert settings.template_unused_ttl_days == 14
+
+
 def test_pool_size_seeds_the_default_providers_target(monkeypatch: pytest.MonkeyPatch) -> None:
     env: dict[str, str | None] = {
         **_base_env(),

src/hosts/api.py

@@ -54,6 +54,7 @@ async def create_host(
         return await service.get_or_create_host(
             env=host_create.env,
             image=host_create.image,
+            template=host_create.template,
             expires_at=expires_at,
             idempotency_key=idempotency_key,
             provider=host_create.provider,

src/hosts/janitor.py

@@ -61,14 +61,3 @@ async def reap_expired_hosts() -> list[uuid.UUID]:
             await tailscale.aclose()
 
     return reaped
-
-
-if __name__ == "__main__":
-    # Cron entry point: `python -m hosts.janitor`.
-    import asyncio
-
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s %(levelname)s %(name)s %(message)s",
-    )
-    asyncio.run(reap_expired_hosts())