fix(gateway): boot-seed challenge backends on restart

crates/gateway-registry/src/lib.rs

@@ -371,6 +371,76 @@ fn normalize_base_url(raw: &str) -> Result<String, RegistryError> {
     Ok(s)
 }
 
+/// Parse a boot-seed list: `challenge_id=url` entries separated by commas
+/// and/or newlines. Empty / whitespace-only input yields an empty vec.
+///
+/// Optional `;weight` suffix (default 1): `design=http://design:8093;2`.
+///
+/// # Errors
+///
+/// [`RegistryError::Invalid`] on malformed entries or bad URLs.
+pub fn parse_backend_seed_list(raw: &str) -> Result<Vec<CreateBackend>, RegistryError> {
+    let mut out = Vec::new();
+    for part in raw.split([',', '\n', '\r']) {
+        let part = part.trim();
+        if part.is_empty() {
+            continue;
+        }
+        let (challenge_id, rest) = part.split_once('=').ok_or_else(|| {
+            RegistryError::Invalid(format!(
+                "backend seed entry must be challenge_id=url[,…]; got `{part}`"
+            ))
+        })?;
+        let challenge_id = challenge_id.trim();
+        if challenge_id.is_empty() {
+            return Err(RegistryError::Invalid(
+                "backend seed challenge_id must be non-empty".into(),
+            ));
+        }
+        let (url_raw, weight) = match rest.rsplit_once(';') {
+            Some((url, w)) if w.trim().is_empty() => (url, 1u32),
+            Some((url, w)) if !w.contains("://") => {
+                let weight: u32 = w.trim().parse().map_err(|_| {
+                    RegistryError::Invalid(format!(
+                        "backend seed weight must be u32; got `{}`",
+                        w.trim()
+                    ))
+                })?;
+                (url, weight)
+            }
+            _ => (rest, 1u32),
+        };
+        let base_url = normalize_base_url(url_raw)?;
+        out.push(CreateBackend {
+            challenge_id: challenge_id.to_owned(),
+            base_url,
+            weight,
+        });
+    }
+    Ok(out)
+}
+
+impl Registry {
+    /// Insert seed backends; skip rows already present (`Duplicate`).
+    ///
+    /// Returns how many rows were newly created.
+    ///
+    /// # Errors
+    ///
+    /// Propagates non-duplicate [`RegistryError`] from [`Self::create`].
+    pub fn seed(&self, backends: &[CreateBackend]) -> Result<usize, RegistryError> {
+        let mut created = 0usize;
+        for req in backends {
+            match self.create(req) {
+                Ok(_) => created += 1,
+                Err(RegistryError::Duplicate { .. }) => {}
+                Err(e) => return Err(e),
+            }
+        }
+        Ok(created)
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -544,4 +614,32 @@ mod tests {
         let picked = reg.pick("prism").expect("pick prism");
         assert_eq!(picked.base_url, "http://prism-challenge:8092");
     }
+
+    #[test]
+    fn parse_and_seed_compose_backends() {
+        let list = parse_backend_seed_list(
+            "prism=http://prism-challenge:8092, design=http://design-challenge:8093\n",
+        )
+        .expect("parse");
+        assert_eq!(list.len(), 2);
+        assert_eq!(list[0].challenge_id, "prism");
+        assert_eq!(list[0].base_url, "http://prism-challenge:8092");
+        assert_eq!(list[1].challenge_id, "design");
+        assert_eq!(list[1].weight, 1);
+
+        let reg = Registry::with_defaults();
+        assert_eq!(reg.seed(&list).expect("seed"), 2);
+        assert_eq!(reg.seed(&list).expect("idempotent"), 0);
+        assert_eq!(reg.list(None).len(), 2);
+        assert_eq!(
+            reg.pick("design").unwrap().base_url,
+            "http://design-challenge:8093"
+        );
+    }
+
+    #[test]
+    fn parse_backend_seed_rejects_bad_entry() {
+        let err = parse_backend_seed_list("not-a-pair").unwrap_err();
+        assert!(matches!(err, RegistryError::Invalid(_)));
+    }
 }

crates/gateway/src/gw_config.rs

@@ -6,7 +6,7 @@ use std::net::SocketAddr;
 use std::str::FromStr;
 
 use config::{Config, Role};
-use gateway_registry::RegistryConfig;
+use gateway_registry::{parse_backend_seed_list, CreateBackend, Registry, RegistryConfig};
 
 use crate::tls::TlsConfig;
 use crate::{GatewayError, DEFAULT_LISTEN};
@@ -31,6 +31,13 @@ pub mod keys {
     /// re-applies to `/challenge/*/v1/view/*` responses (defense in depth).
     /// Defaults to [`design_sanitize::default_frame_ancestors`].
     pub const VIEW_FRAME_ANCESTORS: &str = "BASE_GATEWAY_VIEW_FRAME_ANCESTORS";
+    /// Comma/newline-separated boot seed for the in-memory challenge registry:
+    /// `prism=http://prism-challenge:8092,design=http://design-challenge:8093`.
+    /// Applied on every process start so compose/prod restarts do not leave
+    /// `/challenge/*` at 503 until an operator POSTs `/v1/admin/backends`.
+    pub const BACKENDS: &str = "BASE_GATEWAY_BACKENDS";
+    /// Optional file whose contents are parsed like [`BACKENDS`] (wins when set).
+    pub const BACKENDS_FILE: &str = "BASE_GATEWAY_BACKENDS_FILE";
 
     pub use crate::tls::keys as tls;
 }
@@ -121,6 +128,54 @@ fn registry_config_from_env() -> Result<RegistryConfig, GatewayError> {
     Ok(cfg)
 }
 
+/// Load optional boot-seed backends from [`keys::BACKENDS_FILE`] or [`keys::BACKENDS`].
+///
+/// # Errors
+///
+/// Unreadable file or malformed seed list.
+pub fn load_backend_seed_from_env() -> Result<Vec<CreateBackend>, GatewayError> {
+    if let Ok(path) = std::env::var(keys::BACKENDS_FILE) {
+        let path = path.trim();
+        if !path.is_empty() {
+            let raw = std::fs::read_to_string(path).map_err(|e| {
+                GatewayError::Config(format!("read {} `{path}`: {e}", keys::BACKENDS_FILE))
+            })?;
+            return parse_backend_seed_list(&raw)
+                .map_err(|e| GatewayError::Config(format!("{}: {e}", keys::BACKENDS_FILE)));
+        }
+    }
+    match std::env::var(keys::BACKENDS) {
+        Ok(raw) if !raw.trim().is_empty() => parse_backend_seed_list(&raw)
+            .map_err(|e| GatewayError::Config(format!("{}: {e}", keys::BACKENDS))),
+        _ => Ok(Vec::new()),
+    }
+}
+
+/// Apply [`load_backend_seed_from_env`] to an empty (or already-seeded) registry.
+///
+/// # Errors
+///
+/// Seed parse/load failures, or non-duplicate registry insert errors.
+pub fn seed_registry_from_env(registry: &Registry) -> Result<usize, GatewayError> {
+    let backends = load_backend_seed_from_env()?;
+    if backends.is_empty() {
+        return Ok(0);
+    }
+    let created = registry
+        .seed(&backends)
+        .map_err(|e| GatewayError::Config(format!("backend seed: {e}")))?;
+    for b in &backends {
+        tracing::info!(
+            event = "gateway_backend_seed",
+            challenge_id = %b.challenge_id,
+            base_url = %b.base_url,
+            weight = b.weight,
+            "challenge backend present from boot seed"
+        );
+    }
+    Ok(created)
+}
+
 /// Resolve the gateway hotkey from a Bittensor wallet, mnemonic file, or hex.
 ///
 /// Delegates to [`keystore::resolve_public_key_from_env`] with the

crates/gateway/src/lib.rs

@@ -38,13 +38,14 @@ pub use gateway_core::admin_attest::{
     admin_attest_grant_router, AttestGrantRequest, AttestGrantResponse, AttestGrantState,
     ATTEST_GRANT_ROUTE,
 };
+pub use gateway_registry::parse_backend_seed_list;
 pub use gateway_registry::{
     Backend, BackendView, CreateBackend, Registry, RegistryConfig, RegistryError, DEFAULT_COOLDOWN,
     DEFAULT_FAILURE_THRESHOLD,
 };
 pub use gw_config::{
-    hotkey_hex, keys, parse_hotkey_hex, resolve_gateway_hotkey, GatewayConfig, OwnerCheck,
-    REQUIRE_OWNER_ENV,
+    hotkey_hex, keys, load_backend_seed_from_env, parse_hotkey_hex, resolve_gateway_hotkey,
+    seed_registry_from_env, GatewayConfig, OwnerCheck, REQUIRE_OWNER_ENV,
 };
 pub use sealer::{
     admin_seal_router, bundle_router, load_gateway_secret, seal_epoch, BundleStore,
@@ -381,6 +382,16 @@ where
     let metrics = init_metrics()?;
     // Prefer registry knobs from config when the shared handle was default-built.
     let _ = &config.registry;
+    // In-memory registry: seed from BASE_GATEWAY_BACKENDS(_FILE) so compose/prod
+    // restarts never leave /challenge/* at 503 until a manual admin POST.
+    let seeded = seed_registry_from_env(&registry)?;
+    if seeded > 0 {
+        tracing::info!(
+            event = "gateway_backends_seeded",
+            created = seeded,
+            "boot-seeded challenge backends into in-memory registry"
+        );
+    }
     let app = build_app(metrics, registry, chain, &config.tls, stores, extra)?;
 
     let listener = TcpListener::bind(config.listen)

deploy/AGENTS.md

@@ -40,7 +40,7 @@ Compose always runs a digest-pinned `postgres` service (`base-pgdata` volume, he
 | Gateway raw weight leaves + sealed bundles | **Postgres** (`raw_weight_snapshot`, `epoch_bundle`, …) |
 | Validator attestations (when DB configured) | **Postgres** |
 | Design sandbox staging files | volume `${BASE_STATE_DIR}/design/staging` + `design-artifacts` |
-| Gateway challenge **backend registry** | **in-memory** — re-seed after gateway restart (`remote-deploy.sh` does this on master) |
+| Gateway challenge **backend registry** | **in-memory**, boot-seeded from `BASE_GATEWAY_BACKENDS` (compose default: prism+design DNS URLs); `remote-deploy.sh` POST reseed stays idempotent |
 | site-api (`GET /v1/site/*`) | no DB — proxies challenge upstreams via gateway |
 | Unit/integration tests | may construct `Memory*Store` directly; omit `BASE_DATABASE_URL` only there |
 

deploy/scripts/register-challenge-backends.sh

@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 # Register challenge reverse-proxy backends with the gateway registry.
 #
-# The in-memory registry is empty after every gateway restart/redeploy.
-# Call this on master after `docker compose up` (remote-deploy hooks it).
+# Prefer compose `BASE_GATEWAY_BACKENDS` (gateway boot-seeds on start).
+# This script remains the manual / remote-deploy idempotent fallback when the
+# env seed is absent or you need to re-point URLs without restarting.
 #
 # Usage:
 #   GATEWAY_URL=http://127.0.0.1:8080 ./deploy/scripts/register-challenge-backends.sh

deploy/scripts/remote-deploy.sh

@@ -270,6 +270,7 @@ if [[ "$BUILD_FROM" == "prebuilt" ]]; then
     "$ROOT/target/release/prism-challenge" \
     "$ROOT/target/release/design-challenge" \
     "$ROOT/target/release/design-egress-proxy" \
+    "$ROOT/target/release/challenge-review" \
     "$HOST:$REMOTE_DIR/target/release/"
 fi
 
@@ -438,9 +439,9 @@ if [[ '$ROLE' == 'master' ]]; then
   else
     echo "gateway health: probe deferred"
   fi
-  # Registry is in-memory — re-seed challenge backends after every redeploy.
-  # The gateway races this script on boot, so retry until registration sticks,
-  # then prove proxy routing end-to-end: a missed reseed leaves /challenge/*
+  # Registry is in-memory; compose BASE_GATEWAY_BACKENDS boot-seeds on start.
+  # Keep an idempotent POST reseed after redeploy (covers empty override / race),
+  # then prove proxy routing end-to-end: a missed seed leaves /challenge/*
   # at 503 while /healthz stays green. Both must fail the deploy loudly.
   echo "remote-deploy: registering challenge backends"
   reseed_ok=0

docker-compose.yml

@@ -93,6 +93,9 @@ services:
       BASE_TRUST_ROOT_DIR: ${BASE_TRUST_ROOT_DIR:-/etc/base/config}
       # Bundle seal mini-secret (host file, never baked into image)
       BASE_GATEWAY_SK_FILE: ${BASE_GATEWAY_SK_FILE:-/run/secrets/gateway_sk}
+      # Durable boot seed for the in-memory challenge registry (survives
+      # gateway-only restarts; remote-deploy POST reseed stays idempotent).
+      BASE_GATEWAY_BACKENDS: "${BASE_GATEWAY_BACKENDS:-prism=http://prism-challenge:8092,design=http://design-challenge:8093}"
     volumes:
       - ./config:/etc/base/config:ro
       - ./deploy/secrets/gateway_sk:/run/secrets/gateway_sk:ro

docs/SITE_API.md

@@ -43,5 +43,6 @@ challenge API.
 `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle,
 slug, operator, and (for submissions) prompt title / id / run id.
 
-Backends must be registered (same as challenge proxy), e.g.
+Backends come from the gateway registry (same as challenge proxy). Compose
+boot-seeds them via `BASE_GATEWAY_BACKENDS`; manual fallback:
 `deploy/scripts/register-challenge-backends.sh`.

docs/runbooks/staging-testnet-e2e.md

@@ -47,12 +47,15 @@ docker exec $(docker ps -q --filter name=gateway) curl -fsS http://127.0.0.1:808
 # Returns SCALE-encoded sealed bundle
 ```
 
-## Register challenge backends (required after gateway restart)
+## Register challenge backends (boot seed + manual fallback)
 
-The gateway registry is **in-memory**. After every redeploy/restart, challenge
-proxy routes return `503 no healthy backends for challenge_id=…` until backends
-are registered. `remote-deploy.sh` (master) re-seeds automatically; to do it by
-hand:
+The gateway registry is **in-memory**, but compose sets `BASE_GATEWAY_BACKENDS`
+so the master gateway **boot-seeds** `prism` + `design` on every process start.
+A plain `docker compose restart gateway` must not leave `/challenge/*` at 503.
+
+`remote-deploy.sh` (master) still POSTs `/v1/admin/backends` (idempotent 409).
+Manual re-register only if the env seed was overridden empty or you need a
+non-default URL:
 
 ```bash
 # From this repo (against a reachable gateway):