From 657dec858693c5df2fefd36954b2b0b65e9398c5 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:18:58 +0000 Subject: [PATCH 1/2] fix(gateway): boot-seed challenge backends from compose env Survive gateway-only restarts without empty in-memory registry / 503s by seeding prism+design from BASE_GATEWAY_BACKENDS on process start. --- crates/gateway-registry/src/lib.rs | 98 +++++++++++++++++++ crates/gateway/src/gw_config.rs | 57 ++++++++++- crates/gateway/src/lib.rs | 15 ++- deploy/AGENTS.md | 2 +- deploy/scripts/register-challenge-backends.sh | 5 +- deploy/scripts/remote-deploy.sh | 6 +- docker-compose.yml | 3 + docs/SITE_API.md | 3 +- docs/runbooks/staging-testnet-e2e.md | 13 ++- 9 files changed, 187 insertions(+), 15 deletions(-) diff --git a/crates/gateway-registry/src/lib.rs b/crates/gateway-registry/src/lib.rs index a5184ac6d..542eac66f 100644 --- a/crates/gateway-registry/src/lib.rs +++ b/crates/gateway-registry/src/lib.rs @@ -371,6 +371,76 @@ fn normalize_base_url(raw: &str) -> Result { Ok(s) } +/// Parse a boot-seed list: `challenge_id=url` entries separated by commas +/// and/or newlines. Empty / whitespace-only input yields an empty vec. +/// +/// Optional `;weight` suffix (default 1): `design=http://design:8093;2`. +/// +/// # Errors +/// +/// [`RegistryError::Invalid`] on malformed entries or bad URLs. +pub fn parse_backend_seed_list(raw: &str) -> Result, RegistryError> { + let mut out = Vec::new(); + for part in raw.split([',', '\n', '\r']) { + let part = part.trim(); + if part.is_empty() { + continue; + } + let (challenge_id, rest) = part.split_once('=').ok_or_else(|| { + RegistryError::Invalid(format!( + "backend seed entry must be challenge_id=url[,…]; got `{part}`" + )) + })?; + let challenge_id = challenge_id.trim(); + if challenge_id.is_empty() { + return Err(RegistryError::Invalid( + "backend seed challenge_id must be non-empty".into(), + )); + } + let (url_raw, weight) = match rest.rsplit_once(';') { + Some((url, w)) if w.trim().is_empty() => (url, 1u32), + Some((url, w)) if !w.contains("://") => { + let weight: u32 = w.trim().parse().map_err(|_| { + RegistryError::Invalid(format!( + "backend seed weight must be u32; got `{}`", + w.trim() + )) + })?; + (url, weight) + } + _ => (rest, 1u32), + }; + let base_url = normalize_base_url(url_raw)?; + out.push(CreateBackend { + challenge_id: challenge_id.to_owned(), + base_url, + weight, + }); + } + Ok(out) +} + +impl Registry { + /// Insert seed backends; skip rows already present (`Duplicate`). + /// + /// Returns how many rows were newly created. + /// + /// # Errors + /// + /// Propagates non-duplicate [`RegistryError`] from [`Self::create`]. + pub fn seed(&self, backends: &[CreateBackend]) -> Result { + let mut created = 0usize; + for req in backends { + match self.create(req) { + Ok(_) => created += 1, + Err(RegistryError::Duplicate { .. }) => {} + Err(e) => return Err(e), + } + } + Ok(created) + } +} + #[cfg(test)] mod tests { use super::*; @@ -544,4 +614,32 @@ mod tests { let picked = reg.pick("prism").expect("pick prism"); assert_eq!(picked.base_url, "http://prism-challenge:8092"); } + + #[test] + fn parse_and_seed_compose_backends() { + let list = parse_backend_seed_list( + "prism=http://prism-challenge:8092, design=http://design-challenge:8093\n", + ) + .expect("parse"); + assert_eq!(list.len(), 2); + assert_eq!(list[0].challenge_id, "prism"); + assert_eq!(list[0].base_url, "http://prism-challenge:8092"); + assert_eq!(list[1].challenge_id, "design"); + assert_eq!(list[1].weight, 1); + + let reg = Registry::with_defaults(); + assert_eq!(reg.seed(&list).expect("seed"), 2); + assert_eq!(reg.seed(&list).expect("idempotent"), 0); + assert_eq!(reg.list(None).len(), 2); + assert_eq!( + reg.pick("design").unwrap().base_url, + "http://design-challenge:8093" + ); + } + + #[test] + fn parse_backend_seed_rejects_bad_entry() { + let err = parse_backend_seed_list("not-a-pair").unwrap_err(); + assert!(matches!(err, RegistryError::Invalid(_))); + } } diff --git a/crates/gateway/src/gw_config.rs b/crates/gateway/src/gw_config.rs index c685ab9ce..661d9c8ef 100644 --- a/crates/gateway/src/gw_config.rs +++ b/crates/gateway/src/gw_config.rs @@ -6,7 +6,7 @@ use std::net::SocketAddr; use std::str::FromStr; use config::{Config, Role}; -use gateway_registry::RegistryConfig; +use gateway_registry::{parse_backend_seed_list, CreateBackend, Registry, RegistryConfig}; use crate::tls::TlsConfig; use crate::{GatewayError, DEFAULT_LISTEN}; @@ -31,6 +31,13 @@ pub mod keys { /// re-applies to `/challenge/*/v1/view/*` responses (defense in depth). /// Defaults to [`design_sanitize::default_frame_ancestors`]. pub const VIEW_FRAME_ANCESTORS: &str = "BASE_GATEWAY_VIEW_FRAME_ANCESTORS"; + /// Comma/newline-separated boot seed for the in-memory challenge registry: + /// `prism=http://prism-challenge:8092,design=http://design-challenge:8093`. + /// Applied on every process start so compose/prod restarts do not leave + /// `/challenge/*` at 503 until an operator POSTs `/v1/admin/backends`. + pub const BACKENDS: &str = "BASE_GATEWAY_BACKENDS"; + /// Optional file whose contents are parsed like [`BACKENDS`] (wins when set). + pub const BACKENDS_FILE: &str = "BASE_GATEWAY_BACKENDS_FILE"; pub use crate::tls::keys as tls; } @@ -121,6 +128,54 @@ fn registry_config_from_env() -> Result { Ok(cfg) } +/// Load optional boot-seed backends from [`keys::BACKENDS_FILE`] or [`keys::BACKENDS`]. +/// +/// # Errors +/// +/// Unreadable file or malformed seed list. +pub fn load_backend_seed_from_env() -> Result, GatewayError> { + if let Ok(path) = std::env::var(keys::BACKENDS_FILE) { + let path = path.trim(); + if !path.is_empty() { + let raw = std::fs::read_to_string(path).map_err(|e| { + GatewayError::Config(format!("read {} `{path}`: {e}", keys::BACKENDS_FILE)) + })?; + return parse_backend_seed_list(&raw) + .map_err(|e| GatewayError::Config(format!("{}: {e}", keys::BACKENDS_FILE))); + } + } + match std::env::var(keys::BACKENDS) { + Ok(raw) if !raw.trim().is_empty() => parse_backend_seed_list(&raw) + .map_err(|e| GatewayError::Config(format!("{}: {e}", keys::BACKENDS))), + _ => Ok(Vec::new()), + } +} + +/// Apply [`load_backend_seed_from_env`] to an empty (or already-seeded) registry. +/// +/// # Errors +/// +/// Seed parse/load failures, or non-duplicate registry insert errors. +pub fn seed_registry_from_env(registry: &Registry) -> Result { + let backends = load_backend_seed_from_env()?; + if backends.is_empty() { + return Ok(0); + } + let created = registry + .seed(&backends) + .map_err(|e| GatewayError::Config(format!("backend seed: {e}")))?; + for b in &backends { + tracing::info!( + event = "gateway_backend_seed", + challenge_id = %b.challenge_id, + base_url = %b.base_url, + weight = b.weight, + "challenge backend present from boot seed" + ); + } + Ok(created) +} + /// Resolve the gateway hotkey from a Bittensor wallet, mnemonic file, or hex. /// /// Delegates to [`keystore::resolve_public_key_from_env`] with the diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index d0501218f..3c4d4fe1b 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -38,13 +38,14 @@ pub use gateway_core::admin_attest::{ admin_attest_grant_router, AttestGrantRequest, AttestGrantResponse, AttestGrantState, ATTEST_GRANT_ROUTE, }; +pub use gateway_registry::parse_backend_seed_list; pub use gateway_registry::{ Backend, BackendView, CreateBackend, Registry, RegistryConfig, RegistryError, DEFAULT_COOLDOWN, DEFAULT_FAILURE_THRESHOLD, }; pub use gw_config::{ - hotkey_hex, keys, parse_hotkey_hex, resolve_gateway_hotkey, GatewayConfig, OwnerCheck, - REQUIRE_OWNER_ENV, + hotkey_hex, keys, load_backend_seed_from_env, parse_hotkey_hex, resolve_gateway_hotkey, + seed_registry_from_env, GatewayConfig, OwnerCheck, REQUIRE_OWNER_ENV, }; pub use sealer::{ admin_seal_router, bundle_router, load_gateway_secret, seal_epoch, BundleStore, @@ -381,6 +382,16 @@ where let metrics = init_metrics()?; // Prefer registry knobs from config when the shared handle was default-built. let _ = &config.registry; + // In-memory registry: seed from BASE_GATEWAY_BACKENDS(_FILE) so compose/prod + // restarts never leave /challenge/* at 503 until a manual admin POST. + let seeded = seed_registry_from_env(®istry)?; + if seeded > 0 { + tracing::info!( + event = "gateway_backends_seeded", + created = seeded, + "boot-seeded challenge backends into in-memory registry" + ); + } let app = build_app(metrics, registry, chain, &config.tls, stores, extra)?; let listener = TcpListener::bind(config.listen) diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 2e56f21fc..ce37ee16c 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -40,7 +40,7 @@ Compose always runs a digest-pinned `postgres` service (`base-pgdata` volume, he | Gateway raw weight leaves + sealed bundles | **Postgres** (`raw_weight_snapshot`, `epoch_bundle`, …) | | Validator attestations (when DB configured) | **Postgres** | | Design sandbox staging files | volume `${BASE_STATE_DIR}/design/staging` + `design-artifacts` | -| Gateway challenge **backend registry** | **in-memory** — re-seed after gateway restart (`remote-deploy.sh` does this on master) | +| Gateway challenge **backend registry** | **in-memory**, boot-seeded from `BASE_GATEWAY_BACKENDS` (compose default: prism+design DNS URLs); `remote-deploy.sh` POST reseed stays idempotent | | site-api (`GET /v1/site/*`) | no DB — proxies challenge upstreams via gateway | | Unit/integration tests | may construct `Memory*Store` directly; omit `BASE_DATABASE_URL` only there | diff --git a/deploy/scripts/register-challenge-backends.sh b/deploy/scripts/register-challenge-backends.sh index 9d05f8d52..ab03c7858 100755 --- a/deploy/scripts/register-challenge-backends.sh +++ b/deploy/scripts/register-challenge-backends.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Register challenge reverse-proxy backends with the gateway registry. # -# The in-memory registry is empty after every gateway restart/redeploy. -# Call this on master after `docker compose up` (remote-deploy hooks it). +# Prefer compose `BASE_GATEWAY_BACKENDS` (gateway boot-seeds on start). +# This script remains the manual / remote-deploy idempotent fallback when the +# env seed is absent or you need to re-point URLs without restarting. # # Usage: # GATEWAY_URL=http://127.0.0.1:8080 ./deploy/scripts/register-challenge-backends.sh diff --git a/deploy/scripts/remote-deploy.sh b/deploy/scripts/remote-deploy.sh index 0dbb25aed..7090738c7 100755 --- a/deploy/scripts/remote-deploy.sh +++ b/deploy/scripts/remote-deploy.sh @@ -438,9 +438,9 @@ if [[ '$ROLE' == 'master' ]]; then else echo "gateway health: probe deferred" fi - # Registry is in-memory — re-seed challenge backends after every redeploy. - # The gateway races this script on boot, so retry until registration sticks, - # then prove proxy routing end-to-end: a missed reseed leaves /challenge/* + # Registry is in-memory; compose BASE_GATEWAY_BACKENDS boot-seeds on start. + # Keep an idempotent POST reseed after redeploy (covers empty override / race), + # then prove proxy routing end-to-end: a missed seed leaves /challenge/* # at 503 while /healthz stays green. Both must fail the deploy loudly. echo "remote-deploy: registering challenge backends" reseed_ok=0 diff --git a/docker-compose.yml b/docker-compose.yml index e88037693..e933e9dae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,6 +93,9 @@ services: BASE_TRUST_ROOT_DIR: ${BASE_TRUST_ROOT_DIR:-/etc/base/config} # Bundle seal mini-secret (host file, never baked into image) BASE_GATEWAY_SK_FILE: ${BASE_GATEWAY_SK_FILE:-/run/secrets/gateway_sk} + # Durable boot seed for the in-memory challenge registry (survives + # gateway-only restarts; remote-deploy POST reseed stays idempotent). + BASE_GATEWAY_BACKENDS: "${BASE_GATEWAY_BACKENDS:-prism=http://prism-challenge:8092,design=http://design-challenge:8093}" volumes: - ./config:/etc/base/config:ro - ./deploy/secrets/gateway_sk:/run/secrets/gateway_sk:ro diff --git a/docs/SITE_API.md b/docs/SITE_API.md index 433eb176d..00dd73b3a 100644 --- a/docs/SITE_API.md +++ b/docs/SITE_API.md @@ -43,5 +43,6 @@ challenge API. `?q=` — case-insensitive substring over miner hotkey (SS58 or hex), handle, slug, operator, and (for submissions) prompt title / id / run id. -Backends must be registered (same as challenge proxy), e.g. +Backends come from the gateway registry (same as challenge proxy). Compose +boot-seeds them via `BASE_GATEWAY_BACKENDS`; manual fallback: `deploy/scripts/register-challenge-backends.sh`. diff --git a/docs/runbooks/staging-testnet-e2e.md b/docs/runbooks/staging-testnet-e2e.md index 8e14b35d1..ace9719a5 100644 --- a/docs/runbooks/staging-testnet-e2e.md +++ b/docs/runbooks/staging-testnet-e2e.md @@ -47,12 +47,15 @@ docker exec $(docker ps -q --filter name=gateway) curl -fsS http://127.0.0.1:808 # Returns SCALE-encoded sealed bundle ``` -## Register challenge backends (required after gateway restart) +## Register challenge backends (boot seed + manual fallback) -The gateway registry is **in-memory**. After every redeploy/restart, challenge -proxy routes return `503 no healthy backends for challenge_id=…` until backends -are registered. `remote-deploy.sh` (master) re-seeds automatically; to do it by -hand: +The gateway registry is **in-memory**, but compose sets `BASE_GATEWAY_BACKENDS` +so the master gateway **boot-seeds** `prism` + `design` on every process start. +A plain `docker compose restart gateway` must not leave `/challenge/*` at 503. + +`remote-deploy.sh` (master) still POSTs `/v1/admin/backends` (idempotent 409). +Manual re-register only if the env seed was overridden empty or you need a +non-default URL: ```bash # From this repo (against a reachable gateway): From 687ad22a30a2f4cc426b738775eca74752214786 Mon Sep 17 00:00:00 2001 From: echobt <154886644+echobt@users.noreply.github.com> Date: Sat, 8 Aug 2026 10:23:12 +0000 Subject: [PATCH 2/2] fix(deploy): rsync challenge-review in prebuilt mode Prebuilt docker builds COPY challenge-review; without syncing it, remote-deploy --build-from prebuilt fails the image context. --- deploy/scripts/remote-deploy.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/scripts/remote-deploy.sh b/deploy/scripts/remote-deploy.sh index 7090738c7..2bb275594 100755 --- a/deploy/scripts/remote-deploy.sh +++ b/deploy/scripts/remote-deploy.sh @@ -270,6 +270,7 @@ if [[ "$BUILD_FROM" == "prebuilt" ]]; then "$ROOT/target/release/prism-challenge" \ "$ROOT/target/release/design-challenge" \ "$ROOT/target/release/design-egress-proxy" \ + "$ROOT/target/release/challenge-review" \ "$HOST:$REMOTE_DIR/target/release/" fi