diff --git a/bins/design-challenge/src/main.rs b/bins/design-challenge/src/main.rs index 309f9821e..3bae9d447 100644 --- a/bins/design-challenge/src/main.rs +++ b/bins/design-challenge/src/main.rs @@ -546,6 +546,7 @@ fn build_app_state( staging_root: cli.staging_root.clone(), stage_delay, auto_retry_max: cli.auto_retry_max, + emit_poll: Duration::from_secs(15), }, Arc::clone(&store), sandbox, @@ -673,6 +674,7 @@ async fn cmd_serve(cli: Cli) -> Result<(), String> { } tokio::spawn(Arc::clone(&orch).run_round_loop()); tokio::spawn(Arc::clone(&orch).run_sweeper()); + tokio::spawn(Arc::clone(&orch).run_emitter()); let app = design_router(state); let listener = TcpListener::bind(cli.bind) diff --git a/crates/design-challenge/src/orchestrator.rs b/crates/design-challenge/src/orchestrator.rs index e99789a75..c7d11f38f 100644 --- a/crates/design-challenge/src/orchestrator.rs +++ b/crates/design-challenge/src/orchestrator.rs @@ -149,6 +149,8 @@ pub struct OrchestratorConfig { /// Auto-retry budget for infra-class failures (initial attempt + this /// many retries). Default 3; `cheat` / `rejected` are always terminal. pub auto_retry_max: u32, + /// Poll interval for the late-tempo D24 filler. + pub emit_poll: Duration, } impl Default for OrchestratorConfig { @@ -161,6 +163,7 @@ impl Default for OrchestratorConfig { staging_root: PathBuf::from("/var/lib/design/staging"), stage_delay: Duration::ZERO, auto_retry_max: 3, + emit_poll: Duration::from_secs(15), } } } @@ -177,6 +180,8 @@ pub struct Orchestrator { gating: Option>, /// Shared chain-epoch cache (HTTP `AppState.epoch` + sweeper clock). epoch_cache: Option>, + /// Last epoch successfully covered by [`Self::emit_leaves`]. + emitted_epoch: AtomicU64, } impl std::fmt::Debug for Orchestrator { @@ -209,6 +214,7 @@ impl Orchestrator { sk, gating: None, epoch_cache: None, + emitted_epoch: AtomicU64::new(0), } } @@ -279,6 +285,42 @@ impl Orchestrator { } } + /// Late-tempo D24 filler: `NotAttempted` coverage when no admin award fired + /// (waits ~last 48 blocks so `award_round` can land Score leaves first). + pub async fn run_emitter(self: Arc) + where + C: Sync, + { + loop { + if let Err(e) = self.emitter_tick().await { + warn!(error = %e, "design emitter tick error"); + } + sleep(self.cfg.emit_poll).await; + } + } + + /// One emitter tick. `Ok(true)` when a leaf set was submitted this tick. + /// + /// # Errors + /// Chain / sign / submit failures (retried next tick). + pub async fn emitter_tick(&self) -> Result + where + C: Sync, + { + let state = chain::gather_schedule_state(self.chain.as_ref(), self.cfg.netuid) + .map_err(|e| format!("schedule: {e}"))?; + let epoch = state.subnet_epoch_index; + if epoch == 0 || self.emitted_epoch.load(Ordering::Relaxed) >= epoch { + return Ok(false); + } + let tempo = u64::from(state.tempo.max(1)); + if state.blocks_since_last_step.saturating_add(48) < tempo { + return Ok(false); + } + self.emit_leaves().await?; + Ok(true) + } + async fn sweep_unscored_timeouts(&self) -> Result<(), String> { let current = self.current_epoch(); if current == 0 { @@ -1089,6 +1131,13 @@ impl Orchestrator { submit_signed_leaf_set(self.gateway.as_ref(), &signed) .await .map_err(|e| e.to_string())?; + self.emitted_epoch.store(epoch, Ordering::Relaxed); + info!( + epoch, + participants = expected_set.len(), + last_epoch_block = state.last_epoch_block, + "design leaf set submitted" + ); Ok(()) } } diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 2e56f21fc..66335f688 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -19,8 +19,8 @@ Terraform: [`terraform/`](terraform/). Firewall: SSH from operator IP; CI uses e | File | Purpose | |------|---------| -| `compose/role-master.yml` | gateway profile, VPC publish | -| `compose/role-validator.yml` | no gateway; external gateway endpoint | +| `compose/role-master.yml` | gateway profile, VPC publish; **no validator** (avoids dual CRV4 submit) | +| `compose/role-validator.yml` | no gateway; external gateway endpoint; sole on-chain submitter | | `compose/env-staging.yml` | testnet 541, faster coordination | | `compose/env-prod.yml` | mainnet, conservative intervals | | `compose/env-local.yml` | **local only** — ports/smoke knobs/tunnel env; always on top of `env-staging` | @@ -120,7 +120,9 @@ Prod (`env-prod.yml` + `base-burn-seal.service`): onfinality `public-ws` primary Validator logs should show `Match epoch=` then `Match → submit_intent` / `submit_timelocked ok`. Keep legacy Python weight submit **stopped** to avoid double-commit. -**Legacy Python agents (mainnet):** `validator-5gzi` (`95.133.252.120`) may point `master_url` / `weights_url` / `registry_url` at `https://chain.joinbase.ai` with **`submit_on_chain_enabled: false`**. Coordination shims live in `gateway-compat` (`/v1/validators/*`, `/v1/registry`, empty assignments). `GET /v1/weights/latest` refreshes `computed_at` / `expires_at` at serve time so Python pydantic clients accept sealed vectors older than 720s. Sole on-chain submitter for hotkey `5Gzi…` is the Rust validator on `192.81.218.11` — do **not** start `base-weight-submitter-5gzi` on `validator-root` unless CR ownership is moved off Rust. +**Sole on-chain submitter (mainnet hotkey `5Gzi…`):** Rust `base-validator-1` on **`base-prod-validator` (`192.81.218.11`) only**. `role-master.yml` profiles the validator under `never`; `remote-deploy.sh --role master` force-removes any leftover container. Do **not** run a second validator (or Python weight submitter) with the same wallet — dual submitters fight `WeightsSetRateLimit` and can leave CRV4 commits stuck while incentive still shows a prior monopoly UID. + +**Legacy Python agents (mainnet):** `validator-5gzi` (`95.133.252.120`) may point `master_url` / `weights_url` / `registry_url` at `https://chain.joinbase.ai` with **`submit_on_chain_enabled: false`**. Coordination shims live in `gateway-compat` (`/v1/validators/*`, `/v1/registry`, empty assignments). `GET /v1/weights/latest` refreshes `computed_at` / `expires_at` at serve time so Python pydantic clients accept sealed vectors older than 720s. Do **not** start `base-weight-submitter-5gzi` on `validator-root` unless CR ownership is moved off Rust. **Challenge verification:** on **master** only (validator has **no challenge exec**). Simulate submissions end-to-end — submit **baseline** + submit **cheat**, poll `/v1/runs/{id}` + `/events` + `/logs`, probe edges (bad harness, sanitize, quota, routes), then **admin winners** (`GET/POST /v1/admin/rounds/{id}/…` with bearer from `deploy/secrets/design/annotator_tokens`) and confirm leaf → seal → `GET /v1/weights/latest` **`sealed: true`**. **Never host Sim in staging/prod** (`BASE_ALLOW_HOST_SIM` / host `SimSandbox` are CI/local only). Healthz alone is insufficient. diff --git a/deploy/compose/env-local.yml b/deploy/compose/env-local.yml index 4672975a3..968d2ecb7 100644 --- a/deploy/compose/env-local.yml +++ b/deploy/compose/env-local.yml @@ -39,6 +39,9 @@ services: - ./.local/trust-root:/etc/base/config:ro validator: + # role-master disables on-chain submit on droplet masters; local-e2e needs + # a co-located validator for Match/CRV4 smoke against the local gateway. + profiles: !override [] # Avoid role-master 18080 — often taken by staging SSH tunnels on laptops. ports: !override - "127.0.0.1:${LOCAL_VALIDATOR_HOST_PORT:-28080}:8080" diff --git a/deploy/compose/role-master.yml b/deploy/compose/role-master.yml index aacb0dfe5..3633dd1b2 100644 --- a/deploy/compose/role-master.yml +++ b/deploy/compose/role-master.yml @@ -1,10 +1,12 @@ # Master role — gateway enabled, published on VPC for validator peers. # Used with: docker compose -f docker-compose.yml -f deploy/compose/role-master.yml --profile master +# +# On-chain weight submit must NOT run here (sole submitter = validator host). +# Co-located validator is local-e2e only — env-local.yml clears the profile. services: gateway: profiles: ["master"] ports: - "8080:8080" validator: - ports: - - "127.0.0.1:18080:8080" + profiles: ["never"] diff --git a/deploy/scripts/assert-compose-matrix.sh b/deploy/scripts/assert-compose-matrix.sh index 3d53439a9..04500cc3a 100755 --- a/deploy/scripts/assert-compose-matrix.sh +++ b/deploy/scripts/assert-compose-matrix.sh @@ -58,7 +58,7 @@ for banned in design-challenge design-egress-proxy prism-challenge socket-proxy; done echo "OK: validator role does not render gateway or challenge services" -# --- master role: gateway + challenge services present --- +# --- master role: gateway + challenge services present; no on-chain validator --- services=$(render \ -f docker-compose.yml \ -f deploy/compose/role-master.yml \ @@ -72,7 +72,10 @@ for required in design-challenge design-egress-proxy prism-challenge socket-prox fail "master role does not render $required (must)" fi done -echo "OK: master role renders gateway and challenge services" +if echo "$services" | grep -qx "validator"; then + fail "master role renders validator (dual submitter; must not — use validator host)" +fi +echo "OK: master role renders gateway and challenge services (no validator)" # --- evil-gateway not in default or master --- services=$(render \ @@ -206,6 +209,8 @@ local_services=$(render \ config --services) echo "$local_services" | grep -qx "gateway" \ || fail "env-local master stack does not render gateway" +echo "$local_services" | grep -qx "validator" \ + || fail "env-local master stack does not render co-located validator" echo "$local_services" | grep -qx "prism-challenge" \ || fail "env-local master stack does not render prism-challenge" echo "$local_services" | grep -qx "design-challenge" \ diff --git a/deploy/scripts/remote-deploy.sh b/deploy/scripts/remote-deploy.sh index bc17f0d99..203cde0dc 100755 --- a/deploy/scripts/remote-deploy.sh +++ b/deploy/scripts/remote-deploy.sh @@ -416,23 +416,35 @@ fi docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} \$UP_PROFILE "\${UP_ARGS[@]}" # Profile-disabled services are not started, but an older compose project may # still be running them. On validator, force-remove master-only challenge -# surfaces so smoke health does not see stale unhealthy containers. +# surfaces so smoke health does not see stale unhealthy containers. On master, +# force-remove the validator so a prior dual-submitter cannot fight the +# validator-host wallet for WeightsSetRateLimit / CRV4 commits. if [[ '$ROLE' == 'validator' ]]; then docker compose ${COMPOSE_FILES[*]} rm -sf \ prism-challenge design-challenge design-egress-proxy socket-proxy \ >/dev/null 2>&1 || true +elif [[ '$ROLE' == 'master' ]]; then + docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} rm -sf validator \ + >/dev/null 2>&1 || true fi docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} \$UP_PROFILE ps # Local health probes via published tunnels if present, else container exec. sleep 5 -if curl -fsS -m 5 http://127.0.0.1:18080/healthz >/dev/null 2>&1; then - echo "validator tunnel health: \$(curl -fsS -m 5 http://127.0.0.1:18080/healthz)" -elif docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} exec -T validator curl -fsS -m 5 http://127.0.0.1:8080/healthz >/dev/null 2>&1; then - echo "validator health: ok (in-container)" -else - echo "validator health: probe deferred (container may still be starting)" +if [[ '$ROLE' == 'validator' ]]; then + if curl -fsS -m 5 http://127.0.0.1:18080/healthz >/dev/null 2>&1; then + echo "validator tunnel health: \$(curl -fsS -m 5 http://127.0.0.1:18080/healthz)" + elif docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} exec -T validator curl -fsS -m 5 http://127.0.0.1:8080/healthz >/dev/null 2>&1; then + echo "validator health: ok (in-container)" + else + echo "validator health: probe deferred (container may still be starting)" + fi fi if [[ '$ROLE' == 'master' ]]; then + if docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} ps --status running --services 2>/dev/null | grep -qx validator; then + echo "remote-deploy: ERROR: validator still running on master (dual submitter)" >&2 + exit 1 + fi + echo "master: validator absent (sole on-chain submitter is validator host)" if docker compose ${COMPOSE_FILES[*]} ${PROFILE_ARGS[*]} exec -T gateway curl -fsS -m 5 http://127.0.0.1:8080/healthz >/dev/null 2>&1; then echo "gateway health: ok" else diff --git a/docs/DESIGN_CHALLENGE.md b/docs/DESIGN_CHALLENGE.md index 5d3eb7ee5..7af125cfa 100644 --- a/docs/DESIGN_CHALLENGE.md +++ b/docs/DESIGN_CHALLENGE.md @@ -479,7 +479,9 @@ Expected set `E` = all metagraph hotkeys for the pinned epoch (policy - Exactly one signed leaf per `h ∈ E` - **Refuses subset and superset** — Silence is a bug -- Emit at round close and at each epoch boundary via `POST /v1/weights/raw` +- Emit at round close (scored) and near each epoch boundary via `POST /v1/weights/raw` + (`Orchestrator::run_emitter` fills `NotAttempted` when no admin award fired, so + D24 seals keep advancing under 50/50 emission shares) Absence codes used on this path include `NotAttempted`, `Timeout`, `InvalidResponse`, `MinerError`, `RateLimited`, `ChallengeInternal` (bundle enum).