feat(model,api,taskworker): bootstrap the egress prober's credential without a human

api/api.go

@@ -75,6 +75,18 @@ func Routes() []*router.Route {
 		router.NewRoute("GET", "/network/provider-blackhole-due", handlers.ProviderBlackholeCheckDue),
 		router.NewRoute("POST", "/network/provider-blackhole-checks", handlers.SubmitProviderBlackholeChecks),
 		router.NewRoute("POST", "/network/provider-egress-attempt", handlers.ProviderEgressLocationAttempt),
+		// operator-to-server, same operator secret as the egress routes above:
+		// the prober fetching the network client jwt that the bootstrap task
+		// minted for it. This is what makes the credential arrive without a
+		// human -- until it existed the task stored a jwt nothing ever read, and
+		// an operator still had to hand-carry one into the prober's environment.
+		// Returns the jwt and the client id it names, and nothing else -- but do
+		// NOT read that narrowness as containment. The jwt itself carries
+		// network_id, user_id and network_name as readable claims, and holding
+		// it is enough to regenerate this account's seedphrase. The operator
+		// secret checked in the handler is the actual gate. See
+		// ProberCredentialResult, which spells this out.
+		router.NewRoute("GET", "/network/prober-credential", handlers.ProberCredential),
 		// operator-to-server, same operator secret: the certificate pins this
 		// server observed DIRECTLY for the geolocation source hosts. The
 		// prober fetches them here instead of carrying a compile-time

api/handlers/prober_credential_handlers.go

@@ -0,0 +1,160 @@
+package handlers
+
+import (
+	"crypto/hmac"
+	"encoding/json"
+	"net/http"
+
+	"github.com/urnetwork/glog"
+
+	"github.com/urnetwork/server"
+	"github.com/urnetwork/server/model"
+)
+
+// ProberCredentialResult is the response body of ProberCredential.
+//
+// It is NARROWER than the prober_identity row behind it. Exactly two things
+// leave this server:
+//
+//   - by_client_jwt, the *delivery* credential. It is revocable and re-mintable:
+//     model.clearProberIdentityClient drops it and the bootstrap task mints
+//     another against the same network. A leaked one is recovered by forcing a
+//     re-mint, and costs nothing else.
+//   - client_id, which names the client the jwt already names. It discloses
+//     nothing the jwt does not, and the prober needs it to identify itself in
+//     the routes it goes on to call.
+//
+// Do NOT read the omission of network_id, user_id and network_name as a
+// security measure. An earlier version of this comment claimed they were left
+// out ON PURPOSE, as a "root identity" that a holder could otherwise re-derive
+// the account from. That claim was false. ByJwt.Client copies all three into
+// the very token this endpoint hands out (jwt/by_jwt.go:658-662), and the
+// struct tags serialise them as network_id/user_id/network_name
+// (jwt/by_jwt.go:188-190), so anyone holding this response can base64-decode
+// the token's payload segment and read them. Keeping them out of the JSON body
+// hides nothing from the only party that ever sees it.
+//
+// The ids are inert in any case. /auth/regenerate-seedphrase takes an EMPTY
+// args struct (controller/seedphrase_controller.go:8-9) and keys on
+// session.ByJwt.UserId (:36 and :62) -- the id from the authenticated token,
+// never one the caller supplies. Knowing a user_id buys nothing.
+//
+// Holding the TOKEN is the whole capability, and it reaches further than
+// "delivery credential" suggests. session.Auth parses it for the api audience
+// (session/client_session.go:139), which a client jwt carries
+// (jwt/by_jwt.go:670, via newRegisteredClaims at :209), then calls
+// jwt.ValidateByJwtState(ctx, byJwt, false) -- requireClient=false
+// (session/client_session.go:143) -- so a CLIENT jwt authenticates as the
+// account. POST /auth/regenerate-seedphrase (routed at api/api.go:68) is
+// WrapWithInputRequireAuth (api/handlers/seedphrase_handlers.go:11), and that
+// wrapper's auth is the same session.Auth (router/handler_utils.go:307). So
+// whoever holds this response can mint the prober network a fresh seedphrase.
+// That leaves them a login credential this server cannot read back, because
+// only a salted hash is stored (model/seedphrase_auth_model.go:139-141,
+// model/auth_model_identity.go:136), and cannot revoke, because
+// RegenerateSeedphrase does not touch credential_change_time. Re-minting or
+// dropping the jwt afterwards does not take it back, and the same call
+// invalidates whatever phrase an operator had recorded.
+//
+// The real gate is therefore the operator secret checked in ProberCredential
+// below. It is the only thing standing between a caller and that capability;
+// the shape of this struct is not part of it. Keep the response narrow anyway
+// -- sending the minimum is right on its own terms, and a field set that is
+// pinned by test cannot drift into "whatever the model struct happens to have"
+// -- but do not mistake it for the protection.
+//
+// A seedphrase cannot appear here even by accident: it is never persisted.
+// createProberNetwork calls NetworkCreate and keeps only the network id, the
+// admin user id and the name (model/prober_identity_model.go:502, and the note
+// above that call explaining why the phrase is dropped), so there is no column
+// on prober_identity that could carry one. That is a property worth preserving
+// -- do not add one.
+type ProberCredentialResult struct {
+	ByClientJwt string    `json:"by_client_jwt"`
+	ClientId    server.Id `json:"client_id"`
+}
+
+// ProberCredential hands the operator's prober the network client jwt that the
+// bootstrap task minted for it (see model/prober_identity_model.go and
+// taskworker/work/prober_bootstrap_work.go).
+//
+// This is the last leg of that bootstrap. The task already creates the prober's
+// account, funds it and mints the credential into prober_identity, but nothing
+// read that column, so the credential still had to reach the prober process by
+// hand -- an env file written by an operator. A prober that can fetch its own
+// credential closes the loop: no human step remains between a fresh deployment
+// and a probing prober.
+//
+// Same auth as the provider-egress endpoints it sits beside: operator-to-server,
+// the shared X-UR-Operator-Secret header rather than a network jwt, fail-closed
+// when the vault resource is missing. One secret, one mechanism -- this route
+// hands out a credential, which is the strongest possible reason not to invent
+// a second, less-examined way in.
+func ProberCredential(w http.ResponseWriter, r *http.Request) {
+	secret := operatorIngestSecret()
+	provided := r.Header.Get(operatorSecretHeader)
+
+	// The two failure branches are logged SEPARATELY, and loudly, which is the
+	// one place this deviates from the endpoints next door (they 401 in
+	// silence). The deviation is the point: a credential endpoint that rejects
+	// everything produces a prober that probes nothing, and a fleet that probes
+	// nothing is indistinguishable from a fleet of unhealthy providers. That
+	// exact misreading has already cost this system eight hours. Which side is
+	// misconfigured is the first question anyone asks, so the log answers it
+	// before it is asked.
+	if secret == "" {
+		// SERVER side: no vault resource, or no ingest_secret in it. Every
+		// request is rejected regardless of what the caller sends.
+		glog.Errorf(
+			"[probercred]this server has no operator ingest secret " +
+				"(provider_egress.yml/ingest_secret); the prober cannot fetch its " +
+				"credential, so egress probing will not run at all\n",
+		)
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+	if provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) {
+		// CALLER side: the prober's configured secret is absent or does not
+		// match this server's. Neither the provided value nor any prefix of it
+		// is logged -- a rejected secret is still a secret, and log shipping is
+		// a wider audience than the vault.
+		glog.Errorf(
+			"[probercred]rejected a prober credential request: missing or wrong %s header; "+
+				"the caller's operator secret does not match this server's\n",
+			operatorSecretHeader,
+		)
+		http.Error(w, "Unauthorized", http.StatusUnauthorized)
+		return
+	}
+
+	identity := model.GetProberIdentity(r.Context())
+
+	// 404 -- not a 500, and not an empty 200. The prober polls this before the
+	// bootstrap task has necessarily finished, and it must be able to tell "not
+	// ready yet, keep polling" from "broken, wake someone". An empty 200 makes
+	// those two the same response.
+	//
+	// A missing row is NOT the only not-ready state, and checking only for it
+	// would produce exactly that empty 200. The row is committed by
+	// createProberNetwork before mintProberClientJwt runs, so it legitimately
+	// exists with by_client_jwt still NULL; clearProberIdentityClient also
+	// returns it to that state when a client has to be re-provisioned. All
+	// three are the same answer to this caller: there is no credential yet.
+	//
+	// Not logged: the bootstrap task already reports its own failures at
+	// Errorf, and a poll during normal startup is not an error.
+	if identity == nil || identity.ClientId == nil || identity.ByClientJwt == "" {
+		http.Error(w, "Not found", http.StatusNotFound)
+		return
+	}
+
+	result := &ProberCredentialResult{
+		ByClientJwt: identity.ByClientJwt,
+		ClientId:    *identity.ClientId,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(result); err != nil {
+		glog.Infof("[probercred]could not write response. err = %s\n", err)
+	}
+}

api/handlers/prober_credential_handlers_test.go

@@ -0,0 +1,144 @@
+package handlers
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"reflect"
+	"slices"
+	"strings"
+	"testing"
+
+	"github.com/urnetwork/server"
+)
+
+func TestProberCredentialRejectsMissingSecret(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil)
+	w := httptest.NewRecorder()
+
+	ProberCredential(w, req)
+
+	if w.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code)
+	}
+}
+
+func TestProberCredentialRejectsWrongSecret(t *testing.T) {
+	defer withStubOperatorIngestSecret("correct-operator-secret-0123456789")()
+
+	req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil)
+	req.Header.Set(operatorSecretHeader, "definitely-not-the-secret")
+	w := httptest.NewRecorder()
+
+	ProberCredential(w, req)
+
+	if w.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code)
+	}
+	if strings.Contains(w.Body.String(), "jwt") {
+		t.Fatalf("a rejected request must not be told anything about the credential: %q", w.Body.String())
+	}
+}
+
+// TestProberCredentialRejectsAlteredSecret is the same-length, one-byte-off
+// case. A comparison written with strings.HasPrefix, or one that only checked
+// length, would pass the two tests above and fail this one.
+func TestProberCredentialRejectsAlteredSecret(t *testing.T) {
+	const secret = "correct-operator-secret-0123456789"
+	defer withStubOperatorIngestSecret(secret)()
+
+	altered := []byte(secret)
+	altered[len(altered)-1] ^= 0x01
+
+	req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil)
+	req.Header.Set(operatorSecretHeader, string(altered))
+	w := httptest.NewRecorder()
+
+	ProberCredential(w, req)
+
+	if w.Code != http.StatusUnauthorized {
+		t.Fatalf("status = %d, want 401 on a secret differing in one byte", w.Code)
+	}
+}
+
+// TestProberCredentialResultExposesOnlyTheDeliveryCredential pins the response
+// SHAPE, so the handler keeps returning a deliberate, documented field set
+// rather than whatever a model struct happens to have.
+//
+// The failure it exists to catch is a quiet one: someone "simplifies" the
+// handler by encoding model.ProberIdentity directly, or adds a field here for
+// debugging. Either compiles, builds green, and passes every status-code test
+// above. Asserting the exact field set is the only check that fails.
+//
+// It is NOT what makes this endpoint safe, and it is not a substitute for the
+// auth tests above. See the note on ProberCredentialResult: network_id, user_id
+// and network_name are already inside the by_client_jwt this response carries,
+// so omitting them withholds nothing from a caller who got past the operator
+// secret. That secret is the gate; this test is change detection.
+func TestProberCredentialResultExposesOnlyTheDeliveryCredential(t *testing.T) {
+	want := []string{"by_client_jwt", "client_id"}
+
+	var got []string
+	resultType := reflect.TypeOf(ProberCredentialResult{})
+	for i := range resultType.NumField() {
+		tag, _, _ := strings.Cut(resultType.Field(i).Tag.Get("json"), ",")
+		got = append(got, tag)
+	}
+	slices.Sort(got)
+
+	if !slices.Equal(got, want) {
+		t.Fatalf(
+			"ProberCredentialResult fields = %v, want exactly %v; "+
+				"the response carries the revocable credential ONLY, never the account identity behind it",
+			got,
+			want,
+		)
+	}
+
+	// belt and braces: whatever the field set becomes, these must never be in
+	// it, under any spelling
+	for _, forbidden := range []string{"seed", "network", "user", "password", "secret"} {
+		if slices.ContainsFunc(got, func(f string) bool { return strings.Contains(f, forbidden) }) {
+			t.Fatalf("ProberCredentialResult must never expose a %q field: %v", forbidden, got)
+		}
+	}
+}
+
+// TestProberCredentialNotReadyIs404 proves the auth gate can ACCEPT, and that
+// the accepted-but-not-yet-bootstrapped case is a 404.
+//
+// Both halves matter. Without the accept half, a handler replaced wholesale by
+// an unconditional 401 would pass all three reject tests above. And the 404 is
+// the contract the prober polls against: it must be able to tell "the
+// bootstrap task has not finished yet, keep polling" from "something is
+// broken, wake someone". An empty 200 collapses those two into one answer.
+func TestProberCredentialNotReadyIs404(t *testing.T) {
+	t.Setenv("WARP_ENV", "local")
+	server.DefaultTestEnv().Run(t, func(t testing.TB) {
+		const secret = "correct-operator-secret-0123456789"
+		defer withStubOperatorIngestSecret(secret)()
+
+		// no bootstrap has run against this fresh test database, so
+		// prober_identity is empty
+		req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil)
+		req.Header.Set(operatorSecretHeader, secret)
+		req = req.WithContext(context.Background())
+		w := httptest.NewRecorder()
+
+		ProberCredential(w, req)
+
+		if w.Code == http.StatusUnauthorized {
+			t.Fatalf("the correct operator secret was rejected; the auth gate is broken shut")
+		}
+		if w.Code != http.StatusNotFound {
+			t.Fatalf("status = %d, want 404 before the bootstrap task has minted a credential", w.Code)
+		}
+
+		// and specifically NOT a decodable credential body
+		var result ProberCredentialResult
+		if err := json.Unmarshal(w.Body.Bytes(), &result); err == nil && result.ByClientJwt != "" {
+			t.Fatalf("a not-ready response must carry no credential, got %q", result.ByClientJwt)
+		}
+	})
+}

db_migrations.go

@@ -5720,6 +5720,67 @@ var migrations = []any{
         CREATE INDEX IF NOT EXISTS provider_blackhole_check_checked_at
             ON provider_blackhole_check (checked_at ASC, client_id ASC)
     `),
+	// The egress prober's own network identity, as a single row.
+	//
+	// The prober authenticates to the platform with a network CLIENT jwt. Until
+	// now an operator minted that by hand -- create a network, POST
+	// /network/auth-client, paste the by_client_jwt into the prober's
+	// environment -- so a deployment that had not had that done had no egress
+	// probing at all, silently. taskworker/work/prober_bootstrap_work.go does it
+	// instead, and this table is what makes a job that re-runs every six hours
+	// FOREVER safe: it is the only record that the account already exists.
+	//
+	// Looking the network up by name could not replace it. The seedphrase branch
+	// of model.NetworkCreate ignores the requested name and calls
+	// generateRandomNetworkName() unconditionally, so the name is not knowable in
+	// advance and there is nothing to search for on the next run.
+	//
+	// `singleton bool PRIMARY KEY DEFAULT true CHECK (singleton)` makes "at most
+	// one prober identity" a fact of the schema rather than a convention every
+	// future caller has to remember: every insert names the same row and
+	// therefore collides with it, which is exactly what the create-claim upsert
+	// in model/prober_identity_model.go is built on. client_id is stored so each
+	// re-mint re-auths the SAME client instead of accumulating one client per
+	// refresh, and create_attempts bounds account creation if the create keeps
+	// failing (see MaxProberBootstrapAttempts).
+	//
+	// NOTE ON POSITION: this entry is placed before the competition control-plane
+	// block rather than at the end of the list, and on an ALREADY-DEPLOYED
+	// database that means it does not run. ApplyDbMigrationsUpTo iterates
+	// `for i := DbVersion(ctx); i < upTo; i += 1`, so a migration executes only
+	// when its list INDEX is at or past the deployed version, and a database
+	// already past this index skips it.
+	//
+	// Hence IF NOT EXISTS: the statement has to be safe to apply out of band and
+	// safe to re-apply. `bringyourctl db audit --fix` emits CREATE TABLE for a
+	// missing table and is what actually creates this one on a deployed
+	// database.
+	//
+	// SO: after deploying this to an existing database, run
+	//   bringyourctl db audit --fix
+	// or prober_identity will not exist and the bootstrap task, GetProberIdentity
+	// and GET /network/prober-credential all fail with
+	// `relation "prober_identity" does not exist`.
+	//
+	// The durable fix is not to shuffle this entry: it is to resolve migrations
+	// by identity rather than by list index. Until then every new table added
+	// ahead of the competition block hits this.
+	newSqlMigration(`
+		CREATE TABLE IF NOT EXISTS prober_identity (
+			singleton bool PRIMARY KEY DEFAULT true CHECK (singleton),
+
+			network_id uuid NULL,
+			user_id uuid NULL,
+			network_name varchar(256) NULL,
+
+			client_id uuid NULL,
+			by_client_jwt text NULL,
+
+			create_attempts int NOT NULL DEFAULT 0,
+			create_time timestamp NULL,
+			last_mint_time timestamp NULL
+		)
+	`),
 
 	// Durable sim-latency competition control plane. The queue is deliberately
 	// independent of pending_task: untrusted submissions are claimed by a