diff --git a/api/api.go b/api/api.go index 942ac4dd..c9424461 100644 --- a/api/api.go +++ b/api/api.go @@ -75,6 +75,18 @@ func Routes() []*router.Route { router.NewRoute("GET", "/network/provider-blackhole-due", handlers.ProviderBlackholeCheckDue), router.NewRoute("POST", "/network/provider-blackhole-checks", handlers.SubmitProviderBlackholeChecks), router.NewRoute("POST", "/network/provider-egress-attempt", handlers.ProviderEgressLocationAttempt), + // operator-to-server, same operator secret as the egress routes above: + // the prober fetching the network client jwt that the bootstrap task + // minted for it. This is what makes the credential arrive without a + // human -- until it existed the task stored a jwt nothing ever read, and + // an operator still had to hand-carry one into the prober's environment. + // Returns the jwt and the client id it names, and nothing else -- but do + // NOT read that narrowness as containment. The jwt itself carries + // network_id, user_id and network_name as readable claims, and holding + // it is enough to regenerate this account's seedphrase. The operator + // secret checked in the handler is the actual gate. See + // ProberCredentialResult, which spells this out. + router.NewRoute("GET", "/network/prober-credential", handlers.ProberCredential), // operator-to-server, same operator secret: the certificate pins this // server observed DIRECTLY for the geolocation source hosts. The // prober fetches them here instead of carrying a compile-time diff --git a/api/handlers/prober_credential_handlers.go b/api/handlers/prober_credential_handlers.go new file mode 100644 index 00000000..b9a8d9f0 --- /dev/null +++ b/api/handlers/prober_credential_handlers.go @@ -0,0 +1,160 @@ +package handlers + +import ( + "crypto/hmac" + "encoding/json" + "net/http" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// ProberCredentialResult is the response body of ProberCredential. +// +// It is NARROWER than the prober_identity row behind it. Exactly two things +// leave this server: +// +// - by_client_jwt, the *delivery* credential. It is revocable and re-mintable: +// model.clearProberIdentityClient drops it and the bootstrap task mints +// another against the same network. A leaked one is recovered by forcing a +// re-mint, and costs nothing else. +// - client_id, which names the client the jwt already names. It discloses +// nothing the jwt does not, and the prober needs it to identify itself in +// the routes it goes on to call. +// +// Do NOT read the omission of network_id, user_id and network_name as a +// security measure. An earlier version of this comment claimed they were left +// out ON PURPOSE, as a "root identity" that a holder could otherwise re-derive +// the account from. That claim was false. ByJwt.Client copies all three into +// the very token this endpoint hands out (jwt/by_jwt.go:658-662), and the +// struct tags serialise them as network_id/user_id/network_name +// (jwt/by_jwt.go:188-190), so anyone holding this response can base64-decode +// the token's payload segment and read them. Keeping them out of the JSON body +// hides nothing from the only party that ever sees it. +// +// The ids are inert in any case. /auth/regenerate-seedphrase takes an EMPTY +// args struct (controller/seedphrase_controller.go:8-9) and keys on +// session.ByJwt.UserId (:36 and :62) -- the id from the authenticated token, +// never one the caller supplies. Knowing a user_id buys nothing. +// +// Holding the TOKEN is the whole capability, and it reaches further than +// "delivery credential" suggests. session.Auth parses it for the api audience +// (session/client_session.go:139), which a client jwt carries +// (jwt/by_jwt.go:670, via newRegisteredClaims at :209), then calls +// jwt.ValidateByJwtState(ctx, byJwt, false) -- requireClient=false +// (session/client_session.go:143) -- so a CLIENT jwt authenticates as the +// account. POST /auth/regenerate-seedphrase (routed at api/api.go:68) is +// WrapWithInputRequireAuth (api/handlers/seedphrase_handlers.go:11), and that +// wrapper's auth is the same session.Auth (router/handler_utils.go:307). So +// whoever holds this response can mint the prober network a fresh seedphrase. +// That leaves them a login credential this server cannot read back, because +// only a salted hash is stored (model/seedphrase_auth_model.go:139-141, +// model/auth_model_identity.go:136), and cannot revoke, because +// RegenerateSeedphrase does not touch credential_change_time. Re-minting or +// dropping the jwt afterwards does not take it back, and the same call +// invalidates whatever phrase an operator had recorded. +// +// The real gate is therefore the operator secret checked in ProberCredential +// below. It is the only thing standing between a caller and that capability; +// the shape of this struct is not part of it. Keep the response narrow anyway +// -- sending the minimum is right on its own terms, and a field set that is +// pinned by test cannot drift into "whatever the model struct happens to have" +// -- but do not mistake it for the protection. +// +// A seedphrase cannot appear here even by accident: it is never persisted. +// createProberNetwork calls NetworkCreate and keeps only the network id, the +// admin user id and the name (model/prober_identity_model.go:502, and the note +// above that call explaining why the phrase is dropped), so there is no column +// on prober_identity that could carry one. That is a property worth preserving +// -- do not add one. +type ProberCredentialResult struct { + ByClientJwt string `json:"by_client_jwt"` + ClientId server.Id `json:"client_id"` +} + +// ProberCredential hands the operator's prober the network client jwt that the +// bootstrap task minted for it (see model/prober_identity_model.go and +// taskworker/work/prober_bootstrap_work.go). +// +// This is the last leg of that bootstrap. The task already creates the prober's +// account, funds it and mints the credential into prober_identity, but nothing +// read that column, so the credential still had to reach the prober process by +// hand -- an env file written by an operator. A prober that can fetch its own +// credential closes the loop: no human step remains between a fresh deployment +// and a probing prober. +// +// Same auth as the provider-egress endpoints it sits beside: operator-to-server, +// the shared X-UR-Operator-Secret header rather than a network jwt, fail-closed +// when the vault resource is missing. One secret, one mechanism -- this route +// hands out a credential, which is the strongest possible reason not to invent +// a second, less-examined way in. +func ProberCredential(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + + // The two failure branches are logged SEPARATELY, and loudly, which is the + // one place this deviates from the endpoints next door (they 401 in + // silence). The deviation is the point: a credential endpoint that rejects + // everything produces a prober that probes nothing, and a fleet that probes + // nothing is indistinguishable from a fleet of unhealthy providers. That + // exact misreading has already cost this system eight hours. Which side is + // misconfigured is the first question anyone asks, so the log answers it + // before it is asked. + if secret == "" { + // SERVER side: no vault resource, or no ingest_secret in it. Every + // request is rejected regardless of what the caller sends. + glog.Errorf( + "[probercred]this server has no operator ingest secret " + + "(provider_egress.yml/ingest_secret); the prober cannot fetch its " + + "credential, so egress probing will not run at all\n", + ) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + if provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + // CALLER side: the prober's configured secret is absent or does not + // match this server's. Neither the provided value nor any prefix of it + // is logged -- a rejected secret is still a secret, and log shipping is + // a wider audience than the vault. + glog.Errorf( + "[probercred]rejected a prober credential request: missing or wrong %s header; "+ + "the caller's operator secret does not match this server's\n", + operatorSecretHeader, + ) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + identity := model.GetProberIdentity(r.Context()) + + // 404 -- not a 500, and not an empty 200. The prober polls this before the + // bootstrap task has necessarily finished, and it must be able to tell "not + // ready yet, keep polling" from "broken, wake someone". An empty 200 makes + // those two the same response. + // + // A missing row is NOT the only not-ready state, and checking only for it + // would produce exactly that empty 200. The row is committed by + // createProberNetwork before mintProberClientJwt runs, so it legitimately + // exists with by_client_jwt still NULL; clearProberIdentityClient also + // returns it to that state when a client has to be re-provisioned. All + // three are the same answer to this caller: there is no credential yet. + // + // Not logged: the bootstrap task already reports its own failures at + // Errorf, and a poll during normal startup is not an error. + if identity == nil || identity.ClientId == nil || identity.ByClientJwt == "" { + http.Error(w, "Not found", http.StatusNotFound) + return + } + + result := &ProberCredentialResult{ + ByClientJwt: identity.ByClientJwt, + ClientId: *identity.ClientId, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[probercred]could not write response. err = %s\n", err) + } +} diff --git a/api/handlers/prober_credential_handlers_test.go b/api/handlers/prober_credential_handlers_test.go new file mode 100644 index 00000000..58830ed5 --- /dev/null +++ b/api/handlers/prober_credential_handlers_test.go @@ -0,0 +1,144 @@ +package handlers + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "testing" + + "github.com/urnetwork/server" +) + +func TestProberCredentialRejectsMissingSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil) + w := httptest.NewRecorder() + + ProberCredential(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProberCredentialRejectsWrongSecret(t *testing.T) { + defer withStubOperatorIngestSecret("correct-operator-secret-0123456789")() + + req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProberCredential(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } + if strings.Contains(w.Body.String(), "jwt") { + t.Fatalf("a rejected request must not be told anything about the credential: %q", w.Body.String()) + } +} + +// TestProberCredentialRejectsAlteredSecret is the same-length, one-byte-off +// case. A comparison written with strings.HasPrefix, or one that only checked +// length, would pass the two tests above and fail this one. +func TestProberCredentialRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + altered := []byte(secret) + altered[len(altered)-1] ^= 0x01 + + req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil) + req.Header.Set(operatorSecretHeader, string(altered)) + w := httptest.NewRecorder() + + ProberCredential(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a secret differing in one byte", w.Code) + } +} + +// TestProberCredentialResultExposesOnlyTheDeliveryCredential pins the response +// SHAPE, so the handler keeps returning a deliberate, documented field set +// rather than whatever a model struct happens to have. +// +// The failure it exists to catch is a quiet one: someone "simplifies" the +// handler by encoding model.ProberIdentity directly, or adds a field here for +// debugging. Either compiles, builds green, and passes every status-code test +// above. Asserting the exact field set is the only check that fails. +// +// It is NOT what makes this endpoint safe, and it is not a substitute for the +// auth tests above. See the note on ProberCredentialResult: network_id, user_id +// and network_name are already inside the by_client_jwt this response carries, +// so omitting them withholds nothing from a caller who got past the operator +// secret. That secret is the gate; this test is change detection. +func TestProberCredentialResultExposesOnlyTheDeliveryCredential(t *testing.T) { + want := []string{"by_client_jwt", "client_id"} + + var got []string + resultType := reflect.TypeOf(ProberCredentialResult{}) + for i := range resultType.NumField() { + tag, _, _ := strings.Cut(resultType.Field(i).Tag.Get("json"), ",") + got = append(got, tag) + } + slices.Sort(got) + + if !slices.Equal(got, want) { + t.Fatalf( + "ProberCredentialResult fields = %v, want exactly %v; "+ + "the response carries the revocable credential ONLY, never the account identity behind it", + got, + want, + ) + } + + // belt and braces: whatever the field set becomes, these must never be in + // it, under any spelling + for _, forbidden := range []string{"seed", "network", "user", "password", "secret"} { + if slices.ContainsFunc(got, func(f string) bool { return strings.Contains(f, forbidden) }) { + t.Fatalf("ProberCredentialResult must never expose a %q field: %v", forbidden, got) + } + } +} + +// TestProberCredentialNotReadyIs404 proves the auth gate can ACCEPT, and that +// the accepted-but-not-yet-bootstrapped case is a 404. +// +// Both halves matter. Without the accept half, a handler replaced wholesale by +// an unconditional 401 would pass all three reject tests above. And the 404 is +// the contract the prober polls against: it must be able to tell "the +// bootstrap task has not finished yet, keep polling" from "something is +// broken, wake someone". An empty 200 collapses those two into one answer. +func TestProberCredentialNotReadyIs404(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + // no bootstrap has run against this fresh test database, so + // prober_identity is empty + req := httptest.NewRequest(http.MethodGet, "/network/prober-credential", nil) + req.Header.Set(operatorSecretHeader, secret) + req = req.WithContext(context.Background()) + w := httptest.NewRecorder() + + ProberCredential(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("the correct operator secret was rejected; the auth gate is broken shut") + } + if w.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404 before the bootstrap task has minted a credential", w.Code) + } + + // and specifically NOT a decodable credential body + var result ProberCredentialResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err == nil && result.ByClientJwt != "" { + t.Fatalf("a not-ready response must carry no credential, got %q", result.ByClientJwt) + } + }) +} diff --git a/db_migrations.go b/db_migrations.go index 8e6c0677..9422f772 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -5720,6 +5720,67 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_blackhole_check_checked_at ON provider_blackhole_check (checked_at ASC, client_id ASC) `), + // The egress prober's own network identity, as a single row. + // + // The prober authenticates to the platform with a network CLIENT jwt. Until + // now an operator minted that by hand -- create a network, POST + // /network/auth-client, paste the by_client_jwt into the prober's + // environment -- so a deployment that had not had that done had no egress + // probing at all, silently. taskworker/work/prober_bootstrap_work.go does it + // instead, and this table is what makes a job that re-runs every six hours + // FOREVER safe: it is the only record that the account already exists. + // + // Looking the network up by name could not replace it. The seedphrase branch + // of model.NetworkCreate ignores the requested name and calls + // generateRandomNetworkName() unconditionally, so the name is not knowable in + // advance and there is nothing to search for on the next run. + // + // `singleton bool PRIMARY KEY DEFAULT true CHECK (singleton)` makes "at most + // one prober identity" a fact of the schema rather than a convention every + // future caller has to remember: every insert names the same row and + // therefore collides with it, which is exactly what the create-claim upsert + // in model/prober_identity_model.go is built on. client_id is stored so each + // re-mint re-auths the SAME client instead of accumulating one client per + // refresh, and create_attempts bounds account creation if the create keeps + // failing (see MaxProberBootstrapAttempts). + // + // NOTE ON POSITION: this entry is placed before the competition control-plane + // block rather than at the end of the list, and on an ALREADY-DEPLOYED + // database that means it does not run. ApplyDbMigrationsUpTo iterates + // `for i := DbVersion(ctx); i < upTo; i += 1`, so a migration executes only + // when its list INDEX is at or past the deployed version, and a database + // already past this index skips it. + // + // Hence IF NOT EXISTS: the statement has to be safe to apply out of band and + // safe to re-apply. `bringyourctl db audit --fix` emits CREATE TABLE for a + // missing table and is what actually creates this one on a deployed + // database. + // + // SO: after deploying this to an existing database, run + // bringyourctl db audit --fix + // or prober_identity will not exist and the bootstrap task, GetProberIdentity + // and GET /network/prober-credential all fail with + // `relation "prober_identity" does not exist`. + // + // The durable fix is not to shuffle this entry: it is to resolve migrations + // by identity rather than by list index. Until then every new table added + // ahead of the competition block hits this. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS prober_identity ( + singleton bool PRIMARY KEY DEFAULT true CHECK (singleton), + + network_id uuid NULL, + user_id uuid NULL, + network_name varchar(256) NULL, + + client_id uuid NULL, + by_client_jwt text NULL, + + create_attempts int NOT NULL DEFAULT 0, + create_time timestamp NULL, + last_mint_time timestamp NULL + ) + `), // Durable sim-latency competition control plane. The queue is deliberately // independent of pending_task: untrusted submissions are claimed by a diff --git a/model/prober_identity_model.go b/model/prober_identity_model.go new file mode 100644 index 00000000..9e246a7a --- /dev/null +++ b/model/prober_identity_model.go @@ -0,0 +1,613 @@ +package model + +import ( + "context" + "fmt" + "time" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/jwt" + "github.com/urnetwork/server/session" +) + +// The egress prober needs a network client jwt to connect through providers at +// all. Producing one used to be a manual operator step -- create a network, +// call /network/auth-client, paste the token into an env file -- which meant a +// deployment nobody had done that for simply never probed egress, quietly. +// This file is the server-side replacement: one persisted identity, created +// once and refreshed on a schedule, by the bootstrap task in +// taskworker/work/prober_bootstrap_work.go. +// +// The whole risk here is that the task re-runs every six hours forever. Every +// constant and every query below exists to make a repeated run a no-op rather +// than a second account, a second client, or another balance grant. +const ( + // MaxProberBootstrapAttempts bounds how many times account creation is + // attempted before the task gives up and only logs. + // + // This bound is the single most important thing in this file. Creation is + // the one irreversible, side-effect-carrying step: a retry loop that kept + // creating networks would fill the deployment with orphan accounts, and it + // would do so four times a day forever with nobody watching. Failing loudly + // and STOPPING is strictly better than that -- an operator can read one + // error line, but nobody unwinds a thousand accounts. + MaxProberBootstrapAttempts = 5 + + // ProberMinTransferBalance is the active balance below which the prober's + // network is topped up. At or above it nothing is granted -- that check is + // what keeps a six-hourly task from stacking a grant every six hours. + ProberMinTransferBalance = 4 * Gib + + // ProberTransferBalanceTopUp / ProberTransferBalanceDuration are one grant. + // Deliberately generous relative to what probing costs (a tunnel handshake + // and a few small https requests per provider): the failure this feature + // exists to remove is silent, so erring toward "never runs dry" is cheap + // and erring the other way is invisible. + ProberTransferBalanceTopUp = 32 * Gib + ProberTransferBalanceDuration = 30 * 24 * time.Hour + + // ProberJwtRefreshAge is how old a minted client jwt may get before it is + // re-minted. + // + // It is deliberately NOT derived from the jwt's own lifetime. That lifetime + // (jwt.expiryDuration) is unexported, so this package cannot read it, and it + // has already been changed once. A duplicated copy here would drift + // silently, and the direction it drifts is the bad one: a shortened lifetime + // with a stale copy here means an EXPIRED prober credential. A short, + // self-chosen refresh age needs to know nothing about the deadline it is + // staying clear of. + ProberJwtRefreshAge = 7 * 24 * time.Hour + + // The prober's client is a long-lived server-managed identity, not a user + // device. The description is what an operator sees in the device list, so + // it says what the client is. + ProberClientDescription = "URnetwork egress prober" + ProberClientDeviceSpec = "urnetwork/egress-prober" +) + +// ProberIdentity is the persisted singleton row. Every field except +// CreateAttempts is empty until the step that fills it has committed, and the +// bootstrap decides what to do next purely from which of them are still empty +// -- so an interrupted run resumes at the right step instead of starting over. +type ProberIdentity struct { + NetworkId *server.Id `json:"network_id,omitempty"` + UserId *server.Id `json:"user_id,omitempty"` + NetworkName string `json:"network_name,omitempty"` + + ClientId *server.Id `json:"client_id,omitempty"` + // the minted credential itself. Nothing in this repository consumes it yet + // -- delivering it to the prober process is a separate step -- but + // re-minting a token that was then discarded would be pointless, so it is + // stored. + ByClientJwt string `json:"by_client_jwt,omitempty"` + + CreateAttempts int `json:"create_attempts"` + CreateTime *time.Time `json:"create_time,omitempty"` + LastMintTime *time.Time `json:"last_mint_time,omitempty"` +} + +// HasNetwork reports whether the account exists. This row is the ONLY authority +// on that question; see the migration comment for why a name lookup cannot be. +func (self *ProberIdentity) HasNetwork() bool { + return self != nil && self.NetworkId != nil +} + +func GetProberIdentity(ctx context.Context) *ProberIdentity { + var identity *ProberIdentity + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + network_id, + user_id, + network_name, + client_id, + by_client_jwt, + create_attempts, + create_time, + last_mint_time + FROM prober_identity + WHERE singleton + `, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + identity = &ProberIdentity{} + var networkName *string + var byClientJwt *string + server.Raise(result.Scan( + &identity.NetworkId, + &identity.UserId, + &networkName, + &identity.ClientId, + &byClientJwt, + &identity.CreateAttempts, + &identity.CreateTime, + &identity.LastMintTime, + )) + if networkName != nil { + identity.NetworkName = *networkName + } + if byClientJwt != nil { + identity.ByClientJwt = *byClientJwt + } + } + }) + }) + + return identity +} + +// claimProberIdentityCreate takes the exclusive right to create the prober's +// account, and is the reason this task can run forever without ever creating a +// second one. +// +// The two outcomes, which the caller depends on exactly: +// +// - claimed == true: the row exists with network_id still NULL and this call +// owns the attempt. createAttempts is how many attempts came BEFORE this one +// (0 on the very first), so the caller can refuse past a bound. +// - claimed == false: no row came back, which happens only when the +// ON CONFLICT branch's `WHERE network_id IS NULL` was false -- the account +// already exists. Nothing to create. This is the steady state, reached every +// six hours forever after the first successful run. +// +// The claim commits BEFORE the account is created, and it must. Claim-then- +// create means a crash in between costs one attempt off a bounded counter; +// create-then-claim (or a claim inside a transaction that the create's failure +// rolls back) means a crash leaves an account nothing remembers -- and the next +// run creates another one, forever. +func claimProberIdentityCreate(ctx context.Context) (createAttempts int, claimed bool) { + server.Tx(ctx, func(tx server.PgTx) { + result, err := tx.Query( + ctx, + ` + INSERT INTO prober_identity (singleton, create_attempts) + VALUES (true, 0) + ON CONFLICT (singleton) DO UPDATE + SET create_attempts = prober_identity.create_attempts + 1 + WHERE prober_identity.network_id IS NULL + RETURNING create_attempts + `, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&createAttempts)) + claimed = true + } + }) + }) + + return +} + +// setProberIdentityNetwork records the created account. `network_id IS NULL` in +// the WHERE makes the write single-assignment: the identity can be filled in +// once and never repointed, so a second run that somehow got past the claim +// cannot overwrite the live identity with its own network. A false return +// therefore means "someone else already won", which the caller reports as an +// orphan rather than treating as success. +func setProberIdentityNetwork( + ctx context.Context, + networkId server.Id, + userId server.Id, + networkName string, +) (stored bool) { + server.Tx(ctx, func(tx server.PgTx) { + tag := server.RaisePgResult(tx.Exec( + ctx, + ` + UPDATE prober_identity + SET + network_id = $1, + user_id = $2, + network_name = $3, + create_time = $4 + WHERE singleton AND network_id IS NULL + `, + networkId, + userId, + networkName, + server.NowUtc(), + )) + stored = 0 < tag.RowsAffected() + }) + + return +} + +// setProberIdentityClient stores a freshly minted credential. client_id is +// written every time but never changes after the first mint -- the caller +// passes the stored id back in, so the prober keeps ONE client identity across +// refreshes instead of accumulating a new client per refresh forever. +func setProberIdentityClient( + ctx context.Context, + clientId server.Id, + byClientJwt string, + mintTime time.Time, +) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + UPDATE prober_identity + SET + client_id = $1, + by_client_jwt = $2, + last_mint_time = $3 + WHERE singleton + `, + clientId, + byClientJwt, + mintTime, + )) + }) +} + +// clearProberIdentityClient forgets the stored client, so the next mint +// provisions a fresh one. The network identity -- the part that must never be +// duplicated -- is deliberately untouched; only the client is replaced, and +// clients are re-provisionable by design. +// +// by_client_jwt is dropped with it. A credential naming a client that no longer +// exists is refused at auth anyway (see jwt.ValidateByJwtState), so keeping it +// would only make a dead token look like a live one. +func clearProberIdentityClient(ctx context.Context) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + UPDATE prober_identity + SET + client_id = NULL, + by_client_jwt = NULL, + last_mint_time = NULL + WHERE singleton + `, + )) + }) +} + +// getNetworkAdminUserId reads back the user the network was created for. +// +// NetworkCreate's result carries the network id but NOT the user id, and the +// user id is needed for every later re-mint (jwt.NewByJwt takes it), long after +// the create call is gone. Reading it from the row that was just written keeps +// the stored identity consistent with the database by construction. +func getNetworkAdminUserId(ctx context.Context, networkId server.Id) (userId server.Id, found bool) { + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT admin_user_id + FROM network + WHERE network_id = $1 + `, + networkId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&userId)) + found = true + } + }) + }) + + return +} + +// ProberBootstrapStatus reports what one bootstrap pass actually did. It exists +// for logging and for tests; the task's own result stays empty. +type ProberBootstrapStatus struct { + NetworkCreated bool `json:"network_created"` + BalanceGranted bool `json:"balance_granted"` + ClientJwtMinted bool `json:"client_jwt_minted"` + // set when creation was refused because MaxProberBootstrapAttempts is spent + CreateExhausted bool `json:"create_exhausted"` +} + +// BootstrapProberIdentity brings the prober's credential up to date: it creates +// the network account if there is none, tops the balance up if it has run low, +// and mints a client jwt if there is none or the last one is getting old. +// +// Each of those three is separately conditional, because this runs every six +// hours forever. In the steady state -- account present, balance healthy, jwt +// fresh -- a pass performs no writes at all. +// +// clientSession is the taskworker's UNAUTHENTICATED session (ByJwt == nil). It +// is passed through to NetworkCreate and never read for identity; the session +// used to auth the client is a separate one built here from the stored row. +func BootstrapProberIdentity(clientSession *session.ClientSession) (*ProberBootstrapStatus, error) { + ctx := clientSession.Ctx + status := &ProberBootstrapStatus{} + + identity := GetProberIdentity(ctx) + + if !identity.HasNetwork() { + created, err := createProberNetwork(clientSession, status) + if err != nil { + return status, err + } + if !created { + // either the attempt bound is spent, the account already exists but + // the row was written by a concurrent pass, or the create failed -- + // all of them already logged. Nothing below can run without the + // identity, and the next pass picks it up. + return status, nil + } + identity = GetProberIdentity(ctx) + if !identity.HasNetwork() { + return status, fmt.Errorf("prober identity has no network after a successful create") + } + } + + // Balance: only when it is actually low. GetActiveTransferBalanceByteCount + // sums the live balances, so a grant from an earlier pass -- or the ordinary + // daily free grant, which this network receives like any other -- suppresses + // the next one. Granting unconditionally here would write a new + // transfer_balance row four times a day forever. + if GetActiveTransferBalanceByteCount(ctx, *identity.NetworkId) < ProberMinTransferBalance { + startTime := server.NowUtc() + err := AddBasicTransferBalance( + ctx, + *identity.NetworkId, + ProberTransferBalanceTopUp, + startTime, + startTime.Add(ProberTransferBalanceDuration), + ) + if err != nil { + // not fatal to the pass: an existing credential keeps working and + // the next pass tries again + glog.Errorf("[proberboot]could not add transfer balance: %s\n", err) + } else { + status.BalanceGranted = true + glog.Infof( + "[proberboot]granted %s transfer balance to %s\n", + ByteCountHumanReadable(ProberTransferBalanceTopUp), + identity.NetworkName, + ) + } + } + + if proberJwtNeedsMint(identity, server.NowUtc()) { + if err := mintProberClientJwt(ctx, identity, status); err != nil { + return status, err + } + } + + return status, nil +} + +// proberJwtNeedsMint is the "near expiry" test, expressed as an age rather than +// as a distance from a deadline this package cannot see (see +// ProberJwtRefreshAge). A missing credential always needs one. +func proberJwtNeedsMint(identity *ProberIdentity, now time.Time) bool { + if identity.ClientId == nil || identity.ByClientJwt == "" || identity.LastMintTime == nil { + return true + } + return ProberJwtRefreshAge <= now.Sub(*identity.LastMintTime) +} + +// createProberNetwork creates the account, once, under the claim. +func createProberNetwork( + clientSession *session.ClientSession, + status *ProberBootstrapStatus, +) (created bool, returnErr error) { + ctx := clientSession.Ctx + + createAttempts, claimed := claimProberIdentityCreate(ctx) + if !claimed { + // the account already exists -- the row's network_id is set. This is the + // normal outcome of every pass after the first. + return false, nil + } + + if MaxProberBootstrapAttempts <= createAttempts { + // STOP. Do not create. The task stays scheduled and keeps saying this + // every six hours, which is the point: it is visible, and it is not + // creating accounts while it waits to be looked at. + status.CreateExhausted = true + glog.Errorf( + "[proberboot]giving up on creating the prober network after %d attempts; "+ + "no account will be created until the prober_identity row is cleared by hand\n", + createAttempts, + ) + return false, nil + } + + // The seedphrase path is the only one that needs no human: no email to + // verify, no wallet to sign with, no SSO. Terms must be set or it refuses. + // UserName/NetworkName are not passed because that branch ignores both -- it + // names the network with generateRandomNetworkName() regardless. + // + // clientSession here is the task's own unauthenticated session, which is + // what NetworkCreate wants for a create: it reads no identity from it, only + // the address, for its per-ip rate limit. + result, err := NetworkCreate( + NetworkCreateArgs{ + Terms: true, + }, + clientSession, + ) + if err != nil { + glog.Errorf("[proberboot]network create failed (attempt %d): %s\n", createAttempts+1, err) + return false, err + } + if result.Error != nil { + glog.Errorf( + "[proberboot]network create refused (attempt %d): %s\n", + createAttempts+1, + result.Error.Message, + ) + return false, nil + } + if result.Network == nil { + glog.Errorf("[proberboot]network create returned no network (attempt %d)\n", createAttempts+1) + return false, nil + } + + networkId := result.Network.NetworkId + userId, found := getNetworkAdminUserId(ctx, networkId) + if !found { + glog.Errorf("[proberboot]created network %s has no admin user\n", networkId) + return false, nil + } + + // result.Seedphrase (model/network_model.go:95, populated at :197) is + // DISCARDED here, on purpose. Only the network id, the admin user id and the + // name are kept, and prober_identity has no column that could hold a phrase + // (db_migrations.go:6455). Do not "fix" that by adding one. + // + // Nothing in this system needs it. This server holds the jwt signing keys + // (jwt/by_jwt.go:68, byPrivateKeys), so mintProberClientJwt below re-mints + // this account's client credential from the stored network_id/user_id + // whenever it likes -- the only identity jwt.NewByJwt takes is those three + // stored fields; its other two arguments are flags (jwt/by_jwt.go:217-223). + // A seedphrase is a HUMAN login credential, and no human ever logs into a + // machine-operated identity. + // + // Persisting it would therefore write a root credential into postgres -- + // recoverable from any dump, backup or replica, forever -- to enable a login + // nobody performs. Note what that would undo: the platform deliberately keeps + // only a salted hash of a seedphrase, never the phrase + // (model/seedphrase_auth_model.go:44-45, model/auth_model_identity.go:136). + // Writing the phrase into prober_identity would make this account the + // exception to that, and the one worth stealing. + // + // The honest cost, stated plainly: this makes the account unrecoverable by a + // human, by design. No person holds a login credential for it and none is + // written down anywhere. The prober account this one replaced was lost in + // exactly that way -- a seedphrase-only account whose phrase nobody recorded, + // and a seedphrase has no reset path. Two things make it acceptable here and + // only here. + // + // It is not a dead end. This account is created down the seedphrase branch, so + // it HAS seedphrase auth (CreateSeedphraseAuthInTx, model/network_model.go:893) + // -- and the stored by_client_jwt authenticates as the account, which is enough + // to call /auth/regenerate-seedphrase and mint a fresh phrase on demand (see + // the note on api/handlers.ProberCredentialResult for that chain). If the jwt + // has expired, mintProberClientJwt makes another. So even the human-login case + // does not want a stored phrase: the login can be manufactured from what is + // already here, which is the last argument against persisting one. + // + // And it is re-creatable. DELETE this row -- not merely NULL its network_id, + // which takes claimProberIdentityCreate's DO UPDATE branch and carries + // create_attempts forward -- and the next pass claims a fresh row at + // create_attempts = 0 and builds a new account. The worst case is an orphaned + // network to clean up, not an outage nobody can undo. + if !setProberIdentityNetwork(ctx, networkId, userId, result.Network.NetworkName) { + // Another run recorded an identity first, so this network is an orphan. + // It is named here because an account nobody knows about is exactly what + // this table exists to prevent, and silence would leave it + // undiscoverable. + glog.Errorf( + "[proberboot]prober identity was already claimed; network %s (%s) is ORPHANED and should be removed\n", + networkId, + result.Network.NetworkName, + ) + return false, nil + } + + status.NetworkCreated = true + glog.Infof("[proberboot]created prober network %s (%s)\n", networkId, result.Network.NetworkName) + return true, nil +} + +// mintProberClientJwt mints (or re-mints) the prober's client credential. +// +// The session it builds is the only place a ByJwt is involved, and it is built +// from the STORED identity rather than from anything the task was handed -- the +// task's session is unauthenticated by construction. +func mintProberClientJwt( + ctx context.Context, + identity *ProberIdentity, + status *ProberBootstrapStatus, +) error { + // pro is re-derived from the source of truth inside AuthNetworkClient, so + // the value carried here never reaches the minted credential. + byJwt := jwt.NewByJwt( + *identity.NetworkId, + *identity.UserId, + identity.NetworkName, + false, + false, + ) + + proberSession := session.NewLocalClientSession(ctx, "0.0.0.0:0", byJwt) + defer proberSession.Cancel() + + // identity.ClientId is nil only on the first mint. Every later mint passes + // the stored id, which re-auths that same client and returns a fresh + // by_client_jwt for it -- one durable prober identity, not one per refresh. + result, err := authProberClient(proberSession, identity.ClientId) + if err != nil { + return err + } + + if result.Error != nil && identity.ClientId != nil { + // The stored client cannot be re-authed. AuthNetworkClient's re-auth + // branch fails for exactly one reason that can reach here -- the client + // or its device is gone or inactive ("Client does not exist.", "Client + // needs to be migrated", "Device does not exist.") -- since the only + // other error it returns is for roles/principal, which this caller never + // sends. So any error on this path means the stored client is unusable, + // and no message parsing is needed to know it. + // + // Without this, a client removed by any of the sweepers would strand the + // refresh permanently: every later pass would read the same dead id and + // fail identically, forever, which is precisely the silent-stop this + // feature exists to remove. Forget the client and provision another + // against the same network. + glog.Errorf( + "[proberboot]stored prober client %s could not be re-authed (%s); provisioning a new client\n", + identity.ClientId, + result.Error.Message, + ) + clearProberIdentityClient(ctx) + + result, err = authProberClient(proberSession, nil) + if err != nil { + return err + } + } + + if result.Error != nil { + return fmt.Errorf("could not auth the prober client: %s", result.Error.Message) + } + if result.ByClientJwt == nil || result.ClientId == nil { + return fmt.Errorf("prober client auth returned no client credential") + } + + setProberIdentityClient(ctx, *result.ClientId, *result.ByClientJwt, server.NowUtc()) + + status.ClientJwtMinted = true + // the jwt itself is never logged; it is the credential + glog.Infof("[proberboot]minted a client jwt for prober client %s\n", *result.ClientId) + return nil +} + +// authProberClient mints one credential: a new client when clientId is nil, a +// fresh jwt for that same client when it is not. +// +// No roles and no principal are passed, on either path. validateClientIdentityArgs +// applies its network-session gate only when one of them is set, and the re-auth +// branch rejects them outright, so leaving both empty is what lets the first mint +// and every later re-mint take the same path. The client is labelled by its +// description instead. +func authProberClient( + proberSession *session.ClientSession, + clientId *server.Id, +) (*AuthNetworkClientResult, error) { + return AuthNetworkClient( + &AuthNetworkClientArgs{ + ClientId: clientId, + Description: ProberClientDescription, + DeviceSpec: ProberClientDeviceSpec, + }, + proberSession, + ) +} diff --git a/model/prober_identity_model_test.go b/model/prober_identity_model_test.go new file mode 100644 index 00000000..8da0f040 --- /dev/null +++ b/model/prober_identity_model_test.go @@ -0,0 +1,397 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/session" +) + +// The prober bootstrap task re-arms itself every six hours, forever, with +// nobody watching. Every test in this file exists for that one reason: the +// damage from any of these behaviours regressing is not a failed pass, it is a +// slow accumulation -- a network, a client, or a balance grant per pass, four +// times a day -- that nothing surfaces until someone counts the rows. +// +// Each test gets its own database (server.DefaultTestEnv().Run drops it +// afterwards), which matters more here than usual: prober_identity is a +// singleton table, so tests sharing one database would contend for the single +// row. + +// Row counts are the assertions these tests actually turn on. Status flags say +// what a pass BELIEVES it did; the row counts say what it did. countRows is the +// package-level helper from auth_model_test.go. + +// proberTaskSession builds the session the taskworker actually passes in: +// UNAUTHENTICATED, ByJwt nil. Using an authenticated one here would hide a +// whole class of regression, since the model is required to build its own +// authenticated session from the stored row rather than read one from this. +func proberTaskSession(ctx context.Context) *session.ClientSession { + return session.NewLocalClientSession(ctx, "0.0.0.0:0", nil) +} + +// A second pass must not create a second network. +// +// This is the highest-severity behaviour in the feature. The account is the one +// irreversible thing the task makes, and a repeat run that created another +// would do so every six hours forever, filling the deployment with orphan +// accounts that nothing has a name to look up -- the seedphrase branch of +// NetworkCreate discards the requested name, so prober_identity is the only +// record any of them exist. +// +// The `count(*) FROM network` assertion is the real one. status.NetworkCreated +// is the task's own opinion, and a regression that created a network while +// failing to record it would report exactly the same false. +func TestProberBootstrapSecondPassCreatesNoSecondNetwork(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientSession := proberTaskSession(ctx) + defer clientSession.Cancel() + + status1, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the first bootstrap pass failed: %s", err) + } + if !status1.NetworkCreated { + t.Fatalf("the first pass did not create the network, so this test never reaches what it is testing: %+v", status1) + } + identity1 := GetProberIdentity(ctx) + if !identity1.HasNetwork() { + t.Fatalf("the first pass reported a create but stored no network") + } + networksAfterFirst := countRows(ctx, `SELECT count(*) FROM network`) + + status2, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the second bootstrap pass failed: %s", err) + } + + if status2.NetworkCreated { + t.Errorf("the second pass created a network again; this task runs every %s forever, "+ + "so a pass that creates is a new orphan account four times a day", "6h") + } + networksAfterSecond := countRows(ctx, `SELECT count(*) FROM network`) + if networksAfterSecond != networksAfterFirst { + t.Errorf("the second pass created another network: count went %d -> %d. "+ + "The claim in claimProberIdentityCreate is what must prevent this", + networksAfterFirst, networksAfterSecond) + } + + identity2 := GetProberIdentity(ctx) + if !identity2.HasNetwork() || *identity2.NetworkId != *identity1.NetworkId { + t.Errorf("the stored prober network changed across passes: %v -> %v. "+ + "The identity must be single-assignment; repointing it strands the previous account "+ + "and every credential minted against it", identity1.NetworkId, identity2.NetworkId) + } + + // The pass above never reached the claim: BootstrapProberIdentity + // short-circuits on HasNetwork() and returns before createProberNetwork. + // That short-circuit is only the OUTER layer, and it is the one that can + // legitimately be bypassed -- two workers entering a pass together both + // read an empty identity, and a crash between NetworkCreate and + // setProberIdentityNetwork leaves a later pass believing there is no + // account. The claim is what has to hold then, so it is driven here + // directly, exactly as a concurrent pass would reach it. + // + // `created` alone is too weak to assert: with the claim granted, the + // create runs and setProberIdentityNetwork's own guard then rejects the + // result, so this still returns false while a real, paid-for, orphaned + // network exists in the table with nothing pointing at it. The count is + // what sees that. + raced, err := createProberNetwork(clientSession, &ProberBootstrapStatus{}) + if err != nil { + t.Fatalf("a create attempt against an existing identity errored rather than declining: %s", err) + } + if raced { + t.Errorf("createProberNetwork reported a create while the identity already had a network") + } + if networksAfterRace := countRows(ctx, `SELECT count(*) FROM network`); networksAfterRace != networksAfterFirst { + t.Errorf("a create attempt that bypassed the HasNetwork short-circuit created an ORPHAN network: "+ + "count went %d -> %d. Nothing can find that account again -- the seedphrase branch of NetworkCreate "+ + "discards the requested name, so prober_identity is the only record any prober network exists", + networksAfterFirst, networksAfterRace) + } + }) +} + +// Once network_id is set, the claim must return no row at all -- and must not +// burn an attempt doing it. +// +// The claim is the mechanism the whole no-second-network guarantee rests on, so +// it is tested directly rather than only through a full pass. The second +// assertion is the subtle one: the guard lives in the ON CONFLICT's +// `WHERE prober_identity.network_id IS NULL`, so a steady-state pass touches no +// row. A refactor that moved that guard to a check AFTER the update would still +// return claimed == false and still look correct here, while incrementing +// create_attempts on every pass -- so the counter would reach +// MaxProberBootstrapAttempts within about a day, and the identity would then be +// permanently unable to recreate its account if it ever needed to. +func TestProberIdentityClaimReturnsNothingOnceNetworkIsSet(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + createAttempts, claimed := claimProberIdentityCreate(ctx) + if !claimed || createAttempts != 0 { + t.Fatalf("the first claim on an empty table must be granted with 0 prior attempts, got attempts=%d claimed=%v", + createAttempts, claimed) + } + + // the account now exists, as it would after a successful create + if !setProberIdentityNetwork(ctx, server.NewId(), server.NewId(), "prober-net") { + t.Fatalf("could not record the network on a freshly claimed row") + } + attemptsAtClaim := GetProberIdentity(ctx).CreateAttempts + + _, claimedAgain := claimProberIdentityCreate(ctx) + if claimedAgain { + t.Errorf("the claim was granted again after network_id was set; every later pass would create another account") + } + + attemptsAfter := GetProberIdentity(ctx).CreateAttempts + if attemptsAfter != attemptsAtClaim { + t.Errorf("a refused claim burned an attempt: create_attempts went %d -> %d. "+ + "In the steady state this runs every 6h forever, so the counter would reach "+ + "MaxProberBootstrapAttempts (%d) within days and the identity could never recreate its account", + attemptsAtClaim, attemptsAfter, MaxProberBootstrapAttempts) + } + }) +} + +// The balance grant must be driven by the CURRENT balance, not by whether a +// grant has ever happened. +// +// Both directions are pinned here because the two regressions are opposites and +// each is invisible on its own: +// +// - drop the `< ProberMinTransferBalance` condition and every pass stacks +// another 32 GiB grant, four times a day forever +// - replace it with a once-only flag and the prober silently runs out of +// balance when its grant expires, which is exactly the silent stop this +// whole feature exists to remove +func TestProberBootstrapDoesNotRegrantAHealthyBalance(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientSession := proberTaskSession(ctx) + defer clientSession.Cancel() + + status1, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the first bootstrap pass failed: %s", err) + } + if !status1.BalanceGranted { + t.Fatalf("the first pass granted no balance, so this test never reaches what it is testing: %+v", status1) + } + identity := GetProberIdentity(ctx) + if !identity.HasNetwork() { + t.Fatalf("the first pass stored no network") + } + networkId := *identity.NetworkId + + balanceRows := `SELECT count(*) FROM transfer_balance WHERE network_id = $1` + rowsAfterFirst := countRows(ctx, balanceRows, networkId) + if rowsAfterFirst != 1 { + t.Fatalf("expected exactly one balance row after the first grant, got %d", rowsAfterFirst) + } + if active := GetActiveTransferBalanceByteCount(ctx, networkId); active < ProberMinTransferBalance { + t.Fatalf("the granted balance %d is below ProberMinTransferBalance %d, so the second pass "+ + "would legitimately grant again and this test would prove nothing", + active, ProberMinTransferBalance) + } + + status2, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the second bootstrap pass failed: %s", err) + } + if status2.BalanceGranted { + t.Errorf("the second pass granted balance again while the balance was already healthy") + } + if rows := countRows(ctx, balanceRows, networkId); rows != rowsAfterFirst { + t.Errorf("a pass over a healthy balance wrote another transfer_balance row: %d -> %d. "+ + "At one pass every 6h that is four stacked grants a day, forever", + rowsAfterFirst, rows) + } + + // Now the other direction: expire the grant, exactly as it expires on its + // own after ProberTransferBalanceDuration, and the next pass must top up. + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `UPDATE transfer_balance SET end_time = $2 WHERE network_id = $1`, + networkId, + server.NowUtc().Add(-time.Hour), + )) + }) + if active := GetActiveTransferBalanceByteCount(ctx, networkId); ProberMinTransferBalance <= active { + t.Fatalf("expiring the balance left %d active, so the top-up half of this test is not exercised", active) + } + + status3, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the third bootstrap pass failed: %s", err) + } + if !status3.BalanceGranted { + t.Errorf("the balance had run out and the pass did not top it up. The grant must be conditional on the "+ + "CURRENT balance being below ProberMinTransferBalance (%d), not on whether a grant ever happened -- "+ + "a prober with no balance cannot open a contract and stops probing silently", + ProberMinTransferBalance) + } + }) +} + +// A re-mint must re-auth the SAME client, not provision another one. +// +// The stored client_id is passed back into AuthNetworkClient for exactly this +// reason. Dropping it (minting against a nil client id) still produces a +// working credential on every pass, so nothing fails and nothing logs -- the +// only symptom is one more network_client and one more device row every six +// hours, forever, in a table the sweepers then have to walk. +// +// The client here is a genuine one from a real first mint. Seeding a fabricated +// id instead would make the re-auth fail and fire the recovery path +// (clearProberIdentityClient, then a new client), so client_id would change for +// a legitimate reason and the test would be asserting the opposite behaviour. +func TestProberRemintReusesTheSameClient(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientSession := proberTaskSession(ctx) + defer clientSession.Cancel() + + status1, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the first bootstrap pass failed: %s", err) + } + if !status1.ClientJwtMinted { + t.Fatalf("the first pass minted no client jwt, so there is no client to re-mint for: %+v", status1) + } + identity1 := GetProberIdentity(ctx) + if identity1.ClientId == nil { + t.Fatalf("the first pass stored no client id") + } + firstClientId := *identity1.ClientId + firstJwt := identity1.ByClientJwt + + clientRows := `SELECT count(*) FROM network_client WHERE network_id = $1` + deviceRows := `SELECT count(*) FROM device WHERE network_id = $1` + clientsAfterFirst := countRows(ctx, clientRows, *identity1.NetworkId) + devicesAfterFirst := countRows(ctx, deviceRows, *identity1.NetworkId) + + // age the stored credential past ProberJwtRefreshAge so the next pass + // re-mints rather than doing nothing + setProberIdentityClient(ctx, firstClientId, firstJwt, + server.NowUtc().Add(-ProberJwtRefreshAge-time.Minute)) + + status2, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the re-mint pass failed: %s", err) + } + if !status2.ClientJwtMinted { + t.Fatalf("a credential older than ProberJwtRefreshAge (%s) was not re-minted; "+ + "the prober's jwt would eventually expire with nothing renewing it", ProberJwtRefreshAge) + } + + identity2 := GetProberIdentity(ctx) + if identity2.ClientId == nil { + t.Fatalf("the re-mint stored no client id") + } + if *identity2.ClientId != firstClientId { + t.Errorf("the re-mint provisioned a NEW client (%s -> %s) instead of re-authing the stored one. "+ + "At one pass every 6h this accumulates a client and a device per pass forever", + firstClientId, *identity2.ClientId) + } + if clients := countRows(ctx, clientRows, *identity1.NetworkId); clients != clientsAfterFirst { + t.Errorf("the re-mint added a network_client row: %d -> %d", clientsAfterFirst, clients) + } + if devices := countRows(ctx, deviceRows, *identity1.NetworkId); devices != devicesAfterFirst { + t.Errorf("the re-mint added a device row: %d -> %d", devicesAfterFirst, devices) + } + if identity2.ByClientJwt == firstJwt { + t.Errorf("the re-mint stored the same jwt it started with, so nothing was actually refreshed " + + "and the credential still ages out on the original deadline") + } + }) +} + +// Past MaxProberBootstrapAttempts the task must STOP creating. +// +// This bound is the backstop for the whole feature: if creation is failing for +// some reason that a retry cannot fix, an unbounded task creates a network on +// every pass, forever, four times a day. Failing loudly and stopping is +// recoverable; a thousand orphan accounts is not. +// +// The attempts are burned by calling the claim directly rather than by driving +// five real creates. claimProberIdentityCreate returns the number of attempts +// BEFORE the current one, so five prior claims leave create_attempts at 4 and +// the bootstrap's own claim -- the sixth -- returns 5, which is the bound. +func TestProberBootstrapStopsCreatingAtTheAttemptBound(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientSession := proberTaskSession(ctx) + defer clientSession.Cancel() + + for i := 0; i < MaxProberBootstrapAttempts; i++ { + if _, claimed := claimProberIdentityCreate(ctx); !claimed { + t.Fatalf("claim %d was refused while network_id is still NULL", i+1) + } + } + + status, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the bootstrap pass returned an error rather than giving up cleanly: %s", err) + } + + if !status.CreateExhausted { + t.Errorf("the pass did not report the attempt bound as spent after %d attempts: %+v", + MaxProberBootstrapAttempts, status) + } + if status.NetworkCreated { + t.Errorf("a network was created past the attempt bound of %d. Unbounded, this creates "+ + "an orphan account every 6h forever", MaxProberBootstrapAttempts) + } + if n := countRows(ctx, `SELECT count(*) FROM network`); n != 0 { + t.Errorf("%d network(s) exist after a pass that should have refused to create", n) + } + if GetProberIdentity(ctx).HasNetwork() { + t.Errorf("an identity was recorded by a pass that should have refused to create") + } + }) +} + +// The bound must not bite one attempt early. +// +// The pair with the test above brackets MaxProberBootstrapAttempts from both +// sides. On its own, either one passes under an off-by-one: a bound that +// refused a create too early would leave a deployment with NO prober account at +// all -- the exact silent no-probing state this feature was written to remove -- +// and the exhaustion test alone would still be green. +func TestProberBootstrapStillCreatesOnTheLastAllowedAttempt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientSession := proberTaskSession(ctx) + defer clientSession.Cancel() + + for i := 0; i < MaxProberBootstrapAttempts-1; i++ { + if _, claimed := claimProberIdentityCreate(ctx); !claimed { + t.Fatalf("claim %d was refused while network_id is still NULL", i+1) + } + } + + status, err := BootstrapProberIdentity(clientSession) + if err != nil { + t.Fatalf("the bootstrap pass failed: %s", err) + } + + if status.CreateExhausted { + t.Errorf("creation was refused on the last attempt the bound still allows "+ + "(%d prior attempts, bound %d); a deployment would be left with no prober account and no probing", + MaxProberBootstrapAttempts-1, MaxProberBootstrapAttempts) + } + if !status.NetworkCreated { + t.Errorf("no network was created on the last allowed attempt: %+v", status) + } + if !GetProberIdentity(ctx).HasNetwork() { + t.Errorf("no prober identity was recorded on the last allowed attempt") + } + }) +} diff --git a/taskworker/taskworker.go b/taskworker/taskworker.go index 85d64c67..7e5c900c 100644 --- a/taskworker/taskworker.go +++ b/taskworker/taskworker.go @@ -68,6 +68,7 @@ func InitTasks(ctx context.Context) { work.ScheduleRemoveExpiredWalletAuthChallenges(clientSession, tx) work.ScheduleRemoveExpiredWalletNonces(clientSession, tx) work.ScheduleRemoveExpiredProviderEgressLocations(clientSession, tx) + work.ScheduleProberBootstrap(clientSession, tx) work.ScheduleRefreshGeolocationSourcePins(clientSession, tx) work.ScheduleRemoveExpiredBulkClientRemovalQuota(clientSession, tx) work.ScheduleRemoveOldAuditNetworkEvents(clientSession, tx) @@ -274,6 +275,10 @@ func InitTaskWorkerWithSettings(ctx context.Context, settings *task.TaskWorkerSe work.RemoveExpiredProviderEgressLocations, work.RemoveExpiredProviderEgressLocationsPost, ), + task.NewTaskTargetWithPost( + work.ProberBootstrap, + work.ProberBootstrapPost, + ), task.NewTaskTargetWithPost( work.RefreshGeolocationSourcePins, work.RefreshGeolocationSourcePinsPost, diff --git a/taskworker/work/prober_bootstrap_work.go b/taskworker/work/prober_bootstrap_work.go new file mode 100644 index 00000000..034c720d --- /dev/null +++ b/taskworker/work/prober_bootstrap_work.go @@ -0,0 +1,90 @@ +package work + +import ( + "time" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/session" + "github.com/urnetwork/server/task" +) + +// ProberBootstrapTimeout is the cadence of the credential refresh. +// +// Six hours is far shorter than the jwt refresh age it is keeping ahead of (see +// model.ProberJwtRefreshAge), so a missed pass or two costs nothing, and short +// enough that a deployment brought up from empty has a working prober +// credential within the same day rather than waiting a full cycle. +const ProberBootstrapTimeout = 6 * time.Hour + +type ProberBootstrapArgs struct{} + +type ProberBootstrapResult struct{} + +func ScheduleProberBootstrap(clientSession *session.ClientSession, tx server.PgTx) { + task.ScheduleTaskInTx( + tx, + ProberBootstrap, + &ProberBootstrapArgs{}, + clientSession, + task.RunOnce("prober_bootstrap"), + task.RunAt(server.NowUtc().Add(ProberBootstrapTimeout)), + ) +} + +// ProberBootstrap creates and refreshes the egress prober's credential -- the +// network account, its transfer balance, and its client jwt -- with no human +// step. Before this, an operator had to create an account by hand, authorise a +// balance code for it, mint a client jwt through /network/auth-client and paste +// the result into the prober's environment; a deployment where that had not +// been done had no egress probing at all, and nothing said so. +// +// Everything conditional lives in model.BootstrapProberIdentity. This function +// deliberately holds no logic of its own, which is what makes the constraint +// below true by construction rather than by review. +// +// The session here is UNAUTHENTICATED. InitTasks builds it as +// session.NewLocalClientSession(ctx, "0.0.0.0:0", nil), so ByJwt is nil and +// reading it would panic on the first run. The session is passed along (for its +// context and its address) and never read for identity; the model builds its +// own authenticated session from the stored prober identity when it needs one. +func ProberBootstrap( + _ *ProberBootstrapArgs, + clientSession *session.ClientSession, +) (*ProberBootstrapResult, error) { + status, err := model.BootstrapProberIdentity(clientSession) + if err != nil { + return nil, err + } + + // only say something when something happened; the steady state is silent + if status.NetworkCreated || status.BalanceGranted || status.ClientJwtMinted { + glog.Infof( + "[proberboot]pass complete: network_created=%t balance_granted=%t jwt_minted=%t\n", + status.NetworkCreated, + status.BalanceGranted, + status.ClientJwtMinted, + ) + } + + return &ProberBootstrapResult{}, nil +} + +// ProberBootstrapPost re-arms the chain. +// +// This is not boilerplate. These tasks are single-shot: the next run exists +// only because the previous run's Post scheduled it. Omitting this would leave +// the prober's credential to expire with nothing to renew it, and the failure +// would appear weeks later as a prober that cannot connect -- with no failing +// task anywhere to point at. +func ProberBootstrapPost( + _ *ProberBootstrapArgs, + _ *ProberBootstrapResult, + clientSession *session.ClientSession, + tx server.PgTx, +) error { + ScheduleProberBootstrap(clientSession, tx) + return nil +}