test(e2e): embedded-postgres smoke leg and WithEmbeddedPostgres (RIG-2906)

docs/designs/infra/ci/compass-test-harness-tiers/design.md

@@ -491,8 +491,7 @@ docs.
       (depends T1).
 - [x] T3 — `WithOwnStack()` + leg migration + shared canned-stub
       `LoadScript`. Satisfied on main by `sharedFixture`; see T3 errata.
-- [ ] T4 — `TestEmbeddedPostgresSmoke` + `WithEmbeddedPostgres()`. Owner:
-      implement lane (depends T3).
+- [x] T4 — `TestEmbeddedPostgresSmoke` + `WithEmbeddedPostgres()`.
 - [x] T5 — forge-injection convergence (`mountForgeForTest`). Satisfied on
       main by `newForgeE2EWire`; see T5 errata.
 - [ ] T6 — tier-pyramid contributor doc. Owner: implement lane (depends

go/e2e/embedded_postgres_test.go

@@ -0,0 +1,134 @@
+//go:build podman
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+
+	"connectrpc.com/connect"
+
+	compassv1 "github.com/RigelBuild/compass/go/gen/compass/v1"
+	"github.com/RigelBuild/compass/go/internal/stack"
+)
+
+// TestEmbeddedPostgresSmoke is the one e2e leg that runs the product-default
+// postgres: the stack spawns and owns it, so this is the only dedicated e2e
+// coverage of the supervisor-owns-postgres lifecycle and its pgid-record
+// teardown. Every other leg attaches to the suite postgres instead.
+func TestEmbeddedPostgresSmoke(t *testing.T) {
+	if !podmanUsable(t.Context()) {
+		t.Skip("rootless podman cannot run compass-agent:latest here; skipping the real-stack e2e")
+	}
+	ctx := context.Background()
+	f := NewFixture(ctx, t, WithEmbeddedPostgres())
+
+	// The stack recorded the postgres it spawned, and that group is alive.
+	pgid := recordedPostgresPgid(t, f.StateDir())
+	if err := syscall.Kill(-pgid, 0); err != nil {
+		t.Fatalf("recorded postgres group %d is not alive after Up: %v", pgid, err)
+	}
+
+	health, err := f.Stack().Health(ctx)
+	if err != nil {
+		t.Fatalf("Health: %v", err)
+	}
+	if health.State != stack.StatusReady {
+		t.Fatalf("stack state = %v (%s), want Ready", health.State, health.Detail)
+	}
+
+	rctx, cancel := context.WithTimeout(ctx, rpcTimeout)
+	defer cancel()
+	resp, err := f.Compass().GetServerInfo(rctx, connect.NewRequest(&compassv1.GetServerInfoRequest{}))
+	if err != nil {
+		t.Fatalf("authed GetServerInfo over the embedded postgres: %v", err)
+	}
+	if resp.Msg.GetVersion() == "" {
+		t.Fatal("GetServerInfo returned an empty version")
+	}
+
+	if err := f.Stack().Down(ctx); err != nil {
+		t.Fatalf("Down: %v", err)
+	}
+	post, err := f.Stack().Health(ctx)
+	if err != nil {
+		t.Fatalf("Health after Down: %v", err)
+	}
+	if post.State == stack.StatusReady {
+		t.Fatalf("stack still Ready after Down (%s)", post.Detail)
+	}
+	// A clean Down reaps the postgres group and drops the teardown record.
+	if err := syscall.Kill(-pgid, 0); !errors.Is(err, syscall.ESRCH) {
+		t.Fatalf("postgres group %d after Down: kill(0) = %v, want ESRCH", pgid, err)
+	}
+	if _, err := os.Stat(filepath.Join(f.StateDir(), "stack.pgids")); !errors.Is(err, os.ErrNotExist) {
+		t.Fatalf("pgid record after a clean Down: stat = %v, want not-exist", err)
+	}
+}
+
+// fatalRecorder is a testing.TB whose Fatalf records the call and stops the
+// goroutine, so a test can observe a helper's fatal without failing itself.
+type fatalRecorder struct {
+	testing.TB
+	fataled bool
+}
+
+func (r *fatalRecorder) Helper() {}
+
+func (r *fatalRecorder) Fatalf(string, ...any) {
+	r.fataled = true
+	runtime.Goexit()
+}
+
+// TestWithEmbeddedPostgresRejectsExternalDSN has no podmanUsable skip, so it runs
+// even where the stack cannot: it checks only the option guard, which must
+// refuse a fixture asked for both postgres arms.
+func TestWithEmbeddedPostgresRejectsExternalDSN(t *testing.T) {
+	for name, opts := range map[string][]fixtureOption{
+		"conflict":      {WithEmbeddedPostgres(), withExternalDatabase("host=/tmp dbname=x")},
+		"embedded only": {WithEmbeddedPostgres()},
+	} {
+		t.Run(name, func(t *testing.T) {
+			rec := &fatalRecorder{TB: t}
+			done := make(chan struct{})
+			go func() {
+				defer close(done)
+				buildFixtureConfig(rec, opts)
+			}()
+			<-done
+			if want := name == "conflict"; rec.fataled != want {
+				t.Fatalf("buildFixtureConfig fataled = %v, want %v", rec.fataled, want)
+			}
+		})
+	}
+}
+
+// recordedPostgresPgid reads the postgres process-group id from the stack's
+// pgid record ("proc postgres <pgid> <start>" in the v2 format).
+func recordedPostgresPgid(t *testing.T, stateDir string) int {
+	t.Helper()
+	path := filepath.Join(stateDir, "stack.pgids")
+	raw, err := os.ReadFile(path)
+	if err != nil {
+		t.Fatalf("read pgid record: %v", err)
+	}
+	for _, line := range strings.Split(string(raw), "\n")[1:] {
+		f := strings.Fields(line)
+		if len(f) == 4 && f[0] == "proc" && f[1] == "postgres" {
+			pgid, err := strconv.Atoi(f[2])
+			if err != nil || pgid <= 1 {
+				t.Fatalf("pgid record postgres entry %q has a bad pgid", line)
+			}
+			return pgid
+		}
+	}
+	t.Fatalf("pgid record %q has no proc postgres entry; the stack did not spawn its own postgres:\n%s", path, raw)
+	return 0
+}

go/e2e/fixture.go

@@ -80,17 +80,7 @@ func configureForgeStub(tb testing.TB, secretsPath string) *forgeStub {
 	// Scrub every ambient knob that could redirect the leg off the loopback
 	// stub. The S3 and OTLP names are env-only fallbacks the server reads
 	// directly, so an operator's shell would otherwise reach a real endpoint.
-	for _, name := range []string{
-		"COMPASS_FORGE_REPOS", "COMPASS_FORGE_HOST", "COMPASS_FORGE_APP_ID", "COMPASS_FORGE_INSTALLATION_ID",
-		"COMPASS_FORGE_APP_KEY_SECRET", "COMPASS_FORGE_APP_WEBHOOK_SECRET", "COMPASS_FORGE_REVIEWER_APP_ID",
-		"COMPASS_FORGE_REVIEWER_APP_INSTALLATION_ID", "COMPASS_FORGE_REVIEWER_APP_KEY_SECRET",
-		"COMPASS_FORGE_LINEAR_CLIENT_ID", "COMPASS_FORGE_LINEAR_CLIENT_SECRET", "COMPASS_FORGE_LINEAR_WEBHOOK_SECRET",
-		"COMPASS_FORGE_DISABLE_SCOPE_ENFORCEMENT", "COMPASS_SECRET_PROVIDER", "LINEAR_FORGE_CLIENT_ID", "LINEAR_FORGE_CLIENT_SECRET", "LINEAR_FORGE_WEBHOOK_SECRET",
-		"COMPASS_S3_ENDPOINT", "COMPASS_S3_BUCKET", "COMPASS_S3_ACCESS_KEY", "COMPASS_S3_SECRET_KEY",
-		"COMPASS_S3_REGION", "COMPASS_S3_USE_TLS", "OTEL_EXPORTER_OTLP_ENDPOINT",
-	} {
-		tb.Setenv(name, "")
-	}
+	scrubForgeEnv(tb)
 	// Dogfood forge writes use the single-trust-domain posture, with no grant seed.
 	tb.Setenv("COMPASS_FORGE_DISABLE_SCOPE_ENFORCEMENT", "true")
 	for k, v := range map[string]string{"COMPASS_FORGE_HOST": stub.Host(), "COMPASS_FORGE_APP_ID": "1001", "COMPASS_FORGE_INSTALLATION_ID": "1", "COMPASS_FORGE_APP_KEY_SECRET": "FORGE_APP_PRIVATE_KEY", "COMPASS_FORGE_APP_WEBHOOK_SECRET": "FORGE_APP_WEBHOOK_SECRET", "COMPASS_FORGE_REVIEWER_APP_ID": "1002", "COMPASS_FORGE_REVIEWER_APP_INSTALLATION_ID": "2", "COMPASS_FORGE_REVIEWER_APP_KEY_SECRET": "FORGE_REVIEWER_APP_PRIVATE_KEY", "COMPASS_FORGE_CA": stub.CAPath()} { //nolint:gosec // G101 false positive: secret-store key names and stub app IDs, not credentials
@@ -99,6 +89,25 @@ func configureForgeStub(tb testing.TB, secretsPath string) *forgeStub {
 	return stub
 }
 
+// forgeEnvNames are the ambient knobs the server reads for forge, S3 and OTLP.
+var forgeEnvNames = []string{
+	"COMPASS_FORGE_REPOS", "COMPASS_FORGE_HOST", "COMPASS_FORGE_APP_ID", "COMPASS_FORGE_INSTALLATION_ID",
+	"COMPASS_FORGE_APP_KEY_SECRET", "COMPASS_FORGE_APP_WEBHOOK_SECRET", "COMPASS_FORGE_REVIEWER_APP_ID",
+	"COMPASS_FORGE_REVIEWER_APP_INSTALLATION_ID", "COMPASS_FORGE_REVIEWER_APP_KEY_SECRET",
+	"COMPASS_FORGE_LINEAR_CLIENT_ID", "COMPASS_FORGE_LINEAR_CLIENT_SECRET", "COMPASS_FORGE_LINEAR_WEBHOOK_SECRET",
+	"COMPASS_FORGE_DISABLE_SCOPE_ENFORCEMENT", "COMPASS_FORGE_SCOPE_GRANTS", "COMPASS_FORGE_CA", "COMPASS_SECRET_PROVIDER", "LINEAR_FORGE_CLIENT_ID", "LINEAR_FORGE_CLIENT_SECRET", "LINEAR_FORGE_WEBHOOK_SECRET",
+	"COMPASS_S3_ENDPOINT", "COMPASS_S3_BUCKET", "COMPASS_S3_ACCESS_KEY", "COMPASS_S3_SECRET_KEY",
+	"COMPASS_S3_REGION", "COMPASS_S3_USE_TLS", "OTEL_EXPORTER_OTLP_ENDPOINT",
+}
+
+// scrubForgeEnv blanks forgeEnvNames for tb's lifetime.
+func scrubForgeEnv(tb testing.TB) {
+	tb.Helper()
+	for _, name := range forgeEnvNames {
+		tb.Setenv(name, "")
+	}
+}
+
 // agentImage is the REAL agent image the dogfood stack runs — present in the
 // local containers-storage on the dev/CI box, never a public stand-in. The
 // runner refuses to boot without it present; the stack pulls/validates it but
@@ -136,6 +145,8 @@ type Fixture struct {
 	// forwarded to the runner as --runtime-dir. Exposed so a process-table
 	// assertion can scope its match to this fixture's own runner.
 	runtimeDir string
+	// stateDir holds the stack's TLS anchor, postgres data and pgid record.
+	stateDir string
 	// stub is the canned model backend when the fixture was built with
 	// WithCannedModel, else nil. Its lifecycle rides a t.Cleanup registered at
 	// startup, so a consumer never closes it directly.
@@ -183,6 +194,8 @@ type fixtureConfig struct {
 	// externalDSN, when set, points the stack at an already-running postgres
 	// (ExternalDatabase) instead of spawning its own.
 	externalDSN string
+	// embeddedPostgres pins the stack-owned postgres arm (WithEmbeddedPostgres).
+	embeddedPostgres bool
 }
 
 // fixtureOption mutates a fixtureConfig. Variadic options keep NewFixture's
@@ -297,6 +310,13 @@ func withExternalDatabase(dsn string) fixtureOption {
 	return func(fc *fixtureConfig) { fc.externalDSN = dsn }
 }
 
+// WithEmbeddedPostgres states that a leg needs the stack-owned postgres. Plain
+// NewFixture already takes that wrapper-process arm; this option sets nothing on
+// stack.Config and only fails the test if an external DSN is also set.
+func WithEmbeddedPostgres() fixtureOption {
+	return func(fc *fixtureConfig) { fc.embeddedPostgres = true }
+}
+
 // WithForgeStub makes NewFixture stand up the deterministic forge stub backend.
 func WithForgeStub() fixtureOption {
 	return func(fc *fixtureConfig) { fc.forge = true }
@@ -318,10 +338,7 @@ func WithForgeStub() fixtureOption {
 func NewFixture(ctx context.Context, tb testing.TB, opts ...fixtureOption) *Fixture {
 	tb.Helper()
 
-	var fc fixtureConfig
-	for _, opt := range opts {
-		opt(&fc)
-	}
+	fc := buildFixtureConfig(tb, opts)
 	// The three stack child binaries are built once per run by TestMain and
 	// exported on PATH there (see main_test.go); the ProcessSupervisor resolves
 	// each Component to a bare binary name via exec.LookPath against that entry.
@@ -342,6 +359,10 @@ func NewFixture(ctx context.Context, tb testing.TB, opts ...fixtureOption) *Fixt
 	var forgeStub *forgeStub
 	if fc.forge {
 		forgeStub = configureForgeStub(tb, secretsPath)
+	} else {
+		// The shared stack sets the forge env process-wide for the whole run, and
+		// a server that inherits it declares forge secrets this file lacks.
+		scrubForgeEnv(tb)
 	}
 
 	var garage *garageFixture
@@ -459,6 +480,7 @@ func NewFixture(ctx context.Context, tb testing.TB, opts ...fixtureOption) *Fixt
 		serverURL:   serverURL,
 		adminToken:  adminToken,
 		runtimeDir:  runtimeDir,
+		stateDir:    stateDir,
 		stub:        stub,
 		forgeStub:   forgeStub,
 		agentModel:  cfg.AgentModel,
@@ -500,6 +522,19 @@ func fixtureDSN(fc fixtureConfig, pgSockDir string, pgPort int) string {
 	return "host=" + pgSockDir + " port=" + strconv.Itoa(pgPort) + " dbname=compass sslmode=disable"
 }
 
+// buildFixtureConfig applies opts and rejects a contradictory set.
+func buildFixtureConfig(tb testing.TB, opts []fixtureOption) fixtureConfig {
+	tb.Helper()
+	var fc fixtureConfig
+	for _, opt := range opts {
+		opt(&fc)
+	}
+	if fc.embeddedPostgres && fc.externalDSN != "" {
+		tb.Fatalf("NewFixture: WithEmbeddedPostgres conflicts with an external database DSN")
+	}
+	return fc
+}
+
 func acquireFixtureRoot(tb testing.TB, ctx context.Context, fc fixtureConfig) (root, stateDir string, pgPort int) {
 	tb.Helper()
 	// Acquire the root/stateDir/ports either fresh (the default ephemeral
@@ -581,6 +616,9 @@ func (f *Fixture) ForgeStub() *forgeStub { return f.forgeStub }
 // child processes rather than matching unrelated host processes.
 func (f *Fixture) RuntimeDir() string { return f.runtimeDir }
 
+// StateDir is the stack's state dir: TLS anchor, postgres data and pgid record.
+func (f *Fixture) StateDir() string { return f.stateDir }
+
 // AsObserver mints a NON-ADMIN bearer for an existing account and returns the
 // two Connect clients scoped to it, so a leg can assert what that account CAN
 // and CANNOT see over the real TLS door (RIG-3528 T1). Every other fixture RPC

go/e2e/harness_test.go

@@ -162,7 +162,7 @@ func assertRunnerHasConfiguredFlags(t *testing.T, f *Fixture) {
 // binds, so the data dir is a lasting trace even if the process has exited.
 func assertNoStackPostgres(t *testing.T, f *Fixture) {
 	t.Helper()
-	dataDir := filepath.Join(filepath.Dir(f.caPath), "postgres")
+	dataDir := filepath.Join(f.StateDir(), "postgres")
 	if _, err := os.Stat(dataDir); !errors.Is(err, fs.ErrNotExist) {
 		t.Fatalf("the shared stack ran its own postgres: %s exists (stat err %v)", dataDir, err)
 	}