Merge pull request #123 from open-source-cloud/feat/standard-db-ports-and-engines

feat: standard DB host ports, expose-by-default, unified provisioning + up/down fixes

Author: gustavobertoi

internal/cli/template_new_test.go

@@ -123,15 +123,15 @@ func TestTemplateNewEngineHasNoBuildTree(t *testing.T) {
 	t.Setenv("DEVSTACK_HOME", t.TempDir())
 	dir := t.TempDir()
 	if out, err := runCmd(t, "template", "new", "--no-input", "--dir", dir,
-		"--kind", "engine", "--name", "mariadb", "--base-image", "mariadb:11",
-		"--provides", "mariadb", "--exports", "host,port,user", "--port", "3306"); err != nil {
+		"--kind", "engine", "--name", "couchdb", "--base-image", "couchdb:3",
+		"--provides", "couchdb", "--exports", "host,port,user", "--port", "5984"); err != nil {
 		t.Fatalf("author engine: %v\n%s", err, out)
 	}
-	if _, err := os.Stat(filepath.Join(dir, "mariadb", "build")); !os.IsNotExist(err) {
+	if _, err := os.Stat(filepath.Join(dir, "couchdb", "build")); !os.IsNotExist(err) {
 		t.Errorf("engine template must have no build/ tree, stat err = %v", err)
 	}
-	manifest, _ := os.ReadFile(filepath.Join(dir, "mariadb", "template.yaml"))
-	for _, want := range []string{"image: mariadb:11", "provides: mariadb"} {
+	manifest, _ := os.ReadFile(filepath.Join(dir, "couchdb", "template.yaml"))
+	for _, want := range []string{"image: couchdb:3", "provides: couchdb"} {
 		if !strings.Contains(string(manifest), want) {
 			t.Errorf("engine template.yaml missing %q:\n%s", want, manifest)
 		}

internal/cli/up.go

@@ -33,6 +33,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
 		noHooks       bool
 		noPreflight   bool
 		noProvision   bool
+		noExpose      bool
 		profiles      []string
 		healthTimeout time.Duration
 	)
@@ -56,6 +57,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
 			d.NoHooks = noHooks
 			d.NoPreflight = noPreflight
 			d.NoProvision = noProvision
+			d.NoExpose = noExpose
 			d.Profiles = profiles
 			d.HealthTimeout = healthTimeout
 
@@ -112,6 +114,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
 	cmd.Flags().BoolVar(&noHooks, "no-hooks", false, "skip lifecycle hooks")
 	cmd.Flags().BoolVar(&noPreflight, "no-preflight", false, "skip the preflight checks")
 	cmd.Flags().BoolVar(&noProvision, "no-provision", false, "skip per-project Postgres role/db provisioning")
+	cmd.Flags().BoolVar(&noExpose, "no-expose", false, "do not auto-publish shared engines on their standard 127.0.0.1 ports")
 	cmd.Flags().StringArrayVarP(&profiles, "profile", "p", nil,
 		"service slice(s) to start — repeatable & comma-separated (spec 12); empty → defaultProfile or all")
 	return cmd
@@ -174,8 +177,39 @@ func newDownCmd(g *GlobalOpts) *cobra.Command {
 					fmt.Fprintf(w, "[ok]      down %s\n", p)
 				}
 			}
+			// Tear down anything now left running: after the project refs are dropped,
+			// stop every shared service that fell to zero refs (`shared gc --stop`).
+			// This is what makes `down` actually bring the workspace DOWN instead of
+			// leaving warm engines behind, while still respecting cross-workspace
+			// sharing — an engine another workspace still references keeps running.
+			var stopped []string
+			if gc, err := d.Manager.GC(ctx, true); err != nil {
+				if firstErr == nil {
+					firstErr = err
+				}
+			} else {
+				stopped = gc.Stopped
+				// When shared services were actually stopped, their saga phases are no
+				// longer satisfied — clear them so the next `up` restarts the shared
+				// stack (and re-publishes its ports) instead of skipping.
+				if len(stopped) > 0 {
+					_ = lock.WithLock(ctx, d.LockPath, func() error {
+						for _, ph := range []string{"shared", "provision", "resources"} {
+							if e := d.DB.ClearPhase(d.Model.Workspace.Name, "", ph); e != nil {
+								return e
+							}
+						}
+						return nil
+					})
+				}
+				if !g.JSON && !g.Quiet {
+					for _, s := range stopped {
+						fmt.Fprintf(w, "[ok]      stopped shared %s (0 refs)\n", s)
+					}
+				}
+			}
 			if g.JSON {
-				if err := writeJSON(cmd, map[string]any{"down": results}); err != nil {
+				if err := writeJSON(cmd, map[string]any{"down": results, "sharedStopped": stopped}); err != nil {
 					return err
 				}
 			}
@@ -216,7 +250,13 @@ func downProject(ctx context.Context, d orchestrate.UpDeps, project string) erro
 	if _, err := d.Manager.RegisterDown(ctx, project); err != nil {
 		return err
 	}
-	return nil
+	// The compose-up phase is no longer satisfied — its containers were just
+	// removed. Clear the saga record so the NEXT `up` re-runs it instead of
+	// skipping on a stale fingerprint (the "re-up after down is a no-op" bug).
+	// firstRun/hooks are intentionally NOT cleared (they keep run-once semantics).
+	return lock.WithLock(ctx, d.LockPath, func() error {
+		return d.DB.ClearPhase(d.Model.Workspace.Name, project, "compose-up")
+	})
 }
 
 // buildUpDeps assembles the up/down dependencies from the current directory. It

internal/migrate/migrate.go

@@ -30,6 +30,11 @@ const APIVersion = "devstack/v1"
 // knownEngines maps a devdock template/image keyword to a devstack shared engine.
 var knownEngines = map[string]string{
 	"postgres": "postgres", "postgresql": "postgres", "postgis": "postgres",
+	"mysql": "mysql", "percona": "mysql",
+	"mariadb": "mariadb",
+	"mongo":   "mongodb", "mongodb": "mongodb",
+	"cassandra": "cassandra",
+	"arango":    "arangodb", "arangodb": "arangodb",
 	"redis": "redis", "valkey": "redis",
 	"minio": "minio",
 }

internal/orchestrate/expose.go

@@ -19,11 +19,16 @@ import (
 // is an explicit opt-in that, like provisioning, is an UP-TIME compose overlay —
 // it never touches the deterministic, golden-asserted generated compose.
 //
-// Exposure uses its OWN host-port range (55xxx/58xxx…), distinct from the
-// provisioning range (45xxx), so the expose overlay and the provision overlay
-// never publish the same host port and can both be applied without a duplicate
-// binding. Ports are ledger-allocated (FreeHostPort), so the same engine keeps
-// the same host port across runs and two terminals never collide.
+// Exposure publishes each engine on its OWN WELL-KNOWN host port — the same port
+// the template advertises in-network (postgres→5432, mysql→3306, redis→6379, …) —
+// so a GUI client's default connection settings just work and there is no gap
+// between what the template's `defaultPort` says and what the host sees. That
+// deliberately differs from the provisioning range (45xxx): the two overlays map
+// different host ports onto the same container port, so both can be applied
+// without a duplicate binding. Ports remain ledger-allocated (FreeHostPort) with
+// the standard port as the search base, so the same engine keeps the same host
+// port across runs, and if a host-native server already holds the standard port
+// the allocator transparently falls back to the next free one in the band.
 
 const exposeFile = "compose.expose.yaml"
 
@@ -37,19 +42,26 @@ type exposePort struct {
 }
 
 // exposeEngines maps a shared engine (template name) to the ports `shared expose`
-// publishes on 127.0.0.1. Bases sit in the 5xxxx range so they never collide with
-// the 4xxxx provisioning overlay. Kafka is the exception: host clients MUST reach
-// the broker on 127.0.0.1:49092 (the fixed advertised external listener from the
-// template), so it reuses the kafka provision port rather than a 5xxxx one.
+// publishes on 127.0.0.1. The search base is the engine's WELL-KNOWN port (equal
+// to the in-container port), so clients connect on the port they already expect
+// and the allocator only drifts off it when a host-native server already holds it.
+// Kafka is the exception: host clients MUST reach the broker on 127.0.0.1:49092
+// (the fixed advertised external listener from the template), so it keeps that
+// base and reuses the kafka provision port rather than the broker's 19092.
 var exposeEngines = map[string][]exposePort{
-	"postgres":   {{5432, "postgres", "pg-expose", 55432, true}},
-	"redis":      {{6379, "redis", "redis-expose", 56379, true}},
-	"minio":      {{9000, "s3", "minio-expose", 59000, true}, {9001, "console", "minio-console-expose", 59001, false}},
-	"localstack": {{4566, "aws", "localstack-expose", 54566, true}},
-	"ministack":  {{4566, "aws", "ministack-expose", 54567, true}},
-	"nats":       {{4222, "nats", "nats-expose", 54222, true}, {8222, "monitor", "nats-monitor-expose", 58222, false}},
+	"postgres":   {{5432, "postgres", "pg-expose", 5432, true}},
+	"mysql":      {{3306, "mysql", "mysql-expose", 3306, true}},
+	"mariadb":    {{3306, "mariadb", "mariadb-expose", 3306, true}},
+	"mongodb":    {{27017, "mongodb", "mongodb-expose", 27017, true}},
+	"cassandra":  {{9042, "cassandra", "cassandra-expose", 9042, true}},
+	"arangodb":   {{8529, "arangodb", "arangodb-expose", 8529, true}},
+	"redis":      {{6379, "redis", "redis-expose", 6379, true}},
+	"minio":      {{9000, "s3", "minio-expose", 9000, true}, {9001, "console", "minio-console-expose", 9001, false}},
+	"localstack": {{4566, "aws", "localstack-expose", 4566, true}},
+	"ministack":  {{4566, "aws", "ministack-expose", 4566, true}},
+	"nats":       {{4222, "nats", "nats-expose", 4222, true}, {8222, "monitor", "nats-monitor-expose", 8222, false}},
 	"kafka":      {{19092, "kafka", "kafka-provision", 49092, true}},
-	"rabbitmq":   {{5672, "amqp", "rmq-expose", 55672, true}, {15672, "management", "rmq-mgmt-expose", 55673, false}},
+	"rabbitmq":   {{5672, "amqp", "rmq-expose", 5672, true}, {15672, "management", "rmq-mgmt-expose", 15672, false}},
 }
 
 // ExposableEngine reports whether an engine has a defined host-expose port set.
@@ -58,6 +70,95 @@ func ExposableEngine(engine string) bool {
 	return ok
 }
 
+// primaryExposePort returns an engine's PRIMARY host-published port — the one a
+// client (and devstack's own host-side provisioning) connects the engine's main
+// protocol on. This is the single source of truth for "the host port of engine
+// X": provisioning, reset, snapshot and resource ops all resolve their admin
+// endpoint from it, so there is exactly ONE host port per engine (the standard
+// one), never a separate provisioning band.
+func primaryExposePort(engine string) (exposePort, bool) {
+	for _, ep := range exposeEngines[engine] {
+		if ep.primary {
+			return ep, true
+		}
+	}
+	return exposePort{}, false
+}
+
+// exposableUnion returns the shared instances to publish: the requested set
+// unioned with any already-exposed instance (so writing the overlay never drops
+// another instance's ports), filtered to engines that support exposure. Sorted
+// for a byte-stable overlay.
+func exposableUnion(d UpDeps, want []string) []string {
+	set := map[string]bool{}
+	for _, i := range want {
+		set[i] = true
+	}
+	for _, i := range exposedInstances(d.Model.Root) {
+		set[i] = true
+	}
+	var insts []string
+	for i := range set {
+		if s, ok := d.Model.Workspace.Shared[i]; ok && ExposableEngine(s.Template) {
+			insts = append(insts, i)
+		}
+	}
+	sort.Strings(insts)
+	return insts
+}
+
+// exposeOverlayFor allocates the standard host ports for the exposable instances
+// among want (unioned with the currently-exposed set) and WRITES the single
+// expose overlay, returning its path ("" when there is nothing to expose). It does
+// NOT run compose — the caller (the shared phase) folds the returned path into its
+// own `compose up` so ports are published as the services come up.
+func exposeOverlayFor(ctx context.Context, d UpDeps, want []string) (string, error) {
+	insts := exposableUnion(d, want)
+	if len(insts) == 0 {
+		return "", nil
+	}
+	_, pub, err := allocateExposePorts(ctx, d, insts)
+	if err != nil {
+		return "", err
+	}
+	return writeExposeOverlay(d.Model.Root, pub)
+}
+
+// ensureExposed is the unified host-reachability primitive for callers that need
+// the ports published NOW (provisioning, reset, snapshot, resource ops): it writes
+// the single expose overlay for the exposable instances among want (unioned with
+// the already-exposed set, so it never drops another instance's ports) and applies
+// it via `compose up`. It is idempotent — the ledger returns the same standard
+// ports and the overlay bytes are unchanged, so compose does not recreate the
+// container on repeat calls. Returns instance→primary host port. Because both
+// auto-expose and every host-side admin op go through this one overlay, they can
+// never fight over a container's `ports:`.
+func ensureExposed(ctx context.Context, d UpDeps, want []string) (map[string]int, error) {
+	insts := exposableUnion(d, want)
+	if len(insts) == 0 {
+		return map[string]int{}, nil
+	}
+	out, pub, err := allocateExposePorts(ctx, d, insts)
+	if err != nil {
+		return nil, err
+	}
+	overlay, err := writeExposeOverlay(d.Model.Root, pub)
+	if err != nil {
+		return nil, err
+	}
+	outDir := filepath.Join(d.Model.Root, generate.GenDir, "shared")
+	if err := composeUpShared(ctx, d, outDir, []string{overlay}, insts); err != nil {
+		return nil, fmt.Errorf("apply host-port overlay: %w", err)
+	}
+	ports := map[string]int{}
+	for _, ep := range out {
+		if ep.Primary {
+			ports[ep.Instance] = ep.Port
+		}
+	}
+	return ports, nil
+}
+
 // ExposedPort is one host-published shared-service port with a client-ready
 // connection hint (the `--json` schema + the plain-table source).
 type ExposedPort struct {
@@ -284,6 +385,18 @@ func connectionURL(engine string, ep exposePort, params map[string]any, port int
 		user := paramString(params, "rootUser", "devstack")
 		pass := paramString(params, "rootPassword", "devstack")
 		return fmt.Sprintf("postgres://%s:%s@%s/postgres?sslmode=disable", user, pass, host)
+	case "mysql", "mariadb":
+		user := paramString(params, "rootUser", "devstack")
+		pass := paramString(params, "rootPassword", "devstack")
+		return fmt.Sprintf("mysql://%s:%s@%s/%s", user, pass, host, user)
+	case "mongodb":
+		user := paramString(params, "rootUser", "devstack")
+		pass := paramString(params, "rootPassword", "devstack")
+		return fmt.Sprintf("mongodb://%s:%s@%s/?authSource=admin", user, pass, host)
+	case "cassandra":
+		return host // contact point host:9042 (CQL native transport)
+	case "arangodb":
+		return "http://" + host // HTTP API + web UI (root / rootPassword)
 	case "redis":
 		return "redis://" + host
 	case "minio":

internal/orchestrate/expose_test.go

@@ -93,16 +93,21 @@ func TestConnectionURL(t *testing.T) {
 		port   int
 		want   string
 	}{
-		{"postgres", exposePort{5432, "postgres", "", 0, true}, pgParams, 55432, "postgres://admin:s3cret@127.0.0.1:55432/postgres?sslmode=disable"},
-		{"postgres", exposePort{5432, "postgres", "", 0, true}, nil, 55432, "postgres://devstack:devstack@127.0.0.1:55432/postgres?sslmode=disable"},
-		{"redis", exposePort{6379, "redis", "", 0, true}, nil, 56379, "redis://127.0.0.1:56379"},
-		{"minio", exposePort{9000, "s3", "", 0, true}, nil, 59000, "http://127.0.0.1:59000"},
-		{"localstack", exposePort{4566, "aws", "", 0, true}, nil, 54566, "http://127.0.0.1:54566"},
-		{"nats", exposePort{8222, "monitor", "", 0, false}, nil, 58222, "http://127.0.0.1:58222"},
-		{"nats", exposePort{4222, "nats", "", 0, true}, nil, 54222, "nats://127.0.0.1:54222"},
+		{"postgres", exposePort{5432, "postgres", "", 0, true}, pgParams, 5432, "postgres://admin:s3cret@127.0.0.1:5432/postgres?sslmode=disable"},
+		{"postgres", exposePort{5432, "postgres", "", 0, true}, nil, 5432, "postgres://devstack:devstack@127.0.0.1:5432/postgres?sslmode=disable"},
+		{"mysql", exposePort{3306, "mysql", "", 0, true}, nil, 3306, "mysql://devstack:devstack@127.0.0.1:3306/devstack"},
+		{"mariadb", exposePort{3306, "mariadb", "", 0, true}, nil, 3306, "mysql://devstack:devstack@127.0.0.1:3306/devstack"},
+		{"mongodb", exposePort{27017, "mongodb", "", 0, true}, nil, 27017, "mongodb://devstack:devstack@127.0.0.1:27017/?authSource=admin"},
+		{"cassandra", exposePort{9042, "cassandra", "", 0, true}, nil, 9042, "127.0.0.1:9042"},
+		{"arangodb", exposePort{8529, "arangodb", "", 0, true}, nil, 8529, "http://127.0.0.1:8529"},
+		{"redis", exposePort{6379, "redis", "", 0, true}, nil, 6379, "redis://127.0.0.1:6379"},
+		{"minio", exposePort{9000, "s3", "", 0, true}, nil, 9000, "http://127.0.0.1:9000"},
+		{"localstack", exposePort{4566, "aws", "", 0, true}, nil, 4566, "http://127.0.0.1:4566"},
+		{"nats", exposePort{8222, "monitor", "", 0, false}, nil, 8222, "http://127.0.0.1:8222"},
+		{"nats", exposePort{4222, "nats", "", 0, true}, nil, 4222, "nats://127.0.0.1:4222"},
 		{"kafka", exposePort{19092, "kafka", "", 0, true}, nil, 49092, "127.0.0.1:49092"},
-		{"rabbitmq", exposePort{15672, "management", "", 0, false}, nil, 55673, "http://127.0.0.1:55673"},
-		{"rabbitmq", exposePort{5672, "amqp", "", 0, true}, nil, 55672, "amqp://devstack@127.0.0.1:55672"},
+		{"rabbitmq", exposePort{15672, "management", "", 0, false}, nil, 15672, "http://127.0.0.1:15672"},
+		{"rabbitmq", exposePort{5672, "amqp", "", 0, true}, nil, 5672, "amqp://devstack@127.0.0.1:5672"},
 	}
 	for _, tc := range cases {
 		if got := connectionURL(tc.engine, tc.ep, tc.params, tc.port); got != tc.want {
@@ -111,34 +116,48 @@ func TestConnectionURL(t *testing.T) {
 	}
 }
 
-// TestExposePortsNeverCollideWithProvision is the load-bearing invariant: the
-// expose overlay and the provision overlay must never publish the SAME host port
-// (base) for the SAME engine, or applying both recreates the container with a
-// duplicate binding. Kafka is the deliberate exception — its host clients MUST
-// use the fixed advertised 49092, so it reuses the provision port.
-func TestExposePortsNeverCollideWithProvision(t *testing.T) {
-	provBase := map[string]int{}
-	for engine, ov := range engineOverlays {
-		provBase[engine] = ov.portBase
-	}
+// TestExposeUsesStandardPorts locks in the unification: there is exactly ONE host
+// port per engine — the well-known one — and it equals the in-container port. That
+// is what lets provisioning/reset/snapshot and `expose` share a single overlay
+// instead of two fighting bands. Kafka is the deliberate exception: its broker
+// advertises a fixed 127.0.0.1:49092 external listener, so its host base is 49092
+// while the container port is 19092.
+func TestExposeUsesStandardPorts(t *testing.T) {
 	for engine, ports := range exposeEngines {
 		for _, ep := range ports {
-			if pb, ok := provBase[engine]; ok && ep.base == pb && engine != "kafka" {
-				t.Errorf("engine %q expose base %d collides with provision base %d", engine, ep.base, pb)
+			if engine == "kafka" {
+				continue
+			}
+			if ep.base != ep.container {
+				t.Errorf("engine %q port %q: host base %d must equal container port %d (standard-port unification)",
+					engine, ep.label, ep.base, ep.container)
 			}
 		}
 	}
-	// Every expose base must be unique across all engines/ports (no two services
-	// fight for the same host port at allocation time either).
-	seen := map[int]string{}
-	for _, ports := range exposeEngines {
+	// Every provisionable engine must have a PRIMARY expose port, since provisioning
+	// now resolves its host-reachable admin endpoint from that single overlay.
+	for _, engine := range []string{"postgres", "redis", "minio", "nats", "kafka", "localstack"} {
+		if _, ok := primaryExposePort(engine); !ok {
+			t.Errorf("engine %q has no primary expose port — provisioning cannot reach it", engine)
+		}
+	}
+	// Within a SINGLE engine, its ports must not share a base (else a two-port
+	// engine like minio/nats/rabbitmq would self-collide on the same host port).
+	for engine, ports := range exposeEngines {
+		seen := map[int]string{}
 		for _, ep := range ports {
-			if prev, ok := seen[ep.base]; ok && prev != ep.purpose {
-				t.Errorf("expose base %d reused across purposes %q and %q", ep.base, prev, ep.purpose)
+			if prev, ok := seen[ep.base]; ok {
+				t.Errorf("engine %q reuses expose base %d across purposes %q and %q", engine, ep.base, prev, ep.purpose)
 			}
 			seen[ep.base] = ep.purpose
 		}
 	}
+	// Across DIFFERENT engines the base MAY repeat on purpose: two engines that
+	// speak the same wire protocol want the same well-known port (mysql/mariadb on
+	// 3306, localstack/ministack on 4566). That is safe because the ledger's
+	// AllocatePort skips every already-allocated port (AllocatedPorts spans all
+	// owners), so a lone engine lands on the standard port and, when both are
+	// exposed, the second transparently deconflicts to base+1.
 }
 
 func TestFileExists(t *testing.T) {