ci+test: consolidated fail-fast CI + tests/ folder with functional & e2e CLI tests (#18)

Per request: a dedicated tests/ folder driving the built binary, a consolidated
fail-fast CI, Docker for the tests that need it, and dependency caching.

tests/ (new):
- tests/e2e — drives the compiled `devstack` CLI. All `//go:build e2e` (never in
  the fast unit lane; builds the binary once via TestMain).
  - functional (no daemon): generate + --check, config validate, template list,
    status, version, and `up` outside a workspace errors.
  - daemon e2e (Docker): a full `up → status → re-up(idempotent) → down` against
    a real Engine, gated on DEVSTACK_E2E=1 (it mutates the shared stack) with
    t.Cleanup teardown of the stack + network. Verified locally (14s, green) and
    leaves the machine clean.
- tests/README.md documents the layout + how to run.

CI (.github/workflows/ci.yml): collapse 7 jobs → 2.
- `ci`: one lane ordered cheap → expensive so a lint/unit failure fails fast
  before the costly Docker + cross-compile work: gofmt → vet → installer lint →
  static build → unit -race → smoke → determinism → cross-compile (4 targets in
  one step) → govulncheck → integration (-tags=integration -race) → e2e
  (-tags=e2e, DEVSTACK_E2E=1). setup-go caches GOMODCACHE+GOCACHE (keyed by
  go.sum); ubuntu-latest's Docker backs the integration + e2e steps.
- `release-dryrun`: kept separate (expensive, independent, needs full history) so
  it never gates the fast feedback.
- concurrency group cancels superseded runs.

Makefile: `make integration` + `make e2e` targets; help regex now matches names
with digits (e2e).

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gustavobertoi

.github/workflows/ci.yml

@@ -8,98 +8,84 @@ on:
 permissions:
   contents: read
 
+# Cancel superseded runs on the same ref (fail-fast + CI economy).
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   # Single enforced Go toolchain floor (DECISIONS, ARCHITECTURE §7.8).
   GO_VERSION: "1.25"
 
 jobs:
-  build-test:
+  # One consolidated lane ordered cheap → expensive so a lint/unit failure stops
+  # before the costly Docker + cross-compile work. actions/setup-go caches the
+  # module + build cache (keyed by go.sum). ubuntu-latest ships Docker, used by
+  # the integration + e2e steps.
+  ci:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
           check-latest: true
+          cache: true # GOMODCACHE + GOCACHE, keyed by go.sum
 
+      # --- cheap, high-signal (fail fast) ---
       - name: gofmt
         run: |
           unformatted=$(gofmt -l .)
           if [ -n "$unformatted" ]; then
             echo "These files are not gofmt-clean:"; echo "$unformatted"; exit 1
           fi
-
       - name: go vet
         run: go vet ./...
-
+      - name: installer lint (shellcheck + POSIX sh)
+        run: |
+          shellcheck --severity=warning install.sh
+          sh -n install.sh
       - name: build (CGO disabled — static binary invariant)
         run: CGO_ENABLED=0 go build ./...
-
-      - name: test -race
+      - name: unit tests -race
         env:
           CGO_ENABLED: "1" # the race detector requires cgo (see Makefile)
         run: go test -race ./...
+      - name: smoke (built binary, end-to-end in an XDG sandbox)
+        run: make smoke
+      - name: determinism (byte-identical generation)
+        run: make determinism
 
-  govulncheck:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          check-latest: true
+      # --- medium ---
+      - name: cross-compile (4 CGO-free release targets)
+        env:
+          CGO_ENABLED: "0"
+        run: |
+          for t in linux/amd64 linux/arm64 darwin/amd64 darwin/arm64; do
+            echo "→ $t"
+            GOOS=${t%/*} GOARCH=${t#*/} go build -o /dev/null ./cmd/devstack
+          done
       - name: govulncheck
         run: |
           go install golang.org/x/vuln/cmd/govulncheck@latest
           govulncheck ./...
 
-  cross-compile:
-    # Proves the 4 release targets build CGO-free from one Linux runner.
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        goos: [linux, darwin]
-        goarch: [amd64, arm64]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          check-latest: true
-      - name: build ${{ matrix.goos }}/${{ matrix.goarch }}
+      # --- expensive, real-daemon (Docker on ubuntu-latest) ---
+      - name: docker available
+        run: docker version
+      - name: integration tests (-tags=integration -race)
         env:
-          CGO_ENABLED: "0"
-          GOOS: ${{ matrix.goos }}
-          GOARCH: ${{ matrix.goarch }}
-        run: go build -o /dev/null ./cmd/devstack
-
-  determinism:
-    # Asserts the generation pipeline is byte-identical across runs/paths
-    # (spec 02 acceptance #3, ARCHITECTURE §3) — the rebuild-hash and any
-    # "commit generated artifacts" decision depend on it.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          check-latest: true
-      - name: determinism (byte-identical generation)
-        run: make determinism
-
-  installer:
-    # Lints the curl|sh installer so a broken install path is caught before release.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: shellcheck install.sh
-        run: shellcheck --severity=warning install.sh
-      - name: POSIX sh syntax
-        run: sh -n install.sh
+          CGO_ENABLED: "1"
+        run: go test -tags=integration -race ./...
+      - name: e2e tests (-tags=e2e, real up/down via the CLI)
+        env:
+          DEVSTACK_E2E: "1" # ephemeral runner: safe to mutate the shared stack
+        run: go test -tags=e2e ./tests/e2e/...
 
+  # The full release pipeline (4 CGO-free targets + archives + checksums +
+  # .deb/.rpm) — expensive and independent, so it runs in parallel and never
+  # gates the fast feedback above. Needs full history for goreleaser's versioning.
   release-dryrun:
-    # Proves the full release pipeline (4 CGO-free targets + archives + checksums
-    # + .deb/.rpm) builds, without tagging — so a tag push never fails late.
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -109,31 +95,12 @@ jobs:
         with:
           go-version: ${{ env.GO_VERSION }}
           check-latest: true
+          cache: true
       - uses: goreleaser/goreleaser-action@v6
         with:
           version: "~> v2"
           args: release --snapshot --clean
 
-  integration:
-    # Real-daemon lane (G1): runs the //go:build integration tests against
-    # ubuntu-latest's Docker (read-only inspect/logs, health polling, compose
-    # exec). Tests self-isolate with per-run names (devstack-it-<pid>) + t.Cleanup
-    # and skip gracefully if the daemon is unreachable.
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          check-latest: true
-      - name: docker available
-        run: docker version
-      - name: integration tests (-tags=integration -race)
-        env:
-          CGO_ENABLED: "1" # race detector needs cgo
-        run: go test -tags=integration -race ./...
-
   # Placeholder lanes wired as their milestones land:
-  #  - schema-drift: JSON-Schema ↔ Go-struct round-trip check (M1)
   #  - macos: macos-14 arm64 preflight-only (G2)
   #  - config-conformance: golden workspace exercising every schema field

Makefile

@@ -21,7 +21,7 @@ BINDIR      ?= $(or $(XDG_BIN_HOME),$(PREFIX)/bin)
 
 # The release binary MUST be CGO-free (static), but `go test -race` REQUIRES cgo.
 # So CGO is set per-target, never globally.
-.PHONY: build run test test-race test-one vet fmt fmt-check lint tidy vuln clean snapshot ci determinism install uninstall smoke help
+.PHONY: build run test test-race integration e2e test-one vet fmt fmt-check lint tidy vuln clean snapshot ci determinism install uninstall smoke help
 
 build: ## Build the static binary into ./dist
 	CGO_ENABLED=0 go build -trimpath -ldflags '$(LDFLAGS)' -o dist/$(BINARY) ./cmd/devstack
@@ -35,6 +35,12 @@ test: ## Run unit tests
 test-race: ## Run unit tests with the race detector (needs cgo)
 	CGO_ENABLED=1 go test -race ./...
 
+integration: ## Run the real-daemon integration tests (needs Docker + cgo)
+	CGO_ENABLED=1 go test -tags=integration -race ./...
+
+e2e: ## Run the CLI end-to-end tests (builds the binary; mutates Docker)
+	DEVSTACK_E2E=1 go test -tags=e2e ./tests/e2e/...
+
 # Run a single test, e.g.: make test-one RUN=TestSerializesAcquire PKG=./internal/lock
 test-one:
 	CGO_ENABLED=1 go test -race -run '$(RUN)' -v $(PKG)
@@ -125,4 +131,4 @@ clean:
 	rm -rf dist
 
 help:
-	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-12s\033[0m %s\n", $$1, $$2}'
+	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-12s\033[0m %s\n", $$1, $$2}'

tests/README.md

@@ -0,0 +1,28 @@
+# tests/
+
+Cross-cutting tests that exercise the **built `devstack` binary** end to end,
+complementing the per-package unit + `//go:build integration` tests under
+`internal/`.
+
+## Layout
+
+- **`e2e/`** — drives the compiled CLI in an isolated XDG sandbox + temp
+  workspace. Two tiers, both behind `//go:build e2e` (so they never run in the
+  fast unit lane and only build the binary when asked):
+  - **functional** (no daemon): `generate`, `config validate`, `template list`,
+    `status`, `version`, and error paths like `up` outside a workspace.
+  - **daemon e2e** (Docker): a full `up → status → re-up(idempotent) → down`
+    against a real Engine. Gated on `DEVSTACK_E2E=1` because it mutates Docker
+    (the shared stack + `devstack_shared` network); each test cleans up after
+    itself with `t.Cleanup`.
+
+## Running
+
+```bash
+make e2e          # DEVSTACK_E2E=1 go test -tags=e2e ./tests/e2e/...   (needs Docker)
+go test -tags=e2e ./tests/e2e/...                                      # functional only (daemon tests skip)
+make integration  # the internal/ -tags=integration suite (needs Docker)
+```
+
+CI runs all of these in the consolidated `ci` lane (cheap → expensive, fail-fast)
+on a Docker-enabled runner; see `.github/workflows/ci.yml`.

tests/e2e/cli_test.go

@@ -0,0 +1,114 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+const wsSharedPG = `apiVersion: devstack/v1
+kind: Workspace
+name: e2e
+shared:
+  postgres: { template: postgres, params: { version: "16" } }
+projects:
+  - { name: app, path: app }
+`
+
+const projNodeUsesPG = `apiVersion: devstack/v1
+kind: Project
+name: app
+services:
+  web:
+    template: node.vite
+    uses: [workspace.shared.postgres]
+`
+
+func appWorkspace() map[string]string {
+	return map[string]string{"workspace.yaml": wsSharedPG, "app/devstack.yaml": projNodeUsesPG}
+}
+
+// --- functional (no daemon) -------------------------------------------------
+
+func TestFunctional_Version(t *testing.T) {
+	s := newSandbox(t, appWorkspace())
+	if out := s.run(t, "version"); !strings.Contains(out, "commit") {
+		t.Errorf("version output = %q, want it to mention commit", out)
+	}
+}
+
+func TestFunctional_GenerateValidateTemplate(t *testing.T) {
+	s := newSandbox(t, appWorkspace())
+
+	s.run(t, "generate")
+	// Shared + project compose materialized.
+	for _, rel := range []string{".devstack/shared/docker-compose.yaml", "app/.devstack/docker-compose.yaml"} {
+		if _, err := exec.Command("test", "-f", s.ws+"/"+rel).Output(); err != nil {
+			t.Errorf("expected generated file %s", rel)
+		}
+	}
+	// --check is idempotent right after generate.
+	s.run(t, "generate", "--check")
+	// config validate passes; template list shows a built-in.
+	s.run(t, "config", "validate")
+	if out := s.run(t, "template", "list"); !strings.Contains(out, "php.laravel.nginx") {
+		t.Errorf("template list missing a built-in:\n%s", out)
+	}
+}
+
+func TestFunctional_StatusAndDoctor(t *testing.T) {
+	s := newSandbox(t, appWorkspace())
+	if out := s.run(t, "status"); !strings.Contains(out, "SHARED") {
+		t.Errorf("status output missing SHARED section:\n%s", out)
+	}
+	// doctor may exit non-zero when the daemon is down; we only assert the JSON
+	// contract is emitted.
+	out, _ := s.tryRun("doctor", "--json")
+	if !strings.Contains(out, `"checks"`) {
+		t.Errorf("doctor --json missing checks:\n%s", out)
+	}
+}
+
+func TestFunctional_UpOutsideWorkspaceErrors(t *testing.T) {
+	// A sandbox with no workspace.yaml.
+	s := newSandbox(t, map[string]string{"README": "no workspace here"})
+	if out, err := s.tryRun("up"); err == nil {
+		t.Errorf("up outside a workspace should fail; got:\n%s", out)
+	}
+}
+
+// --- daemon e2e (Docker; DEVSTACK_E2E=1) ------------------------------------
+
+func TestE2E_UpStatusDown(t *testing.T) {
+	requireDaemon(t)
+	s := newSandbox(t, appWorkspace())
+	t.Cleanup(func() {
+		_, _ = s.tryRun("down")
+		dockerComposeDown("devstack-shared")
+		dockerComposeDown("devstack-app")
+		_ = exec.Command("docker", "network", "rm", "devstack_shared").Run()
+	})
+
+	// First up: full saga, shared-postgres health-gated, project started.
+	out := s.run(t, "up")
+	if !strings.Contains(out, "[ok]") || strings.Contains(out, "[failed]") {
+		t.Fatalf("up did not complete cleanly:\n%s", out)
+	}
+
+	// Status shows the shared service with a ref from app.
+	st := s.run(t, "status")
+	if !strings.Contains(st, "shared-postgres") {
+		t.Errorf("status missing shared-postgres:\n%s", st)
+	}
+
+	// Re-run is idempotent: satisfied phases skip.
+	reup := s.run(t, "up")
+	if !strings.Contains(reup, "[skipped]") {
+		t.Errorf("re-run should skip satisfied phases:\n%s", reup)
+	}
+
+	// down stops the project + releases its ref; shared keeps running.
+	s.run(t, "down")
+}