From 01b06b777ff1faac091152a6ef02e10fff44baa6 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 12:09:23 -0500 Subject: [PATCH 01/21] test: Validate internal DNS on the shared Kubernetes environment --- .github/workflows/e2e.yml | 22 +- .github/workflows/internal-dns-package.yml | 26 + .gitignore | 3 + Taskfile.yaml | 53 +- config/crd/kustomization.yaml | 12 + config/internal-dns/README.md | 25 +- config/internal-dns/fleet.example.yaml | 8 +- config/internal-dns/nats-server.conf.example | 5 +- internal/internaldns/model/model.go | 8 +- internal/internaldns/model/model_test.go | 21 + internal/internaldns/platform/planner.go | 22 +- .../internaldns/platform/platform_test.go | 70 ++- .../runtime/deployment_permissions_test.go | 12 + internal/internaldns/serving/agent.go | 40 +- .../serving/binding_fences_test.go | 92 ++- internal/internaldns/serving/render.go | 8 +- internal/internaldns/serving/render_test.go | 25 + internal/internaldns/serving/state.go | 5 +- internal/internaldns/serving/verify_test.go | 2 +- test/internaldns/README.md | 115 ++++ test/internaldns/colima-env.sh | 24 + .../fixtures/validating-webhook.yaml.tmpl | 37 ++ test/internaldns/kubernetes/Dockerfile | 26 + .../kubernetes/Dockerfile.dockerignore | 6 + test/internaldns/kubernetes/collect.sh | 20 + .../kubernetes/commands/internal-dns-daemons | 30 + .../commands/internal-dns-flush-bind | 3 + .../kubernetes/commands/internal-dns-gate | 6 + .../commands/internal-dns-reload-bind | 11 + .../commands/internal-dns-reload-dnsdist | 10 + .../commands/internal-dns-validate-bind | 4 + .../commands/internal-dns-validate-dnsdist | 3 + test/internaldns/kubernetes/deploy.sh | 82 +++ test/internaldns/kubernetes/prepare-kind.sh | 18 + test/internaldns/kubernetes/qualify/main.go | 590 ++++++++++++++++++ .../kubernetes/qualify/scenarios.go | 467 ++++++++++++++ test/internaldns/kubernetes/render/main.go | 447 +++++++++++++ .../kubernetes/render/main_test.go | 74 +++ test/internaldns/kubernetes/reset.sh | 40 ++ test/internaldns/production-parity.md | 188 ++++++ test/internaldns/qualify-package.sh | 50 ++ test/internaldns/run.sh | 11 + 42 files changed, 2675 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/internal-dns-package.yml create mode 100644 test/internaldns/README.md create mode 100644 test/internaldns/colima-env.sh create mode 100644 test/internaldns/fixtures/validating-webhook.yaml.tmpl create mode 100644 test/internaldns/kubernetes/Dockerfile create mode 100644 test/internaldns/kubernetes/Dockerfile.dockerignore create mode 100755 test/internaldns/kubernetes/collect.sh create mode 100755 test/internaldns/kubernetes/commands/internal-dns-daemons create mode 100755 test/internaldns/kubernetes/commands/internal-dns-flush-bind create mode 100755 test/internaldns/kubernetes/commands/internal-dns-gate create mode 100755 test/internaldns/kubernetes/commands/internal-dns-reload-bind create mode 100755 test/internaldns/kubernetes/commands/internal-dns-reload-dnsdist create mode 100755 test/internaldns/kubernetes/commands/internal-dns-validate-bind create mode 100755 test/internaldns/kubernetes/commands/internal-dns-validate-dnsdist create mode 100755 test/internaldns/kubernetes/deploy.sh create mode 100755 test/internaldns/kubernetes/prepare-kind.sh create mode 100644 test/internaldns/kubernetes/qualify/main.go create mode 100644 test/internaldns/kubernetes/qualify/scenarios.go create mode 100644 test/internaldns/kubernetes/render/main.go create mode 100644 test/internaldns/kubernetes/render/main_test.go create mode 100755 test/internaldns/kubernetes/reset.sh create mode 100644 test/internaldns/production-parity.md create mode 100755 test/internaldns/qualify-package.sh create mode 100755 test/internaldns/run.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 91c613e5..c1e1a3ba 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -13,7 +13,7 @@ jobs: # Three test-infra kind clusters (upstream + control + edge), each with # Flux/cert-manager/Kyverno/Envoy Gateway, plus RustFS and a Lightningstream # sync interval the federation/full-chain suites poll for - give it headroom. - timeout-minutes: 60 + timeout-minutes: 90 env: # The Taskfile pulls in the shared datum-cloud/test-infra Taskfile as a # remote include; go-task gates that behind this experiment flag, and @@ -58,6 +58,11 @@ jobs: make chainsaw echo "$GITHUB_WORKSPACE/bin" >> "$GITHUB_PATH" + - name: Bring up private DNS on the shared Kubernetes foundation + run: | + go test ./test/internaldns/kubernetes/... + task --yes env:internal-dns:up + - name: Bring up the full DNS environment # Creates the upstream (replicator) + control (downstream agent + # PowerDNS + Lightningstream + RustFS) + edge clusters and wires the @@ -65,6 +70,9 @@ jobs: # env:up) so the suites own all zones on the shared control PowerDNS. run: task --yes env:stack-up + - name: Qualify private DNS publication and serving + run: task --yes env:internal-dns:qualify + - name: Run Chainsaw tests # Runs every suite under test/e2e against the environment: the # upstream->control replicator suites (alias, display-annotations, @@ -100,6 +108,18 @@ jobs: echo "=== RustFS logs (control) ===" kubectl --context kind-dns-control -n rustfs-system logs -l app.kubernetes.io/instance=rustfs --tail=100 || true + - name: Collect private DNS deployment evidence + if: always() + run: test/internaldns/kubernetes/collect.sh + + - name: Retain private DNS qualification evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: internal-dns-kubernetes + path: test/internaldns/results/kubernetes* + if-no-files-found: warn + - name: Tear down the environment if: always() run: task --yes env:down || true diff --git a/.github/workflows/internal-dns-package.yml b/.github/workflows/internal-dns-package.yml new file mode 100644 index 00000000..ffd47f39 --- /dev/null +++ b/.github/workflows/internal-dns-package.yml @@ -0,0 +1,26 @@ +name: Internal DNS package + +on: + pull_request: + push: + branches: [main] + +jobs: + package: + name: Qualify internal DNS runtime package + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Build and qualify the runtime package + run: test/internaldns/qualify-package.sh + - name: Retain package evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: internal-dns-package + path: test/internaldns/results/package + if-no-files-found: warn diff --git a/.gitignore b/.gitignore index b48f9e6e..7f4c1fa4 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ go.work.sum .claude/settings.local.json .claude/worktrees/ .mcp.json + +/.internal-dns-e2e/ +/test/internaldns/results/ diff --git a/Taskfile.yaml b/Taskfile.yaml index d8e56347..91136f35 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -11,9 +11,9 @@ version: '3' vars: - # Bump to the merged tag once datum-cloud/test-infra#35 lands (it removes - # the hardcoded NodePorts that block running two clusters concurrently). - TEST_INFRA_REPO_REF: 'fix/remove-hardcoded-nodeports' + # Shared bootstrap with dynamic NodePorts and an immutable revision for + # consistent qualification across environments. + TEST_INFRA_REPO_REF: '7622c925d5bcf2fa5bc01823775973cf6daf51ca' # Upstream (control-plane) cluster runs the replicator; it pushes DNSZone/ # DNSRecordSet resources to the control cluster, which plays the "downstream" # agent role for the replicator and the "control" (Lightningstream source) @@ -56,6 +56,36 @@ includes: REPO_REF: "{{.TEST_INFRA_REPO_REF}}" tasks: + env:internal-dns:up: + desc: "Deploy private DNS pods on the shared upstream/control/edge environment" + cmds: + - task: test-infra:ensure-repo + - task: test-infra:ensure-tools + - test/internaldns/kubernetes/prepare-kind.sh + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.CONTROL_CLUSTER_NAME}}" + KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.UPSTREAM_CLUSTER_NAME}}" + KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml + - task: env:test-infra-cluster-up + vars: + CLUSTER_NAME: "{{.EDGE_CLUSTER_NAME}}" + KIND_CFG: .internal-dns-e2e/kubernetes/kind-config.yaml + - test/internaldns/kubernetes/deploy.sh + + env:internal-dns:qualify: + desc: "Qualify private DNS through scoped product APIs and Kubernetes serving pods" + cmds: + - go run ./test/internaldns/kubernetes/qualify --environment .internal-dns-e2e/kubernetes/environment.json --results test/internaldns/results/kubernetes.json + + env:internal-dns:package: + desc: "Build and qualify the internal DNS runtime image and deployment contracts" + cmds: + - test/internaldns/qualify-package.sh + default: desc: "Show available commands" silent: true @@ -65,6 +95,8 @@ tasks: - echo " task env:stack-up Bring up the full chain with no sample data (used by CI)" - echo " task env:chainsaw Run the full Chainsaw e2e suite against the environment" - echo " task env:down Tear down all clusters" + - echo " task env:internal-dns:up Deploy private DNS on the shared clusters" + - echo " task env:internal-dns:qualify Validate private DNS publication and isolation" - echo " task env:dns-check Query control + edge PowerDNS for the sample record" - echo "" - echo "Individual pieces:" @@ -139,18 +171,8 @@ tasks: # internal: brings up a test-infra-managed kind cluster (kind cluster + # Flux + cert-manager + Kyverno + Envoy Gateway) for CLUSTER_NAME. # - # NOTE: this deliberately calls the individual test-infra sub-tasks - # (ensure-repo/ensure-tools/create-kind/install-components) instead of the - # composite `test-infra:cluster-up`/`up` task. As of test-infra's - # fix/remove-hardcoded-nodeports branch (and go-task 3.49/3.50), - # `cluster-up`'s trailing informational `echo "... # ..."` lines panic - # go-task's command parser ("reached EOF without closing quote") because - # go-task strips ` #...` as a shell comment even inside double quotes on - # single-line (non `|` block) cmd entries - a go-task parsing quirk, not - # a shell issue. Calling the sub-tasks directly gets the same cluster - # setup while skipping the broken trailing echoes. If that upstream - # Taskfile is fixed (or go-task fixes the quoting bug), this can go back - # to a single `task: test-infra:cluster-up` call. + # Invoke bootstrap sub-tasks directly to pass cluster variables explicitly + # and avoid the composite task's informational command parsing errors. env:test-infra-cluster-up: internal: true cmds: @@ -163,6 +185,7 @@ tasks: - task: test-infra:create-kind vars: CLUSTER_NAME: "{{.CLUSTER_NAME}}" + KIND_CFG: '{{.KIND_CFG | default ".test-infra/cluster/kind-config.yaml"}}' - task: test-infra:install-components vars: CLUSTER_NAME: "{{.CLUSTER_NAME}}" diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 760dc379..e25fc9e1 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -7,17 +7,29 @@ resources: - bases/dns.networking.miloapis.com_dnsrecordsets.yaml - bases/dns.networking.miloapis.com_dnszonediscoveries.yaml - bases/dns.networking.miloapis.com_dnscontributiongrants.yaml + - bases/dns.networking.miloapis.com_dnsmanagednamespaces.yaml + - bases/dns.networking.miloapis.com_dnsnamingpolicies.yaml + - bases/dns.networking.miloapis.com_dnsrecordcontributions.yaml + - bases/dns.networking.miloapis.com_dnsregistrations.yaml + - bases/dns.networking.miloapis.com_dnsresolveraccessbindings.yaml + - bases/dns.networking.miloapis.com_dnsresolvercontexts.yaml + - bases/dns.networking.miloapis.com_dnszoneassociations.yaml + - bases/dns.networking.miloapis.com_dnspublicationchunks.yaml + - bases/dns.networking.miloapis.com_dnspublicationmanifests.yaml + - bases/dns.networking.miloapis.com_dnspublicationownerships.yaml + - bases/dns.networking.miloapis.com_dnsresolverbindings.yaml + - bases/dns.networking.miloapis.com_dnstransportoutboxes.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/config/internal-dns/README.md b/config/internal-dns/README.md index 307f97d3..a1573c32 100644 --- a/config/internal-dns/README.md +++ b/config/internal-dns/README.md @@ -15,9 +15,9 @@ The fixed fleet has three roles: exact version over UDP and TCP, and publish a leased ACK. Contexts, access bindings, and zones add destinations, views, and zone files to -these processes. They never create workloads per VPC or zone. The legacy -`cluster` and `both` render roles remain development fallbacks and should not be -used for the multi-member production layout. +these processes. They never create workloads per VPC or zone. Use `node`, +`regional-dnsdist`, and `regional-bind` roles for the multi-member production +layout. The `cluster` and `both` roles are limited to development. Each member needs a distinct member ID, NATS identity and durable consumers, checkpoint PVC, configuration PVC, and independent watchdog lease. Treat that @@ -79,11 +79,16 @@ Regional publication proof connects directly to that member's BIND listener, supplies the exact context destination in a protected PROXYv2 header over both UDP and TCP, and verifies the installed SOA serial and fingerprint. Restrict `proxyPeers` to regional dnsdist and the member's local publication agent -network. There are no PowerDNS views, network variants, or per-context source -markers. +network. ## Transactions, health, and expiry +An attached zone that has no active publication gates its context until the +configuration is complete. Pending contexts deny queries, recursion, and cache +answers while retaining the original access authorization. Configuration +completion can restore service without renewing authorization; an actual access +withdrawal still requires a newer source authorization to reactivate. + The agent writes a complete candidate set into its stage directory. Regional zone files are siblings of `cluster-bind.conf`; validation rewrites file references to the staged paths and runs `named-checkconf -z`. Installation is @@ -116,6 +121,11 @@ filesystem whose fsync latency can consume the watchdog budget. Replication is provided by the durable publication stream and independent regional members; each member still has its own crash-safe checkpoint volume. +The transaction stage and active configuration must share one filesystem: +installation uses atomic rename. The examples put the stage at +`/etc/internal-dns/stage` on the configuration PVC and keep checkpoints on the +separate state PVC. + The regional BIND reload helper must reload existing primary zones, for example with `rndc reload`. `rndc reconfig` only guarantees configuration and new-zone loading, so it is insufficient when an existing private apex moves to @@ -151,3 +161,8 @@ The webhook validates live references and lease limits. It also verifies that changes to a publisher's contribution and freshness observation use that publisher's current grant. Publication-result updates cannot renew a publisher's observation. Resource permissions do not depend on webhook principal lists. +The supported internal envelope version is `v1alpha3` and the serving checkpoint +format is 4. Unsupported versions fail closed. Deploy matching controller and +fleet versions with fresh internal DNS transport state and checkpoints. Preserve +any existing state for recovery. The deployment requires a coordinated rollout; +it does not support mixed versions or automatic state conversion. diff --git a/config/internal-dns/fleet.example.yaml b/config/internal-dns/fleet.example.yaml index 19ef501f..66c96aa9 100644 --- a/config/internal-dns/fleet.example.yaml +++ b/config/internal-dns/fleet.example.yaml @@ -12,7 +12,7 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"node","nodeDNSDist":{"path":"/etc/internal-dns/node-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:100::/64"],"poolName":"node_shared","backends":[{"memberID":"node-bind-0","address":"fd70:10::10","port":5300}]},"nodeBIND":{"path":"/etc/internal-dns/node-bind.conf","listenAddress":"fd70:10::10","port":5300,"proxyPeers":["fd70:10::20"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["node","{stageDir}/node-dnsdist.conf"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]},{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["node","{stageDir}/node-bind.conf"],"whenChanged":["/etc/internal-dns/node-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["node"],"whenChanged":["/etc/internal-dns/node-bind.conf"]},{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["node"],"whenChanged":["/etc/internal-dns/node-dnsdist.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["node","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"node-us-central1-0","replicaID":"node-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["node"]}]} @@ -29,7 +29,7 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"resolver", "render":{"role":"regional-dnsdist","clusterDNSDist":{"path":"/etc/internal-dns/cluster-dnsdist.conf","listenAddress":"[::]:53","acls":["fd71:200::/64"],"poolName":"regional_shared","backends":[{"memberID":"regional-bind-us-central1-0","address":"fd70:20::10","port":5300},{"memberID":"regional-bind-us-central1-1","address":"fd70:20::11","port":5300}]}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-dnsdist","args":["regional","{stageDir}/cluster-dnsdist.conf"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-dnsdist","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-dnsdist.conf"]}]}, "expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"localServingLease":90000000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"resolverDNSProbe":{"timeout":3000000000},"resolverVerifyConcurrency":16 }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-front-us-central1-0","replicaID":"regional-front-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-front"]}]} @@ -46,7 +46,7 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::10","port":5300,"proxyPeers":["fd70:20::20","fd70:20::30"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::10]:5300","timeout":3000000000} }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-0","replicaID":"regional-bind-us-central1-0","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} @@ -66,7 +66,7 @@ data: "agent":{ "region":"us-central1","shard":"shared-0","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","stateDir":"/var/lib/internal-dns","mode":"combined", "render":{"role":"regional-bind","clusterBIND":{"path":"/etc/internal-dns/cluster-bind.conf","listenAddress":"fd70:20::11","port":5300,"proxyPeers":["fd70:20::20","fd70:20::31"],"defaultCacheSize":"2M","maxNegativeCacheTTLSeconds":5,"dnssecValidation":"auto","rndcIncludePath":"/etc/bind/rndc.key","controlAddress":"127.0.0.1","controlPort":9953,"controlKeyName":"internal-dns"}}, - "transaction":{"stageDir":"/var/lib/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, + "transaction":{"stageDir":"/etc/internal-dns/stage","validate":[{"path":"/usr/local/sbin/internal-dns-validate-bind","args":["regional","-z","{stageDir}/cluster-bind.conf"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}],"reload":[{"path":"/usr/local/sbin/internal-dns-reload-bind","args":["regional"],"whenChanged":["/etc/internal-dns/cluster-bind.conf"]}]}, "cacheFlush":[{"path":"/usr/local/sbin/internal-dns-flush-bind","args":["regional","{name}"]}],"expiryInterval":1000000000,"retryDelay":1000000000,"ackInterval":30000000000,"ackLease":90000000000,"ackPublishTimeout":500000000,"maxBindings":200,"watchdogLeasePath":"/run/internal-dns/lease.json","watchdogLease":5000000000,"externalWatchdog":true,"publicationDNSProbe":{"server":"[fd70:20::11]:5300","timeout":3000000000} }, "watchdog":{"leasePath":"/run/internal-dns/lease.json","memberID":"regional-bind-us-central1-1","replicaID":"regional-bind-us-central1-1","maxLeaseSeconds":5,"startupGraceSeconds":0,"intervalMillis":250,"failClosed":[{"path":"/usr/local/sbin/internal-dns-gate","args":["regional-bind"]}]} diff --git a/config/internal-dns/nats-server.conf.example b/config/internal-dns/nats-server.conf.example index c75850a0..7ef46515 100644 --- a/config/internal-dns/nats-server.conf.example +++ b/config/internal-dns/nats-server.conf.example @@ -20,7 +20,10 @@ authorization { permissions: { publish: { allow: [ "dns.private.serving.>", "dns.private.records.>", - "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", "_INBOX.>" + "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", + # Match the control-plane ACK durable for this region and shard. + "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>", + "_INBOX.>" ] } subscribe: { allow: ["dns.private.acks.>", "_INBOX.>"] } } diff --git a/internal/internaldns/model/model.go b/internal/internaldns/model/model.go index ae4785fa..aef57230 100644 --- a/internal/internaldns/model/model.go +++ b/internal/internaldns/model/model.go @@ -17,7 +17,7 @@ import ( "github.com/miekg/dns" ) -const APIVersion = "internal.dns.networking.miloapis.com/v1alpha2" +const APIVersion = "internal.dns.networking.miloapis.com/v1alpha3" type Kind string @@ -112,6 +112,9 @@ type Binding struct { ClusterBackends []Backend `json:"clusterBackends"` Authorization Authorization `json:"authorization"` Tombstone bool `json:"tombstone,omitempty"` + // ConfigurationPending gates queries while attached publications are absent. + // It does not revoke or advance the source authorization. + ConfigurationPending bool `json:"configurationPending,omitempty"` } func (b Binding) ViewName() string { @@ -192,6 +195,9 @@ func (s ServingSnapshot) Validate() error { if b.Port == 0 || len(b.Transports) == 0 || b.Authorization.IssuerEpoch == 0 || b.Authorization.Revision == 0 || b.Authorization.ValidUntil.IsZero() { return fmt.Errorf("binding %q has incomplete transport or authorization", b.BindingUID) } + if b.ConfigurationPending && len(b.Zones) != 0 { + return fmt.Errorf("binding %q pending configuration must not expose partial zones", b.BindingUID) + } if err := validateTransports(b.Transports); err != nil { return fmt.Errorf("binding %q: %w", b.BindingUID, err) } diff --git a/internal/internaldns/model/model_test.go b/internal/internaldns/model/model_test.go index ef88d0fb..8cdef63c 100644 --- a/internal/internaldns/model/model_test.go +++ b/internal/internaldns/model/model_test.go @@ -91,3 +91,24 @@ func TestEnvelopeRejectsObsoleteWireVersion(t *testing.T) { t.Fatal("obsolete transport version accepted") } } + +func TestPendingConfigurationCannotExposePartialZoneAttachments(t *testing.T) { + now := time.Now().UTC() + binding := Binding{BindingUID: "binding", ProjectUID: "project", ContextUID: "context", BindingGeneration: 1, ConfigurationRevision: 1, ConsumerAddress: "fd53::a", ClusterAddress: "fd54::a", Port: 53, Transports: []Transport{TransportUDP}, Authorization: Authorization{IssuerEpoch: 1, Revision: 1, ValidUntil: now.Add(time.Minute)}, ConfigurationPending: true, Zones: []ZoneAttachment{{ZoneUID: "zone", Apex: "private.internal", RequiredPublicationEpoch: 1, RequiredPublicationRevision: 1}}} + snapshot := ServingSnapshot{Region: "east", Shard: "shared", ConfigurationEpoch: 1, ConfigurationRevision: 1, GeneratedAt: now, Bindings: []Binding{binding}} + if err := snapshot.Validate(); err == nil { + t.Fatal("partial pending context accepted") + } + snapshot.Bindings[0].Zones = nil + if err := snapshot.Validate(); err != nil { + t.Fatal(err) + } + event, err := NewEnvelope(KindServingSnapshot, "pending", "east", "shared", "shard", 1, 1, now, snapshot) + if err != nil { + t.Fatal(err) + } + event.APIVersion = "internal.dns.networking.miloapis.com/v1alpha2" + if err := event.Validate(); err == nil { + t.Fatal("pre-pending wire format accepted") + } +} diff --git a/internal/internaldns/platform/planner.go b/internal/internaldns/platform/planner.go index 1c7ddd2f..e5c974b6 100644 --- a/internal/internaldns/platform/planner.go +++ b/internal/internaldns/platform/planner.go @@ -150,15 +150,19 @@ func (p *Planner) Step(ctx context.Context) error { view.Transports = append(view.Transports, model.Transport(t)) } complete := true + bindingDependencies := map[string]bool{} for _, zoneRef := range b.Configuration.ZoneRefs { uid := zoneRef.UID o, ok := byZone[string(uid)] if !ok || o.Spec.ActiveManifestName == "" { complete = false - break + continue } var m dnsv1alpha1.DNSPublicationManifest - if err := p.Client.Get(ctx, client.ObjectKey{Namespace: c.Namespace, Name: o.Spec.ActiveManifestName}, &m); err != nil { + if err := p.Client.Get(ctx, client.ObjectKey{Namespace: c.Namespace, Name: o.Spec.ActiveManifestName}, &m); apierrors.IsNotFound(err) { + complete = false + continue + } else if err != nil { return err } targeted := false @@ -170,7 +174,7 @@ func (p *Planner) Step(ctx context.Context) error { } if m.Spec.ZoneRef.UID != uid || m.Spec.Tombstone || !targeted { complete = false - break + continue } attachment := model.ZoneAttachment{ZoneUID: string(uid), Apex: m.Spec.ZoneApex, RequiredPublicationEpoch: uint64(m.Spec.WriterEpoch), RequiredPublicationRevision: uint64(m.Spec.Revision)} // Record updates do not change a view's minimum activation dependency. @@ -184,10 +188,18 @@ func (p *Planner) Step(ctx context.Context) error { } activationName := model.PublicationActivationName(m.Name, c.Region, c.Shard) view.Zones = append(view.Zones, attachment) - dependencies[activationName] = true + bindingDependencies[activationName] = true } if !complete { - continue + // Missing configuration is not a withdrawal of source authorization. + // Retain the identity but expose no partial or removed zone set. The + // pending snapshot can publish without waiting on this tenant's outbox. + view.ConfigurationPending = true + view.Zones = nil + } else { + for name := range bindingDependencies { + dependencies[name] = true + } } sort.Slice(view.Zones, func(i, j int) bool { return view.Zones[i].ZoneUID < view.Zones[j].ZoneUID }) snapshot.Bindings = append(snapshot.Bindings, view) diff --git a/internal/internaldns/platform/platform_test.go b/internal/internaldns/platform/platform_test.go index 52221e90..b14b1767 100644 --- a/internal/internaldns/platform/platform_test.go +++ b/internal/internaldns/platform/platform_test.go @@ -142,6 +142,41 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { if len(out.Items) != 1 { t.Fatal("unchanged desired state generated a new configuration revision") } + var changing dnsv1alpha1.DNSResolverBinding + if err := cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "binding-1"}, &changing); err != nil { + t.Fatal(err) + } + originalAuthorization := changing.Spec.Authorization + changing.Spec.Configuration.Revision++ + changing.Spec.Configuration.ZoneRefs = append(changing.Spec.Configuration.ZoneRefs, dnsv1alpha1.DNSObjectReference{UID: "zone-extra"}) + if err := cl.Update(context.Background(), &changing); err != nil { + t.Fatal(err) + } + if err := p.Step(context.Background()); err != nil { + t.Fatal(err) + } + pending := latestPlannerSnapshot(t, &p) + if len(pending.Bindings) != 2 || !pending.Bindings[0].ConfigurationPending || pending.Bindings[0].Tombstone || len(pending.Bindings[0].Zones) != 0 || pending.Bindings[1].ConfigurationPending { + t.Fatal("incomplete zone addition withdrew authorization, exposed partial zones, or gated the healthy project") + } + if pending.Bindings[0].Authorization.Revision != uint64(originalAuthorization.Sequence) || !pending.Bindings[0].Authorization.ValidUntil.Equal(originalAuthorization.ValidUntil.Time) { + t.Fatal("pending configuration renewed the original authorization") + } + for _, obj := range []client.Object{ + &dnsv1alpha1.DNSPublicationOwnership{ObjectMeta: metav1.ObjectMeta{Name: "owner-" + model.OpaqueToken("zone-extra")[:20], Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationOwnershipSpec{ZoneUID: "zone-extra", ActiveManifestName: "manifest-extra"}}, + &dnsv1alpha1.DNSPublicationManifest{ObjectMeta: metav1.ObjectMeta{Name: "manifest-extra", Namespace: ns}, Spec: dnsv1alpha1.DNSPublicationManifestSpec{ZoneRef: dnsv1alpha1.DNSObjectReference{UID: "zone-extra"}, ZoneApex: "apps.internal", WriterEpoch: 1, Revision: 1, ServingTargets: []dnsv1alpha1.DNSPublicationTarget{{Region: region, Shard: shard}}}}, + } { + if err := cl.Create(context.Background(), obj); err != nil { + t.Fatal(err) + } + } + if err := p.Step(context.Background()); err != nil { + t.Fatal(err) + } + ready := latestPlannerSnapshot(t, &p) + if ready.Bindings[0].ConfigurationPending || len(ready.Bindings[0].Zones) != 2 || ready.Bindings[0].Authorization != pending.Bindings[0].Authorization { + t.Fatal("completed zone addition did not recover with the original authorization") + } var untargeted dnsv1alpha1.DNSPublicationManifest if err := cl.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: "manifest-1"}, &untargeted); err != nil { t.Fatal(err) @@ -154,7 +189,7 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { t.Fatal(err) } _ = cl.List(context.Background(), &out) - if len(out.Items) != 2 { + if len(out.Items) != 4 { t.Fatal("regional export removal did not change the shared snapshot") } var updatedOwner corev1.ConfigMap @@ -169,19 +204,46 @@ func TestSharedPlannerPreservesOtherProjects(t *testing.T) { } _ = json.Unmarshal(current.Spec.Payload, &env) _ = json.Unmarshal(env.Payload, &snapshot) - if len(snapshot.Bindings) != 1 || snapshot.Bindings[0].ContextUID != "vpc-2" { - t.Fatal("untargeted zone entered snapshot or removed another project") + if len(snapshot.Bindings) != 2 || !snapshot.Bindings[0].ConfigurationPending || len(snapshot.Bindings[0].Zones) != 0 || snapshot.Bindings[1].ContextUID != "vpc-2" || snapshot.Bindings[1].ConfigurationPending { + t.Fatal("untargeted zone remained attached, withdrew access, or gated another project") } p.Config.Identity = "retired-other-owner" if err := p.Step(context.Background()); err != nil { t.Fatal(err) } _ = cl.List(context.Background(), &out) - if len(out.Items) != 2 { + if len(out.Items) != 4 { t.Fatal("another holder published during a current lease") } } +func latestPlannerSnapshot(t *testing.T, p *Planner) model.ServingSnapshot { + t.Helper() + ctx := context.Background() + var owner corev1.ConfigMap + key := client.ObjectKey{Namespace: p.Config.Namespace, Name: "dns-shard-owner-" + model.OpaqueToken(p.Config.Region+"/"+p.Config.Shard)} + if err := p.Client.Get(ctx, key, &owner); err != nil { + t.Fatal(err) + } + var state shardState + if err := json.Unmarshal([]byte(owner.Data["state"]), &state); err != nil { + t.Fatal(err) + } + var out dnsv1alpha1.DNSTransportOutbox + if err := p.Client.Get(ctx, client.ObjectKey{Namespace: p.Config.Namespace, Name: state.ActiveOutbox}, &out); err != nil { + t.Fatal(err) + } + var env model.Envelope + if err := json.Unmarshal(out.Spec.Payload, &env); err != nil { + t.Fatal(err) + } + var snapshot model.ServingSnapshot + if err := json.Unmarshal(env.Payload, &snapshot); err != nil { + t.Fatal(err) + } + return snapshot +} + type recordingPublisher struct{ events []model.Envelope } func (p *recordingPublisher) Publish(_ context.Context, _ string, e model.Envelope) (transport.PublishAck, error) { diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index 093eee6c..ac3b622b 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -178,6 +178,18 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { t.Fatal(err) } contents := string(data) + start := strings.Index(contents, `user: "internal-dns-control-plane"`) + if start < 0 { + t.Fatal("missing control-plane broker identity") + } + controlPolicy := strings.SplitN(contents[start:], "\n },", 2)[0] + controlACK := "$JS.ACK.DNS_PRIVATE." + durable(roleControlPlane, "us-central1", "shared-0", "acks") + ".>" + if !strings.Contains(controlPolicy, `"`+controlACK+`"`) { + t.Errorf("control plane cannot acknowledge its member-ACK consumer: %s", controlACK) + } + if strings.Contains(controlPolicy, `"$JS.ACK.DNS_PRIVATE.>"`) { + t.Error("control plane can acknowledge another serving member's consumers") + } for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-bind-us-central1-0", "regional-bind-us-central1-1"} { start := strings.Index(contents, `user: "`+member+`"`) if start < 0 { diff --git a/internal/internaldns/serving/agent.go b/internal/internaldns/serving/agent.go index 18c3b9f8..dbd40ed7 100644 --- a/internal/internaldns/serving/agent.go +++ b/internal/internaldns/serving/agent.go @@ -512,6 +512,10 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving if err != nil { return err } + if err := a.flushRemovedBindingZones(ctx, snapshot); err != nil { + _ = a.failClosed(ctx) + return err + } a.state.BindingFences = bindingFences a.state.ServingFence = fence{snapshot.ConfigurationEpoch, snapshot.ConfigurationRevision} a.state.Snapshot = &snapshot @@ -525,6 +529,33 @@ func (a *Agent) applyServingSnapshot(ctx context.Context, snapshot model.Serving return a.finishEvent(ctx) } +// A configuration change can detach a zone without changing its publication. +// Clear its old cache tree before accepting the mapping, including when the +// context is temporarily pending. The cache hook must flush descendants too. +func (a *Agent) flushRemovedBindingZones(ctx context.Context, snapshot model.ServingSnapshot) error { + if a.state.Snapshot == nil { + return nil + } + attached := map[string]bool{} + for _, binding := range snapshot.Bindings { + for _, zone := range binding.Zones { + attached[binding.BindingUID+"/"+zone.ZoneUID+"/"+zone.Apex] = true + } + } + flushed := map[string]bool{} + for _, binding := range a.state.Snapshot.Bindings { + for _, zone := range binding.Zones { + if !attached[binding.BindingUID+"/"+zone.ZoneUID+"/"+zone.Apex] && !flushed[zone.Apex] { + if err := a.flushName(ctx, zone.Apex); err != nil { + return err + } + flushed[zone.Apex] = true + } + } + } + return nil +} + func (a *Agent) handleChunk(env model.Envelope) error { var c model.PublicationChunk if err := json.Unmarshal(env.Payload, &c); err != nil { @@ -968,7 +999,7 @@ func (a *Agent) reconcile(ctx context.Context) error { } } for _, b := range a.state.Snapshot.Bindings { - if b.Tombstone { + if b.Tombstone || b.ConfigurationPending { continue } p := model.AckVerified @@ -1071,6 +1102,9 @@ func (a *Agent) verifyResolverBindings(ctx context.Context, bindings []model.Bin } func (a *Agent) publicationsReady(b model.Binding) bool { + if b.ConfigurationPending { + return false + } if len(b.Zones) == 0 { return true } @@ -1090,7 +1124,7 @@ func (a *Agent) publicationsReady(b model.Binding) bool { // plan version this resolver holds. One zone per different member is unsafe. func (a *Agent) eligibleRegionalMembers(b model.Binding) map[string]bool { out := map[string]bool{} - if a.state.Snapshot == nil { + if a.state.Snapshot == nil || b.ConfigurationPending { return out } if a.cfg.Mode == ModeCombined { @@ -1155,7 +1189,7 @@ func (a *Agent) publicationViewsAt(p publicationState, now time.Time) []Publicat } var out []PublicationView for _, b := range a.state.Snapshot.Bindings { - if b.Tombstone || !now.Before(b.Authorization.ValidUntil) { + if b.Tombstone || b.ConfigurationPending || !now.Before(b.Authorization.ValidUntil) { continue } for _, z := range b.Zones { diff --git a/internal/internaldns/serving/binding_fences_test.go b/internal/internaldns/serving/binding_fences_test.go index b398cc6e..bc065561 100644 --- a/internal/internaldns/serving/binding_fences_test.go +++ b/internal/internaldns/serving/binding_fences_test.go @@ -149,7 +149,7 @@ func TestBindingOmissionFenceSurvivesRestartAndRequiresNewSourceAuthority(t *tes func TestCheckpointRejectsPreBindingFenceFormatWithoutDiscardingIt(t *testing.T) { t.Parallel() - for _, version := range []int{2, checkpointFormatVersion} { + for _, version := range []int{2, 3, checkpointFormatVersion} { t.Run(string(rune('0'+version)), func(t *testing.T) { path := filepath.Join(t.TempDir(), "checkpoint.json") data, err := json.Marshal(map[string]any{"formatVersion": version, "contributionFences": map[string]any{"original-observation": map[string]any{"epoch": 1, "sequence": 2}}}) @@ -169,3 +169,93 @@ func TestCheckpointRejectsPreBindingFenceFormatWithoutDiscardingIt(t *testing.T) }) } } + +func TestPendingConfigurationGatesQueriesAndRecoversWithoutAuthorizationRenewal(t *testing.T) { + t.Parallel() + now := time.Now().UTC() + cfg := regionalTestConfig(t) + cfg.CacheFlush = []Command{{Path: "flush", Args: []string{"{name}"}}} + runner := &recordingRunner{} + dependencies := Dependencies{Runner: runner, Clock: func() time.Time { return now }} + agent, err := NewAgent(cfg, dependencies) + if err != nil { + t.Fatal(err) + } + active := testSnapshot(now) + healthy := active.Bindings[0] + healthy.BindingUID, healthy.ProjectUID, healthy.ContextUID = "binding-b", "project-b", "context-b" + healthy.ConsumerAddress, healthy.ClusterAddress = "10.253.0.52", "10.253.0.42" + healthy.Zones = nil + active.Bindings = append(active.Bindings, healthy) + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 1, active)); err != nil { + t.Fatal(err) + } + chunk, manifest := testPublication(t, now) + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationChunk, "zone-a", 2, 4, chunk)); err != nil { + t.Fatal(err) + } + if err := agent.Handle(context.Background(), envelope(t, model.KindPublicationManifest, "zone-a", 2, 4, manifest)); err != nil { + t.Fatal(err) + } + original := active.Bindings[0].Authorization + pending := active + pending.Bindings = append([]model.Binding(nil), active.Bindings...) + pending.ConfigurationRevision = 2 + pending.Bindings[0].ConfigurationRevision = 2 + pending.Bindings[0].ConfigurationPending = true + pending.Bindings[0].Zones = nil + priorCalls := len(runner.calls) + if err := agent.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, pending)); err != nil { + t.Fatal(err) + } + if agent.state.BindingFences["binding-a"].Tombstone || agent.publicationsReady(pending.Bindings[0]) || !agent.publicationsReady(healthy) { + t.Fatal("pending configuration revoked access, activated incomplete data, or gated another context") + } + flushed := false + for _, command := range runner.calls[priorCalls:] { + if command.Path == "flush" && len(command.Args) == 1 && command.Args[0] == "prod.internal." { + flushed = true + } + } + if !flushed { + t.Fatal("detached zone retained its cached names") + } + config, err := os.ReadFile(cfg.Render.ClusterBIND.Path) + if err != nil { + t.Fatal(err) + } + start := strings.Index(string(config), `view "`+pending.Bindings[0].ViewName()+`"`) + if start < 0 { + t.Fatal("pending context missing") + } + view := strings.SplitN(string(config)[start:], "};\n\n", 2)[0] + for _, denied := range []string{"recursion no;", "allow-query { none; };", "allow-recursion { none; };", "allow-query-cache { none; };"} { + if !strings.Contains(view, denied) { + t.Fatalf("pending view can answer queries: missing %s", denied) + } + } + if strings.Contains(view, `zone "prod.internal"`) { + t.Fatal("pending configuration retained a private zone") + } + restarted, err := NewAgent(cfg, dependencies) + if err != nil { + t.Fatal(err) + } + ready := active + ready.Bindings = append([]model.Binding(nil), active.Bindings...) + ready.ConfigurationRevision = 3 + ready.Bindings[0].ConfigurationRevision = 2 + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 3, ready)); err != nil { + t.Fatalf("completion required a new authorization: %v", err) + } + if restarted.state.BindingFences["binding-a"].Authorization != original || !restarted.publicationsReady(ready.Bindings[0]) { + t.Fatal("completion changed authority or remained gated") + } + // The older pending snapshot cannot undo the completed configuration. + if err := restarted.Handle(context.Background(), envelope(t, model.KindServingSnapshot, "shard", 1, 2, pending)); err != nil { + t.Fatal(err) + } + if restarted.state.Snapshot.Bindings[0].ConfigurationPending { + t.Fatal("stale pending replay gated the recovered context") + } +} diff --git a/internal/internaldns/serving/render.go b/internal/internaldns/serving/render.go index 4037ae69..05d0a25a 100644 --- a/internal/internaldns/serving/render.go +++ b/internal/internaldns/serving/render.go @@ -234,7 +234,7 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m fmt.Fprintf(&out, "local server = newServer({address=%q, name=%q, pool=%q, useProxyProtocol=true, checkName=\".\", checkType=\"SOA\", mustResolve=false})\n", net.JoinHostPort(b.Address, strconv.Itoa(int(b.Port))), "b_"+model.OpaqueToken(b.MemberID), c.PoolName) if cluster { for _, binding := range bindings { - if active[binding.BindingUID] && (eligible == nil || eligible[binding.BindingUID][b.MemberID]) { + if !binding.ConfigurationPending && active[binding.BindingUID] && (eligible == nil || eligible[binding.BindingUID][b.MemberID]) { fmt.Fprintf(&out, "server:addPool(%q)\n", binding.ViewName()) } } @@ -246,7 +246,7 @@ func renderDNSDist(c DNSDistRenderConfig, backends []model.Backend, bindings []m if cluster { addr = b.ClusterAddress } - fmt.Fprintf(&out, " [%q] = {context=%q, validUntil=%d, ready=%t},\n", addr, b.ViewName(), b.Authorization.ValidUntil.Unix(), active[b.BindingUID]) + fmt.Fprintf(&out, " [%q] = {context=%q, validUntil=%d, ready=%t},\n", addr, b.ViewName(), b.Authorization.ValidUntil.Unix(), !b.ConfigurationPending && active[b.BindingUID]) } fmt.Fprint(&out, "}\n\nfunction classifyVPC(dq)\n local binding = bindings[dq.localaddr:toString()]\n if binding == nil or os.time() >= binding.validUntil then\n return DNSAction.Refused, \"\"\n end\n dq:setTag(\"vpc-context\", binding.context)\n if not binding.ready then dq:setTag(\"context-unready\", \"true\") end\n return DNSAction.None, \"\"\nend\n\n") fmt.Fprint(&out, "addAction(AllRule(), LuaAction(classifyVPC))\n") @@ -337,6 +337,10 @@ func renderClusterBIND(c BINDRenderConfig, bindings []model.Binding, publication } func renderViewStart(out *bytes.Buffer, b model.Binding, dest, defaultCache string, maxNegativeCacheTTL uint32) { + if b.ConfigurationPending { + fmt.Fprintf(out, "view %q {\n match-clients { any; };\n match-destinations { %s; };\n recursion no;\n allow-query { none; };\n allow-recursion { none; };\n allow-query-cache { none; };\n", b.ViewName(), dest) + return + } cache := defaultCache if cache == "" { cache = "16M" diff --git a/internal/internaldns/serving/render_test.go b/internal/internaldns/serving/render_test.go index e2b8d0e7..0b2914ec 100644 --- a/internal/internaldns/serving/render_test.go +++ b/internal/internaldns/serving/render_test.go @@ -157,3 +157,28 @@ func bindingReadyInConfig(config, address string) bool { } return false } + +func TestPendingConfigurationCannotBeActivatedByReadinessMap(t *testing.T) { + t.Parallel() + config, snapshot := renderFixture(t) + snapshot.Bindings[0].ConfigurationPending = true + snapshot.Bindings[0].Zones = nil + rendered, err := Render(config, snapshot, map[string]bool{"bind-a": true}) + if err != nil { + t.Fatal(err) + } + for _, path := range []string{config.NodeDNSDist.Path, config.ClusterDNSDist.Path} { + address := snapshot.Bindings[0].ConsumerAddress + if path == config.ClusterDNSDist.Path { + address = snapshot.Bindings[0].ClusterAddress + } + if bindingReadyInConfig(string(rendered.Files[path]), address) { + t.Fatal("readiness activated a pending context") + } + } + for _, path := range []string{config.NodeBIND.Path, config.ClusterBIND.Path} { + if !strings.Contains(string(rendered.Files[path]), "allow-recursion { none; };") { + t.Fatal("pending BIND context can recurse") + } + } +} diff --git a/internal/internaldns/serving/state.go b/internal/internaldns/serving/state.go index 52a4faf5..d2386829 100644 --- a/internal/internaldns/serving/state.go +++ b/internal/internaldns/serving/state.go @@ -17,8 +17,7 @@ import ( const legacyObservationGrantUID = "legacy-checkpoint" // contributionFence persists the complete immutable observation identity. -// LegacyRevision only exists to decode checkpoints written before Sequence -// and the original deadline were persisted. +// The optional revision field supplies the sequence when the decoded fence omits it. type contributionFence struct { GrantUID string `json:"grantUID,omitempty"` Epoch uint64 `json:"epoch"` @@ -33,7 +32,7 @@ type replicaAck struct { ValidUntil time.Time `json:"validUntil"` } -const checkpointFormatVersion = 3 +const checkpointFormatVersion = 4 type checkpoint struct { FormatVersion int `json:"formatVersion"` diff --git a/internal/internaldns/serving/verify_test.go b/internal/internaldns/serving/verify_test.go index be86fd68..e44120b6 100644 --- a/internal/internaldns/serving/verify_test.go +++ b/internal/internaldns/serving/verify_test.go @@ -122,7 +122,7 @@ func TestPublicationVerifierProvesMemberContextAndFingerprintOverBothTransports( return } defer func() { _ = conn.Close() }() - header := make([]byte, 28) + header := make([]byte, 28, 30) if _, err := io.ReadFull(conn, header); err != nil { return } diff --git a/test/internaldns/README.md b/test/internaldns/README.md new file mode 100644 index 00000000..93e75daa --- /dev/null +++ b/test/internaldns/README.md @@ -0,0 +1,115 @@ +# Internal DNS end-to-end qualification + +Run private DNS on the repository's shared Kubernetes test environment. Task +reuses the pinned `datum-cloud/test-infra` bootstrap, cluster names, add-ons, and +cleanup. The environment also supports the existing public DNS regression chain. +The qualification uses Go clients and Kubernetes workloads. + +## Run locally + +Install Docker, Kind, kubectl, Go, and Task. On macOS, select the suite-owned +Colima runtime before starting the environment: + +```sh +colima --profile internal-dns-e2e start --activate=false --cpu 6 --memory 16 --disk 60 +source test/internaldns/colima-env.sh +export TASK_X_REMOTE_TASKFILES=1 +task --yes env:internal-dns:up +task --yes env:internal-dns:qualify +``` + +`env:internal-dns:up` resets only the owned private DNS fixtures, then rebuilds +and deploys them on the existing clusters. Run it again before repeating the +qualification. It retains the shared bootstrap and public DNS workloads. + +Start private DNS first on fresh clusters to select the shared dual-stack Kind +configuration. If an existing public-only environment uses IPv4, the preflight +stops before resetting private fixtures. When ready to replace that environment, +run `env:down`, then start `env:internal-dns:up` before `env:stack-up`. + +To validate public DNS alongside private DNS, use the same lifecycle as CI: + +```sh +task --yes env:internal-dns:up +task --yes env:stack-up +task --yes env:internal-dns:qualify +task --yes env:chainsaw +test/internaldns/kubernetes/collect.sh +task --yes env:down +``` + +`env:down` removes the three shared test clusters. Qualification results and +sanitized deployment logs remain under `test/internaldns/results/kubernetes*`. +Generated kubeconfigs and Secrets stay in `.internal-dns-e2e/kubernetes/` and +are excluded from collected evidence. The qualifier restores controllers and +broker replicas after fault injection, including a failed scenario. + +## Deployed path + +```mermaid +flowchart LR + A[Project A API / scoped publisher] --> C[Two DNS controller Deployments] + B[Project B API / scoped publisher] --> C + C --> J[Three TLS JetStream replicas / PVCs] + J --> N[Shared node dnsdist / BIND] + J --> R[Shared regional dnsdist / two BIND members] + P[Separate consumer probe] -->|Service destination fixture| N --> R +``` + +The `dns-upstream` and `dns-edge` clusters provide independent project APIs. +The `dns-control` cluster hosts the DNS service API, admission Service, broker, +controllers, and fixed serving fleet. All three use the shared Flux, +cert-manager, Kyverno, and Envoy bootstrap. Private and public DNS use separate +namespaces on this foundation. + +The manifest renderer reads the shipped runtime configuration and RBAC. Each +controller uses scoped project worker credentials; product, grant issuer, and +VPC integration credentials are separate. The simulated Compute publisher +creates registrations and contributions through the project API. It discovers +managed names from `DNSResolverContext.status.managedNamespace`. + +The fleet contains one node member, one regional frontend, and two regional +BIND members, each with independent persistent state. Agents validate and +activate configuration, prove UDP/TCP publication, and renew watchdog leases. +The broker uses three persistent replicas, client and route TLS, and scoped +subject permissions. cert-manager issues broker, client, and admission +certificates. ACK timing matches the shipped 30-second interval and 90-second +lease; the fixture shortens the ownership lease to 10 seconds and pod +termination grace to 2 seconds for fault tests. + +## Qualification scenarios + +The Go qualifier covers overlapping tenant names over UDP and TCP, A/AAAA and +negative-cache isolation, multiple zones, managed name discovery, scoped +publication and admission denial, record updates and deletion, health +withdrawal and recovery, fixed fleet size, regional publication proof, +controller takeover, local record/access expiry while controllers and the +broker are stopped, and publication after broker/controller restoration using +the retained broker PVCs. Each assertion records its result; skipped scenarios do +not count as passes. + +The separate package gate checks the shipped runtime image, all CRD schemas, +deployment rendering, configuration, and Kubernetes/NATS permissions: + +```sh +task --yes env:internal-dns:package +``` + +## Remaining production boundaries + +The deployment validates live Kubernetes APIs, admission, credentials, durable +transport, serving pods, and DNS responses. It does not deploy Milo project +discovery, Karmada, Galactic private +service networking, or the actual Compute service. The independent project APIs +and service-side IPv6 routes are explicit fixtures. Exact route/NAT exemptions +preserve serving pod peers without widening their ACLs. The probe does not share a +trusted serving network namespace or receive a Kubernetes credential. + +The [production qualification proposal](production-parity.md) covers the next +stage: queries from attached consumer VPCs through authorized private endpoints +into the separate DNS service VPC. Kind cannot establish physical fabric, +microVM attachment, host-loss durability, regional independence, or production +capacity. + +The suite requires internal wire version `v1alpha3`, fresh format 4 checkpoints, +and matching controller and fleet versions. diff --git a/test/internaldns/colima-env.sh b/test/internaldns/colima-env.sh new file mode 100644 index 00000000..fe5c54a8 --- /dev/null +++ b/test/internaldns/colima-env.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Keep every internal-DNS qualification resource in its suite-owned Colima VM. +# This deliberately does not activate or change the user's default Docker +# context. Start the profile separately with: +# colima --profile internal-dns-e2e start --activate=false + +set -euo pipefail + +profile=${INTERNAL_DNS_COLIMA_PROFILE:-internal-dns-e2e} +socket=${INTERNAL_DNS_DOCKER_SOCKET:-"$HOME/.colima/$profile/docker.sock"} +expected="unix://$socket" + +if [[ -n ${DOCKER_HOST:-} && $DOCKER_HOST != "$expected" ]]; then + echo "internal DNS qualification requires DOCKER_HOST=$expected (got $DOCKER_HOST)" >&2 + exit 2 +fi +if [[ ! -S $socket ]]; then + echo "internal DNS Colima socket is unavailable: $socket" >&2 + echo "start it with: colima --profile $profile start --activate=false" >&2 + exit 2 +fi + +export INTERNAL_DNS_COLIMA_PROFILE=$profile +export DOCKER_HOST=$expected diff --git a/test/internaldns/fixtures/validating-webhook.yaml.tmpl b/test/internaldns/fixtures/validating-webhook.yaml.tmpl new file mode 100644 index 00000000..af30f491 --- /dev/null +++ b/test/internaldns/fixtures/validating-webhook.yaml.tmpl @@ -0,0 +1,37 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: internal-dns-e2e + labels: + internal-dns.datum.net/qualification: "true" +webhooks: + - name: internal-dns.dns.networking.miloapis.com + admissionReviewVersions: [v1] + sideEffects: None + failurePolicy: Fail + timeoutSeconds: 5 + matchPolicy: Equivalent + clientConfig: + url: https://host.docker.internal:9443/validate-internal-dns + caBundle: ${CA_BUNDLE} + rules: + - operations: [CREATE, UPDATE, DELETE] + apiGroups: [dns.networking.miloapis.com] + apiVersions: [v1alpha1] + resources: + - dnszoneassociations + - dnsregistrations + - dnscontributiongrants + - dnsrecordcontributions + - dnsrecordcontributions/status + - dnsnamingpolicies + - dnsmanagednamespaces + - dnsresolvercontexts + - dnsresolvercontexts/status + - dnsresolveraccessbindings + - dnsresolveraccessbindings/status + - dnsresolverbindings + - dnspublicationownerships + - dnspublicationmanifests + - dnspublicationchunks + - dnstransportoutboxes diff --git a/test/internaldns/kubernetes/Dockerfile b/test/internaldns/kubernetes/Dockerfile new file mode 100644 index 00000000..f8ed6b93 --- /dev/null +++ b/test/internaldns/kubernetes/Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1.7 +FROM --platform=$BUILDPLATFORM golang:1.26 AS build +ARG TARGETOS=linux +ARG TARGETARCH +WORKDIR /src +COPY go.mod go.sum ./ +RUN --mount=type=cache,target=/go/pkg/mod go mod download +COPY api ./api +COPY internal ./internal +COPY cmd/internal-dns ./cmd/internal-dns +COPY test/internaldns/kubernetes/qualify ./test/internaldns/kubernetes/qualify +RUN --mount=type=cache,target=/root/.cache/go-build \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/internal-dns ./cmd/internal-dns && \ + CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/dns-qualify ./test/internaldns/kubernetes/qualify + +# Debian trixie supplies BIND 9.20 with PROXYv2 and dnsdist. The qualification +# records the resolved image ID and daemon versions rather than relying on tags. +FROM debian:trixie-slim +RUN apt-get update && apt-get install --no-install-recommends -y \ + bind9 bind9-utils dnsdist iproute2 iptables procps netcat-openbsd ca-certificates && \ + rm -rf /var/lib/apt/lists/* +COPY --from=build /out/internal-dns /internal-dns +COPY --from=build /out/dns-qualify /dns-qualify +COPY test/internaldns/kubernetes/commands/ /usr/local/sbin/ +USER 65532:65532 +ENTRYPOINT ["/internal-dns"] diff --git a/test/internaldns/kubernetes/Dockerfile.dockerignore b/test/internaldns/kubernetes/Dockerfile.dockerignore new file mode 100644 index 00000000..23a4512e --- /dev/null +++ b/test/internaldns/kubernetes/Dockerfile.dockerignore @@ -0,0 +1,6 @@ +** +!**/*.go +**/*_test.go +!go.mod +!go.sum +!test/internaldns/kubernetes/commands/** diff --git a/test/internaldns/kubernetes/collect.sh b/test/internaldns/kubernetes/collect.sh new file mode 100755 index 00000000..527f07e1 --- /dev/null +++ b/test/internaldns/kubernetes/collect.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cd "$root" +work="$root/.internal-dns-e2e/kubernetes" +evidence="$root/test/internaldns/results/kubernetes-artifacts" +mkdir -p "$evidence" +for file in platform project-a project-b; do + [[ -f $work/$file.kubeconfig ]] || continue + cli=(kubectl --kubeconfig "$work/$file.kubeconfig" --request-timeout=10s) + namespace=project-e2e + if [[ $file == platform ]]; then namespace=internal-dns-system; fi + "${cli[@]}" -n "$namespace" get pods,deployments,statefulsets,pvc,services,configmaps -o yaml > "$evidence/$file-deployment.yaml" 2>&1 || true + "${cli[@]}" -n "$namespace" get events -o yaml > "$evidence/$file-events.yaml" 2>&1 || true + "${cli[@]}" -n "$namespace" get dnszones,dnszoneassociations,dnsresolvercontexts,dnsresolveraccessbindings,dnsnamingpolicies,dnsmanagednamespaces,dnsregistrations,dnscontributiongrants,dnsrecordcontributions,dnsresolverbindings,dnspublicationmanifests,dnspublicationchunks,dnspublicationownerships,dnstransportoutboxes -o yaml > "$evidence/$file-dns.yaml" 2>&1 || true + while IFS= read -r pod; do + [[ -z $pod ]] || "${cli[@]}" -n "$namespace" logs "$pod" --all-containers --prefix --tail=1000 > "$evidence/$file-${pod#pod/}.log" 2>&1 || true + done < <("${cli[@]}" -n "$namespace" get pods -o name 2>/dev/null) +done +# Do not archive generated bootstrap manifests, Secrets, or kubeconfigs. diff --git a/test/internaldns/kubernetes/commands/internal-dns-daemons b/test/internaldns/kubernetes/commands/internal-dns-daemons new file mode 100755 index 00000000..d530d7c8 --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-daemons @@ -0,0 +1,30 @@ +#!/bin/sh +# One member's daemon supervisor. Only its local agent can enable a listener. +set -eu +role=$1 +supervise() { + daemon=$1 + config=$2 + shift 2 + while :; do + if [ -f "/run/internal-dns/allow-$daemon" ] && [ -s "$config" ]; then + "$@" || true + fi + sleep 0.05 + done +} +trap 'pkill -TERM -x named || true; pkill -TERM -x dnsdist || true; exit 0' INT TERM +case "$role" in + node) + supervise named /etc/internal-dns/node-bind.conf named -g -c /etc/internal-dns/node-bind.conf -n 2 & + supervise dnsdist /etc/internal-dns/node-dnsdist.conf dnsdist --supervised -C /etc/internal-dns/node-dnsdist.conf & + ;; + regional-dnsdist) + supervise dnsdist /etc/internal-dns/cluster-dnsdist.conf dnsdist --supervised -C /etc/internal-dns/cluster-dnsdist.conf & + ;; + regional-bind) + supervise named /etc/internal-dns/cluster-bind.conf named -g -c /etc/internal-dns/cluster-bind.conf -n 2 & + ;; + *) exit 2 ;; +esac +wait diff --git a/test/internaldns/kubernetes/commands/internal-dns-flush-bind b/test/internaldns/kubernetes/commands/internal-dns-flush-bind new file mode 100755 index 00000000..21bbe10c --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-flush-bind @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec rndc -s 127.0.0.1 -p 9953 -k /etc/bind/rndc.key flushtree "$2" diff --git a/test/internaldns/kubernetes/commands/internal-dns-gate b/test/internaldns/kubernetes/commands/internal-dns-gate new file mode 100755 index 00000000..c2ad3563 --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-gate @@ -0,0 +1,6 @@ +#!/bin/sh +# Clearing enable files before stopping processes prevents automatic restart. +set -eu +rm -f /run/internal-dns/allow-named /run/internal-dns/allow-dnsdist +pkill -TERM -x named || true +pkill -TERM -x dnsdist || true diff --git a/test/internaldns/kubernetes/commands/internal-dns-reload-bind b/test/internaldns/kubernetes/commands/internal-dns-reload-bind new file mode 100755 index 00000000..d05937de --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-reload-bind @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +touch /run/internal-dns/allow-named +for attempt in $(seq 1 35); do + if rndc -s 127.0.0.1 -p 9953 -k /etc/bind/rndc.key status >/dev/null 2>&1; then + exec rndc -s 127.0.0.1 -p 9953 -k /etc/bind/rndc.key reload + fi + sleep 0.05 +done +echo 'BIND control listener did not become ready' >&2 +exit 1 diff --git a/test/internaldns/kubernetes/commands/internal-dns-reload-dnsdist b/test/internaldns/kubernetes/commands/internal-dns-reload-dnsdist new file mode 100755 index 00000000..562b584d --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-reload-dnsdist @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +touch /run/internal-dns/allow-dnsdist +pkill -TERM -x dnsdist || true +for attempt in $(seq 1 35); do + if nc -z -w 1 ::1 53 >/dev/null 2>&1; then exit 0; fi + sleep 0.05 +done +echo 'dnsdist listener did not become ready' >&2 +exit 1 diff --git a/test/internaldns/kubernetes/commands/internal-dns-validate-bind b/test/internaldns/kubernetes/commands/internal-dns-validate-bind new file mode 100755 index 00000000..989cd2d5 --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-validate-bind @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +shift +exec named-checkconf "$@" diff --git a/test/internaldns/kubernetes/commands/internal-dns-validate-dnsdist b/test/internaldns/kubernetes/commands/internal-dns-validate-dnsdist new file mode 100755 index 00000000..20ecff4e --- /dev/null +++ b/test/internaldns/kubernetes/commands/internal-dns-validate-dnsdist @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec dnsdist --check-config -C "$2" diff --git a/test/internaldns/kubernetes/deploy.sh b/test/internaldns/kubernetes/deploy.sh new file mode 100755 index 00000000..431b5fc9 --- /dev/null +++ b/test/internaldns/kubernetes/deploy.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Deploy onto the repository's shared test-infra clusters; never create a +# second Kind lifecycle or select the user's current Kubernetes context. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cd "$root" +work="$root/.internal-dns-e2e/kubernetes" +mkdir -p "$work" +chmod 700 "$work" + +for pair in 'dns-control platform' 'dns-upstream project-a' 'dns-edge project-b'; do + read -r cluster file <<< "$pair" + kind export kubeconfig --name "$cluster" --kubeconfig "$work/$file.kubeconfig" + kind get kubeconfig --name "$cluster" --internal > "$work/$file-internal.kubeconfig" + chmod 600 "$work/$file"*.kubeconfig +done +platform=(kubectl --kubeconfig "$work/platform.kubeconfig") +# Existing public-only clusters can use IPv4. Check the serving network before +# resetting any private fixtures; changing Kind networking requires fresh nodes. +pod_cidrs=$("${platform[@]}" get nodes -o jsonpath='{range .items[*]}{.spec.podCIDRs}{"\n"}{end}') +if ! grep -q ':' <<< "$pod_cidrs"; then + echo 'Private DNS requires dual-stack dns-control nodes. Start env:internal-dns:up before env:stack-up on fresh shared clusters; existing fixtures were not reset.' >&2 + exit 2 +fi +test/internaldns/kubernetes/reset.sh + +make kustomize +bin/kustomize build config/crd > "$work/crds.yaml" +for file in platform project-a project-b; do + kubectl --kubeconfig "$work/$file.kubeconfig" apply --server-side -f "$work/crds.yaml" +done + +docker build -f test/internaldns/kubernetes/Dockerfile \ + --iidfile "$work/image.id" -t internal-dns-kubernetes:e2e . +kind load docker-image internal-dns-kubernetes:e2e --name dns-control +go run ./test/internaldns/kubernetes/render --phase bootstrap --out "$work" +"${platform[@]}" apply -f "$work/platform-bootstrap.json" +for project in project-a project-b; do + kubectl --kubeconfig "$work/$project.kubeconfig" apply -f "$work/$project-bootstrap.json" +done +"${platform[@]}" -n internal-dns-system wait certificates --all --for=condition=Ready --timeout=180s +# Wait for network assignment before rendering concrete member addresses. The +# processes wait for ConfigMaps, so no placeholder serving configuration runs. +for pod in internal-dns-node internal-dns-regional-front internal-dns-regional-bind-0 internal-dns-regional-bind-1 internal-dns-probe; do + "${platform[@]}" -n internal-dns-system wait --for=jsonpath='{.status.podIPs}' "pod/$pod" --timeout=180s +done +go run ./test/internaldns/kubernetes/render --phase configure --out "$work" \ + --platform "$work/platform.kubeconfig" \ + --source-a "$work/project-a.kubeconfig" --source-b "$work/project-b.kubeconfig" \ + --source-a-internal "$work/project-a-internal.kubeconfig" --source-b-internal "$work/project-b-internal.kubeconfig" +"${platform[@]}" apply -f "$work/platform-configure.json" +"${platform[@]}" -n internal-dns-system wait pod/internal-dns-network-fixture --for=jsonpath='{.status.phase}'=Succeeded --timeout=60s +"${platform[@]}" -n internal-dns-system rollout status statefulset/internal-dns-nats --timeout=180s +for controller in control-a control-b; do + "${platform[@]}" -n internal-dns-system rollout status "deployment/$controller" --timeout=180s +done + +# The source API servers reach a real TLS admission Service over the isolated +# Kind network. +ca_bundle=$("${platform[@]}" -n internal-dns-system get secret internal-dns-admission-tls -o jsonpath='{.data.ca\.crt}') +for project in project-a project-b; do + sed -e "s|\${CA_BUNDLE}|$ca_bundle|g" \ + -e 's|host.docker.internal:9443|dns-control-control-plane:30443|g' \ + test/internaldns/fixtures/validating-webhook.yaml.tmpl > "$work/$project-webhook.yaml" + kubectl --kubeconfig "$work/$project.kubeconfig" apply -f "$work/$project-webhook.yaml" +done + +evidence="$root/test/internaldns/results/kubernetes-artifacts" +mkdir -p "$evidence" +cp "$work/crds.yaml" "$evidence/crds.yaml" +cp "$work/environment.json" "$evidence/environment.json" +cp "$work/image.id" "$evidence/image.id" +"${platform[@]}" -n internal-dns-system exec internal-dns-probe -- named -V > "$evidence/bind-version.txt" +"${platform[@]}" -n internal-dns-system exec internal-dns-probe -- dnsdist --version > "$evidence/dnsdist-version.txt" +git rev-parse HEAD > "$evidence/source.commit" +git status --short > "$evidence/source.status" +if [[ -s $evidence/source.status ]]; then + printf 'dirty checkout; source.commit identifies the base, not the complete built source\n' > "$evidence/source.tree-state" +else + printf 'clean checkout\n' > "$evidence/source.tree-state" +fi +printf 'Kubernetes private DNS environment ready: %s\n' "$work/environment.json" diff --git a/test/internaldns/kubernetes/prepare-kind.sh b/test/internaldns/kubernetes/prepare-kind.sh new file mode 100755 index 00000000..b1017271 --- /dev/null +++ b/test/internaldns/kubernetes/prepare-kind.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +cd "$root" +mkdir -p .internal-dns-e2e/kubernetes +# Kind runs against the selected Docker host. A remote VM may not share this +# checkout's filesystem, so stage the common audit policy on that Docker host. +docker run --rm -i \ + -v /tmp/datum-internal-dns-audit:/audit \ + alpine@sha256:5291449c3df73caf6ed85e649dec1b9e818b39a5d8c871e97afc13e9cd5e8fa8 \ + sh -ec 'cat > /audit/audit-policy.yaml; chmod 644 /audit/audit-policy.yaml' \ + < .test-infra/cluster/audit-policy.yaml +sed 's|${REPO_DIR}/cluster/audit-policy.yaml|/tmp/datum-internal-dns-audit/audit-policy.yaml|g' \ + .test-infra/cluster/kind-config.yaml > .internal-dns-e2e/kubernetes/kind-config.yaml +cat >> .internal-dns-e2e/kubernetes/kind-config.yaml <<'EOF' +networking: + ipFamily: dual +EOF diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go new file mode 100644 index 00000000..29d9e93a --- /dev/null +++ b/test/internaldns/kubernetes/qualify/main.go @@ -0,0 +1,590 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Command qualify publishes through project credentials and queries the shared +// Kubernetes serving fleet. The same executable runs DNS probes inside a pod. +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "reflect" + "sort" + "strings" + "time" + + "github.com/miekg/dns" + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type project struct { + Name string `json:"name"` + ProjectUID string `json:"projectUID"` + SourceClusterUID string `json:"sourceClusterUID"` + Namespace string `json:"namespace"` + AdminKubeconfig string `json:"adminKubeconfig"` + ProductKubeconfig string `json:"productKubeconfig"` + IssuerKubeconfig string `json:"issuerKubeconfig"` + IntegrationKubeconfig string `json:"integrationKubeconfig"` + PublisherSubject string `json:"publisherSubject"` + ConsumerAddress string `json:"consumerAddress"` + admin, product, issuer, integration client.Client + resolverContext dnsv1.DNSResolverContext + zone dnsv1.DNSZone + access dnsv1.DNSResolverAccessBinding +} + +type environment struct { + PlatformKubeconfig string `json:"platformKubeconfig"` + Namespace string `json:"namespace"` + Region string `json:"region"` + Shard string `json:"shard"` + ProbePod string `json:"probePod"` + ProbeContainer string `json:"probeContainer"` + ProbeBinary string `json:"probeBinary"` + ControllerDeployments []string `json:"controllerDeployments"` + BrokerStatefulSet string `json:"brokerStatefulSet"` + RegionalMembers []string `json:"regionalMembers"` + Projects []project `json:"projects"` +} + +type check struct { + Name string `json:"name"` + Passed bool `json:"passed"` + Time time.Time `json:"time"` + Evidence any `json:"evidence,omitempty"` + Error string `json:"error,omitempty"` +} + +type report struct { + Suite string `json:"suite"` + Started time.Time `json:"started"` + Completed time.Time `json:"completed"` + Passed bool `json:"passed"` + Checks []check `json:"checks"` + Images map[string]string `json:"images"` + Limitations []string `json:"limitations"` + Error string `json:"error,omitempty"` +} + +type probeResult struct { + Rcode int `json:"rcode"` + Answers []string `json:"answers"` + Addresses []string `json:"addresses"` + TTL uint32 `json:"ttl"` + Error string `json:"error,omitempty"` +} + +type publication struct { + registration dnsv1.DNSRegistration + grant dnsv1.DNSContributionGrant + contribution dnsv1.DNSRecordContribution + sequence int64 +} + +type suite struct { + env environment + platform client.Client + report report + ctx context.Context +} + +func main() { + envPath := flag.String("environment", "", "Generated environment JSON") + results := flag.String("results", "test/internaldns/results/kubernetes.json", "Qualification report") + probe := flag.Bool("probe", false, "Run one DNS query from a workload pod") + server := flag.String("server", "", "DNS server address and port") + name := flag.String("name", "", "DNS query name") + rrType := flag.String("type", "A", "DNS record type") + tcp := flag.Bool("tcp", false, "Use DNS over TCP") + flag.Parse() + if *probe { + if err := runProbe(*server, *name, *rrType, *tcp); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + s := &suite{ctx: context.Background(), report: report{ + Suite: "internal-dns-kubernetes", Started: time.Now().UTC(), Images: map[string]string{}, + Limitations: []string{ + "Kind clusters share one host; this run does not qualify regional or host failure.", + "Project APIs use independent Kubernetes servers; Milo discovery and Karmada are not exercised.", + "IPv6 context routes are a network fixture; Galactic private service connectivity and Compute attachment are not exercised.", + }, + }} + err := s.run(*envPath) + s.report.Completed = time.Now().UTC() + s.report.Passed = err == nil + if err != nil { + s.report.Error = err.Error() + s.report.Checks = append(s.report.Checks, check{Name: "qualification completed", Passed: false, Time: time.Now().UTC(), Error: err.Error()}) + } + if writeErr := writeReport(*results, s.report); writeErr != nil { + fmt.Fprintln(os.Stderr, writeErr) + os.Exit(1) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Printf("Passed %d Kubernetes qualification checks; evidence: %s\n", len(s.report.Checks), *results) +} + +func runProbe(server, name, rrType string, tcp bool) error { + t, ok := dns.StringToType[strings.ToUpper(rrType)] + if !ok || server == "" || name == "" { + return errors.New("probe requires a server, name, and valid record type") + } + message := new(dns.Msg) + message.SetQuestion(dns.Fqdn(name), t) + transport := "udp" + if tcp { + transport = "tcp" + } + response, _, err := (&dns.Client{Net: transport, Timeout: 3 * time.Second}).Exchange(message, server) + if err != nil { + return json.NewEncoder(os.Stdout).Encode(probeResult{Error: err.Error()}) + } + result := probeResult{Rcode: response.Rcode, Answers: []string{}, Addresses: []string{}} + for _, answer := range response.Answer { + result.Answers = append(result.Answers, answer.String()) + if result.TTL == 0 || answer.Header().Ttl < result.TTL { + result.TTL = answer.Header().Ttl + } + switch record := answer.(type) { + case *dns.A: + result.Addresses = append(result.Addresses, record.A.String()) + case *dns.AAAA: + result.Addresses = append(result.Addresses, record.AAAA.String()) + } + } + sort.Strings(result.Addresses) + return json.NewEncoder(os.Stdout).Encode(result) +} + +func writeReport(path string, value any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o600) +} + +func newClient(path string) (client.Client, error) { + cfg, err := clientcmd.BuildConfigFromFlags("", path) + if err != nil { + return nil, err + } + cfg.Timeout = 8 * time.Second + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, appsv1.AddToScheme, dnsv1.AddToScheme} { + if err := add(scheme); err != nil { + return nil, err + } + } + return client.New(cfg, client.Options{Scheme: scheme}) +} + +func (s *suite) pass(name string, evidence any) { + s.report.Checks = append(s.report.Checks, check{Name: name, Passed: true, Time: time.Now().UTC(), Evidence: evidence}) + fmt.Println("PASS", name) +} + +func (s *suite) wait(label string, duration time.Duration, test func() (bool, error)) error { + deadline := time.Now().Add(duration) + var last error + for time.Now().Before(deadline) { + passed, err := test() + if passed && err == nil { + return nil + } + if err != nil { + last = err + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("timed out waiting for %s: %v", label, last) +} + +func objectRef(object client.Object) dnsv1.DNSObjectReference { + return dnsv1.DNSObjectReference{Name: object.GetName(), UID: object.GetUID(), Generation: object.GetGeneration()} +} + +func condition(conditions []metav1.Condition, name string, generation int64) bool { + c := apimeta.FindStatusCondition(conditions, name) + return c != nil && c.Status == metav1.ConditionTrue && c.ObservedGeneration == generation +} + +func (s *suite) run(path string) error { + data, err := os.ReadFile(path) + if err != nil { + return err + } + if err := json.Unmarshal(data, &s.env); err != nil { + return err + } + if len(s.env.Projects) != 2 || len(s.env.ControllerDeployments) == 0 || s.env.BrokerStatefulSet == "" { + return errors.New("qualification requires two project APIs, controllers, and a broker") + } + if s.env.ProbeBinary == "" { + s.env.ProbeBinary = "/dns-qualify" + } + if len(s.env.RegionalMembers) != 2 { + return errors.New("qualification requires two independent regional BIND members") + } + s.platform, err = newClient(s.env.PlatformKubeconfig) + if err != nil { + return err + } + for i := range s.env.Projects { + p := &s.env.Projects[i] + for _, pair := range []struct { + path string + target *client.Client + }{ + {p.AdminKubeconfig, &p.admin}, {p.ProductKubeconfig, &p.product}, + {p.IssuerKubeconfig, &p.issuer}, {p.IntegrationKubeconfig, &p.integration}, + } { + *pair.target, err = newClient(pair.path) + if err != nil { + return fmt.Errorf("%s client: %w", p.Name, err) + } + } + if net.ParseIP(p.ConsumerAddress) == nil || p.ProjectUID == "" || p.SourceClusterUID == "" || p.PublisherSubject == "" { + return fmt.Errorf("%s has incomplete identity or resolver address", p.Name) + } + } + before, err := s.workloads() + if err != nil { + return err + } + if err := s.captureImages(); err != nil { + return err + } + for i := range s.env.Projects { + p := &s.env.Projects[i] + if err := s.permissions(p); err != nil { + return err + } + if err := s.prepare(p); err != nil { + return err + } + } + a, b := &s.env.Projects[0], &s.env.Projects[1] + if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.Suffix == b.resolverContext.Status.ManagedNamespace.Suffix { + return errors.New("source projects lost their distinct context, zone, or managed namespace identity") + } + s.pass("independent projects retain distinct context, zone, and managed namespace identities", map[string]string{"a": string(a.resolverContext.UID), "b": string(b.resolverContext.UID)}) + if err := s.scenarios(a, b); err != nil { + return err + } + after, err := s.workloads() + if err != nil { + return err + } + if !reflect.DeepEqual(before, after) { + return fmt.Errorf("contexts changed the shared workload count: before=%v after=%v", before, after) + } + s.pass("adding contexts and zones preserves the fixed shared workload count", after) + return nil +} + +func (s *suite) permissions(p *project) error { + grant := &dnsv1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: "unauthorized", Namespace: p.Namespace}} + if err := p.product.Create(s.ctx, grant); !apierrors.IsForbidden(err) { + return fmt.Errorf("%s product grant creation must be forbidden: %v", p.Name, err) + } + contextObject := &dnsv1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "unauthorized", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverContextSpec{ConsumerID: "unauthorized"}} + if err := p.product.Create(s.ctx, contextObject); !apierrors.IsForbidden(err) { + return fmt.Errorf("%s product network authorization must be forbidden: %v", p.Name, err) + } + access := &dnsv1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "unauthorized", Namespace: p.Namespace}} + if err := p.product.Create(s.ctx, access); !apierrors.IsForbidden(err) { + return fmt.Errorf("%s product access binding creation must be forbidden: %v", p.Name, err) + } + s.pass(p.Name+" product credentials cannot issue grants or authorize resolver access", nil) + return nil +} + +func (s *suite) prepare(p *project) error { + p.resolverContext = dnsv1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "application-network", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverContextSpec{ConsumerID: "qualification/" + p.Name, ManagedNamespace: dnsv1.DNSResolverContextManagedNamespace{Enabled: true}}} + if err := p.integration.Create(s.ctx, &p.resolverContext); err != nil { + return err + } + if err := s.wait(p.Name+" managed namespace", 120*time.Second, func() (bool, error) { + err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&p.resolverContext), &p.resolverContext) + return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID != "", err + }); err != nil { + return err + } + s.pass(p.Name+" product discovers its managed namespace through the context API", p.resolverContext.Status.ManagedNamespace) + p.zone = dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: p.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "prod.internal", DNSZoneClassName: "private-bind", Visibility: dnsv1.DNSZoneVisibilityPrivate}} + if err := p.admin.Create(s.ctx, &p.zone); err != nil { + return err + } + if err := s.associate(p, &p.zone, "prod"); err != nil { + return err + } + p.access = dnsv1.DNSResolverAccessBinding{ObjectMeta: metav1.ObjectMeta{Name: "application-network-access", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverAccessBindingSpec{ + ContextRef: objectRef(&p.resolverContext), Region: s.env.Region, + QueryIdentity: dnsv1.DNSResolverQueryIdentity{Type: dnsv1.DNSResolverQueryIdentityDestinationAddress, Value: p.ConsumerAddress}, Port: 53, Transports: []string{"UDP", "TCP"}, + Authorization: dnsv1.DNSResolverAccessAuthorization{WriterEpoch: p.resolverContext.Status.AccessWriterEpoch, Sequence: 1, ValidUntil: metav1.NewTime(time.Now().Add(5 * time.Minute).UTC())}, + }} + if err := p.integration.Create(s.ctx, &p.access); err != nil { + return err + } + if err := s.waitAccess(p); err != nil { + return err + } + s.pass(p.Name+" authorized destination becomes ready through shared Kubernetes members", p.ConsumerAddress) + return nil +} + +func (s *suite) associate(p *project, zone *dnsv1.DNSZone, name string) error { + association := &dnsv1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: p.Namespace}, Spec: dnsv1.DNSZoneAssociationSpec{DNSZoneRef: objectRef(zone), ResolverContextRef: objectRef(&p.resolverContext)}} + if err := p.admin.Create(s.ctx, association); err != nil { + return err + } + return s.wait("private zone association "+p.Name+"/"+name, 60*time.Second, func() (bool, error) { + err := p.admin.Get(s.ctx, client.ObjectKeyFromObject(association), association) + return condition(association.Status.Conditions, "Accepted", association.Generation), err + }) +} + +func (s *suite) waitAccess(p *project) error { + return s.wait(p.Name+" resolver access ready", 120*time.Second, func() (bool, error) { + err := p.integration.Get(s.ctx, client.ObjectKeyFromObject(&p.access), &p.access) + if err != nil { + return false, err + } + if !condition(p.access.Status.Conditions, "Ready", p.access.Generation) { + return false, fmt.Errorf("current access conditions: %v", p.access.Status.Conditions) + } + return true, nil + }) +} + +func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { + sets := []dnsv1.DNSContributionRecordSet{{RecordType: dnsv1.RRTypeA, Records: []dnsv1.RecordEntry{{Name: name, A: &dnsv1.ARecordSpec{Content: address}}}}} + if ipv6 != "" { + sets = append(sets, dnsv1.DNSContributionRecordSet{RecordType: dnsv1.RRTypeAAAA, Records: []dnsv1.RecordEntry{{Name: name, AAAA: &dnsv1.AAAARecordSpec{Content: ipv6}}}}) + } + return sets +} + +func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string, lifetime time.Duration) (*publication, error) { + pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, PublicationPolicy: dnsv1.DNSPublicationPolicyEligibleContributions, TTLSeconds: 2}}} + if ipv6 != "" { + pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) + } + if err := p.product.Create(s.ctx, &pub.registration); err != nil { + return nil, err + } + pub.grant = dnsv1.DNSContributionGrant{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-compute", Namespace: p.Namespace}, Spec: dnsv1.DNSContributionGrantSpec{RegistrationRef: objectRef(&pub.registration), ProducerID: "compute", Principal: dnsv1.DNSProducerPrincipal{ClusterUID: p.SourceClusterUID, Subject: p.PublisherSubject}, RecordTypes: pub.registration.Spec.RecordTypes}} + if err := p.issuer.Create(s.ctx, &pub.grant); err != nil { + return nil, err + } + if err := s.wait("grant writer epoch", 60*time.Second, func() (bool, error) { + err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.grant), &pub.grant) + return pub.grant.Status.ActiveWriterEpoch > 0, err + }); err != nil { + return nil, err + } + pub.contribution = dnsv1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-endpoint", Namespace: p.Namespace}, Spec: dnsv1.DNSRecordContributionSpec{RegistrationRef: objectRef(&pub.registration), GrantRef: objectRef(&pub.grant), RecordSets: recordSets(name, address, ipv6)}} + if err := p.product.Create(s.ctx, &pub.contribution); err != nil { + return nil, err + } + if err := s.wait("bound contribution epoch", 60*time.Second, func() (bool, error) { + err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution) + return pub.contribution.Status.WriterEpoch == pub.grant.Status.ActiveWriterEpoch, err + }); err != nil { + return nil, err + } + if err := s.observe(p, pub, true, lifetime); err != nil { + return nil, err + } + return pub, nil +} + +func (s *suite) observe(p *project, pub *publication, eligible bool, lifetime time.Duration) error { + if err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution); err != nil { + return err + } + before := pub.contribution.DeepCopy() + pub.sequence++ + pub.contribution.Status.Sequence = pub.sequence + pub.contribution.Status.ObservedGeneration = pub.contribution.Generation + pub.contribution.Status.Eligible = eligible + pub.contribution.Status.Reason = "EndpointReady" + if !eligible { + pub.contribution.Status.Reason = "EndpointNotReady" + } + deadline := metav1.NewTime(time.Now().Add(lifetime).UTC()) + pub.contribution.Status.ValidUntil = &deadline + if err := p.product.Status().Patch(s.ctx, &pub.contribution, client.MergeFrom(before)); err != nil { + return err + } + return s.waitPublication(pub) +} + +func (s *suite) waitPublication(pub *publication) error { + return s.wait("verified publication for "+string(pub.contribution.UID), 90*time.Second, func() (bool, error) { + var publications dnsv1.DNSPublicationManifestList + if err := s.platform.List(s.ctx, &publications, client.InNamespace(s.env.Namespace)); err != nil { + return false, err + } + for _, manifest := range publications.Items { + matched := false + for _, fence := range manifest.Spec.ContributionFences { + if fence.UID == pub.contribution.UID && fence.Sequence == pub.sequence { + matched = true + } + } + if !matched { + continue + } + members := map[string]bool{} + for _, ack := range manifest.Status.ReplicaAcknowledgements { + if ack.Phase == "Verified" && ack.WriterEpoch == manifest.Spec.WriterEpoch && ack.Revision == manifest.Spec.Revision && ack.ValidUntil != nil && ack.ValidUntil.After(time.Now()) { + members[ack.MemberID] = true + } + } + if members[s.env.RegionalMembers[0]] && members[s.env.RegionalMembers[1]] { + return true, nil + } + } + return false, fmt.Errorf("no fresh proof from both regional members for contribution %s sequence %d", pub.contribution.UID, pub.sequence) + }) +} + +func (s *suite) query(p *project, name, rrType string, tcp bool) (probeResult, error) { + ctx, cancel := context.WithTimeout(s.ctx, 12*time.Second) + defer cancel() + args := []string{"--kubeconfig", s.env.PlatformKubeconfig, "-n", s.env.Namespace, "exec", s.env.ProbePod} + if s.env.ProbeContainer != "" { + args = append(args, "-c", s.env.ProbeContainer) + } + args = append(args, "--", s.env.ProbeBinary, "--probe", "--server", net.JoinHostPort(p.ConsumerAddress, "53"), "--name", name, "--type", rrType) + if tcp { + args = append(args, "--tcp") + } + cmd := exec.CommandContext(ctx, "kubectl", args...) + var stderr strings.Builder + cmd.Stderr = &stderr + output, err := cmd.Output() + if err != nil { + return probeResult{}, fmt.Errorf("query %s: %w: %s", name, err, stderr.String()) + } + var result probeResult + if err := json.Unmarshal(output, &result); err != nil { + return result, err + } + return result, nil +} + +func (s *suite) expect(p *project, name, rrType string, tcp bool, rcode int, addresses ...string) error { + sort.Strings(addresses) + var result probeResult + err := s.wait("DNS answer "+p.Name+" "+name, 60*time.Second, func() (bool, error) { + var err error + result, err = s.query(p, name, rrType, tcp) + if err != nil { + return false, err + } + if result.Error != "" { + return false, fmt.Errorf("DNS transport: %s", result.Error) + } + if result.Rcode != rcode { + return false, fmt.Errorf("rcode=%s answers=%v", dns.RcodeToString[result.Rcode], result.Answers) + } + if len(addresses) == 0 { + if len(result.Answers) != 0 { + return false, fmt.Errorf("unexpected answers=%v", result.Answers) + } + return true, nil + } + if !reflect.DeepEqual(result.Addresses, addresses) { + return false, fmt.Errorf("addresses=%v want=%v", result.Addresses, addresses) + } + return true, nil + }) + if err != nil { + return err + } + transport := "UDP" + if tcp { + transport = "TCP" + } + s.pass(p.Name+" "+transport+" "+rrType+" "+name, result) + return nil +} + +func (s *suite) workloads() (map[string]int32, error) { + values := map[string]int32{} + var deployments appsv1.DeploymentList + if err := s.platform.List(s.ctx, &deployments, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, value := range deployments.Items { + if value.Spec.Replicas != nil { + values["deployment/"+value.Name] = *value.Spec.Replicas + } + } + var sets appsv1.StatefulSetList + if err := s.platform.List(s.ctx, &sets, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, value := range sets.Items { + if value.Spec.Replicas != nil { + values["statefulset/"+value.Name] = *value.Spec.Replicas + } + } + var daemonSets appsv1.DaemonSetList + if err := s.platform.List(s.ctx, &daemonSets, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, value := range daemonSets.Items { + values["daemonset/"+value.Name] = value.Status.DesiredNumberScheduled + } + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace)); err != nil { + return nil, err + } + for _, pod := range pods.Items { + if len(pod.OwnerReferences) == 0 { + values["pod/"+pod.Name] = 1 + } + } + return values, nil +} + +func (s *suite) captureImages() error { + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace)); err != nil { + return err + } + for _, pod := range pods.Items { + for _, c := range pod.Status.ContainerStatuses { + s.report.Images[pod.Name+"/"+c.Name] = c.ImageID + } + } + return nil +} diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go new file mode 100644 index 00000000..0ba1f0d6 --- /dev/null +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -0,0 +1,467 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "errors" + "fmt" + "strings" + "time" + + "github.com/miekg/dns" + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func (s *suite) scenarios(a, b *project) error { + for _, p := range []*project{a, b} { + if err := s.expect(p, "only-a.prod.internal", "A", false, dns.RcodeNameError); err != nil { + return err + } + } + pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10", 90*time.Second) + if err != nil { + return err + } + pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20", 90*time.Second) + if err != nil { + return err + } + if pa.registration.UID == pb.registration.UID || pa.grant.UID == pb.grant.UID || pa.contribution.UID == pb.contribution.UID { + return fmt.Errorf("same-named project publications lost their distinct identities") + } + s.pass("same-named registrations, grants, and contributions retain independent source identities", nil) + foreign := &dnsv1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "foreign-project-endpoint", Namespace: a.Namespace}, Spec: dnsv1.DNSRecordContributionSpec{RegistrationRef: objectRef(&pb.registration), GrantRef: objectRef(&pb.grant), RecordSets: recordSets("common", "10.42.0.99", "")}} + if err := a.product.Create(s.ctx, foreign); !apierrors.IsForbidden(err) && !apierrors.IsInvalid(err) { + return fmt.Errorf("another project's publication references must be rejected by admission: %v", err) + } + s.pass("product admission rejects publication references from another source project", nil) + for _, tcp := range []bool{false, true} { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.10"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", tcp, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + if err := s.expect(a, "common.prod.internal", "AAAA", tcp, dns.RcodeSuccess, "fd42::10"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "AAAA", tcp, dns.RcodeSuccess, "fd42::20"); err != nil { + return err + } + } + for _, p := range []*project{a, b} { + if err := s.expect(p, "common.prod.internal", "TXT", false, dns.RcodeSuccess); err != nil { + return err + } + } + unknown := *a + unknown.ConsumerAddress = "fd53::dead" + for _, tcp := range []bool{false, true} { + answer, err := s.query(&unknown, "common.prod.internal", "A", tcp) + if err != nil { + return err + } + if answer.Error == "" && answer.Rcode != dns.RcodeRefused && answer.Rcode != dns.RcodeServerFailure { + return fmt.Errorf("unknown destination must not resolve a tenant's record: %+v", answer) + } + } + if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + s.pass("an unbound destination cannot resolve either tenant's private records", nil) + if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", "", 90*time.Second); err != nil { + return err + } + if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + if err := s.expect(b, "only-a.prod.internal", "A", true, dns.RcodeNameError); err != nil { + return err + } + s.pass("negative caches and overlapping endpoint address spaces stay isolated", nil) + // The product discovers the zone from context status rather than accepting a + // zone selection on the Compute resource. + if _, err := s.publish(a, a.resolverContext.Status.ManagedNamespace.DNSZoneRef, "managed", "web.instances", "10.42.0.11", "", 90*time.Second); err != nil { + return err + } + managedName := "web.instances." + a.resolverContext.Status.ManagedNamespace.Suffix + if err := s.expect(a, managedName, "A", false, dns.RcodeSuccess, "10.42.0.11"); err != nil { + return err + } + apps := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "apps", Namespace: a.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "apps.internal", DNSZoneClassName: "private-bind", Visibility: dnsv1.DNSZoneVisibilityPrivate}} + if err := a.admin.Create(s.ctx, apps); err != nil { + return err + } + if err := s.associate(a, apps, "apps"); err != nil { + return err + } + if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", "", 90*time.Second); err != nil { + return err + } + if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { + return err + } + s.pass("one resolver context serves its managed namespace and multiple custom zones", nil) + if err := s.observe(b, pb, true, 90*time.Second); err != nil { + return err + } + if err := s.update(a, pa, "10.42.0.30", "fd42::30"); err != nil { + return err + } + if err := s.expect(a, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + if err := s.observe(a, pa, false, 90*time.Second); err != nil { + return err + } + for _, tcp := range []bool{false, true} { + if err := s.expect(a, "common.prod.internal", "A", tcp, dns.RcodeSuccess); err != nil { + return err + } + } + s.pass("an unhealthy endpoint stops producing DNS answers", nil) + if err := s.observe(a, pa, true, 90*time.Second); err != nil { + return err + } + if err := s.expect(a, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + if err := a.product.Get(s.ctx, client.ObjectKeyFromObject(&pa.contribution), &pa.contribution); err != nil { + return err + } + stale := pa.contribution.DeepCopy() + stale.Status.Sequence-- + if err := a.product.Status().Update(s.ctx, stale); err == nil { + return fmt.Errorf("stale producer sequence was accepted") + } else if !apierrors.IsForbidden(err) && !apierrors.IsInvalid(err) { + return fmt.Errorf("stale producer sequence failed for an unrelated reason: %w", err) + } + s.pass("admission rejects a stale producer observation", nil) + if err := s.takeover(a, pa); err != nil { + return err + } + if err := s.observe(b, pb, true, 90*time.Second); err != nil { + return err + } + deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "", 90*time.Second) + if err != nil { + return err + } + if err := s.expect(a, "deleted.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.14"); err != nil { + return err + } + if err := a.product.Delete(s.ctx, &deleted.contribution); err != nil { + return err + } + if err := s.expect(a, "deleted.prod.internal", "A", true, dns.RcodeSuccess); err != nil { + return err + } + s.pass("deleting a contribution withdraws its answer and preserves registration ownership", nil) + if err := s.offlineExpiry(a, b, pa, pb); err != nil { + return err + } + if err := s.observe(b, pb, true, 90*time.Second); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + s.pass("restored controllers and persistent broker publish a fresh observation", nil) + return nil +} + +func (s *suite) update(p *project, pub *publication, address, ipv6 string) error { + if err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution); err != nil { + return err + } + before := pub.contribution.DeepCopy() + pub.contribution.Spec.RecordSets = recordSets(pub.registration.Spec.Name, address, ipv6) + if err := p.product.Patch(s.ctx, &pub.contribution, client.MergeFrom(before)); err != nil { + return err + } + return s.observe(p, pub, true, 90*time.Second) +} + +func (s *suite) owner(zoneUID string) (dnsv1.DNSPublicationOwnership, error) { + var values dnsv1.DNSPublicationOwnershipList + if err := s.platform.List(s.ctx, &values, client.InNamespace(s.env.Namespace)); err != nil { + return dnsv1.DNSPublicationOwnership{}, err + } + for _, value := range values.Items { + if string(value.Spec.ZoneUID) == zoneUID { + return value, nil + } + } + return dnsv1.DNSPublicationOwnership{}, fmt.Errorf("zone %s has no publication owner", zoneUID) +} + +func (s *suite) scaleController(name string, replicas int32) error { + deployment := &appsv1.Deployment{} + key := client.ObjectKey{Namespace: s.env.Namespace, Name: name} + if err := s.platform.Get(s.ctx, key, deployment); err != nil { + return err + } + before := deployment.DeepCopy() + deployment.Spec.Replicas = &replicas + if err := s.platform.Patch(s.ctx, deployment, client.MergeFrom(before)); err != nil { + return err + } + return s.wait("controller scale "+name, 60*time.Second, func() (bool, error) { + var pods corev1.PodList + err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels(deployment.Spec.Selector.MatchLabels)) + if err != nil { + return false, err + } + if replicas == 0 { + return len(pods.Items) == 0, nil + } + if err := s.platform.Get(s.ctx, key, deployment); err != nil { + return false, err + } + return deployment.Status.ObservedGeneration >= deployment.Generation && + deployment.Status.AvailableReplicas == replicas && readyPods(pods.Items, replicas), nil + }) +} + +func (s *suite) takeover(p *project, pub *publication) (resultErr error) { + initial, err := s.owner(string(p.zone.UID)) + if err != nil { + return err + } + var target string + for _, name := range s.env.ControllerDeployments { + deployment := &appsv1.Deployment{} + if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: name}, deployment); err != nil { + return err + } + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), client.MatchingLabels(deployment.Spec.Selector.MatchLabels)); err != nil { + return err + } + for _, pod := range pods.Items { + if strings.HasPrefix(initial.Spec.HolderIdentity, pod.Name+"-") { + target = name + } + } + } + if target == "" { + return fmt.Errorf("cannot identify current controller owner %s", initial.Spec.HolderIdentity) + } + defer func() { + if err := s.scaleController(target, 1); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("controller restoration: %w", err)) + } + }() + if err := s.scaleController(target, 0); err != nil { + return err + } + var current dnsv1.DNSPublicationOwnership + if err := s.wait("ownership takeover by surviving controller", 90*time.Second, func() (bool, error) { + var err error + current, err = s.owner(string(p.zone.UID)) + return current.Spec.WriterEpoch > initial.Spec.WriterEpoch && current.Spec.HolderIdentity != initial.Spec.HolderIdentity, err + }); err != nil { + return err + } + if err := s.observe(p, pub, true, 90*time.Second); err != nil { + return err + } + if err := s.expect(p, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.30"); err != nil { + return err + } + s.pass("surviving Kubernetes controller takes ownership at a higher writer epoch", map[string]any{"previous": initial.Spec.HolderIdentity, "current": current.Spec.HolderIdentity, "writerEpoch": current.Spec.WriterEpoch}) + return nil +} + +func (s *suite) scaleBroker(replicas int32) error { + set := &appsv1.StatefulSet{} + key := client.ObjectKey{Namespace: s.env.Namespace, Name: s.env.BrokerStatefulSet} + if err := s.platform.Get(s.ctx, key, set); err != nil { + return err + } + before := set.DeepCopy() + set.Spec.Replicas = &replicas + if err := s.platform.Patch(s.ctx, set, client.MergeFrom(before)); err != nil { + return err + } + return s.wait("broker scale", 90*time.Second, func() (bool, error) { + var pods corev1.PodList + if err := s.platform.List(s.ctx, &pods, client.InNamespace(s.env.Namespace), + client.MatchingLabels(set.Spec.Selector.MatchLabels)); err != nil { + return false, err + } + if replicas == 0 { + return len(pods.Items) == 0, nil + } + if err := s.platform.Get(s.ctx, key, set); err != nil { + return false, err + } + return set.Status.ObservedGeneration >= set.Generation && + set.Status.ReadyReplicas == replicas && readyPods(pods.Items, replicas), nil + }) +} + +func (s *suite) offlineExpiry(a, b *project, pa, pb *publication) (resultErr error) { + // Keep the healthy tenant's original record and access lease long enough to + // cover the outage. Only the expired tenant receives the short access lease. + // Finish access readiness before starting the record leases: serving ACKs + // follow the production 30-second interval and can consume that setup time. + if err := s.renewAccess(b, 5*time.Minute); err != nil { + return err + } + if err := s.observe(a, pa, true, 90*time.Second); err != nil { + return err + } + if err := s.observe(b, pb, true, 90*time.Second); err != nil { + return err + } + expiring, err := s.publish(a, objectRef(&a.zone), "expires", "expires", "10.42.0.15", "", 45*time.Second) + if err != nil { + return err + } + if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.15"); err != nil { + return err + } + if err := a.integration.Get(s.ctx, client.ObjectKeyFromObject(&a.access), &a.access); err != nil { + return err + } + before := a.access.DeepCopy() + a.access.Spec.Authorization.Sequence++ + a.access.Spec.Authorization.ValidUntil = metav1.NewTime(time.Now().Add(50 * time.Second).UTC()) + if err := a.integration.Patch(s.ctx, &a.access, client.MergeFrom(before)); err != nil { + return err + } + if err := s.waitAccess(a); err != nil { + return err + } + var broker appsv1.StatefulSet + if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: s.env.BrokerStatefulSet}, &broker); err != nil { + return err + } + brokerReplicas := int32(1) + if broker.Spec.Replicas != nil { + brokerReplicas = *broker.Spec.Replicas + } + defer func() { + if err := s.scaleBroker(brokerReplicas); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("broker restoration: %w", err)) + } + for _, name := range s.env.ControllerDeployments { + if err := s.scaleController(name, 1); err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("controller restoration: %w", err)) + } + } + }() + for _, name := range s.env.ControllerDeployments { + if err := s.scaleController(name, 0); err != nil { + return err + } + } + if err := s.scaleBroker(0); err != nil { + return err + } + if expiring.contribution.Status.ValidUntil == nil || !expiring.contribution.Status.ValidUntil.After(time.Now().Add(3*time.Second)) { + return fmt.Errorf("record expired before the outage; no offline expiry proof") + } + if !a.access.Spec.Authorization.ValidUntil.After(time.Now().Add(10 * time.Second)) { + return fmt.Errorf("access expired before the outage; no offline authorization proof") + } + if !a.access.Spec.Authorization.ValidUntil.After(expiring.contribution.Status.ValidUntil.Add(5 * time.Second)) { + return fmt.Errorf("record and access deadlines overlap; no independent expiry proof") + } + if pb.contribution.Status.ValidUntil == nil || + !pb.contribution.Status.ValidUntil.After(a.access.Spec.Authorization.ValidUntil.Add(15*time.Second)) { + return fmt.Errorf("healthy record lease cannot cover access expiry; no outage positive control") + } + if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.15"); err != nil { + return err + } + s.pass("Kubernetes controllers and broker stop without stopping the serving pods", nil) + if deadline := expiring.contribution.Status.ValidUntil; deadline != nil { + if remaining := time.Until(deadline.Add(3 * time.Second)); remaining > 0 { + time.Sleep(remaining) + } + } + if err := s.expect(a, "expires.prod.internal", "A", false, dns.RcodeSuccess); err != nil { + return err + } + if err := s.expect(b, "common.prod.internal", "A", true, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + s.pass("local record expiry preserves the original deadline during a broker and controller outage", expiring.contribution.Status.ValidUntil) + if remaining := time.Until(a.access.Spec.Authorization.ValidUntil.Add(-3 * time.Second)); remaining > 0 { + time.Sleep(remaining) + } + answer, err := s.query(a, "expires.prod.internal", "A", false) + if err != nil { + return err + } + if answer.Error != "" || answer.Rcode != dns.RcodeSuccess || len(answer.Answers) != 0 || + !a.access.Spec.Authorization.ValidUntil.After(time.Now()) { + return fmt.Errorf("access was not usable just before its original deadline: %+v", answer) + } + s.pass("unexpired access remains usable after record expiry", map[string]any{ + "validUntil": a.access.Spec.Authorization.ValidUntil, "answer": answer, + }) + if remaining := time.Until(a.access.Spec.Authorization.ValidUntil.Add(3 * time.Second)); remaining > 0 { + time.Sleep(remaining) + } + for _, tcp := range []bool{false, true} { + if err := s.wait("expired access rejected", 15*time.Second, func() (bool, error) { + answer, err := s.query(a, "common.prod.internal", "A", tcp) + if err != nil { + return false, err + } + return answer.Error != "" || answer.Rcode == dns.RcodeRefused || answer.Rcode == dns.RcodeServerFailure, nil + }); err != nil { + return err + } + } + if err := s.expect(b, "common.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { + return err + } + s.pass("expired access fails closed while another tenant keeps resolving", a.access.Spec.Authorization.ValidUntil) + return nil +} + +func (s *suite) renewAccess(p *project, lifetime time.Duration) error { + if err := p.integration.Get(s.ctx, client.ObjectKeyFromObject(&p.access), &p.access); err != nil { + return err + } + before := p.access.DeepCopy() + p.access.Spec.Authorization.Sequence++ + p.access.Spec.Authorization.ValidUntil = metav1.NewTime(time.Now().Add(lifetime).UTC()) + if err := p.integration.Patch(s.ctx, &p.access, client.MergeFrom(before)); err != nil { + return err + } + return s.waitAccess(p) +} + +func readyPods(pods []corev1.Pod, replicas int32) bool { + if len(pods) != int(replicas) { + return false + } + for _, pod := range pods { + if pod.DeletionTimestamp != nil { + return false + } + ready := false + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + ready = true + } + } + if !ready { + return false + } + } + return true +} diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go new file mode 100644 index 00000000..11797354 --- /dev/null +++ b/test/internaldns/kubernetes/render/main.go @@ -0,0 +1,447 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Render the Kubernetes qualification from the shipped DNS configuration and RBAC. +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "net/netip" + "os" + "path/filepath" + "strings" + + "go.miloapis.com/dns-operator/internal/internaldns/model" + runtime "go.miloapis.com/dns-operator/internal/internaldns/runtime" + authv1 "k8s.io/api/authentication/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8syaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +type object = map[string]any + +const ns = "internal-dns-system" +const projectNS = "project-e2e" +const image = "internal-dns-kubernetes:e2e" + +var out, phase, platform, a, b, ai, bi string +var memberNames = []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-bind-us-central1-0", "regional-bind-us-central1-1"} +var podNames = []string{"internal-dns-node", "internal-dns-regional-front", "internal-dns-regional-bind-0", "internal-dns-regional-bind-1"} + +func main() { + flag.StringVar(&out, "out", ".internal-dns-e2e/kubernetes", "Private generated environment directory") + flag.StringVar(&phase, "phase", "bootstrap", "bootstrap or configure") + flag.StringVar(&platform, "platform", "", "Platform admin kubeconfig") + flag.StringVar(&a, "source-a", "", "Source A admin kubeconfig") + flag.StringVar(&b, "source-b", "", "Source B admin kubeconfig") + flag.StringVar(&ai, "source-a-internal", "", "Source A internal admin kubeconfig") + flag.StringVar(&bi, "source-b-internal", "", "Source B internal admin kubeconfig") + flag.Parse() + var err error + out, err = filepath.Abs(out) + must(err) + must(os.MkdirAll(out, 0700)) + switch phase { + case "bootstrap": + bootstrap() + case "configure": + configure() + default: + panic("unknown phase") + } +} +func must(err error) { + if err != nil { + panic(err) + } +} +func obj(api, kind, name, namespace string) object { + m := object{"name": name} + if namespace != "" { + m["namespace"] = namespace + } + if kind == "Namespace" && (name == ns || name == projectNS) { + m["labels"] = object{"internal-dns.datum.net/qualification": "true"} + } + return object{"apiVersion": api, "kind": kind, "metadata": m} +} +func write(name string, value any) { + data, err := json.MarshalIndent(value, "", " ") + must(err) + must(os.WriteFile(filepath.Join(out, name), append(data, '\n'), 0600)) +} +func list(name string, items []object) { + write(name, object{"apiVersion": "v1", "kind": "List", "items": items}) +} +func appendShippedRBAC(items []object) []object { + data, err := os.ReadFile("config/internal-dns/rbac.yaml") + must(err) + d := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + for { + var o object + err = d.Decode(&o) + if err == io.EOF { + break + } + must(err) + u := unstructured.Unstructured{Object: o} + if u.GetNamespace() == "example-project" { + u.SetNamespace(projectNS) + } + items = append(items, u.Object) + } + return items +} +func certificate(name string, dns []string, usages []string, ca bool, issuer string) object { + o := obj("cert-manager.io/v1", "Certificate", name, ns) + spec := object{"secretName": name, "commonName": name, "isCA": ca, "issuerRef": object{"name": issuer, "kind": "Issuer"}, "privateKey": object{"algorithm": "ECDSA", "size": 256}, "duration": "24h", "renewBefore": "8h"} + if len(dns) > 0 { + spec["dnsNames"] = dns + } + if len(usages) > 0 { + spec["usages"] = usages + } + o["spec"] = spec + return o +} +func bootstrap() { + items := []object{obj("v1", "Namespace", ns, ""), obj("v1", "Namespace", projectNS, "")} + items = appendShippedRBAC(items) + self := obj("cert-manager.io/v1", "Issuer", "internal-dns-selfsigned", ns) + self["spec"] = object{"selfSigned": object{}} + items = append(items, self, certificate("internal-dns-ca", nil, []string{"cert sign", "crl sign"}, true, "internal-dns-selfsigned")) + issuer := obj("cert-manager.io/v1", "Issuer", "internal-dns-ca", ns) + issuer["spec"] = object{"ca": object{"secretName": "internal-dns-ca"}} + items = append(items, issuer) + items = append(items, certificate("internal-dns-nats-server", []string{"nats." + ns + ".svc", "nats." + ns + ".svc.cluster.local", "*.internal-dns-nats." + ns + ".svc.cluster.local"}, []string{"server auth", "client auth"}, false, "internal-dns-ca")) + items = append(items, certificate("internal-dns-admission-tls", []string{"internal-dns-admission." + ns + ".svc", "dns-control-control-plane"}, []string{"server auth"}, false, "internal-dns-ca")) + identities := append([]string{"control-a", "control-b"}, memberNames...) + for _, name := range identities { + items = append(items, certificate(name+"-nats", nil, []string{"client auth"}, false, "internal-dns-ca")) + } + // Kubernetes Secrets carry the broker ACL passwords and RNDC key; none are + // emitted into ConfigMaps or test reports. + users := make([]object, 0, len(identities)) + for _, name := range identities { + password := randomSecret() + secret := obj("v1", "Secret", name+"-password", ns) + secret["stringData"] = object{"password": password} + items = append(items, secret) + allow := []string{"dns.private.serving.>", "dns.private.records.>", "$KV.DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.>", "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>", "_INBOX.>"} + subscribe := []string{"dns.private.acks.>", "_INBOX.>"} + if !strings.HasPrefix(name, "control-") { + allow = make([]string, 0, 24) + allow = append(allow, model.AckSubject("us-central1", "shared-0", name), "$JS.API.STREAM.INFO.DNS_PRIVATE", "$JS.API.DIRECT.GET.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.STREAM.INFO.KV_DNS_PRIVATE_SNAPSHOTS", "$JS.API.CONSUMER.CREATE.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.CONSUMER.INFO.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.CONSUMER.MSG.NEXT.KV_DNS_PRIVATE_SNAPSHOTS.>", "$JS.API.CONSUMER.DELETE.KV_DNS_PRIVATE_SNAPSHOTS.>", "_INBOX.>") + subscribe = []string{"_INBOX.>"} + for _, suffix := range []string{"serving", "records", "acks"} { + c := name + "-" + suffix + allow = append(allow, "$JS.API.CONSUMER.INFO.DNS_PRIVATE."+c, "$JS.API.CONSUMER.CREATE.DNS_PRIVATE."+c, "$JS.API.CONSUMER.CREATE.DNS_PRIVATE."+c+".>", "$JS.API.CONSUMER.MSG.NEXT.DNS_PRIVATE."+c, "$JS.ACK.DNS_PRIVATE."+c+".>") + } + } + users = append(users, object{"user": name, "password": password, "permissions": object{"publish": object{"allow": allow}, "subscribe": object{"allow": subscribe}}}) + } + tls := object{"cert_file": "/tls/tls.crt", "key_file": "/tls/tls.key", "ca_file": "/tls/ca.crt", "verify": true} + cfg := object{"port": 4222, "http_port": 8222, "jetstream": object{"store_dir": "/data"}, "tls": tls, "authorization": object{"users": users}, "cluster": object{"name": "internal-dns-e2e", "port": 6222, "tls": tls, "routes": []string{"nats-route://internal-dns-nats-0.internal-dns-nats." + ns + ".svc.cluster.local:6222", "nats-route://internal-dns-nats-1.internal-dns-nats." + ns + ".svc.cluster.local:6222", "nats-route://internal-dns-nats-2.internal-dns-nats." + ns + ".svc.cluster.local:6222"}}} + var encoded bytes.Buffer + encoder := json.NewEncoder(&encoded) + // NATS accepts JSON-style configuration but not JSON's HTML Unicode escapes. + encoder.SetEscapeHTML(false) + must(encoder.Encode(cfg)) + secret := obj("v1", "Secret", "internal-dns-nats-config", ns) + secret["stringData"] = object{"nats.conf": encoded.String()} + items = append(items, secret) + rndc := obj("v1", "Secret", "internal-dns-rndc", ns) + rndc["stringData"] = object{"rndc.key": fmt.Sprintf("key \"internal-dns\" { algorithm hmac-sha256; secret \"%s\"; };\n", randomSecret())} + items = append(items, rndc) + for _, name := range []string{"nats", "internal-dns-nats"} { + s := obj("v1", "Service", name, ns) + spec := object{"selector": object{"app": "internal-dns-nats"}, "ports": []object{{"name": "client", "port": 4222}, {"name": "cluster", "port": 6222}}} + if name == "internal-dns-nats" { + spec["clusterIP"] = "None" + spec["publishNotReadyAddresses"] = true + } + s["spec"] = spec + items = append(items, s) + } + broker := obj("apps/v1", "StatefulSet", "internal-dns-nats", ns) + broker["spec"] = object{"replicas": 3, "serviceName": "internal-dns-nats", "podManagementPolicy": "Parallel", "selector": object{"matchLabels": object{"app": "internal-dns-nats"}}, "template": object{"metadata": object{"labels": object{"app": "internal-dns-nats"}}, "spec": object{"terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "containers": []object{{"name": "nats", "image": "nats@sha256:e4bf19f15fd3218814a4e3c9e0064e1334bd8aa20d5984b9f1a0afd084f8cc00", "command": []string{"nats-server"}, "args": []string{"-c", "/config/nats.conf", "-n", "$(POD_NAME)"}, "env": []object{{"name": "POD_NAME", "valueFrom": object{"fieldRef": object{"fieldPath": "metadata.name"}}}}, "volumeMounts": []object{{"name": "config", "mountPath": "/config", "readOnly": true}, {"name": "tls", "mountPath": "/tls", "readOnly": true}, {"name": "data", "mountPath": "/data"}}, "readinessProbe": object{"httpGet": object{"path": "/healthz?js-enabled-only=true", "port": 8222}}, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}}, "volumes": []object{{"name": "config", "secret": object{"secretName": "internal-dns-nats-config"}}, {"name": "tls", "secret": object{"secretName": "internal-dns-nats-server"}}}}}, "volumeClaimTemplates": []object{{"metadata": object{"name": "data"}, "spec": object{"accessModes": []string{"ReadWriteOnce"}, "resources": object{"requests": object{"storage": "1Gi"}}}}}} + items = append(items, broker) + for i, name := range podNames { + role := []string{"node", "regional-dnsdist", "regional-bind", "regional-bind"}[i] + items = append(items, fleetPod(name, memberNames[i], role)) + for _, vol := range []string{"state", "config"} { + pvc := obj("v1", "PersistentVolumeClaim", name+"-"+vol, ns) + pvc["spec"] = object{"accessModes": []string{"ReadWriteOnce"}, "resources": object{"requests": object{"storage": "512Mi"}}} + items = append(items, pvc) + } + } + // Admission is a real Kubernetes TLS Service, reached by the independent + // source API servers through a Kind-network NodePort. + s := obj("v1", "Service", "internal-dns-admission", ns) + s["spec"] = object{"type": "NodePort", "selector": object{"internal-dns-role": "control-plane"}, "ports": []object{{"name": "https", "port": 443, "targetPort": 9443, "nodePort": 30443}}} + items = append(items, s) + for _, name := range []string{"control-a", "control-b"} { + items = append(items, controller(name)) + } + probe := obj("v1", "Pod", "internal-dns-probe", ns) + probe["spec"] = object{"terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "securityContext": object{"runAsUser": 65532, "runAsGroup": 65532}, "containers": []object{{"name": "probe", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-c", "sleep infinity"}, "securityContext": object{"allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true, "capabilities": object{"drop": []string{"ALL"}}}}}} + items = append(items, probe) + list("platform-bootstrap.json", items) + for _, project := range []string{"project-a", "project-b"} { + list(project+"-bootstrap.json", sourceObjects(project)) + } +} +func randomSecret() string { + data := make([]byte, 32) + _, err := rand.Read(data) + must(err) + return base64.StdEncoding.EncodeToString(data) +} +func mount(name, path string, ro bool) object { + return object{"name": name, "mountPath": path, "readOnly": ro} +} +func security() object { + return object{"runAsNonRoot": true, "runAsUser": 65532, "runAsGroup": 65532, "allowPrivilegeEscalation": false, "readOnlyRootFilesystem": true, "capabilities": object{"drop": []string{"ALL"}}} +} +func credentialsVolume(member string) object { + return object{"name": "nats", "projected": object{"sources": []object{{"secret": object{"name": member + "-nats"}}, {"secret": object{"name": member + "-password"}}}}} +} +func fleetPod(name, member, role string) object { + o := obj("v1", "Pod", name, ns) + o["metadata"].(object)["labels"] = object{"internal-dns-role": role, "internal-dns-member": member} + mounts := []object{mount("config", "/config", true), mount("nats", "/var/run/internal-dns/nats", true), mount("state", "/var/lib/internal-dns", false), mount("daemon-config", "/etc/internal-dns", false), mount("lease", "/run/internal-dns", false), mount("runtime", "/tmp", false), mount("rndc", "/etc/bind", true)} + containers := make([]object, 0, 3) + for _, process := range []string{"agent", "watchdog"} { + containers = append(containers, object{"name": process, "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ]; do sleep 1; done; exec /internal-dns --role=" + process + " --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) + } + sec := security() + sec["capabilities"] = object{"drop": []string{"ALL"}, "add": []string{"NET_BIND_SERVICE"}} + containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/usr/local/sbin/internal-dns-daemons", role}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) + volumes := []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(member), {"name": "state", "persistentVolumeClaim": object{"claimName": name + "-state"}}, {"name": "daemon-config", "persistentVolumeClaim": object{"claimName": name + "-config"}}, {"name": "lease", "emptyDir": object{"medium": "Memory"}}, {"name": "runtime", "emptyDir": object{}}, {"name": "rndc", "secret": object{"secretName": "internal-dns-rndc"}}} + // The aliases model the service-side destination contract only. The separate + // probe has no ability to write routes or send trusted PROXY traffic. + init := []object{} + if role == "node" || role == "regional-dnsdist" { + prefix := "fd53::/64" + if role != "node" { + prefix = "fd54::/64" + } + init = append(init, object{"name": "destination-fixture", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "ip -6 route replace local " + prefix + " dev lo"}, "securityContext": object{"runAsUser": 0, "capabilities": object{"drop": []string{"ALL"}, "add": []string{"NET_ADMIN"}}}}) + } + o["spec"] = object{"serviceAccountName": "internal-dns-fleet", "terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "shareProcessNamespace": true, "securityContext": object{"fsGroup": 65532, "fsGroupChangePolicy": "OnRootMismatch", "sysctls": []object{{"name": "net.ipv4.ip_unprivileged_port_start", "value": "0"}}}, "containers": containers, "initContainers": init, "volumes": volumes} + return o +} +func controller(name string) object { + o := obj("apps/v1", "Deployment", name, ns) + labels := object{"app": name, "internal-dns-role": "control-plane"} + mounts := []object{mount("config", "/config", true), mount("nats", "/var/run/internal-dns/nats", true), mount("admission", "/var/run/internal-dns/admission", true), mount("sources", "/sources", true)} + o["spec"] = object{"replicas": 1, "selector": object{"matchLabels": object{"app": name}}, "template": object{"metadata": object{"labels": labels}, "spec": object{"terminationGracePeriodSeconds": 2, "serviceAccountName": "internal-dns-control-plane", "containers": []object{{"name": "control-plane", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ] || [ ! -s /sources/project-a.json ]; do sleep 1; done; exec /internal-dns --role=control-plane --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts, "ports": []object{{"name": "admission", "containerPort": 9443}}, "readinessProbe": object{"tcpSocket": object{"port": 9443}}, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "768Mi"}}}}, "volumes": []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(name), {"name": "admission", "secret": object{"secretName": "internal-dns-admission-tls"}}, {"name": "sources", "secret": object{"secretName": "internal-dns-sources", "optional": true}}}}}} + return o +} +func sourceObjects(project string) []object { + items := []object{obj("v1", "Namespace", ns, ""), obj("v1", "Namespace", projectNS, ""), obj("v1", "Namespace", "compute-system", ""), obj("v1", "Namespace", "network-system", "")} + items = appendShippedRBAC(items) + filtered := items[:0] + for _, item := range items { + if item["kind"] == "ClusterRoleBinding" && item["metadata"].(map[string]any)["name"] == "internal-dns-control-plane" { + continue + } + filtered = append(filtered, item) + } + items = filtered + worker := obj("rbac.authorization.k8s.io/v1", "RoleBinding", "internal-dns-source-worker", projectNS) + worker["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "internal-dns-control-plane"} + worker["subjects"] = []object{{"kind": "ServiceAccount", "name": "internal-dns-control-plane", "namespace": ns}} + sar := obj("rbac.authorization.k8s.io/v1", "ClusterRole", "internal-dns-source-authorization", "") + sar["rules"] = []object{{"apiGroups": []string{"authorization.k8s.io"}, "resources": []string{"subjectaccessreviews"}, "verbs": []string{"create"}}} + sarBinding := obj("rbac.authorization.k8s.io/v1", "ClusterRoleBinding", "internal-dns-source-authorization", "") + sarBinding["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "internal-dns-source-authorization"} + sarBinding["subjects"] = worker["subjects"] + items = append(items, worker, sar, sarBinding) + accounts := []struct{ name, namespace, role string }{{"dns-publisher", "compute-system", "internal-dns-product-writer"}, {"dns-grant-issuer", "compute-system", "internal-dns-grant-issuer"}, {"dns-vpc-integration", "network-system", "internal-dns-vpc-integration"}, {"internal-dns-control-plane", ns, ""}} + for _, account := range accounts { + if account.role != "" { + items = append(items, obj("v1", "ServiceAccount", account.name, account.namespace)) + rb := obj("rbac.authorization.k8s.io/v1", "RoleBinding", account.name, projectNS) + rb["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": account.role} + rb["subjects"] = []object{{"kind": "ServiceAccount", "name": account.name, "namespace": account.namespace}} + items = append(items, rb) + } + // Project identity in Kind is scoped to impersonation of the same + // service account and this fixed project parent. + r := obj("rbac.authorization.k8s.io/v1", "ClusterRole", account.name+"-parent", "") + r["rules"] = []object{{"apiGroups": []string{"authentication.k8s.io"}, "resources": []string{"userextras/iam.miloapis.com/parent-name"}, "resourceNames": []string{project}, "verbs": []string{"impersonate"}}} + rb := obj("rbac.authorization.k8s.io/v1", "ClusterRoleBinding", account.name+"-parent", "") + rb["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": account.name + "-parent"} + rb["subjects"] = []object{{"kind": "ServiceAccount", "name": account.name, "namespace": account.namespace}} + items = append(items, r, rb) + self := obj("rbac.authorization.k8s.io/v1", "Role", account.name+"-self", account.namespace) + self["rules"] = []object{{"apiGroups": []string{""}, "resources": []string{"serviceaccounts"}, "resourceNames": []string{account.name}, "verbs": []string{"impersonate"}}} + binding := obj("rbac.authorization.k8s.io/v1", "RoleBinding", account.name+"-self", account.namespace) + binding["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "Role", "name": account.name + "-self"} + binding["subjects"] = rb["subjects"] + items = append(items, self, binding) + } + c := obj("dns.networking.miloapis.com/v1alpha1", "DNSZoneClass", "private-bind", "") + c["metadata"].(object)["labels"] = object{"internal-dns.datum.net/qualification": "true"} + c["spec"] = object{"controllerName": "bind", "nameServerPolicy": object{"mode": "Static", "static": object{"servers": []string{"ns.internal."}}}} + items = append(items, c) + return items +} +func client(path string) *kubernetes.Clientset { + cfg, err := clientcmd.BuildConfigFromFlags("", path) + must(err) + cl, err := kubernetes.NewForConfig(cfg) + must(err) + return cl +} +func ipv6(cl *kubernetes.Clientset, name string) string { + p, err := cl.CoreV1().Pods(ns).Get(context.Background(), name, metav1.GetOptions{}) + must(err) + for _, ip := range p.Status.PodIPs { + addr, err := netip.ParseAddr(ip.IP) + if err == nil && addr.Is6() { + return ip.IP + } + } + panic("pod has no IPv6 address: " + name) +} +func tokenConfig(admin, internal, project, account, namespace, name string) []byte { + seconds := int64(7200) + req, err := client(admin).CoreV1().ServiceAccounts(namespace).CreateToken(context.Background(), account, &authv1.TokenRequest{Spec: authv1.TokenRequestSpec{ExpirationSeconds: &seconds}}, metav1.CreateOptions{}) + must(err) + cfg, err := clientcmd.LoadFromFile(admin) + must(err) + ctx := cfg.Contexts[cfg.CurrentContext] + cluster := *cfg.Clusters[ctx.Cluster] + subject := "system:serviceaccount:" + namespace + ":" + account + auth := &clientcmdapi.AuthInfo{Token: req.Status.Token, Impersonate: subject, ImpersonateUserExtra: map[string][]string{"iam.miloapis.com/parent-name": {project}}} + result := clientcmdapi.Config{Clusters: map[string]*clientcmdapi.Cluster{"source": &cluster}, Contexts: map[string]*clientcmdapi.Context{"source": {Cluster: "source", AuthInfo: "scoped", Namespace: projectNS}}, CurrentContext: "source", AuthInfos: map[string]*clientcmdapi.AuthInfo{"scoped": auth}} + host, err := clientcmd.Write(result) + must(err) + must(os.WriteFile(filepath.Join(out, project+"-"+name+".kubeconfig"), host, 0600)) + if internal != "" { + raw, err := clientcmd.LoadFromFile(internal) + must(err) + ic := raw.Contexts[raw.CurrentContext] + result.Clusters["source"] = raw.Clusters[ic.Cluster] + } + data, err := clientcmd.Write(result) + must(err) + return data +} +func exampleConfigs(filename string) map[string]runtime.Config { + data, err := os.ReadFile("config/internal-dns/" + filename) + must(err) + dec := k8syaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + result := map[string]runtime.Config{} + for { + var o object + err = dec.Decode(&o) + if err == io.EOF { + break + } + must(err) + if o["kind"] != "ConfigMap" { + continue + } + m := o["metadata"].(map[string]any) + d := o["data"].(map[string]any) + raw, ok := d["config.json"].(string) + if !ok { + continue + } + var cfg runtime.Config + must(json.Unmarshal([]byte(raw), &cfg)) + result[m["name"].(string)] = cfg + } + return result +} +func configMap(name string, cfg runtime.Config) object { + raw, err := json.Marshal(cfg) + must(err) + o := obj("v1", "ConfigMap", name, ns) + o["data"] = object{"config.json": string(raw)} + return o +} +func configure() { + cl := client(platform) + nodeIP := ipv6(cl, podNames[0]) + frontIP := ipv6(cl, podNames[1]) + bindIP := []string{ipv6(cl, podNames[2]), ipv6(cl, podNames[3])} + sources := obj("v1", "Secret", "internal-dns-sources", ns) + sourceData := object{} + projectRows := make([]object, 0, 2) + for i, project := range []string{"project-a", "project-b"} { + admin := []string{a, b}[i] + internal := []string{ai, bi}[i] + sourceData[project+".json"] = string(tokenConfig(admin, internal, project, "internal-dns-control-plane", ns, "worker")) + tokenConfig(admin, "", project, "dns-publisher", "compute-system", "product") + tokenConfig(admin, "", project, "dns-grant-issuer", "compute-system", "issuer") + tokenConfig(admin, "", project, "dns-vpc-integration", "network-system", "integration") + projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "sourceClusterUID": []string{"source-cluster-a-uid", "source-cluster-b-uid"}[i], "namespace": projectNS, "adminKubeconfig": admin, "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) + } + sources["stringData"] = sourceData + items := make([]object, 0, 8) + items = append(items, sources) + cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] + cfg.ConsumerPrefix = "fd53::/64" + cfg.ClusterPrefix = "fd54::/64" + cfg.ManagedDomainSuffix = "managed.internal" + cfg.OwnershipLeaseSeconds = 10 + cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", SourceClusterUID: "source-cluster-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", SourceClusterUID: "source-cluster-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} + cfg.NodeBackends = []model.Backend{{MemberID: "node-bind-0", Address: "127.0.0.1", Port: 5300}} + cfg.ClusterBackends = []model.Backend{{MemberID: memberNames[2], Address: bindIP[0], Port: 5300}, {MemberID: memberNames[3], Address: bindIP[1], Port: 5300}} + cfg.Admission.PlatformSubjects = []string{"system:serviceaccount:" + ns + ":internal-dns-control-plane", "kubernetes-admin"} + cfg.Admission.IntegrationSubjects = []string{"system:serviceaccount:network-system:dns-vpc-integration"} + cfg.Admission.MaxAccessLeaseSeconds = 600 + for _, name := range []string{"control-a", "control-b"} { + c := cfg + c.NATS.Name = name + c.NATS.Username = name + must(c.Validate("control-plane")) + items = append(items, configMap(name+"-config", c)) + } + configs := exampleConfigs("fleet.example.yaml") + for i, key := range []string{"internal-dns-node-0-config", "internal-dns-regional-front-0-config", "internal-dns-regional-bind-0-config", "internal-dns-regional-bind-1-config"} { + c := configs[key] + c.Watchdog.StartupGraceSeconds = 30 + if i == 0 { + c.Agent.Render.NodeDNSDist.Backends = cfg.NodeBackends + c.Agent.Render.NodeDNSDist.ACLs = []string{"fd00::/8", "::1/128"} + c.Agent.Render.NodeBIND.ListenAddress = "127.0.0.1" + c.Agent.Render.NodeBIND.ProxyPeers = []string{"127.0.0.1"} + } + if i == 1 { + c.Agent.Render.ClusterDNSDist.Backends = cfg.ClusterBackends + c.Agent.Render.ClusterDNSDist.ACLs = []string{nodeIP + "/128", frontIP + "/128", "::1/128"} + } + if i >= 2 { + c.Agent.Render.ClusterBIND.ListenAddress = bindIP[i-2] + c.Agent.Render.ClusterBIND.ProxyPeers = []string{frontIP, bindIP[i-2]} + c.Agent.PublicationDNSProbe.Server = "[" + bindIP[i-2] + "]:5300" + } + must(c.Validate("agent")) + must(c.Validate("watchdog")) + items = append(items, configMap(podNames[i]+"-config", c)) + } + routes := obj("v1", "Pod", "internal-dns-network-fixture", ns) + // Kind's default masquerading would erase the pod peer identity for service + // destinations outside its PodCIDR. Exempt only these two fixture prefixes; + // never widen the protected BIND/front ACLs to the host's shared address. + routeCommand := fmt.Sprintf("ip -6 route replace fd53::/64 via %s; ip -6 route replace fd54::/64 via %s; for prefix in fd53::/64 fd54::/64; do while ip6tables -t nat -C POSTROUTING -d \"$prefix\" -j RETURN 2>/dev/null; do ip6tables -t nat -D POSTROUTING -d \"$prefix\" -j RETURN; done; ip6tables -t nat -I POSTROUTING 1 -d \"$prefix\" -j RETURN; done", nodeIP, frontIP) + routes["spec"] = object{"hostNetwork": true, "restartPolicy": "Never", "terminationGracePeriodSeconds": 2, "automountServiceAccountToken": false, "containers": []object{{"name": "routes", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", routeCommand}, "securityContext": object{"runAsUser": 0, "capabilities": object{"drop": []string{"ALL"}, "add": []string{"NET_ADMIN"}}}}}} + items = append(items, routes) + list("platform-configure.json", items) + // No tokens or certificate contents are included in the public contract. + write("environment.json", object{"platformKubeconfig": platform, "namespace": ns, "region": "us-central1", "shard": "shared-0", "probePod": "internal-dns-probe", "probeContainer": "probe", "probeBinary": "/dns-qualify", "controllerDeployments": []string{"control-a", "control-b"}, "brokerStatefulSet": "internal-dns-nats", "projects": projectRows, "regionalMembers": memberNames[2:], "members": memberNames, "limitations": []string{"Service-side route fixture, not Galactic VPC authorization", "Independent Kind project APIs, not Milo project discovery", "One host and one region, not independent regional failure"}}) +} diff --git a/test/internaldns/kubernetes/render/main_test.go b/test/internaldns/kubernetes/render/main_test.go new file mode 100644 index 00000000..6d23683e --- /dev/null +++ b/test/internaldns/kubernetes/render/main_test.go @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: AGPL-3.0-only +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// Wildcard subjects must stay literal: NATS rejects JSON's HTML Unicode +// escapes even though a JSON decoder accepts them. +func TestBrokerConfigurationKeepsNATSWildcards(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + previous := out + out = t.TempDir() + t.Cleanup(func() { out = previous }) + bootstrap() + data, err := os.ReadFile(filepath.Join(out, "platform-bootstrap.json")) + if err != nil { + t.Fatal(err) + } + var list struct { + Items []object `json:"items"` + } + if err := json.Unmarshal(data, &list); err != nil { + t.Fatal(err) + } + for _, item := range list.Items { + if item["kind"] != "Secret" || item["metadata"].(map[string]any)["name"] != "internal-dns-nats-config" { + continue + } + cfg := item["stringData"].(map[string]any)["nats.conf"].(string) + if strings.Contains(cfg, `\u003e`) || !strings.Contains(cfg, `dns.private.serving.>`) { + t.Fatal("broker configuration escaped NATS wildcard subjects") + } + if !strings.Contains(cfg, "$JS.ACK.DNS_PRIVATE.control-plane-us-central1-shared-0-acks.>") { + t.Fatal("controller cannot acknowledge its member-ACK durable") + } + return + } + t.Fatal("broker configuration missing") +} + +// Project workers must not inherit the platform's cluster-wide artifact +// privileges when both APIs share the shipped ClusterRole. +func TestSourceWorkerBindingIsProjectScoped(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + scoped, sar := false, false + for _, item := range sourceObjects("project-a") { + metadata := item["metadata"].(map[string]any) + switch { + case item["kind"] == "ClusterRoleBinding" && metadata["name"] == "internal-dns-control-plane": + t.Fatal("source worker inherited platform cluster-wide binding") + case item["kind"] == "RoleBinding" && metadata["name"] == "internal-dns-source-worker": + scoped = metadata["namespace"] == projectNS + case item["kind"] == "ClusterRole" && metadata["name"] == "internal-dns-source-authorization": + rules := item["rules"].([]object) + sar = len(rules) == 1 && rules[0]["resources"].([]string)[0] == "subjectaccessreviews" + } + } + if !scoped || !sar { + t.Fatal("source worker requires project scope and separate authorization access") + } +} diff --git a/test/internaldns/kubernetes/reset.sh b/test/internaldns/kubernetes/reset.sh new file mode 100755 index 00000000..f4e40d9a --- /dev/null +++ b/test/internaldns/kubernetes/reset.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Reset only owned private DNS fixtures; retain the shared bootstrap/public DNS. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +work="$root/.internal-dns-e2e/kubernetes" +# Remove only the owned fixture's fail-closed webhook before stopping its +# service, otherwise admission could prevent namespace cleanup. +for file in project-a project-b; do + cli=(kubectl --kubeconfig "$work/$file.kubeconfig") + owner=$("${cli[@]}" get namespace project-e2e -o jsonpath='{.metadata.labels.internal-dns\.datum\.net/qualification}' 2>/dev/null || true) + if [[ $owner == true ]]; then + "${cli[@]}" delete clusterrolebinding internal-dns-control-plane --ignore-not-found + webhook_owner=$("${cli[@]}" get validatingwebhookconfiguration internal-dns-e2e -o jsonpath='{.metadata.labels.internal-dns\.datum\.net/qualification}' 2>/dev/null || true) + if [[ -n $webhook_owner && $webhook_owner != true ]]; then + echo 'Refusing to delete an unowned admission webhook' >&2 + exit 1 + fi + "${cli[@]}" delete validatingwebhookconfiguration internal-dns-e2e --ignore-not-found + fi +done +for file in platform project-a project-b; do + cli=(kubectl --kubeconfig "$work/$file.kubeconfig") + namespace=project-e2e + if [[ $file == platform ]]; then namespace=internal-dns-system; fi + if "${cli[@]}" get namespace "$namespace" >/dev/null 2>&1; then + owner=$("${cli[@]}" get namespace "$namespace" -o jsonpath='{.metadata.labels.internal-dns\.datum\.net/qualification}') + if [[ $owner != true ]]; then + echo "Refusing to reset unowned namespace $namespace on $file" >&2 + exit 1 + fi + if [[ $file != platform ]]; then + # The fixture may have deliberately stopped both controllers. Release + # only the private-zone finalizers in this owned test namespace. + while IFS= read -r zone; do + [[ -z $zone ]] || "${cli[@]}" -n "$namespace" patch "$zone" --type merge -p '{"metadata":{"finalizers":[]}}' + done < <("${cli[@]}" -n "$namespace" get dnszones -o name) + fi + "${cli[@]}" delete namespace "$namespace" --wait=true --timeout=90s + fi +done diff --git a/test/internaldns/production-parity.md b/test/internaldns/production-parity.md new file mode 100644 index 00000000..84dfb42b --- /dev/null +++ b/test/internaldns/production-parity.md @@ -0,0 +1,188 @@ +# Production-oriented internal DNS qualification + +## Proposal + +Build deployment qualification on the existing `datum-cloud/test-infra` +environment. Use the Task lifecycle, Kubernetes workloads, and Go clients for +assertions and fault injection. CI also runs the +existing public DNS Chainsaw suites on the same foundation. + +The acceptance path starts with a product publishing a record in a project +control plane and ends with a workload querying the well-known resolver address +inside its Galactic VPC. A successful direct query to a generated context +address does not qualify the consumer network path. + +## Evidence and gaps + +This assessment uses tracked infrastructure declarations at +[`datum-cloud/infra@5cc5caba5`](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac). +Declarations establish the intended configuration; they do not establish live +cluster health or that the proposed private DNS fleet is deployed. + +- [DNS control-plane configuration](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/apps/dns-operator/control-plane/base) + discovers Milo projects and uses project credentials, certificate trust, and + an admission service. The Kubernetes qualification substitutes independent + Kind APIs, but uses a TLS admission Service and scoped worker credentials. +- [Karmada](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/infrastructure/karmada/host/base/karmada-helm-release.yaml) + provides a separate federation API. The current lab has no federation or + edge API and creates DNS access intent directly. +- [Galactic edge configuration](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/channels/edge/components/galactic/galactic-system.yaml) + selects the networking deployment independently. The lab uses synthetic routes + and direct context addresses; it bypasses VPC attachment, private service + authorization, service route programming, and workload resolver settings. +- [NATS infrastructure](https://github.com/datum-cloud/infra/blob/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/infrastructure/databases/nats/base/nats-hr.yaml) + declares three broker replicas, persistent JetStream storage, and client and + route TLS. The Kubernetes lab uses that three-replica storage and TLS pattern. + Its full broker outage test proves local expiry and replica restart using + retained PVCs. Quorum loss, disk/host-loss recovery, and regional transport + still need dedicated scenarios. +- [Public edge DNS](https://github.com/datum-cloud/infra/tree/5cc5caba5b7f886ca00e7c3ed90726834dac7fac/apps/dns-operator/downstream/edge) + uses PowerDNS authoritative servers and Lightningstream. Its loopback + recursor supports ALIAS expansion. Reuse that public regression environment; + add the proposed private dnsdist/BIND fleet alongside it. The public recursor + does not establish private tenant isolation. + +The existing [Task environment](../../Taskfile.yaml) already deploys public DNS +across upstream, control, and edge clusters with shared test-infra tooling. +The [CI workflow](../../.github/workflows/e2e.yml) exercises both private DNS +qualification and that public chain. The shared Task include is pinned to +`datum-cloud/test-infra@7622c925d5bcf2fa5bc01823775973cf6daf51ca`. + +## Target topology + +Use three local Kubernetes clusters: management and two edge clusters. Host +the Milo core and Karmada APIs in the management cluster. Exercise two consumer +project APIs and the DNS service's managed project through Milo's project +control-plane routing and distinct credentials; namespaces alone cannot prove +those boundaries. The current independent-source Kind APIs are a distributed +control-plane fixture, rather than the final deployment topology. + +```mermaid +flowchart LR + subgraph A[Project A control plane] + CA[Compute publisher A] --> DA[Project DNS APIs] + end + subgraph B[Project B control plane] + CB[Compute publisher B] --> DB[Project DNS APIs] + end + subgraph M[Management cluster] + CORE[Milo core API / project discovery] + DNS[DNS service managed project API] + CP[Two DNS control-plane replicas] + INT[Trusted VPC / DNS integration] + JS[Three-node TLS JetStream / persistent volumes] + K[Karmada federation API] + end + DA & DB --> CP + CORE --> CP + CP <--> DNS + CP --> JS + K --> E1 & E2 + INT -->|Access intent| DA & DB + subgraph E1[Edge A] + V1[Consumer VPC A workload] + F1[Private endpoint / well-known resolver] + G1[Galactic ServiceRoutePolicy A] + V1 --> F1 --> G1 + end + subgraph E2[Edge B] + V2[Consumer VPC B workload] + F2[Private endpoint / same resolver address] + G2[Galactic ServiceRoutePolicy B] + V2 --> F2 --> G2 + end + subgraph SV[Producer DNS service VPC] + EP1[ServiceEndpoint / node listener A] + EP2[ServiceEndpoint / node listener B] + N1[Edge A shared node dnsdist / BIND] + N2[Edge B shared node dnsdist / BIND] + R[Shared regional dnsdist / two BIND members] + EP1 --> N1 --> R + EP2 --> N2 --> R + end + G1 --> EP1 + G2 --> EP2 + INT -->|Private service authorization| K + JS --> N1 & N2 & R +``` + +The two edge clusters model separate edge control planes on one host in one region. +The DNS service runs in its own producer VPC. Both node and regional resolver +tiers belong to that service VPC; node members run on shared edge capacity, +not in consumer VPCs. Galactic `ServiceEndpoint` resources select the producer +listeners. Consumer `ServiceRoutePolicy` resources expose those listeners as +private endpoints at the well-known resolver address in each consumer VPC. +Galactic owns VPC authorization and destination translation. The DNS integration +publishes short-lived access intent through the DNS APIs; DNS workers do not +watch Galactic resources. Queries carry the authorized service destination to +dnsdist, which supplies PROXYv2 only across protected serving links. Clients +cannot choose another tenant's destination or supply a trusted PROXYv2 header. + +Keep fleet size fixed when adding VPCs or zones. Two contexts must share the +same node and regional workloads, with independent regional member storage and +identities. Deploy controllers, agents, watchdogs, dnsdist, and BIND as real +pods from the runtime package, with the same reload and fail-closed helpers +intended for staging. Do not mount the Docker socket into those workloads. + +## Qualification stages + +1. **Package gate (implemented).** Build the runtime image through + `config/internal-dns/Dockerfile`, smoke-test its executable as the image's + non-root user, render all CRDs and deployment examples with the installer's + pinned Kustomize, and validate their JSON + configuration, CRD schemas installed in an RBAC-enabled test API, Kubernetes + permissions, and NATS subject permissions in CI. + This gate checks packaging; the examples still require deployment overlays + and serving images. It makes no live Kubernetes or consumer-path claim. +2. **Kubernetes deployment (implemented).** Reuse pinned test-infra, serving + images and command helpers, TLS admission, scoped worker/product credentials, + two controllers, three TLS brokers, and persistent serving state. Run + publication, isolation, takeover, and expiry checks against real pods. ACK + timing is 30/90 seconds; ownership lease and termination grace are shortened + to 10 and 2 seconds. CI runs the public regression chain in the same job. + Publication resumes after broker/controller restoration using retained + broker PVCs. Member restart, stale-envelope replay, broker quorum, and API-partition fault + scenarios remain to be added to this deployment suite. +3. **Consumer network path.** Add real Milo project discovery and the DNS + service managed project, Karmada and edge resource propagation, Galactic + VPCs, the separate producer DNS service VPC, its `ServiceEndpoint` listeners, + consumer private endpoints through `ServiceRoutePolicy`, and the trusted + access integration. Run Go publisher jobs with product credentials and query from + attached workloads without manually programming context routes. Also test + an actual feature-flagged Compute build; a publisher fixture alone does not + qualify the product integration. +4. **Staging and regional resilience.** Reuse those overlays in staging-lab + for actual Compute attachments and edge networking. Add a second region only + after defining and deploying its broker and delivery topology; a second edge + cluster connected to the same broker is not regional replication proof. + +## Acceptance and evidence + +Each stage must retain the exact Git commits, rendered manifests, image digests, +topology, non-secret identity metadata, observed resource versions, failure +timestamps, DNS responses, and pod logs. Mark each scenario as passed, failed, +or not exercised. Do not count a skipped integration as a passing check. + +Required deployment and consumer-path scenarios: + +- Overlapping zones, names, and endpoint addresses in two VPCs return distinct + answers through private endpoints into the same producer DNS service VPC over + UDP and TCP, including positive and negative cache isolation. +- Unauthorized access, forged destinations and PROXYv2 headers, and an expired + service route cannot return another VPC's records. Product credentials cannot + issue grants or authorize network access. +- Endpoint health withdrawal and recovery, deletion, source API loss, and + stale-observation replay obey the original deadlines and configured TTLs. +- Either controller can take ownership without stale publication; a healthy + project continues publishing while another project's API is unavailable. +- Broker leader loss retains publication with quorum; quorum loss, agent + restart, member failure, and node failure preserve expiry and fail closed. +- Certificate rotation, webhook unavailability, rolling member replacement, + and PVC reuse do not bypass admission or regress checkpoint authority. +- Adding contexts leaves the workload count fixed, and public authoritative + DNS still resolves its test records without exposing private zones. + +Kind cannot qualify physical fabric, microVM attachment, disk durability across +host loss, production scale, or independent regional failure. Staging-lab owns +those checks. Capacity tests must measure reload/proof latency against the +five-second watchdog budget before increasing shard limits. diff --git a/test/internaldns/qualify-package.sh b/test/internaldns/qualify-package.sh new file mode 100755 index 00000000..c6988973 --- /dev/null +++ b/test/internaldns/qualify-package.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Qualify the shipped runtime artifact without claiming deployment parity. +set -euo pipefail + +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$root" +evidence_dir=${INTERNAL_DNS_PACKAGE_EVIDENCE:-test/internaldns/results/package} +mkdir -p "$evidence_dir" +evidence_dir=$(cd "$evidence_dir" && pwd) + +# The RBAC test skips without a real API server. Require its binaries rather +# than accepting that skip as deployment qualification. +if [[ -z ${KUBEBUILDER_ASSETS:-} ]]; then + make setup-envtest > "$evidence_dir/envtest.log" 2>&1 + k8s_version=$(go list -m -f '{{.Version}}' k8s.io/api | awk -F'[v.]' '{printf "1.%d", $3}') + KUBEBUILDER_ASSETS=$(bin/setup-envtest use "$k8s_version" --bin-dir bin -p path) + export KUBEBUILDER_ASSETS +fi +# go test runs from the package directory, not this script's repository root. +# setup-envtest can return a relative path when --bin-dir is relative. +KUBEBUILDER_ASSETS=$(cd "$KUBEBUILDER_ASSETS" && pwd) +export KUBEBUILDER_ASSETS +test -x "$KUBEBUILDER_ASSETS/kube-apiserver" +test -x "$KUBEBUILDER_ASSETS/etcd" +go test ./internal/internaldns/runtime \ + -run '^(TestDeploymentExampleConfigsValidate|TestDeploymentRolesAuthorizeWorkerAndDiscoveryBoundaries|TestNATSExampleUsesExactMemberAckSubjects|TestEveryMemberCanCreateAllRuntimeConsumers|TestReadOnlyServingImageHasWritableBINDWorkingDirectory)$' \ + -count=1 -v > "$evidence_dir/contracts.log" 2>&1 +# Use the same pinned Kustomize binary as installation and the public E2E job. +make kustomize > "$evidence_dir/kustomize.log" 2>&1 +bin/kustomize build config/crd > "$evidence_dir/crds.yaml" +crd_files=(config/crd/bases/*.yaml) +test "$(grep -c '^kind: CustomResourceDefinition$' "$evidence_dir/crds.yaml")" -eq "${#crd_files[@]}" +bin/kustomize build config/internal-dns > "$evidence_dir/deployment-examples.yaml" + +# Run by immutable local image ID, so an unrelated tag cannot replace the +# artifact between build and verification. Docker selects the host architecture. +docker build --file config/internal-dns/Dockerfile \ + --iidfile "$evidence_dir/image.id" . > "$evidence_dir/build.log" 2>&1 +image_id=$(cat "$evidence_dir/image.id") +docker image inspect "$image_id" > "$evidence_dir/image.json" +test "$(docker image inspect --format '{{.Config.User}}' "$image_id")" = '65532:65532' +docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges "$image_id" --help \ + > "$evidence_dir/help.log" 2>&1 +# Confirm this is the internal DNS entry point rather than an image that only +# happens to exit successfully. +grep -q 'Process role: control-plane, agent, or watchdog' "$evidence_dir/help.log" +git rev-parse HEAD > "$evidence_dir/source.commit" +git status --porcelain > "$evidence_dir/source.status" +printf 'Internal DNS package qualification passed; evidence: %s\n' "$evidence_dir" diff --git a/test/internaldns/run.sh b/test/internaldns/run.sh new file mode 100755 index 00000000..cc910f90 --- /dev/null +++ b/test/internaldns/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Task entry point for the shared Kubernetes qualification. +set -euo pipefail +root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +cd "$root" +if (( $# )); then + echo 'Use Task env:internal-dns:up / env:internal-dns:qualify; this wrapper takes no arguments.' >&2 + exit 2 +fi +task --yes env:internal-dns:up +task --yes env:internal-dns:qualify From da9ae8bfe9cf4c517d08f1e8c64c79ddd49ba139 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 16:36:22 -0500 Subject: [PATCH 02/21] Qualify private DNS using project-scoped service roles --- test/internaldns/kubernetes/render/main.go | 9 +-------- test/internaldns/kubernetes/render/main_test.go | 10 ++++------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 11797354..aee5faf6 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -261,12 +261,7 @@ func sourceObjects(project string) []object { worker := obj("rbac.authorization.k8s.io/v1", "RoleBinding", "internal-dns-source-worker", projectNS) worker["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "internal-dns-control-plane"} worker["subjects"] = []object{{"kind": "ServiceAccount", "name": "internal-dns-control-plane", "namespace": ns}} - sar := obj("rbac.authorization.k8s.io/v1", "ClusterRole", "internal-dns-source-authorization", "") - sar["rules"] = []object{{"apiGroups": []string{"authorization.k8s.io"}, "resources": []string{"subjectaccessreviews"}, "verbs": []string{"create"}}} - sarBinding := obj("rbac.authorization.k8s.io/v1", "ClusterRoleBinding", "internal-dns-source-authorization", "") - sarBinding["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "internal-dns-source-authorization"} - sarBinding["subjects"] = worker["subjects"] - items = append(items, worker, sar, sarBinding) + items = append(items, worker) accounts := []struct{ name, namespace, role string }{{"dns-publisher", "compute-system", "internal-dns-product-writer"}, {"dns-grant-issuer", "compute-system", "internal-dns-grant-issuer"}, {"dns-vpc-integration", "network-system", "internal-dns-vpc-integration"}, {"internal-dns-control-plane", ns, ""}} for _, account := range accounts { if account.role != "" { @@ -401,8 +396,6 @@ func configure() { cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", SourceClusterUID: "source-cluster-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", SourceClusterUID: "source-cluster-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} cfg.NodeBackends = []model.Backend{{MemberID: "node-bind-0", Address: "127.0.0.1", Port: 5300}} cfg.ClusterBackends = []model.Backend{{MemberID: memberNames[2], Address: bindIP[0], Port: 5300}, {MemberID: memberNames[3], Address: bindIP[1], Port: 5300}} - cfg.Admission.PlatformSubjects = []string{"system:serviceaccount:" + ns + ":internal-dns-control-plane", "kubernetes-admin"} - cfg.Admission.IntegrationSubjects = []string{"system:serviceaccount:network-system:dns-vpc-integration"} cfg.Admission.MaxAccessLeaseSeconds = 600 for _, name := range []string{"control-a", "control-b"} { c := cfg diff --git a/test/internaldns/kubernetes/render/main_test.go b/test/internaldns/kubernetes/render/main_test.go index 6d23683e..9e7ad8fd 100644 --- a/test/internaldns/kubernetes/render/main_test.go +++ b/test/internaldns/kubernetes/render/main_test.go @@ -55,7 +55,7 @@ func TestSourceWorkerBindingIsProjectScoped(t *testing.T) { t.Fatal(err) } t.Chdir(root) - scoped, sar := false, false + scoped := false for _, item := range sourceObjects("project-a") { metadata := item["metadata"].(map[string]any) switch { @@ -63,12 +63,10 @@ func TestSourceWorkerBindingIsProjectScoped(t *testing.T) { t.Fatal("source worker inherited platform cluster-wide binding") case item["kind"] == "RoleBinding" && metadata["name"] == "internal-dns-source-worker": scoped = metadata["namespace"] == projectNS - case item["kind"] == "ClusterRole" && metadata["name"] == "internal-dns-source-authorization": - rules := item["rules"].([]object) - sar = len(rules) == 1 && rules[0]["resources"].([]string)[0] == "subjectaccessreviews" + } } - if !scoped || !sar { - t.Fatal("source worker requires project scope and separate authorization access") + if !scoped { + t.Fatal("source worker requires project-scoped permissions") } } From 10611d11a822d3f498ab6dfb08fd3aed3de91f21 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 16:37:14 -0500 Subject: [PATCH 03/21] Use the bounded validation scope in project qualification --- .../internaldns/fixtures/validating-webhook.yaml.tmpl | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/internaldns/fixtures/validating-webhook.yaml.tmpl b/test/internaldns/fixtures/validating-webhook.yaml.tmpl index af30f491..1f7af719 100644 --- a/test/internaldns/fixtures/validating-webhook.yaml.tmpl +++ b/test/internaldns/fixtures/validating-webhook.yaml.tmpl @@ -15,7 +15,7 @@ webhooks: url: https://host.docker.internal:9443/validate-internal-dns caBundle: ${CA_BUNDLE} rules: - - operations: [CREATE, UPDATE, DELETE] + - operations: [CREATE, UPDATE] apiGroups: [dns.networking.miloapis.com] apiVersions: [v1alpha1] resources: @@ -25,13 +25,4 @@ webhooks: - dnsrecordcontributions - dnsrecordcontributions/status - dnsnamingpolicies - - dnsmanagednamespaces - - dnsresolvercontexts - - dnsresolvercontexts/status - dnsresolveraccessbindings - - dnsresolveraccessbindings/status - - dnsresolverbindings - - dnspublicationownerships - - dnspublicationmanifests - - dnspublicationchunks - - dnstransportoutboxes From fb6a5441aeb4af53b3b34c5636e3bcdb439efdb8 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 17:04:14 -0500 Subject: [PATCH 04/21] Qualify customer zone configuration with scoped project credentials --- test/internaldns/kubernetes/qualify/main.go | 36 +++++++++---------- .../kubernetes/qualify/scenarios.go | 2 +- test/internaldns/kubernetes/render/main.go | 5 +-- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index 29d9e93a..e5202f50 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -32,20 +32,20 @@ import ( ) type project struct { - Name string `json:"name"` - ProjectUID string `json:"projectUID"` - SourceClusterUID string `json:"sourceClusterUID"` - Namespace string `json:"namespace"` - AdminKubeconfig string `json:"adminKubeconfig"` - ProductKubeconfig string `json:"productKubeconfig"` - IssuerKubeconfig string `json:"issuerKubeconfig"` - IntegrationKubeconfig string `json:"integrationKubeconfig"` - PublisherSubject string `json:"publisherSubject"` - ConsumerAddress string `json:"consumerAddress"` - admin, product, issuer, integration client.Client - resolverContext dnsv1.DNSResolverContext - zone dnsv1.DNSZone - access dnsv1.DNSResolverAccessBinding + Name string `json:"name"` + ProjectUID string `json:"projectUID"` + SourceClusterUID string `json:"sourceClusterUID"` + Namespace string `json:"namespace"` + CustomerKubeconfig string `json:"customerKubeconfig"` + ProductKubeconfig string `json:"productKubeconfig"` + IssuerKubeconfig string `json:"issuerKubeconfig"` + IntegrationKubeconfig string `json:"integrationKubeconfig"` + PublisherSubject string `json:"publisherSubject"` + ConsumerAddress string `json:"consumerAddress"` + customer, product, issuer, integration client.Client + resolverContext dnsv1.DNSResolverContext + zone dnsv1.DNSZone + access dnsv1.DNSResolverAccessBinding } type environment struct { @@ -260,7 +260,7 @@ func (s *suite) run(path string) error { path string target *client.Client }{ - {p.AdminKubeconfig, &p.admin}, {p.ProductKubeconfig, &p.product}, + {p.CustomerKubeconfig, &p.customer}, {p.ProductKubeconfig, &p.product}, {p.IssuerKubeconfig, &p.issuer}, {p.IntegrationKubeconfig, &p.integration}, } { *pair.target, err = newClient(pair.path) @@ -337,7 +337,7 @@ func (s *suite) prepare(p *project) error { } s.pass(p.Name+" product discovers its managed namespace through the context API", p.resolverContext.Status.ManagedNamespace) p.zone = dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "prod", Namespace: p.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "prod.internal", DNSZoneClassName: "private-bind", Visibility: dnsv1.DNSZoneVisibilityPrivate}} - if err := p.admin.Create(s.ctx, &p.zone); err != nil { + if err := p.customer.Create(s.ctx, &p.zone); err != nil { return err } if err := s.associate(p, &p.zone, "prod"); err != nil { @@ -360,11 +360,11 @@ func (s *suite) prepare(p *project) error { func (s *suite) associate(p *project, zone *dnsv1.DNSZone, name string) error { association := &dnsv1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: p.Namespace}, Spec: dnsv1.DNSZoneAssociationSpec{DNSZoneRef: objectRef(zone), ResolverContextRef: objectRef(&p.resolverContext)}} - if err := p.admin.Create(s.ctx, association); err != nil { + if err := p.customer.Create(s.ctx, association); err != nil { return err } return s.wait("private zone association "+p.Name+"/"+name, 60*time.Second, func() (bool, error) { - err := p.admin.Get(s.ctx, client.ObjectKeyFromObject(association), association) + err := p.customer.Get(s.ctx, client.ObjectKeyFromObject(association), association) return condition(association.Status.Conditions, "Accepted", association.Generation), err }) } diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index 0ba1f0d6..7602eca8 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -94,7 +94,7 @@ func (s *suite) scenarios(a, b *project) error { return err } apps := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "apps", Namespace: a.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "apps.internal", DNSZoneClassName: "private-bind", Visibility: dnsv1.DNSZoneVisibilityPrivate}} - if err := a.admin.Create(s.ctx, apps); err != nil { + if err := a.customer.Create(s.ctx, apps); err != nil { return err } if err := s.associate(a, apps, "apps"); err != nil { diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index aee5faf6..9ff2f81e 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -262,7 +262,7 @@ func sourceObjects(project string) []object { worker["roleRef"] = object{"apiGroup": "rbac.authorization.k8s.io", "kind": "ClusterRole", "name": "internal-dns-control-plane"} worker["subjects"] = []object{{"kind": "ServiceAccount", "name": "internal-dns-control-plane", "namespace": ns}} items = append(items, worker) - accounts := []struct{ name, namespace, role string }{{"dns-publisher", "compute-system", "internal-dns-product-writer"}, {"dns-grant-issuer", "compute-system", "internal-dns-grant-issuer"}, {"dns-vpc-integration", "network-system", "internal-dns-vpc-integration"}, {"internal-dns-control-plane", ns, ""}} + accounts := []struct{ name, namespace, role string }{{"dns-customer", projectNS, "internal-dns-customer"}, {"dns-publisher", "compute-system", "internal-dns-product-writer"}, {"dns-grant-issuer", "compute-system", "internal-dns-grant-issuer"}, {"dns-vpc-integration", "network-system", "internal-dns-vpc-integration"}, {"internal-dns-control-plane", ns, ""}} for _, account := range accounts { if account.role != "" { items = append(items, obj("v1", "ServiceAccount", account.name, account.namespace)) @@ -380,10 +380,11 @@ func configure() { admin := []string{a, b}[i] internal := []string{ai, bi}[i] sourceData[project+".json"] = string(tokenConfig(admin, internal, project, "internal-dns-control-plane", ns, "worker")) + tokenConfig(admin, "", project, "dns-customer", projectNS, "customer") tokenConfig(admin, "", project, "dns-publisher", "compute-system", "product") tokenConfig(admin, "", project, "dns-grant-issuer", "compute-system", "issuer") tokenConfig(admin, "", project, "dns-vpc-integration", "network-system", "integration") - projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "sourceClusterUID": []string{"source-cluster-a-uid", "source-cluster-b-uid"}[i], "namespace": projectNS, "adminKubeconfig": admin, "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) + projectRows = append(projectRows, object{"name": project, "projectUID": project + "-uid", "sourceClusterUID": []string{"source-cluster-a-uid", "source-cluster-b-uid"}[i], "namespace": projectNS, "customerKubeconfig": filepath.Join(out, project+"-customer.kubeconfig"), "productKubeconfig": filepath.Join(out, project+"-product.kubeconfig"), "issuerKubeconfig": filepath.Join(out, project+"-issuer.kubeconfig"), "integrationKubeconfig": filepath.Join(out, project+"-integration.kubeconfig"), "publisherSubject": "system:serviceaccount:compute-system:dns-publisher", "consumerAddress": []string{"fd53::a", "fd53::b"}[i]}) } sources["stringData"] = sourceData items := make([]object, 0, 8) From ea77716eb6db6722fd155439105cbfb0f100ff3b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Mon, 12 Oct 2026 17:54:04 -0500 Subject: [PATCH 05/21] Qualify stale observations without overwriting concurrent publication status --- test/internaldns/kubernetes/qualify/scenarios.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index 7602eca8..25725c68 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -139,7 +139,7 @@ func (s *suite) scenarios(a, b *project) error { } stale := pa.contribution.DeepCopy() stale.Status.Sequence-- - if err := a.product.Status().Update(s.ctx, stale); err == nil { + if err := a.product.Status().Patch(s.ctx, stale, client.MergeFrom(&pa.contribution)); err == nil { return fmt.Errorf("stale producer sequence was accepted") } else if !apierrors.IsForbidden(err) && !apierrors.IsInvalid(err) { return fmt.Errorf("stale producer sequence failed for an unrelated reason: %w", err) From 7694ef3cb4c531f488c64041beae09127bdf1489 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 09:54:10 -0500 Subject: [PATCH 06/21] test: Keep DNS fixtures context scoped --- test/internaldns/kubernetes/render/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 9ff2f81e..d4268b2a 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -390,7 +390,6 @@ func configure() { items := make([]object, 0, 8) items = append(items, sources) cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] - cfg.ConsumerPrefix = "fd53::/64" cfg.ClusterPrefix = "fd54::/64" cfg.ManagedDomainSuffix = "managed.internal" cfg.OwnershipLeaseSeconds = 10 From a2a70d3296fab02805d32de7ba51985c6a8736c1 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 16:26:24 -0500 Subject: [PATCH 07/21] test: Select explicit Kubernetes source APIs --- test/internaldns/kubernetes/render/main.go | 7 +++++- .../kubernetes/render/main_test.go | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 14b7c752..76fe9087 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -392,7 +392,7 @@ func configure() { cfg.ClusterPrefix = "fd54::/64" cfg.ManagedDomainSuffix = "managed.internal" cfg.OwnershipLeaseSeconds = 10 - cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} + configureProjectSources(&cfg) cfg.NodeBackends = []model.Backend{{MemberID: "node-bind-0", Address: "127.0.0.1", Port: 5300}} cfg.ClusterBackends = []model.Backend{{MemberID: memberNames[2], Address: bindIP[0], Port: 5300}, {MemberID: memberNames[3], Address: bindIP[1], Port: 5300}} for _, name := range []string{"control-a", "control-b"} { @@ -436,3 +436,8 @@ func configure() { // No tokens or certificate contents are included in the public contract. write("environment.json", object{"platformKubeconfig": platform, "namespace": ns, "region": "us-central1", "shard": "shared-0", "probePod": "internal-dns-probe", "probeContainer": "probe", "probeBinary": "/dns-qualify", "controllerDeployments": []string{"control-a", "control-b"}, "brokerStatefulSet": "internal-dns-nats", "projects": projectRows, "regionalMembers": memberNames[2:], "members": memberNames, "limitations": []string{"Service-side route fixture, not Galactic VPC authorization", "Independent Kind project APIs, not Milo project discovery", "One host and one region, not independent regional failure"}}) } + +func configureProjectSources(cfg *runtime.Config) { + cfg.ProjectDiscovery = nil + cfg.Projects = []runtime.ProjectConfig{{Name: "project-a", ProjectUID: "project-a-uid", Namespace: projectNS, Kubeconfig: "/sources/project-a.json"}, {Name: "project-b", ProjectUID: "project-b-uid", Namespace: projectNS, Kubeconfig: "/sources/project-b.json"}} +} diff --git a/test/internaldns/kubernetes/render/main_test.go b/test/internaldns/kubernetes/render/main_test.go index 6f63c37d..ece0726e 100644 --- a/test/internaldns/kubernetes/render/main_test.go +++ b/test/internaldns/kubernetes/render/main_test.go @@ -100,3 +100,27 @@ func TestSourceWorkerBindingIsProjectScoped(t *testing.T) { t.Fatal("source worker requires project-scoped permissions") } } + +func TestFixtureSourcesUseIndependentProjectAPIs(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + t.Chdir(root) + cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] + if cfg.ProjectDiscovery == nil { + t.Fatal("deployment example must exercise project discovery") + } + configureProjectSources(&cfg) + if cfg.ProjectDiscovery != nil || len(cfg.Projects) != 2 { + t.Fatal("fixture source selection mixes discovery and direct APIs") + } + for _, project := range cfg.Projects { + if project.Kubeconfig == "" || project.Namespace != projectNS { + t.Fatalf("fixture source scope missing: %#v", project) + } + } + if err := cfg.Validate("control-plane"); err != nil { + t.Fatalf("fixture source config rejected: %v", err) + } +} From f71a81fba74f9028716213e5fd91bcedd0f27fce Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:09:08 -0500 Subject: [PATCH 08/21] Validate DNS continuity and safe activation in Kubernetes --- test/internaldns/README.md | 2 +- test/internaldns/kubernetes/deploy.sh | 3 +++ test/internaldns/kubernetes/render/main.go | 7 +++---- test/internaldns/production-parity.md | 6 +++--- test/internaldns/qualify-package.sh | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/test/internaldns/README.md b/test/internaldns/README.md index a3f79971..92c77253 100644 --- a/test/internaldns/README.md +++ b/test/internaldns/README.md @@ -72,7 +72,7 @@ checks health withdrawal, stale-write conflicts, and DNS-owned status permission The fleet contains one node member, one regional frontend, and two regional BIND members, each with independent persistent state. Agents validate and -activate configuration, prove UDP/TCP publication, and renew watchdog leases. +activate configuration and prove UDP/TCP publication. Local readiness verifies\napproved files and live daemons while healthy DNS survives agent restarts. The broker uses three persistent replicas, client and route TLS, and scoped subject permissions. cert-manager issues broker, client, and admission certificates. ACK timing matches the shipped 30-second interval and 90-second diff --git a/test/internaldns/kubernetes/deploy.sh b/test/internaldns/kubernetes/deploy.sh index 431b5fc9..2bff0417 100755 --- a/test/internaldns/kubernetes/deploy.sh +++ b/test/internaldns/kubernetes/deploy.sh @@ -54,6 +54,9 @@ go run ./test/internaldns/kubernetes/render --phase configure --out "$work" \ for controller in control-a control-b; do "${platform[@]}" -n internal-dns-system rollout status "deployment/$controller" --timeout=180s done +for pod in internal-dns-node internal-dns-regional-front internal-dns-regional-bind-0 internal-dns-regional-bind-1; do + "${platform[@]}" -n internal-dns-system wait --for=condition=Ready "pod/$pod" --timeout=180s +done # The source API servers reach a real TLS admission Service over the isolated # Kind network. diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 76fe9087..0f07fea3 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -220,12 +220,12 @@ func fleetPod(name, member, role string) object { o["metadata"].(object)["labels"] = object{"internal-dns-role": role, "internal-dns-member": member} mounts := []object{mount("config", "/config", true), mount("nats", "/var/run/internal-dns/nats", true), mount("state", "/var/lib/internal-dns", false), mount("daemon-config", "/etc/internal-dns", false), mount("lease", "/run/internal-dns", false), mount("runtime", "/tmp", false), mount("rndc", "/etc/bind", true)} containers := make([]object, 0, 3) - for _, process := range []string{"agent", "watchdog"} { + for _, process := range []string{"agent"} { containers = append(containers, object{"name": process, "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ]; do sleep 1; done; exec /internal-dns --role=" + process + " --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) } sec := security() sec["capabilities"] = object{"drop": []string{"ALL"}, "add": []string{"NET_BIND_SERVICE"}} - containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/usr/local/sbin/internal-dns-daemons", role}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) + containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/internal-dns", "--role=serving", "--config=/config/config.json"}, "readinessProbe": object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) volumes := []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(member), {"name": "state", "persistentVolumeClaim": object{"claimName": name + "-state"}}, {"name": "daemon-config", "persistentVolumeClaim": object{"claimName": name + "-config"}}, {"name": "lease", "emptyDir": object{"medium": "Memory"}}, {"name": "runtime", "emptyDir": object{}}, {"name": "rndc", "secret": object{"secretName": "internal-dns-rndc"}}} // The aliases model the service-side destination contract only. The separate // probe has no ability to write routes or send trusted PROXY traffic. @@ -405,7 +405,6 @@ func configure() { configs := exampleConfigs("fleet.example.yaml") for i, key := range []string{"internal-dns-node-0-config", "internal-dns-regional-front-0-config", "internal-dns-regional-bind-0-config", "internal-dns-regional-bind-1-config"} { c := configs[key] - c.Watchdog.StartupGraceSeconds = 30 if i == 0 { c.Agent.Render.NodeDNSDist.Backends = cfg.NodeBackends c.Agent.Render.NodeDNSDist.ACLs = []string{"fd00::/8", "::1/128"} @@ -422,7 +421,7 @@ func configure() { c.Agent.PublicationDNSProbe.Server = "[" + bindIP[i-2] + "]:5300" } must(c.Validate("agent")) - must(c.Validate("watchdog")) + must(c.Validate("readiness")) items = append(items, configMap(podNames[i]+"-config", c)) } routes := obj("v1", "Pod", "internal-dns-network-fixture", ns) diff --git a/test/internaldns/production-parity.md b/test/internaldns/production-parity.md index 66fd4041..6b94ad31 100644 --- a/test/internaldns/production-parity.md +++ b/test/internaldns/production-parity.md @@ -120,7 +120,7 @@ cannot choose another tenant's destination or supply a trusted PROXYv2 header. Keep fleet size fixed when adding VPCs or zones. Two contexts must share the same node and regional workloads, with independent regional member storage and -identities. Deploy controllers, agents, watchdogs, dnsdist, and BIND as real +identities. Deploy controllers, agents, dnsdist, and BIND as real pods from the runtime package, with the same reload and fail-closed helpers intended for staging. Do not mount the Docker socket into those workloads. @@ -187,5 +187,5 @@ Required deployment and consumer-path scenarios: Kind cannot qualify physical fabric, microVM attachment, disk durability across host loss, production scale, or independent regional failure. Staging-lab owns -those checks. Capacity tests must measure reload/proof latency against the -five-second watchdog budget before increasing shard limits. +those checks. Capacity tests must measure reload/proof latency, query continuity, +and incomplete-activation recovery before increasing shard limits. diff --git a/test/internaldns/qualify-package.sh b/test/internaldns/qualify-package.sh index c6988973..c36065b8 100755 --- a/test/internaldns/qualify-package.sh +++ b/test/internaldns/qualify-package.sh @@ -44,7 +44,7 @@ docker run --rm --network none --read-only --cap-drop ALL \ > "$evidence_dir/help.log" 2>&1 # Confirm this is the internal DNS entry point rather than an image that only # happens to exit successfully. -grep -q 'Process role: control-plane, agent, or watchdog' "$evidence_dir/help.log" +grep -q 'Process role: control-plane, agent, serving, or readiness' "$evidence_dir/help.log" git rev-parse HEAD > "$evidence_dir/source.commit" git status --porcelain > "$evidence_dir/source.status" printf 'Internal DNS package qualification passed; evidence: %s\n' "$evidence_dir" From 52e81056fa062c9386109a40adc21c45a99a203c Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:45:08 -0500 Subject: [PATCH 09/21] Retry private DNS publication after readiness conflicts --- test/internaldns/kubernetes/qualify/main.go | 17 ++--- .../kubernetes/qualify/publisher.go | 23 +++++++ .../kubernetes/qualify/publisher_test.go | 67 +++++++++++++++++++ .../kubernetes/qualify/scenarios.go | 13 ++-- 4 files changed, 102 insertions(+), 18 deletions(-) create mode 100644 test/internaldns/kubernetes/qualify/publisher.go create mode 100644 test/internaldns/kubernetes/qualify/publisher_test.go diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index c8a0a775..51ec4512 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -396,16 +396,13 @@ func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, } func (s *suite) observe(p *project, pub *publication, eligible bool) error { - if err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution); err != nil { - return err - } - before := pub.contribution.DeepCopy() - pub.contribution.Spec.Eligible = eligible - pub.contribution.Spec.Reason = "EndpointReady" - if !eligible { - pub.contribution.Spec.Reason = "EndpointNotReady" - } - if err := p.product.Patch(s.ctx, &pub.contribution, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + if err := mutateContribution(s.ctx, p.product, &pub.contribution, func(spec *dnsv1.DNSRecordContributionSpec) { + spec.Eligible = eligible + spec.Reason = "EndpointReady" + if !eligible { + spec.Reason = "EndpointNotReady" + } + }); err != nil { return err } return s.waitPublication(pub) diff --git a/test/internaldns/kubernetes/qualify/publisher.go b/test/internaldns/kubernetes/qualify/publisher.go new file mode 100644 index 00000000..1fc5f3f0 --- /dev/null +++ b/test/internaldns/kubernetes/qualify/publisher.go @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func mutateContribution(ctx context.Context, source client.Client, contribution *dnsv1.DNSRecordContribution, mutate func(*dnsv1.DNSRecordContributionSpec)) error { + key := client.ObjectKeyFromObject(contribution) + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := source.Get(ctx, key, contribution); err != nil { + return err + } + before := contribution.DeepCopy() + mutate(&contribution.Spec) + return source.Patch(ctx, contribution, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})) + }) +} diff --git a/test/internaldns/kubernetes/qualify/publisher_test.go b/test/internaldns/kubernetes/qualify/publisher_test.go new file mode 100644 index 00000000..c38c0f1f --- /dev/null +++ b/test/internaldns/kubernetes/qualify/publisher_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "errors" + "testing" + + dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +type statusConflictClient struct { + client.Client + patches int +} + +func (c *statusConflictClient) Patch(ctx context.Context, object client.Object, patch client.Patch, opts ...client.PatchOption) error { + c.patches++ + if c.patches == 1 { + var current dnsv1.DNSRecordContribution + if err := c.Client.Get(ctx, client.ObjectKeyFromObject(object), ¤t); err != nil { + return err + } + current.Status.PublishedRevision = 21 + if err := c.Client.Status().Update(ctx, ¤t); err != nil { + return err + } + return apierrors.NewConflict(schema.GroupResource{Group: dnsv1.GroupVersion.Group, Resource: "dnsrecordcontributions"}, object.GetName(), errors.New("DNS readiness changed")) + } + return c.Client.Patch(ctx, object, patch, opts...) +} + +func TestContributionSpecRetryPreservesDNSReadiness(t *testing.T) { + scheme := runtime.NewScheme() + if err := dnsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + contribution := dnsv1.DNSRecordContribution{ObjectMeta: metav1.ObjectMeta{Name: "endpoint", Namespace: "application"}, Spec: dnsv1.DNSRecordContributionSpec{RecordSets: recordSets("web", "10.42.0.10", ""), Eligible: true}} + source := &statusConflictClient{Client: fake.NewClientBuilder().WithScheme(scheme).WithObjects(&contribution).WithStatusSubresource(&dnsv1.DNSRecordContribution{}).Build()} + if err := mutateContribution(context.Background(), source, &contribution, func(spec *dnsv1.DNSRecordContributionSpec) { + spec.RecordSets = recordSets("web", "10.42.0.30", "") + spec.Eligible = false + spec.Reason = "EndpointNotReady" + }); err != nil { + t.Fatal(err) + } + if source.patches != 2 { + t.Fatalf("patch attempts = %d, want 2", source.patches) + } + var stored dnsv1.DNSRecordContribution + if err := source.Get(context.Background(), client.ObjectKeyFromObject(&contribution), &stored); err != nil { + t.Fatal(err) + } + if stored.Status.PublishedRevision != 21 { + t.Fatal("producer write changed DNS readiness status") + } + if stored.Spec.Eligible || stored.Spec.Reason != "EndpointNotReady" || len(stored.Spec.RecordSets) != 1 || len(stored.Spec.RecordSets[0].Records) != 1 || stored.Spec.RecordSets[0].Records[0].A == nil || stored.Spec.RecordSets[0].Records[0].A.Content != "10.42.0.30" { + t.Fatal("records and eligibility were not published together") + } +} diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index e9a493eb..dd6dc8f0 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -177,14 +177,11 @@ func (s *suite) scenarios(a, b *project) error { } func (s *suite) update(p *project, pub *publication, address, ipv6 string) error { - if err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&pub.contribution), &pub.contribution); err != nil { - return err - } - before := pub.contribution.DeepCopy() - pub.contribution.Spec.RecordSets = recordSets(pub.registration.Spec.Name, address, ipv6) - pub.contribution.Spec.Eligible = true - pub.contribution.Spec.Reason = "EndpointReady" - if err := p.product.Patch(s.ctx, &pub.contribution, client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})); err != nil { + if err := mutateContribution(s.ctx, p.product, &pub.contribution, func(spec *dnsv1.DNSRecordContributionSpec) { + spec.RecordSets = recordSets(pub.registration.Spec.Name, address, ipv6) + spec.Eligible = true + spec.Reason = "EndpointReady" + }); err != nil { return err } return s.waitPublication(pub) From d2aa9d231d17ddd13249915f1bd4be40cfb6ed27 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Tue, 13 Oct 2026 17:48:36 -0500 Subject: [PATCH 10/21] Use promoted client methods in the publication fixture --- test/internaldns/kubernetes/qualify/publisher_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/internaldns/kubernetes/qualify/publisher_test.go b/test/internaldns/kubernetes/qualify/publisher_test.go index c38c0f1f..d8818cff 100644 --- a/test/internaldns/kubernetes/qualify/publisher_test.go +++ b/test/internaldns/kubernetes/qualify/publisher_test.go @@ -25,11 +25,11 @@ func (c *statusConflictClient) Patch(ctx context.Context, object client.Object, c.patches++ if c.patches == 1 { var current dnsv1.DNSRecordContribution - if err := c.Client.Get(ctx, client.ObjectKeyFromObject(object), ¤t); err != nil { + if err := c.Get(ctx, client.ObjectKeyFromObject(object), ¤t); err != nil { return err } current.Status.PublishedRevision = 21 - if err := c.Client.Status().Update(ctx, ¤t); err != nil { + if err := c.Status().Update(ctx, ¤t); err != nil { return err } return apierrors.NewConflict(schema.GroupResource{Group: dnsv1.GroupVersion.Group, Resource: "dnsrecordcontributions"}, object.GetName(), errors.New("DNS readiness changed")) From 72faf93003935c1b48b218968dfd43d557d0fd6e Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:05:14 -0500 Subject: [PATCH 11/21] test: qualify the publication API in its stack --- .../runtime/deployment_permissions_test.go | 12 ++++++++++++ test/internaldns/kubernetes/qualify/main.go | 4 ++-- test/internaldns/kubernetes/qualify/scenarios.go | 10 +++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/internal/internaldns/runtime/deployment_permissions_test.go b/internal/internaldns/runtime/deployment_permissions_test.go index e47d4536..224d03de 100644 --- a/internal/internaldns/runtime/deployment_permissions_test.go +++ b/internal/internaldns/runtime/deployment_permissions_test.go @@ -161,6 +161,18 @@ func TestEveryMemberCanCreateAllRuntimeConsumers(t *testing.T) { t.Fatal(err) } contents := string(data) + start := strings.Index(contents, `user: "internal-dns-control-plane"`) + if start < 0 { + t.Fatal("missing control-plane broker identity") + } + controlPolicy := strings.SplitN(contents[start:], "\n },", 2)[0] + controlACK := "$JS.ACK.DNS_PRIVATE." + durable(roleControlPlane, "us-central1", "shared-0", "acks") + ".>" + if !strings.Contains(controlPolicy, `"`+controlACK+`"`) { + t.Errorf("control plane cannot acknowledge its member-ACK consumer: %s", controlACK) + } + if strings.Contains(controlPolicy, `"$JS.ACK.DNS_PRIVATE.>"`) { + t.Error("control plane can acknowledge another serving member's consumers") + } for _, member := range []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} { start := strings.Index(contents, `user: "`+member+`"`) if start < 0 { diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index 50095d2e..985dbd7f 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -354,7 +354,7 @@ func (s *suite) prepare(p *project) error { } func (s *suite) associate(p *project, zone *dnsv1.DNSZone, name string) error { - association := &dnsv1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: p.Namespace}, Spec: dnsv1.DNSZoneAssociationSpec{DNSZoneRef: dnsv1.DNSNameReference{Name: zone.Name}, ResolverContextRef: dnsv1.DNSNameReference{Name: p.resolverContext.Name}}} + association := &dnsv1.DNSZoneAssociation{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: p.Namespace}, Spec: dnsv1.DNSZoneAssociationSpec{DNSZoneRef: objectRef(zone), ResolverContextRef: objectRef(&p.resolverContext)}} if err := p.customer.Create(s.ctx, association); err != nil { return err } @@ -385,7 +385,7 @@ func recordSets(name, address, ipv6 string) []dnsv1.DNSContributionRecordSet { return sets } -func (s *suite) publish(p *project, zone dnsv1.DNSNameReference, prefix, name, address, ipv6 string) (*publication, error) { +func (s *suite) publish(p *project, zone dnsv1.DNSObjectReference, prefix, name, address, ipv6 string) (*publication, error) { pub := &publication{registration: dnsv1.DNSRegistration{ObjectMeta: metav1.ObjectMeta{Name: prefix + "-registration", Namespace: p.Namespace}, Spec: dnsv1.DNSRegistrationSpec{DNSZoneRef: zone, Name: name, RecordTypes: []dnsv1.RRType{dnsv1.RRTypeA}, TTLSeconds: 2}}} if ipv6 != "" { pub.registration.Spec.RecordTypes = append(pub.registration.Spec.RecordTypes, dnsv1.RRTypeAAAA) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index b97c6eee..5945d566 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -35,11 +35,11 @@ func (s *suite) scenarios(a, b *project) error { return err } } - pa, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "common", "common", "10.42.0.10", "fd42::10") + pa, err := s.publish(a, objectRef(&a.zone), "common", "common", "10.42.0.10", "fd42::10") if err != nil { return err } - pb, err := s.publish(b, dnsv1.DNSNameReference{Name: b.zone.Name}, "common", "common", "10.42.0.20", "fd42::20") + pb, err := s.publish(b, objectRef(&b.zone), "common", "common", "10.42.0.20", "fd42::20") if err != nil { return err } @@ -86,7 +86,7 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("an unbound destination cannot resolve either tenant's private records", nil) - if _, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "only-a", "only-a", "10.42.0.20", ""); err != nil { + if _, err := s.publish(a, objectRef(&a.zone), "only-a", "only-a", "10.42.0.20", ""); err != nil { return err } if err := s.expect(a, "only-a.prod.internal", "A", false, dns.RcodeSuccess, "10.42.0.20"); err != nil { @@ -125,7 +125,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.associate(a, apps, "apps"); err != nil { return err } - if _, err := s.publish(a, dnsv1.DNSNameReference{Name: apps.Name}, "apps", "api", "10.42.0.12", ""); err != nil { + if _, err := s.publish(a, objectRef(apps), "apps", "api", "10.42.0.12", ""); err != nil { return err } if err := s.expect(a, "api.apps.internal", "A", true, dns.RcodeSuccess, "10.42.0.12"); err != nil { @@ -173,7 +173,7 @@ func (s *suite) scenarios(a, b *project) error { if err := s.observe(b, pb, true); err != nil { return err } - deleted, err := s.publish(a, dnsv1.DNSNameReference{Name: a.zone.Name}, "deleted", "deleted", "10.42.0.14", "") + deleted, err := s.publish(a, objectRef(&a.zone), "deleted", "deleted", "10.42.0.14", "") if err != nil { return err } From 214fd4a832b6b4485cdd1ccaf48885f2f42c08a2 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:07:15 -0500 Subject: [PATCH 12/21] test: keep domain policy checks in their stack --- test/internaldns/kubernetes/qualify/main.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test/internaldns/kubernetes/qualify/main.go b/test/internaldns/kubernetes/qualify/main.go index 985dbd7f..e3e2beff 100644 --- a/test/internaldns/kubernetes/qualify/main.go +++ b/test/internaldns/kubernetes/qualify/main.go @@ -287,14 +287,9 @@ func (s *suite) run(path string) error { } } a, b := &s.env.Projects[0], &s.env.Projects[1] - if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name == b.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name { + if a.resolverContext.UID == b.resolverContext.UID || a.zone.UID == b.zone.UID || a.resolverContext.Status.ManagedNamespace.Suffix == b.resolverContext.Status.ManagedNamespace.Suffix { return errors.New("source projects lost their distinct context, zone, or managed namespace identity") } - for _, p := range []*project{a, b} { - if p.resolverContext.Status.ManagedNamespace.Suffix != p.resolverContext.Spec.ManagedNamespace.DomainSuffix { - return fmt.Errorf("%s managed suffix does not match its requested domain", p.Name) - } - } s.pass("independent projects retain distinct context, zone, and managed namespace identities", map[string]string{"a": string(a.resolverContext.UID), "b": string(b.resolverContext.UID)}) if err := s.scenarios(a, b); err != nil { return err @@ -324,13 +319,13 @@ func (s *suite) permissions(p *project) error { } func (s *suite) prepare(p *project) error { - p.resolverContext = dnsv1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "application-network", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverContextSpec{ConsumerID: "qualification/" + p.Name, ManagedNamespace: dnsv1.DNSResolverContextManagedNamespace{Enabled: true, DomainSuffix: "datum.internal"}}} + p.resolverContext = dnsv1.DNSResolverContext{ObjectMeta: metav1.ObjectMeta{Name: "application-network", Namespace: p.Namespace}, Spec: dnsv1.DNSResolverContextSpec{ConsumerID: "qualification/" + p.Name, ManagedNamespace: dnsv1.DNSResolverContextManagedNamespace{Enabled: true}}} if err := p.integration.Create(s.ctx, &p.resolverContext); err != nil { return err } if err := s.wait(p.Name+" managed namespace", 120*time.Second, func() (bool, error) { err := p.product.Get(s.ctx, client.ObjectKeyFromObject(&p.resolverContext), &p.resolverContext) - return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.Name != "", err + return condition(p.resolverContext.Status.Conditions, "Ready", p.resolverContext.Generation) && p.resolverContext.Status.ManagedNamespace.DNSZoneRef.UID != "", err }); err != nil { return err } From bf7ad5d98a7cfd3f0f853a67c8ca2e81deb4c2af Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 14 Oct 2026 17:15:39 -0500 Subject: [PATCH 13/21] test: check the CoreDNS command in its image --- test/internaldns/kubernetes/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/test/internaldns/kubernetes/Dockerfile b/test/internaldns/kubernetes/Dockerfile index 4f199460..774183f7 100644 --- a/test/internaldns/kubernetes/Dockerfile +++ b/test/internaldns/kubernetes/Dockerfile @@ -24,4 +24,5 @@ COPY --from=build /out/dns-qualify /dns-qualify COPY --from=build /out/coredns /usr/local/bin/coredns COPY config/internal-dns/commands/ /usr/local/sbin/ USER 65532:65532 +RUN coredns -version ENTRYPOINT ["/internal-dns"] From 80981abe44afb99ae5556412813026c6866cbe4b Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:28:25 -0500 Subject: [PATCH 14/21] test: qualify native dnsdist lifecycle Run the pinned upstream daemon in Kubernetes and verify that routing updates and agent recovery preserve its running container. --- test/internaldns/kubernetes/Dockerfile | 5 ++- .../kubernetes/qualify/scenarios.go | 39 +++++++++++++++++++ test/internaldns/kubernetes/render/main.go | 16 +++++++- test/internaldns/qualify-package.sh | 2 +- 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/test/internaldns/kubernetes/Dockerfile b/test/internaldns/kubernetes/Dockerfile index 774183f7..f059b804 100644 --- a/test/internaldns/kubernetes/Dockerfile +++ b/test/internaldns/kubernetes/Dockerfile @@ -15,9 +15,10 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/coredns ./cmd/internal-dns-coredns && \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/dns-qualify ./test/internaldns/kubernetes/qualify -FROM debian:trixie-slim +FROM powerdns/dnsdist-20:2.0.10@sha256:24ac720af997aff6b98cb5b50d748754f815428e07b5368dfb0eccd5aa8d07e8 +USER 0:0 RUN apt-get update && apt-get install --no-install-recommends -y \ - dnsdist pdns-recursor iproute2 iptables procps netcat-openbsd ca-certificates && \ + pdns-recursor iproute2 iptables procps netcat-openbsd ca-certificates && \ rm -rf /var/lib/apt/lists/* COPY --from=build /out/internal-dns /internal-dns COPY --from=build /out/dns-qualify /dns-qualify diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index 5945d566..ae0d5c08 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -11,6 +11,7 @@ import ( "github.com/miekg/dns" dnsv1 "go.miloapis.com/dns-operator/api/v1alpha1" + "go.miloapis.com/dns-operator/internal/internaldns/serving" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -19,6 +20,10 @@ import ( ) func (s *suite) scenarios(a, b *project) error { + initialDNSDist, err := s.dnsdistProcesses() + if err != nil { + return err + } for _, p := range []*project{a, b} { for _, tcp := range []bool{false, true} { if err := s.wait("validated public recursion", 30*time.Second, func() (bool, error) { @@ -197,6 +202,16 @@ func (s *suite) scenarios(a, b *project) error { return err } s.pass("restored controllers and persistent broker publish a fresh observation", nil) + finalDNSDist, err := s.dnsdistProcesses() + if err != nil { + return err + } + for pod, id := range initialDNSDist { + if finalDNSDist[pod] != id { + return fmt.Errorf("dnsdist restarted during routing updates or agent recovery on %s", pod) + } + } + s.pass("upstream dnsdist processes survive context updates and agent restart", finalDNSDist) return nil } @@ -446,3 +461,27 @@ func readyPods(pods []corev1.Pod, replicas int32) bool { } return true } + +func (s *suite) dnsdistProcesses() (map[string]string, error) { + result := map[string]string{} + for _, name := range []string{"internal-dns-node", "internal-dns-regional-front"} { + var pod corev1.Pod + if err := s.platform.Get(s.ctx, client.ObjectKey{Namespace: s.env.Namespace, Name: name}, &pod); err != nil { + return nil, err + } + for _, container := range pod.Spec.Containers { + if container.Name == "dnsdist" && (container.Image != serving.DNSDistImage || len(container.Command) != 1 || container.Command[0] != "/usr/local/bin/dnsdist") { + return nil, fmt.Errorf("%s must run the upstream dnsdist container directly", name) + } + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "dnsdist" && status.State.Running != nil { + result[name] = status.ContainerID + } + } + if result[name] == "" { + return nil, fmt.Errorf("%s has no running dnsdist process", name) + } + } + return result, nil +} diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 07f9c24e..2441c0eb 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -18,6 +18,7 @@ import ( "go.miloapis.com/dns-operator/internal/internaldns/model" runtime "go.miloapis.com/dns-operator/internal/internaldns/runtime" + "go.miloapis.com/dns-operator/internal/internaldns/serving" authv1 "k8s.io/api/authentication/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -226,11 +227,24 @@ func fleetPod(name, member, role string) object { } sec := security() sec["capabilities"] = object{"drop": []string{"ALL"}, "add": []string{"NET_BIND_SERVICE"}} - containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/internal-dns", "--role=serving", "--config=/config/config.json"}, "readinessProbe": object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) + if role != "regional-dnsdist" { + containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/internal-dns", "--role=serving", "--config=/config/config.json"}, "readinessProbe": object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) + } + if role == "node" || role == "regional-dnsdist" { + file := "node-dnsdist.conf" + if role != "node" { + file = "cluster-dnsdist.conf" + containers[0]["readinessProbe"] = object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2} + } + containers = append(containers, object{"name": "dnsdist", "image": serving.DNSDistImage, "command": []string{"/usr/local/bin/dnsdist"}, "args": []string{"--supervised", "--disable-syslog", "-C", "/etc/internal-dns/" + file}, "securityContext": sec, "volumeMounts": []object{mount("daemon-config", "/etc/internal-dns", true), mount("lease", "/run/internal-dns", true), mount("runtime", "/tmp", false)}, "readinessProbe": object{"tcpSocket": object{"port": 53}, "periodSeconds": 2}, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) + } volumes := []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(member), {"name": "state", "persistentVolumeClaim": object{"claimName": name + "-state"}}, {"name": "daemon-config", "persistentVolumeClaim": object{"claimName": name + "-config"}}, {"name": "lease", "emptyDir": object{"medium": "Memory"}}, {"name": "runtime", "emptyDir": object{}}} // The aliases model the service-side destination contract only. The separate // probe has no ability to write routes or send trusted PROXY traffic. init := []object{} + if role == "node" || role == "regional-dnsdist" { + init = append(init, object{"name": "dnsdist-config", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ]; do sleep 1; done; exec /internal-dns --role=prepare --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts}) + } if role == "node" || role == "regional-dnsdist" { prefix := "fd53::/64" if role != "node" { diff --git a/test/internaldns/qualify-package.sh b/test/internaldns/qualify-package.sh index 6dc6f079..423d5dd6 100755 --- a/test/internaldns/qualify-package.sh +++ b/test/internaldns/qualify-package.sh @@ -44,7 +44,7 @@ docker run --rm --network none --read-only --cap-drop ALL \ > "$evidence_dir/help.log" 2>&1 # Confirm this is the internal DNS entry point rather than an image that only # happens to exit successfully. -grep -q 'Process role: control-plane, agent, serving, or readiness' "$evidence_dir/help.log" +grep -q 'Process role: control-plane, agent, serving, readiness, or prepare' "$evidence_dir/help.log" git rev-parse HEAD > "$evidence_dir/source.commit" git status --porcelain > "$evidence_dir/source.status" printf 'Internal DNS package qualification passed; evidence: %s\n' "$evidence_dir" From a0fa7a519f50949f616620d4ff641b16c9ec7861 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:38:58 -0500 Subject: [PATCH 15/21] build: reuse qualification module downloads Share downloaded modules with the Kubernetes qualification build. --- test/internaldns/kubernetes/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internaldns/kubernetes/Dockerfile b/test/internaldns/kubernetes/Dockerfile index f059b804..0ddff2c4 100644 --- a/test/internaldns/kubernetes/Dockerfile +++ b/test/internaldns/kubernetes/Dockerfile @@ -10,7 +10,7 @@ COPY internal ./internal COPY cmd/internal-dns ./cmd/internal-dns COPY cmd/internal-dns-coredns ./cmd/internal-dns-coredns COPY test/internaldns/kubernetes/qualify ./test/internaldns/kubernetes/qualify -RUN --mount=type=cache,target=/root/.cache/go-build \ +RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/internal-dns ./cmd/internal-dns && \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/coredns ./cmd/internal-dns-coredns && \ CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -trimpath -o /out/dns-qualify ./test/internaldns/kubernetes/qualify From 1e0b13a1c940802ea13d5874b8db7db44ec75928 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:45:14 -0500 Subject: [PATCH 16/21] test: use configured DNS listener readiness Qualify native dnsdist with the shared member query readiness checks. --- test/internaldns/kubernetes/render/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 2441c0eb..2e96ea8d 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -236,7 +236,7 @@ func fleetPod(name, member, role string) object { file = "cluster-dnsdist.conf" containers[0]["readinessProbe"] = object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2} } - containers = append(containers, object{"name": "dnsdist", "image": serving.DNSDistImage, "command": []string{"/usr/local/bin/dnsdist"}, "args": []string{"--supervised", "--disable-syslog", "-C", "/etc/internal-dns/" + file}, "securityContext": sec, "volumeMounts": []object{mount("daemon-config", "/etc/internal-dns", true), mount("lease", "/run/internal-dns", true), mount("runtime", "/tmp", false)}, "readinessProbe": object{"tcpSocket": object{"port": 53}, "periodSeconds": 2}, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) + containers = append(containers, object{"name": "dnsdist", "image": serving.DNSDistImage, "command": []string{"/usr/local/bin/dnsdist"}, "args": []string{"--supervised", "--disable-syslog", "-C", "/etc/internal-dns/" + file}, "securityContext": sec, "volumeMounts": []object{mount("daemon-config", "/etc/internal-dns", true), mount("lease", "/run/internal-dns", true), mount("runtime", "/tmp", false)}, "resources": object{"requests": object{"cpu": "50m", "memory": "64Mi"}, "limits": object{"memory": "512Mi"}}}) } volumes := []object{{"name": "config", "configMap": object{"name": name + "-config", "optional": true}}, credentialsVolume(member), {"name": "state", "persistentVolumeClaim": object{"claimName": name + "-state"}}, {"name": "daemon-config", "persistentVolumeClaim": object{"claimName": name + "-config"}}, {"name": "lease", "emptyDir": object{"medium": "Memory"}}, {"name": "runtime", "emptyDir": object{}}} // The aliases model the service-side destination contract only. The separate From 4a36c7f4ff82ab1c63b8fc52d4358617ab0d06de Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Thu, 15 Oct 2026 07:48:26 -0500 Subject: [PATCH 17/21] style: reuse shared DNS fleet roles Use the serving role definitions in the Kubernetes qualification renderer. --- test/internaldns/kubernetes/render/main.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 2e96ea8d..22437e19 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -33,6 +33,8 @@ type object = map[string]any const ns = "internal-dns-system" const projectNS = "project-e2e" const image = "internal-dns-kubernetes:e2e" +const nodeRole = string(serving.RenderNode) +const regionalDNSDistRole = string(serving.RenderRegionalDNSDist) var out, phase, platform, a, b, ai, bi string var memberNames = []string{"node-us-central1-0", "regional-front-us-central1-0", "regional-coredns-us-central1-0", "regional-coredns-us-central1-1"} @@ -178,7 +180,7 @@ func bootstrap() { public["spec"] = object{"ipFamilies": []string{"IPv6"}, "ipFamilyPolicy": "SingleStack", "selector": object{"app": "internal-dns-public-recursor"}, "ports": []object{{"name": "dns-udp", "port": 5353, "protocol": "UDP"}, {"name": "dns-tcp", "port": 5353, "protocol": "TCP"}}} items = append(items, public) for i, name := range podNames { - role := []string{"node", "regional-dnsdist", "regional-coredns", "regional-coredns"}[i] + role := []string{nodeRole, regionalDNSDistRole, "regional-coredns", "regional-coredns"}[i] items = append(items, fleetPod(name, memberNames[i], role)) for _, vol := range []string{"state", "config"} { pvc := obj("v1", "PersistentVolumeClaim", name+"-"+vol, ns) @@ -227,12 +229,12 @@ func fleetPod(name, member, role string) object { } sec := security() sec["capabilities"] = object{"drop": []string{"ALL"}, "add": []string{"NET_BIND_SERVICE"}} - if role != "regional-dnsdist" { + if role != regionalDNSDistRole { containers = append(containers, object{"name": "serving", "image": image, "imagePullPolicy": "Never", "command": []string{"/internal-dns", "--role=serving", "--config=/config/config.json"}, "readinessProbe": object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2}, "securityContext": sec, "volumeMounts": mounts, "resources": object{"requests": object{"cpu": "100m", "memory": "128Mi"}, "limits": object{"memory": "512Mi"}}}) } - if role == "node" || role == "regional-dnsdist" { + if role == nodeRole || role == regionalDNSDistRole { file := "node-dnsdist.conf" - if role != "node" { + if role != nodeRole { file = "cluster-dnsdist.conf" containers[0]["readinessProbe"] = object{"exec": object{"command": []string{"/internal-dns", "--role=readiness", "--config=/config/config.json"}}, "periodSeconds": 2, "timeoutSeconds": 2} } @@ -242,12 +244,12 @@ func fleetPod(name, member, role string) object { // The aliases model the service-side destination contract only. The separate // probe has no ability to write routes or send trusted PROXY traffic. init := []object{} - if role == "node" || role == "regional-dnsdist" { + if role == nodeRole || role == regionalDNSDistRole { init = append(init, object{"name": "dnsdist-config", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "while [ ! -s /config/config.json ]; do sleep 1; done; exec /internal-dns --role=prepare --config=/config/config.json"}, "securityContext": security(), "volumeMounts": mounts}) } - if role == "node" || role == "regional-dnsdist" { + if role == nodeRole || role == regionalDNSDistRole { prefix := "fd53::/64" - if role != "node" { + if role != nodeRole { prefix = "fd54::/64" } init = append(init, object{"name": "destination-fixture", "image": image, "imagePullPolicy": "Never", "command": []string{"/bin/sh", "-ec", "ip -6 route replace local " + prefix + " dev lo"}, "securityContext": object{"runAsUser": 0, "capabilities": object{"drop": []string{"ALL"}, "add": []string{"NET_ADMIN"}}}}) From ee22f661a36ae09bebe329cf1ecdb200657cf0fb Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 10:26:03 -0500 Subject: [PATCH 18/21] ci: allow DNS candidate qualification --- .github/workflows/e2e.yml | 1 + .github/workflows/internal-dns-package.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c1e1a3ba..1e1b4645 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -5,6 +5,7 @@ on: branches: - main pull_request: + workflow_dispatch: jobs: e2e: diff --git a/.github/workflows/internal-dns-package.yml b/.github/workflows/internal-dns-package.yml index ffd47f39..18b9c8dc 100644 --- a/.github/workflows/internal-dns-package.yml +++ b/.github/workflows/internal-dns-package.yml @@ -2,6 +2,7 @@ name: Internal DNS package on: pull_request: + workflow_dispatch: push: branches: [main] From 893106038aca249adaa19d5524b89c6eb5b7535c Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 10:51:28 -0500 Subject: [PATCH 19/21] ci: prepare the multi-cluster DNS runner --- .github/workflows/e2e.yml | 7 +++++++ hack/collect-kind-diagnostics.sh | 22 ++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100755 hack/collect-kind-diagnostics.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1e1b4645..ac937a90 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -41,6 +41,9 @@ jobs: with: go-version-file: go.mod + - name: Set file watcher limits + run: sudo sysctl -w fs.inotify.max_user_instances=8192 fs.inotify.max_user_watches=1048576 + - name: Setup Kind uses: helm/kind-action@v1 with: @@ -81,6 +84,10 @@ jobs: # (federation), and the end-to-end upstream->edge suite (full-chain). run: task --yes env:chainsaw + - name: Collect Kubernetes bootstrap diagnostics + if: failure() + run: hack/collect-kind-diagnostics.sh + - name: Collect diagnostic logs if: failure() env: diff --git a/hack/collect-kind-diagnostics.sh b/hack/collect-kind-diagnostics.sh new file mode 100755 index 00000000..5ea40470 --- /dev/null +++ b/hack/collect-kind-diagnostics.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +umask 077 + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT + +free -h || true +df -h || true +sysctl fs.inotify.max_user_instances fs.inotify.max_user_watches || true + +while IFS= read -r cluster; do + [[ $cluster == dns-* ]] || continue + echo "::group::$cluster bootstrap" + if kind get kubeconfig --name "$cluster" > "$work/$cluster.kubeconfig"; then + cli=(kubectl --kubeconfig "$work/$cluster.kubeconfig" --request-timeout=10s) + "${cli[@]}" get nodes,pods -A -o wide || true + "${cli[@]}" get events -A --sort-by=.lastTimestamp | tail -60 || true + fi + docker exec "$cluster-control-plane" journalctl -u kubelet --no-pager -n 120 || true + echo "::endgroup::" +done < <(kind get clusters) From 5db442db0699405df1f1ce9693761d2f76a6eead Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 11:27:16 -0500 Subject: [PATCH 20/21] test: configure the managed DNS namespace --- test/internaldns/kubernetes/render/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/internaldns/kubernetes/render/main.go b/test/internaldns/kubernetes/render/main.go index 22437e19..d903a786 100644 --- a/test/internaldns/kubernetes/render/main.go +++ b/test/internaldns/kubernetes/render/main.go @@ -411,6 +411,7 @@ func configure() { items = append(items, recursor) cfg := exampleConfigs("control-plane.example.yaml")["internal-dns-control-plane-config"] cfg.ClusterPrefix = "fd54::/64" + cfg.ManagedDomainSuffix = "datum.internal" cfg.OwnershipLeaseSeconds = 10 cfg.Stream.MaxBytes = 128 << 20 configureProjectSources(&cfg) From 55f1883c6d4da8fa45b33066a9097a25f57182e8 Mon Sep 17 00:00:00 2001 From: Scot Wells Date: Wed, 21 Oct 2026 11:51:00 -0500 Subject: [PATCH 21/21] test: query each context managed namespace --- test/internaldns/kubernetes/qualify/scenarios.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/internaldns/kubernetes/qualify/scenarios.go b/test/internaldns/kubernetes/qualify/scenarios.go index ae0d5c08..1ed1d550 100644 --- a/test/internaldns/kubernetes/qualify/scenarios.go +++ b/test/internaldns/kubernetes/qualify/scenarios.go @@ -106,6 +106,7 @@ func (s *suite) scenarios(a, b *project) error { return err } managedName := "web." + a.resolverContext.Status.ManagedNamespace.Suffix + managedNameB := "web." + b.resolverContext.Status.ManagedNamespace.Suffix for _, tcp := range []bool{false, true} { if err := s.expect(b, managedName, "A", tcp, dns.RcodeNameError); err != nil { return err @@ -118,11 +119,11 @@ func (s *suite) scenarios(a, b *project) error { if err := s.expect(a, managedName, "A", tcp, dns.RcodeSuccess, "10.42.0.11"); err != nil { return err } - if err := s.expect(b, managedName, "A", tcp, dns.RcodeSuccess, "10.42.0.21"); err != nil { + if err := s.expect(b, managedNameB, "A", tcp, dns.RcodeSuccess, "10.42.0.21"); err != nil { return err } } - s.pass("the same hostname in the shared managed domain resolves within its own context over UDP and TCP", managedName) + s.pass("managed names resolve within their owning contexts over UDP and TCP", map[string]string{"a": managedName, "b": managedNameB}) apps := &dnsv1.DNSZone{ObjectMeta: metav1.ObjectMeta{Name: "apps", Namespace: a.Namespace}, Spec: dnsv1.DNSZoneSpec{DomainName: "apps.internal", DNSZoneClassName: "private-coredns", Visibility: dnsv1.DNSZoneVisibilityPrivate}} if err := a.customer.Create(s.ctx, apps); err != nil { return err