From 7fa1cab1f1e923eb1a6c9d1ae907f7d62ce247b9 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 17:49:37 -0700 Subject: [PATCH 01/11] feat: support bounded api key rotation --- README.md | 18 +- deploy/kubernetes/qualification.yaml | 4 +- docs/DEPLOYMENT.md | 17 +- docs/LOAD_BALANCER_BUILD_OUT.md | 10 +- scripts/bench/README.md | 16 +- .../kubernetes-topology-contract-test.sh | 18 +- .../kubernetes-topology-profile.example.json | 12 +- scripts/bench/proxy-kubernetes-topology.sh | 349 +++++++++++++++++- .../api/config/ApiKeyVerifier.java | 48 +++ .../api/config/AuthModeConfiguration.java | 15 +- .../api/config/AuthProperties.java | 4 +- .../api/config/ProdApiKeyFilter.java | 28 +- .../proxy/ReverseProxyStatusController.java | 36 +- .../proxy/ReverseProxyStatusSummaries.java | 5 +- .../api/ApiKeyRotationProtectionTest.java | 66 ++++ .../KubernetesLiveTopologyContractTest.java | 31 +- .../ProdApiKeyMissingConfigurationTest.java | 16 + 17 files changed, 595 insertions(+), 98 deletions(-) create mode 100644 src/main/java/com/richmond423/loadbalancerpro/api/config/ApiKeyVerifier.java create mode 100644 src/test/java/com/richmond423/loadbalancerpro/api/ApiKeyRotationProtectionTest.java diff --git a/README.md b/README.md index 18055647..a860d4c6 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,10 @@ The default posture is conservative: API-key authentication is selected, proxyin loaded rollout/rollback, plus a deployment-equivalent capacity staircase bound to the exact candidate and per-replica telemetry; CI-gated Compose and live two-zone Kubernetes proofs cover distribution, candidate abort, content-distinct image rollout and baseline rollback under load, immutable inbound-TLS identity rotation and rollback, - pod-identity turnover, endpoint continuity, replica loss, planned worker + bounded API-key overlap/commit/rollback, pod-identity turnover, endpoint continuity, replica loss, planned worker removal, operator-remediated abrupt worker loss, degraded service, and worker recovery. -- API-key and OAuth2 resource-server modes with deny-by-default API classification. +- API-key mode with a required primary key and at most one optional rotation key, plus OAuth2 resource-server mode, + with deny-by-default API classification. - Actuator health/readiness, optional Prometheus metrics, and optional OTLP metrics export with endpoint validation. - Capacity-aware, predictive, and evaluation-only allocation APIs in the separate source/Lab Tools runtime. - Deterministic request-level comparison, Decision Explorer/browser cockpit surfaces, Enterprise Lab scenarios, durable @@ -46,9 +47,10 @@ and restores the prior digest. The Kubernetes adapter compiler supplies rollout, certificate-rotation, deployment-inspection, and capacity-sampling executables. A disposable two-worker/two-zone kind lane now deploys the restricted production image and proves live Service distribution, a metadata-only content-distinct candidate rollout and baseline rollback, then rotates between independently rooted certificates through versioned - immutable Secrets and restores the original identity. Both transitions require complete pod-UID turnover, unchanged - runtime image identity, endpoint continuity, and traffic through both replicas and backends before the lane exercises - worker drain/stop, degraded traffic, and + immutable Secrets and restores the original identity. It then rolls through immutable A-only, A+B, and B-only API-key + Secrets and reverses that sequence for rollback. Every credential transition preserves two ready endpoints, turns over + both pod UIDs, keeps the runtime image fixed, and proves the retired key is rejected before the lane exercises worker + drain/stop, degraded traffic, and operator-remediated no-drain worker loss and recovery. The next action remains to compile the adapters from the reviewed staging cluster identity, freeze the observed configuration/ingress hashes into the profiles, then run staging qualification @@ -135,6 +137,7 @@ Important defaults in `application.properties`: | Property | Default | Effect | | --- | --- | --- | | `loadbalancerpro.auth.mode` | `api-key` | Protected API mode | +| `loadbalancerpro.api.rotation-key` | empty | Optional second key accepted only during an operator-bounded rotation overlap | | `loadbalancerpro.proxy.enabled` | `false` | No forwarding until explicitly enabled | | `loadbalancerpro.lase.shadow.enabled` | `false` | No shadow evaluation by default | | `loadbalancerpro.api.max-request-bytes` | `16384` | Bounded API request bodies | @@ -169,6 +172,11 @@ export LOADBALANCERPRO_API_KEY='supply-from-a-secret-manager' java -jar "$(bash scripts/resolve-executable-jar.sh)" --spring.profiles.active=prod ``` +API-key rotation is deliberately bounded to two credentials. Roll out a configuration containing primary A plus +rotation key B, switch clients to B, then roll out B as the sole primary. Rollback reverses the sequence through the +same A+B overlap. A rotation key cannot replace a missing primary, and the process does not dynamically reread mounted +credential files. + Do not commit API keys, OAuth tokens, AWS credentials, telemetry headers, private keys, or production targets. Terminate TLS at a trusted reverse proxy, ingress, managed load balancer, platform edge, or service mesh before shared-network exposure. OTLP metrics are opt-in. When enabled, the endpoint validator rejects blank or malformed URLs, embedded credentials, query strings, fragments, disallowed localhost, and obvious public hosts when private endpoints are required: diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index 41b067ba..4fb7f588 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -322,10 +322,12 @@ spec: volumes: - name: api-key secret: - secretName: loadbalancerpro-api-key + secretName: loadbalancerpro-api-key-a items: - key: api-key path: loadbalancerpro.api.key + - key: rotation-key + path: loadbalancerpro.api.rotation-key - name: server-tls secret: secretName: loadbalancerpro-server-tls-a diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 9e5ee937..209a201d 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -59,7 +59,16 @@ curl --cacert "$LBP_TLS_DIRECTORY/ca.pem" --resolve lbp.local:18443:127.0.0.1 \ docker compose -f deploy/docker-compose.proxy-prod.yml down ``` -The API key is mounted read-only as `/run/secrets/loadbalancerpro.api.key` and imported through Spring config trees. The temporary example uses read-only files inside a private `mktemp` parent so the image's non-root user can read the mounts; for a durable host path, grant read access only to the runtime UID/GID through the host's ownership or ACL mechanism. TLS, trust, client identity, and additional configuration directories are separate read-only mounts. Define backend custom trust or mTLS bundles only in the external configuration directory, for example: +The required primary API key is mounted read-only as `/run/secrets/loadbalancerpro.api.key` and imported through Spring +config trees. During a bounded rotation overlap only, a second key may be supplied as +`/run/secrets/loadbalancerpro.api.rotation-key`; both authenticate, but the rotation key cannot replace a missing primary. +Roll from A-only to A+B, switch clients to B, then roll to B-only. Rollback reverses the sequence. The process snapshots +both values at startup and does not dynamically reread mounted credentials, so use versioned immutable Secrets and a +zero-unavailable pod rollout instead of mutating an in-use Secret. The temporary example uses read-only files inside a +private `mktemp` parent so the image's non-root user can read the mounts; for a durable host path, grant read access only +to the runtime UID/GID through the host's ownership or ACL mechanism. TLS, trust, client identity, and additional +configuration directories are separate read-only mounts. Define backend custom trust or mTLS bundles only in the +external configuration directory, for example: ```properties spring.ssl.bundle.pem.backendtrust.truststore.certificate=file:/run/trust/ca.pem @@ -112,7 +121,11 @@ continuous traffic windows. The TLS exercise uses independently generated roots, fingerprints, and single-CA positive/negative checks; it also requires fresh pod UIDs, unchanged runtime image identity, ready-endpoint continuity, and traffic through both replicas and backends in both directions. It proves application server TLS termination behind the loopback NodePort, not an ingress controller, external issuer, or trust-distribution -system. The lane then proves two-zone Service distribution, planned worker removal, and +system. The lane next proves bounded API-key rotation through immutable A-only, A+B, and B-only Secrets and reverses the +sequence for rollback. Both keys are accepted only in the overlap phases; the retired key must return 401 after each +commit, while zero-unavailable endpoint continuity, fresh pod UIDs, fixed runtime image identity, and traffic through +both replicas and backends remain required. This is startup configuration rollout proof, not dynamic Secret reload or +external secret-manager proof. The lane then proves two-zone Service distribution, planned worker removal, and operator-remediated no-drain worker loss and recovery. The abrupt-loss exercise forcibly stops the kind worker, confirms its container is down, applies the out-of-service `NoExecute` taint, and force-removes the three exact stateless qualification pods from the API. The disposable cluster also pins immediate EndpointSlice-triggered iptables updates diff --git a/docs/LOAD_BALANCER_BUILD_OUT.md b/docs/LOAD_BALANCER_BUILD_OUT.md index 825851f8..041ae091 100644 --- a/docs/LOAD_BALANCER_BUILD_OUT.md +++ b/docs/LOAD_BALANCER_BUILD_OUT.md @@ -165,7 +165,11 @@ an independently rooted immutable candidate Secret and back under continuous tra served-leaf fingerprints prove both identity transitions; fresh pods, unchanged runtime image identity, two-zone continuity, and positive traffic deltas on both replicas and backends are required in both directions. This exercises application server TLS behind the loopback NodePort, not an ingress controller, external issuer, or client -trust-distribution system. One worker is then +trust-distribution system. The restored-certificate deployment then rotates credentials through immutable A-only, A+B, +and B-only API-key Secrets and rolls back through the same bounded overlap. Each of the four zero-unavailable rollouts +must preserve two ready endpoints and both-zone placement, replace both pod UIDs without changing the runtime image, +serve traffic through both replicas/backends, and enforce the expected positive and negative key boundary. This proves +startup credential rollout mechanics, not dynamic Secret reload or an external secret manager. One worker is then drained and stopped under load, degraded traffic must continue through the remaining replica, and the stopped worker and second replica must recover inside the bound, and both recovered replicas and backends must serve new traffic. It then forcibly stops that recovered worker without a drain, confirms the container is down, and applies the documented @@ -200,7 +204,9 @@ turnover and a content-distinct runtime image transition, then proves another co the initial runtime image identity. It restores two-zone placement and requires positive post-transition traffic deltas on both candidate/restored replicas and both backends. It applies the same zero-unavailable and distribution checks to versioned immutable TLS Secret rotation and rollback while proving the served leaf fingerprint changes and returns and -the runtime image identity remains fixed. +the runtime image identity remains fixed. It applies those rollout, pod-turnover, endpoint-continuity, and distribution +checks again to bounded API-key overlap, candidate commit, rollback overlap, and baseline commit, with 401 checks for the +retired credential after each commit. Use an immutable image digest and begin with a small, explicitly approved traffic slice. During every step, compare client success/latency, upstream health, proxy p95/p99, in-flight work, retries, sheds, cooldown trips, CPU, memory, GC, diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 753c6d71..67f37f37 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -129,7 +129,12 @@ Secrets, rotates the Deployment's TLS Secret reference under continuous close-pe rollover bundle, and restores the baseline Secret under a second load window. Positive and negative single-CA checks plus repeated served-leaf SHA-256 fingerprint checks prove the identity changed and returned; both directions also require complete pod turnover, an unchanged runtime image ID, two-zone endpoint continuity, and post-transition traffic -through both replicas and backends. It then drains and stops one worker under load, +through both replicas and backends. Before the worker-loss phases, it creates immutable A-only, A+B overlap, and B-only +API-key Secrets. Four zero-unavailable rollouts prove baseline-key traffic through overlap, candidate-key traffic through +commit, the reverse rollback overlap, and final restoration of A-only. Positive and negative authentication checks prove +both overlap windows and both key-retirement boundaries; every phase also requires fresh pod UIDs, two ready endpoints, +an unchanged runtime image, and traffic through both replicas and backends. The runtime accepts only the required primary +plus one optional rotation key and does not dynamically reload Secret files. It then drains and stops one worker under load, tests the one-replica degraded service, requires both recovered replicas and backends to serve new traffic, then forcibly stops that recovered worker without a drain. After confirming the worker container is down, it applies Kubernetes' out-of-service `NoExecute` remediation and force-removes the three exact stateless workload pods from the @@ -143,11 +148,12 @@ bash scripts/bench/kubernetes-topology-contract-test.sh bash scripts/bench/proxy-kubernetes-topology.sh --mode smoke ``` -Smoke mode requires Docker, kind 0.31.0, kubectl 1.34.3, Vegeta, jq, OpenSSL, and curl. TLS private keys and the API key -live only in a temporary directory; evidence contains only redacted Secret metadata and generated leaf fingerprints +Smoke mode requires Docker, kind 0.31.0, kubectl 1.34.3, Vegeta, jq, OpenSSL, and curl. TLS private keys and both API keys +live only in a temporary directory; evidence contains only redacted Secret metadata, key-slot names, and generated leaf fingerprints and is written beneath `target/kubernetes/`. The result proves disposable Kubernetes content-addressed image, -inbound-server TLS Secret transition/rollback, and worker-loss mechanics. It does not deploy or test an ingress -controller, external certificate authority, or client trust-distribution system. Because the local candidate +inbound-server TLS Secret transition/rollback, bounded API-key rotation/rollback, and worker-loss mechanics. It does not +prove dynamic Secret reload or an external secret manager, and it does not deploy or test an ingress controller, +external certificate authority, or client trust-distribution system. Because the local candidate changes immutable proof metadata but not application layers, it does not prove application-layer release compatibility, registry integrity, deployment capacity, external ingress behavior, automatic infrastructure-failure detection, or an authorized staging diff --git a/scripts/bench/kubernetes-topology-contract-test.sh b/scripts/bench/kubernetes-topology-contract-test.sh index 5862a213..07980357 100644 --- a/scripts/bench/kubernetes-topology-contract-test.sh +++ b/scripts/bench/kubernetes-topology-contract-test.sh @@ -14,7 +14,7 @@ trap cleanup EXIT bash -n "$runner" bash "$runner" --mode validate --profile "$profile" > "$work_dir/valid.log" -grep -Fq 'service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery' \ +grep -Fq 'service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery' \ "$work_dir/valid.log" assert_rejected() { @@ -31,7 +31,8 @@ assert_rejected() { assert_rejected production-status '.review.status = "reviewed"' assert_rejected legacy-schema '.schemaVersion = 1' assert_rejected older-schema '.schemaVersion = 3' -assert_rejected previous-schema '.schemaVersion = 4' +assert_rejected older-schema-four '.schemaVersion = 4' +assert_rejected previous-schema '.schemaVersion = 5' assert_rejected skewed-kubectl '.cluster.kubectlVersion = "v1.32.2"' assert_rejected mutable-node-image '.cluster.nodeImageDigest = "sha256:" + ("f" * 64)' assert_rejected one-worker '.cluster.workers = 1' @@ -52,6 +53,9 @@ assert_rejected certificate-rotation-window-too-short '.workload.certificateRota assert_rejected short-certificate-rollback '.workload.certificateRollbackSeconds = 10' assert_rejected short-post-certificate-rollback '.workload.postCertificateRollbackSeconds = 1' assert_rejected certificate-rollback-window-too-short '.workload.certificateRollbackSeconds = .objectives.maximumCertificateRollbackSeconds' +assert_rejected short-api-key-transition '.workload.apiKeyTransitionSeconds = 10' +assert_rejected short-post-api-key-transition '.workload.postApiKeyTransitionSeconds = 1' +assert_rejected api-key-transition-window-too-short '.workload.apiKeyTransitionSeconds = .objectives.maximumApiKeyTransitionSeconds' assert_rejected short-transition '.workload.transitionSeconds = 5' assert_rejected short-abrupt-transition '.workload.abruptTransitionSeconds = 5' assert_rejected short-abrupt-degraded '.workload.abruptDegradedSeconds = 1' @@ -65,10 +69,13 @@ assert_rejected weak-certificate-rotation-objective '.objectives.minimumCertific assert_rejected weak-post-certificate-rotation-objective '.objectives.minimumPostCertificateRotationSuccessRatio = 0.5' assert_rejected weak-certificate-rollback-objective '.objectives.minimumCertificateRollbackSuccessRatio = 0.5' assert_rejected weak-post-certificate-rollback-objective '.objectives.minimumPostCertificateRollbackSuccessRatio = 0.5' +assert_rejected weak-api-key-transition-objective '.objectives.minimumApiKeyTransitionSuccessRatio = 0.5' +assert_rejected weak-post-api-key-transition-objective '.objectives.minimumPostApiKeyTransitionSuccessRatio = 0.5' assert_rejected long-rollout '.objectives.maximumRolloutSeconds = 180' assert_rejected long-rollback '.objectives.maximumRollbackSeconds = 180' assert_rejected long-certificate-rotation '.objectives.maximumCertificateRotationSeconds = 180' assert_rejected long-certificate-rollback '.objectives.maximumCertificateRollbackSeconds = 180' +assert_rejected long-api-key-transition '.objectives.maximumApiKeyTransitionSeconds = 180' assert_rejected weak-transition-objective '.objectives.minimumTransitionSuccessRatio = 0.5' assert_rejected weak-abrupt-transition-objective '.objectives.minimumAbruptTransitionSuccessRatio = 0.5' assert_rejected weak-abrupt-degraded-objective '.objectives.minimumAbruptDegradedSuccessRatio = 0.5' @@ -80,5 +87,10 @@ assert_rejected long-abrupt-recovery '.objectives.maximumAbruptRecoverySeconds = assert_rejected wrong-tls-hostname '.tlsRotation.hostname = "production.example.com"' assert_rejected wrong-baseline-tls-secret '.tlsRotation.baselineSecret = "production-server-tls"' assert_rejected unchanged-tls-secret '.tlsRotation.candidateSecret = .tlsRotation.baselineSecret' +assert_rejected wrong-baseline-api-key-secret '.apiKeyRotation.baselineSecret = "production-api-key"' +assert_rejected wrong-overlap-api-key-secret '.apiKeyRotation.overlapSecret = "production-api-key-overlap"' +assert_rejected wrong-candidate-api-key-secret '.apiKeyRotation.candidateSecret = "production-api-key-candidate"' +assert_rejected unchanged-overlap-api-key-secret '.apiKeyRotation.overlapSecret = .apiKeyRotation.baselineSecret' +assert_rejected unchanged-candidate-api-key-secret '.apiKeyRotation.candidateSecret = .apiKeyRotation.baselineSecret' -printf 'Kubernetes topology contract rejected 52 unsafe profiles without creating a cluster.\n' +printf 'Kubernetes topology contract rejected 64 unsafe profiles without creating a cluster.\n' diff --git a/scripts/bench/kubernetes-topology-profile.example.json b/scripts/bench/kubernetes-topology-profile.example.json index d10c3f61..37cba791 100644 --- a/scripts/bench/kubernetes-topology-profile.example.json +++ b/scripts/bench/kubernetes-topology-profile.example.json @@ -1,5 +1,5 @@ { - "schemaVersion": 5, + "schemaVersion": 6, "profileId": "local-kind-two-zone-smoke", "review": { "status": "example" @@ -27,6 +27,8 @@ "postCertificateRotationSeconds": 8, "certificateRollbackSeconds": 90, "postCertificateRollbackSeconds": 8, + "apiKeyTransitionSeconds": 50, + "postApiKeyTransitionSeconds": 6, "transitionSeconds": 20, "degradedSeconds": 8, "recoveredSeconds": 8, @@ -44,6 +46,8 @@ "minimumPostCertificateRotationSuccessRatio": 0.999, "minimumCertificateRollbackSuccessRatio": 0.999, "minimumPostCertificateRollbackSuccessRatio": 0.999, + "minimumApiKeyTransitionSuccessRatio": 0.999, + "minimumPostApiKeyTransitionSuccessRatio": 0.999, "minimumTransitionSuccessRatio": 0.97, "minimumDegradedSuccessRatio": 0.999, "minimumRecoveredSuccessRatio": 0.999, @@ -56,6 +60,7 @@ "maximumRollbackSeconds": 75, "maximumCertificateRotationSeconds": 75, "maximumCertificateRollbackSeconds": 75, + "maximumApiKeyTransitionSeconds": 45, "maximumRecoverySeconds": 180, "maximumAbruptEndpointWithdrawalSeconds": 15, "maximumAbruptRecoverySeconds": 180 @@ -64,5 +69,10 @@ "hostname": "lbp-kubernetes.local", "baselineSecret": "loadbalancerpro-server-tls-a", "candidateSecret": "loadbalancerpro-server-tls-b" + }, + "apiKeyRotation": { + "baselineSecret": "loadbalancerpro-api-key-a", + "overlapSecret": "loadbalancerpro-api-key-a-b", + "candidateSecret": "loadbalancerpro-api-key-b" } } diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index fc0b57be..ccab91a8 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -25,7 +25,7 @@ for required_file in "$cluster_config" "$workload_manifest" "$candidate_dockerfi done jq -e ' - .schemaVersion == 5 + .schemaVersion == 6 and (.profileId | type == "string" and test("^[a-z0-9][a-z0-9._-]{0,62}$")) and .review.status == "example" and .cluster.kindVersion == "v0.31.0" @@ -48,6 +48,8 @@ jq -e ' and (.workload.postCertificateRotationSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.certificateRollbackSeconds | type == "number" and . >= 20 and . <= 180 and floor == .) and (.workload.postCertificateRollbackSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) + and (.workload.apiKeyTransitionSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) + and (.workload.postApiKeyTransitionSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.transitionSeconds | type == "number" and . >= 15 and . <= 120 and floor == .) and (.workload.degradedSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.recoveredSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) @@ -63,6 +65,8 @@ jq -e ' and (.objectives.minimumPostCertificateRotationSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumCertificateRollbackSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumPostCertificateRollbackSuccessRatio | type == "number" and . >= 0.95 and . <= 1) + and (.objectives.minimumApiKeyTransitionSuccessRatio | type == "number" and . >= 0.95 and . <= 1) + and (.objectives.minimumPostApiKeyTransitionSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumTransitionSuccessRatio | type == "number" and . >= 0.90 and . <= 1) and (.objectives.minimumDegradedSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.minimumRecoveredSuccessRatio | type == "number" and . >= 0.95 and . <= 1) @@ -75,6 +79,7 @@ jq -e ' and (.objectives.maximumRollbackSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.objectives.maximumCertificateRotationSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.objectives.maximumCertificateRollbackSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) + and (.objectives.maximumApiKeyTransitionSeconds | type == "number" and . >= 20 and . <= 90 and floor == .) and (.objectives.maximumRecoverySeconds | type == "number" and . >= 30 and . <= 300 and floor == .) and (.objectives.maximumAbruptEndpointWithdrawalSeconds | type == "number" and . >= 5 and . <= 30 and floor == .) and (.objectives.maximumAbruptRecoverySeconds | type == "number" and . >= 30 and . <= 300 and floor == .) @@ -82,11 +87,17 @@ jq -e ' and .workload.rollbackSeconds >= (.objectives.maximumRollbackSeconds + 5) and .workload.certificateRotationSeconds >= (.objectives.maximumCertificateRotationSeconds + 5) and .workload.certificateRollbackSeconds >= (.objectives.maximumCertificateRollbackSeconds + 5) + and .workload.apiKeyTransitionSeconds >= (.objectives.maximumApiKeyTransitionSeconds + 5) and .workload.abruptTransitionSeconds >= (.objectives.maximumAbruptEndpointWithdrawalSeconds + 5) and .tlsRotation.hostname == "lbp-kubernetes.local" and .tlsRotation.baselineSecret == "loadbalancerpro-server-tls-a" and .tlsRotation.candidateSecret == "loadbalancerpro-server-tls-b" and .tlsRotation.baselineSecret != .tlsRotation.candidateSecret + and .apiKeyRotation.baselineSecret == "loadbalancerpro-api-key-a" + and .apiKeyRotation.overlapSecret == "loadbalancerpro-api-key-a-b" + and .apiKeyRotation.candidateSecret == "loadbalancerpro-api-key-b" + and ([.apiKeyRotation.baselineSecret, .apiKeyRotation.overlapSecret, + .apiKeyRotation.candidateSecret] | unique | length) == 3 ' "$profile" >/dev/null || { echo "Kubernetes topology profile does not satisfy the executable contract" >&2; exit 2; } for invariant in \ @@ -111,7 +122,9 @@ for invariant in \ 'nodePort: 30443' \ 'kind: PodDisruptionBudget' \ 'kind: NetworkPolicy' \ - 'secretName: loadbalancerpro-server-tls-a'; do + 'secretName: loadbalancerpro-server-tls-a' \ + 'secretName: loadbalancerpro-api-key-a' \ + 'path: loadbalancerpro.api.rotation-key'; do grep -Fq "$invariant" "$workload_manifest" || { echo "Kubernetes workload is missing: $invariant" >&2; exit 2; } done if grep -Eq '(BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|api-key:[[:space:]]+[^[:space:]]+)' "$workload_manifest"; then @@ -121,7 +134,7 @@ fi if [[ "$mode" == "validate" ]]; then printf 'Validated disposable two-worker/two-zone Kubernetes topology contract %s.\n' "$(jq -r '.profileId' "$profile")" - printf 'Validated proof cases: service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery\n' + printf 'Validated proof cases: service-distribution per-replica-metrics content-distinct-rollout endpoint-continuity candidate-pod-identity-turnover post-rollout-distribution baseline-rollback rollback-endpoint-continuity rollback-pod-identity-turnover post-rollback-distribution immutable-certificate-secrets certificate-identity-transition certificate-rotation-continuity certificate-pod-identity-turnover post-certificate-rotation-distribution certificate-identity-rollback certificate-rollback-continuity certificate-rollback-pod-identity-turnover post-certificate-rollback-distribution bounded-api-key-overlap immutable-api-key-secrets api-key-rotation-continuity api-key-retirement api-key-rollback-continuity api-key-rollback-retirement planned-worker-drain stopped-worker degraded-service worker-recovery abrupt-worker-stop out-of-service-remediation abrupt-endpoint-withdrawal abrupt-recovery\n' exit 0 fi @@ -142,6 +155,9 @@ rate="$(jq -r '.workload.ratePerSecond' "$profile")" tls_hostname="$(jq -r '.tlsRotation.hostname' "$profile")" baseline_tls_secret="$(jq -r '.tlsRotation.baselineSecret' "$profile")" candidate_tls_secret="$(jq -r '.tlsRotation.candidateSecret' "$profile")" +baseline_api_key_secret="$(jq -r '.apiKeyRotation.baselineSecret' "$profile")" +overlap_api_key_secret="$(jq -r '.apiKeyRotation.overlapSecret' "$profile")" +candidate_api_key_secret="$(jq -r '.apiKeyRotation.candidateSecret' "$profile")" default_run_id="${GITHUB_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-${BASHPID}}-${GITHUB_RUN_ATTEMPT:-1}" cluster_name="${LBP_KUBERNETES_CLUSTER:-lbp-k8s-${GITHUB_RUN_ID:-local-${BASHPID}}-${GITHUB_RUN_ATTEMPT:-1}}" [[ "$cluster_name" =~ ^lbp-k8s-[a-z0-9][a-z0-9-]{0,48}$ ]] || { @@ -165,7 +181,9 @@ output_dir="$(cd "$output_dir" && pwd -P)" work_dir="$(mktemp -d "${TMPDIR:-/tmp}/lbp-kubernetes.XXXXXX")" kubeconfig="$work_dir/kubeconfig" -api_key_file="$work_dir/loadbalancerpro-api-key" +api_key_file="$work_dir/loadbalancerpro-api-key-a" +candidate_api_key_file="$work_dir/loadbalancerpro-api-key-b" +empty_rotation_key_file="$work_dir/loadbalancerpro-api-key-empty" tls_dir="$work_dir/tls" candidate_tls_dir="$work_dir/tls-candidate" tls_trust_bundle="$work_dir/tls-rollover-ca-bundle.pem" @@ -276,6 +294,11 @@ kubectl label node "${workers[1]}" loadbalancerpro.io/qualification-worker=true kind load docker-image "$proxy_image" "$candidate_image" "$fixture_image" --name "$cluster_name" openssl rand -hex 24 > "$api_key_file" +openssl rand -hex 24 > "$candidate_api_key_file" +: > "$empty_rotation_key_file" +[[ "$(<"$api_key_file")" != "$(<"$candidate_api_key_file")" ]] || { + echo "Generated Kubernetes API keys were not distinct" >&2; exit 1; +} generate_server_identity() { local directory="$1" authority_common_name="$2" serial="$3" local ca_key="$directory/ca-key.pem" @@ -323,11 +346,33 @@ candidate_certificate_fingerprint="$(certificate_fingerprint "$candidate_tls_dir && "$baseline_certificate_fingerprint" != "$candidate_certificate_fingerprint" ]] || { echo "Generated Kubernetes TLS identities were not content-distinct" >&2; exit 1; } -chmod 0600 "$api_key_file" "$tls_dir"/* "$candidate_tls_dir"/* "$tls_trust_bundle" +chmod 0600 "$api_key_file" "$candidate_api_key_file" "$empty_rotation_key_file" \ + "$tls_dir"/* "$candidate_tls_dir"/* "$tls_trust_bundle" kubectl apply --server-side --field-manager=loadbalancerpro-qualification -f "$workload_manifest" -kubectl create secret generic loadbalancerpro-api-key --namespace "$namespace" \ - --from-file=api-key="$api_key_file" +create_immutable_api_key_secret() { + local secret_name="$1" primary_file="$2" rotation_file="$3" + kubectl create secret generic "$secret_name" --namespace "$namespace" \ + --from-file=api-key="$primary_file" --from-file=rotation-key="$rotation_file" \ + --dry-run=client -o json | jq '.immutable = true' | kubectl create -f - +} +create_immutable_api_key_secret "$baseline_api_key_secret" "$api_key_file" "$empty_rotation_key_file" +create_immutable_api_key_secret "$overlap_api_key_secret" "$api_key_file" "$candidate_api_key_file" +create_immutable_api_key_secret "$candidate_api_key_secret" "$candidate_api_key_file" "$empty_rotation_key_file" +kubectl get "secret/$baseline_api_key_secret" "secret/$overlap_api_key_secret" \ + "secret/$candidate_api_key_secret" --namespace "$namespace" -o json \ + | jq '.items |= map(. + {dataKeys: (.data | keys)} + | del(.data) | del(.metadata.managedFields))' \ + > "$output_dir/api-key-secret-metadata.json" +jq -e --arg baseline "$baseline_api_key_secret" --arg overlap "$overlap_api_key_secret" \ + --arg candidate "$candidate_api_key_secret" ' + (.items | length) == 3 + and ([.items[].metadata.name] | sort == ([$baseline, $overlap, $candidate] | sort)) + and all(.items[]; .immutable == true and .type == "Opaque" + and .dataKeys == ["api-key", "rotation-key"])' \ + "$output_dir/api-key-secret-metadata.json" >/dev/null || { + echo "Versioned Kubernetes API-key Secrets were not immutable bounded-overlap objects" >&2; exit 1; +} create_immutable_tls_secret() { local secret_name="$1" directory="$2" kubectl create secret generic "$secret_name" --namespace "$namespace" \ @@ -503,8 +548,9 @@ assert_two_zone_proxy_pods() { prove_post_transition_distribution() { local phase="$1" seconds="$2" minimum_success="$3" proof_field="$4" failure_message="$5" + local attack_targets="${6:-$targets}" collect_distribution "${phase}-before" - run_attack "$phase" "$seconds" "$minimum_success" + run_attack "$phase" "$seconds" "$minimum_success" "$attack_targets" collect_distribution "$phase" jq -n --arg phase "$phase" --arg proofField "$proof_field" \ --slurpfile before "$output_dir/${phase}-before-distribution.json" \ @@ -566,12 +612,37 @@ initial_proxy_runtime_image_ids_json="$(jq '[.[].status.containerStatuses[]? <<< "$initial_ready_proxy_pods_json")" == true ]] || { echo "Initial proxy pods do not reference the immutable baseline TLS Secret" >&2; exit 1; } +[[ "$(jq --arg secret "$baseline_api_key_secret" '[.[] | any(.spec.volumes[]?; + .name == "api-key" and .secret.secretName == $secret)] | all' \ + <<< "$initial_ready_proxy_pods_json")" == true ]] || { + echo "Initial proxy pods do not reference the immutable baseline API-key Secret" >&2; exit 1; +} capture_state initial api_key="$(<"$api_key_file")" targets="$work_dir/targets.txt" +candidate_targets="$work_dir/candidate-targets.txt" printf 'GET https://127.0.0.1:%s/proxy/kubernetes/topology\nX-API-Key: %s\n\n' \ "$host_port" "$api_key" > "$targets" +printf 'GET https://127.0.0.1:%s/proxy/kubernetes/topology\nX-API-Key: %s\n\n' \ + "$host_port" "$(<"$candidate_api_key_file")" > "$candidate_targets" + +api_key_status() { + local key_file="$1" + curl --silent --show-error --cacert "$tls_dir/ca.pem" --connect-timeout 3 --max-time 10 \ + --output /dev/null --write-out '%{http_code}' \ + --header "X-API-Key: $(<"$key_file")" \ + "https://127.0.0.1:${host_port}/api/proxy/status" +} + +assert_api_key_status() { + local key_file="$1" expected="$2" description="$3" observed + observed="$(api_key_status "$key_file")" + [[ "$observed" == "$expected" ]] || { + echo "$description returned HTTP $observed instead of $expected" >&2 + return 1 + } +} curl_with_ca() { local ca_file="$1" @@ -619,6 +690,8 @@ assert_served_certificate() { assert_ca_trusts_endpoint "$tls_dir/ca.pem" "Baseline CA" assert_ca_rejected_by_endpoint "$candidate_tls_dir/ca.pem" "Candidate-only CA before rotation" assert_served_certificate "$baseline_certificate_fingerprint" "$tls_dir/ca.pem" "Baseline TLS identity" +assert_api_key_status "$api_key_file" 200 "Baseline API key before rotation" +assert_api_key_status "$candidate_api_key_file" 401 "Candidate API key before overlap" report_attack() { local name="$1" minimum_success="$2" @@ -642,11 +715,108 @@ report_attack() { run_attack() { local name="$1" seconds="$2" minimum_success="$3" + local attack_targets="${4:-$targets}" vegeta attack -duration="${seconds}s" -rate="${rate}/s" -timeout=5s -keepalive=false -http2=false \ - -root-certs="$tls_trust_bundle" -targets="$targets" > "$work_dir/${name}.bin" + -root-certs="$tls_trust_bundle" -targets="$attack_targets" > "$work_dir/${name}.bin" report_attack "$name" "$minimum_success" } +run_api_key_secret_transition() { + local phase="$1" secret_name="$2" annotation_name="$3" attack_targets="$4" + local prior_pods_json="$5" + local duration_seconds maximum_seconds minimum_success token patch started_epoch elapsed_seconds + local sample_count minimum_ready_pods minimum_ready_endpoints pods_json ready_pods_json + local prior_uids_json current_uids_json retained_prior_uids runtime_image_ids_json pod + duration_seconds="$(jq -r '.workload.apiKeyTransitionSeconds' "$profile")" + maximum_seconds="$(jq -r '.objectives.maximumApiKeyTransitionSeconds' "$profile")" + minimum_success="$(jq -r '.objectives.minimumApiKeyTransitionSuccessRatio' "$profile")" + token="$(printf '%s\n' \ + "$source_revision|$secret_name|$default_run_id|$phase" | sha256sum | awk '{print $1}')" + + rm -f -- "$rollout_stop_file" + sample_transition_continuity "$phase" & + rollout_sampler_pid=$! + vegeta attack -duration="${duration_seconds}s" -rate="${rate}/s" -timeout=5s \ + -keepalive=false -http2=false -root-certs="$tls_trust_bundle" -targets="$attack_targets" \ + > "$work_dir/${phase}.bin" & + attack_pid=$! + sleep 3 + started_epoch="$(date +%s)" + patch="$(jq -cn --arg secret "$secret_name" --arg token "$token" \ + --arg annotation "$annotation_name" \ + '{spec:{template:{metadata:{annotations:{($annotation):$token, + "loadbalancerpro.io/qualification-api-key-secret":$secret}}, + spec:{volumes:[{name:"api-key",secret:{secretName:$secret,items:[ + {key:"api-key",path:"loadbalancerpro.api.key"}, + {key:"rotation-key",path:"loadbalancerpro.api.rotation-key"}]}}]}}}}')" + kubectl patch deployment loadbalancerpro --namespace "$namespace" --type strategic -p "$patch" + kubectl rollout status deployment/loadbalancerpro --namespace "$namespace" \ + --timeout="${maximum_seconds}s" + elapsed_seconds=$(( $(date +%s) - started_epoch )) + (( elapsed_seconds <= maximum_seconds )) || { + echo "$phase exceeded the API-key transition objective" >&2; return 1; + } + if ! wait "$attack_pid"; then + attack_pid="" + echo "$phase traffic attack failed" >&2 + return 1 + fi + attack_pid="" + touch "$rollout_stop_file" + if ! wait "$rollout_sampler_pid"; then + rollout_sampler_pid="" + echo "$phase endpoint-continuity sampler failed" >&2 + return 1 + fi + rollout_sampler_pid="" + report_attack "$phase" "$minimum_success" + sample_count="$(awk -F, 'NR > 1 { count++ } END { print count + 0 }' \ + "$output_dir/${phase}-continuity.csv")" + minimum_ready_pods="$(awk -F, 'NR > 1 && (minimum == "" || $2 < minimum) { minimum = $2 } + END { print minimum + 0 }' "$output_dir/${phase}-continuity.csv")" + minimum_ready_endpoints="$(awk -F, 'NR > 1 && (minimum == "" || $3 < minimum) { minimum = $3 } + END { print minimum + 0 }' "$output_dir/${phase}-continuity.csv")" + (( sample_count >= 5 && minimum_ready_pods >= 2 && minimum_ready_endpoints >= 2 )) || { + echo "$phase continuity evidence was incomplete" >&2; return 1; + } + + pods_json="$(kubectl get pod --namespace "$namespace" \ + -l app.kubernetes.io/name=loadbalancerpro -o json)" + ready_pods_json="$(jq --arg token "$token" --arg annotation "$annotation_name" \ + --arg secret "$secret_name" '[.items[] + | select(.metadata.deletionTimestamp == null) + | select(.status.phase == "Running") + | select(any(.status.conditions[]?; .type == "Ready" and .status == "True")) + | select(.metadata.annotations[$annotation] == $token) + | select(any(.spec.volumes[]?; .name == "api-key" and .secret.secretName == $secret))]' \ + <<< "$pods_json")" + assert_two_zone_proxy_pods "$ready_pods_json" "$phase" + prior_uids_json="$(jq '[.[].metadata.uid] | sort' <<< "$prior_pods_json")" + current_uids_json="$(jq '[.[].metadata.uid] | sort' <<< "$ready_pods_json")" + retained_prior_uids="$(jq -n --argjson prior "$prior_uids_json" \ + --argjson current "$current_uids_json" '[ $current[] | select(. as $uid | $prior | index($uid)) ] | length')" + [[ "$retained_prior_uids" == 0 ]] || { + echo "$phase retained a prior API-key pod UID" >&2; return 1; + } + runtime_image_ids_json="$(jq '[.[].status.containerStatuses[]? + | select(.name == "loadbalancerpro") | .imageID] | unique | sort' <<< "$ready_pods_json")" + [[ "$runtime_image_ids_json" == "$initial_proxy_runtime_image_ids_json" ]] || { + echo "$phase changed the immutable runtime image ID" >&2; return 1; + } + while read -r pod; do + kubectl wait --for=delete "pod/$pod" --namespace "$namespace" --timeout="${maximum_seconds}s" + done < <(jq -r '.[].metadata.name' <<< "$prior_pods_json" | sort) + capture_state "$phase" + + api_key_transition_token="$token" + api_key_transition_elapsed_seconds="$elapsed_seconds" + api_key_transition_sample_count="$sample_count" + api_key_transition_minimum_ready_pods="$minimum_ready_pods" + api_key_transition_minimum_ready_endpoints="$minimum_ready_endpoints" + api_key_transition_ready_pods_json="$ready_pods_json" + api_key_transition_uids_json="$current_uids_json" +} + baseline_seconds="$(jq -r '.workload.baselineSeconds' "$profile")" run_attack baseline "$baseline_seconds" "$(jq -r '.objectives.minimumBaselineSuccessRatio' "$profile")" collect_distribution baseline @@ -1100,7 +1270,144 @@ prove_post_transition_distribution post-certificate-rollback \ bothRestoredCertificateProxyReplicasServed \ "Both restored-certificate proxies and both backends must serve post-certificate-rollback traffic" -failed_node="$(jq -r '.[0].spec.nodeName' <<< "$certificate_rollback_ready_proxy_pods_json")" +post_api_key_transition_seconds="$(jq -r '.workload.postApiKeyTransitionSeconds' "$profile")" +minimum_post_api_key_transition_success_ratio="$(jq -r \ + '.objectives.minimumPostApiKeyTransitionSuccessRatio' "$profile")" + +run_api_key_secret_transition api-key-overlap "$overlap_api_key_secret" \ + loadbalancerpro.io/qualification-api-key-overlap "$targets" \ + "$certificate_rollback_ready_proxy_pods_json" +api_key_overlap_token="$api_key_transition_token" +api_key_overlap_elapsed_seconds="$api_key_transition_elapsed_seconds" +api_key_overlap_sample_count="$api_key_transition_sample_count" +api_key_overlap_minimum_ready_pods="$api_key_transition_minimum_ready_pods" +api_key_overlap_minimum_ready_endpoints="$api_key_transition_minimum_ready_endpoints" +api_key_overlap_pods_json="$api_key_transition_ready_pods_json" +api_key_overlap_uids_json="$api_key_transition_uids_json" +assert_api_key_status "$api_key_file" 200 "Baseline API key during overlap" +assert_api_key_status "$candidate_api_key_file" 200 "Candidate API key during overlap" +prove_post_transition_distribution post-api-key-overlap "$post_api_key_transition_seconds" \ + "$minimum_post_api_key_transition_success_ratio" bothApiKeysAcceptedDuringOverlap \ + "Both overlap proxies and both backends must serve baseline-key traffic" "$targets" + +run_api_key_secret_transition api-key-commit "$candidate_api_key_secret" \ + loadbalancerpro.io/qualification-api-key-commit "$candidate_targets" \ + "$api_key_overlap_pods_json" +api_key_commit_token="$api_key_transition_token" +api_key_commit_elapsed_seconds="$api_key_transition_elapsed_seconds" +api_key_commit_sample_count="$api_key_transition_sample_count" +api_key_commit_minimum_ready_pods="$api_key_transition_minimum_ready_pods" +api_key_commit_minimum_ready_endpoints="$api_key_transition_minimum_ready_endpoints" +api_key_commit_pods_json="$api_key_transition_ready_pods_json" +api_key_commit_uids_json="$api_key_transition_uids_json" +assert_api_key_status "$candidate_api_key_file" 200 "Candidate API key after commit" +assert_api_key_status "$api_key_file" 401 "Retired baseline API key after commit" +prove_post_transition_distribution post-api-key-commit "$post_api_key_transition_seconds" \ + "$minimum_post_api_key_transition_success_ratio" baselineApiKeyRetired \ + "Both committed-key proxies and both backends must serve candidate-key traffic" "$candidate_targets" + +run_api_key_secret_transition api-key-rollback-overlap "$overlap_api_key_secret" \ + loadbalancerpro.io/qualification-api-key-rollback-overlap "$candidate_targets" \ + "$api_key_commit_pods_json" +api_key_rollback_overlap_token="$api_key_transition_token" +api_key_rollback_overlap_elapsed_seconds="$api_key_transition_elapsed_seconds" +api_key_rollback_overlap_sample_count="$api_key_transition_sample_count" +api_key_rollback_overlap_minimum_ready_pods="$api_key_transition_minimum_ready_pods" +api_key_rollback_overlap_minimum_ready_endpoints="$api_key_transition_minimum_ready_endpoints" +api_key_rollback_overlap_pods_json="$api_key_transition_ready_pods_json" +api_key_rollback_overlap_uids_json="$api_key_transition_uids_json" +assert_api_key_status "$api_key_file" 200 "Baseline API key during rollback overlap" +assert_api_key_status "$candidate_api_key_file" 200 "Candidate API key during rollback overlap" +prove_post_transition_distribution post-api-key-rollback-overlap "$post_api_key_transition_seconds" \ + "$minimum_post_api_key_transition_success_ratio" bothApiKeysAcceptedDuringRollbackOverlap \ + "Both rollback-overlap proxies and both backends must serve candidate-key traffic" "$candidate_targets" + +run_api_key_secret_transition api-key-rollback-commit "$baseline_api_key_secret" \ + loadbalancerpro.io/qualification-api-key-rollback-commit "$targets" \ + "$api_key_rollback_overlap_pods_json" +api_key_rollback_commit_token="$api_key_transition_token" +api_key_rollback_commit_elapsed_seconds="$api_key_transition_elapsed_seconds" +api_key_rollback_commit_sample_count="$api_key_transition_sample_count" +api_key_rollback_commit_minimum_ready_pods="$api_key_transition_minimum_ready_pods" +api_key_rollback_commit_minimum_ready_endpoints="$api_key_transition_minimum_ready_endpoints" +api_key_rollback_commit_pods_json="$api_key_transition_ready_pods_json" +api_key_rollback_commit_uids_json="$api_key_transition_uids_json" +assert_api_key_status "$api_key_file" 200 "Restored baseline API key after rollback" +assert_api_key_status "$candidate_api_key_file" 401 "Retired candidate API key after rollback" +prove_post_transition_distribution post-api-key-rollback-commit "$post_api_key_transition_seconds" \ + "$minimum_post_api_key_transition_success_ratio" candidateApiKeyRetiredAfterRollback \ + "Both restored-key proxies and both backends must serve baseline-key traffic" "$targets" + +jq -n \ + --arg baselineSecret "$baseline_api_key_secret" \ + --arg overlapSecret "$overlap_api_key_secret" \ + --arg candidateSecret "$candidate_api_key_secret" \ + --arg overlapToken "$api_key_overlap_token" \ + --arg commitToken "$api_key_commit_token" \ + --arg rollbackOverlapToken "$api_key_rollback_overlap_token" \ + --arg rollbackCommitToken "$api_key_rollback_commit_token" \ + --argjson initialPodUids "$certificate_rollback_proxy_uids_json" \ + --argjson overlapPodUids "$api_key_overlap_uids_json" \ + --argjson commitPodUids "$api_key_commit_uids_json" \ + --argjson rollbackOverlapPodUids "$api_key_rollback_overlap_uids_json" \ + --argjson rollbackCommitPodUids "$api_key_rollback_commit_uids_json" \ + --argjson overlapSeconds "$api_key_overlap_elapsed_seconds" \ + --argjson overlapSamples "$api_key_overlap_sample_count" \ + --argjson overlapMinimumReadyPods "$api_key_overlap_minimum_ready_pods" \ + --argjson overlapMinimumReadyEndpoints "$api_key_overlap_minimum_ready_endpoints" \ + --argjson commitSeconds "$api_key_commit_elapsed_seconds" \ + --argjson commitSamples "$api_key_commit_sample_count" \ + --argjson commitMinimumReadyPods "$api_key_commit_minimum_ready_pods" \ + --argjson commitMinimumReadyEndpoints "$api_key_commit_minimum_ready_endpoints" \ + --argjson rollbackOverlapSeconds "$api_key_rollback_overlap_elapsed_seconds" \ + --argjson rollbackOverlapSamples "$api_key_rollback_overlap_sample_count" \ + --argjson rollbackOverlapMinimumReadyPods "$api_key_rollback_overlap_minimum_ready_pods" \ + --argjson rollbackOverlapMinimumReadyEndpoints "$api_key_rollback_overlap_minimum_ready_endpoints" \ + --argjson rollbackCommitSeconds "$api_key_rollback_commit_elapsed_seconds" \ + --argjson rollbackCommitSamples "$api_key_rollback_commit_sample_count" \ + --argjson rollbackCommitMinimumReadyPods "$api_key_rollback_commit_minimum_ready_pods" \ + --argjson rollbackCommitMinimumReadyEndpoints "$api_key_rollback_commit_minimum_ready_endpoints" \ + --slurpfile overlapDistribution "$output_dir/post-api-key-overlap-distribution-delta.json" \ + --slurpfile commitDistribution "$output_dir/post-api-key-commit-distribution-delta.json" \ + --slurpfile rollbackOverlapDistribution \ + "$output_dir/post-api-key-rollback-overlap-distribution-delta.json" \ + --slurpfile rollbackCommitDistribution \ + "$output_dir/post-api-key-rollback-commit-distribution-delta.json" ' + {verificationModel: "required primary plus at most one operator-bounded rotation key", + dynamicSecretReload: false, + secrets: {immutable: true, baseline: $baselineSecret, overlap: $overlapSecret, + candidate: $candidateSecret}, + assertions: {candidateRejectedBeforeOverlap: true, bothAcceptedDuringOverlap: true, + baselineRejectedAfterCommit: true, bothAcceptedDuringRollbackOverlap: true, + candidateRejectedAfterRollbackCommit: true, secretValuesAbsentFromEvidence: true}, + overlap: {triggerAnnotation: $overlapToken, priorPodUids: $initialPodUids, + podUids: $overlapPodUids, retainedPriorPodUids: 0, runtimeImageUnchanged: true, + transitionSeconds: $overlapSeconds, continuitySamples: $overlapSamples, + minimumReadyProxyPods: $overlapMinimumReadyPods, + minimumReadyServiceEndpoints: $overlapMinimumReadyEndpoints, + traffic: $overlapDistribution[0]}, + commit: {triggerAnnotation: $commitToken, priorPodUids: $overlapPodUids, + podUids: $commitPodUids, retainedPriorPodUids: 0, runtimeImageUnchanged: true, + transitionSeconds: $commitSeconds, continuitySamples: $commitSamples, + minimumReadyProxyPods: $commitMinimumReadyPods, + minimumReadyServiceEndpoints: $commitMinimumReadyEndpoints, + traffic: $commitDistribution[0]}, + rollbackOverlap: {triggerAnnotation: $rollbackOverlapToken, priorPodUids: $commitPodUids, + podUids: $rollbackOverlapPodUids, retainedPriorPodUids: 0, runtimeImageUnchanged: true, + transitionSeconds: $rollbackOverlapSeconds, continuitySamples: $rollbackOverlapSamples, + minimumReadyProxyPods: $rollbackOverlapMinimumReadyPods, + minimumReadyServiceEndpoints: $rollbackOverlapMinimumReadyEndpoints, + traffic: $rollbackOverlapDistribution[0]}, + rollbackCommit: {triggerAnnotation: $rollbackCommitToken, + priorPodUids: $rollbackOverlapPodUids, podUids: $rollbackCommitPodUids, + retainedPriorPodUids: 0, runtimeImageUnchanged: true, + transitionSeconds: $rollbackCommitSeconds, continuitySamples: $rollbackCommitSamples, + minimumReadyProxyPods: $rollbackCommitMinimumReadyPods, + minimumReadyServiceEndpoints: $rollbackCommitMinimumReadyEndpoints, + traffic: $rollbackCommitDistribution[0]}}' \ + > "$output_dir/api-key-rotation.json" + +failed_node="$(jq -r '.[0].spec.nodeName' <<< "$api_key_rollback_commit_pods_json")" [[ "$failed_node" == "${cluster_name}-worker" || "$failed_node" == "${cluster_name}-worker2" ]] || { echo "Refusing to drain unexpected node $failed_node" >&2; exit 1; } @@ -1292,6 +1599,7 @@ post_rollout_distribution_delta_json="$(<"$output_dir/post-rollout-distribution- post_rollback_distribution_delta_json="$(<"$output_dir/post-rollback-distribution-delta.json")" post_certificate_rotation_distribution_delta_json="$(<"$output_dir/post-certificate-rotation-distribution-delta.json")" post_certificate_rollback_distribution_delta_json="$(<"$output_dir/post-certificate-rollback-distribution-delta.json")" +api_key_rotation_json="$(<"$output_dir/api-key-rotation.json")" recovered_distribution_delta_json="$(<"$output_dir/recovered-distribution-delta.json")" abrupt_recovered_distribution_delta_json="$(<"$output_dir/abrupt-recovered-distribution-delta.json")" jq -n \ @@ -1347,9 +1655,10 @@ jq -n \ --argjson postRollbackDistribution "$post_rollback_distribution_delta_json" \ --argjson postCertificateRotationDistribution "$post_certificate_rotation_distribution_delta_json" \ --argjson postCertificateRollbackDistribution "$post_certificate_rollback_distribution_delta_json" \ + --argjson apiKeyRotation "$api_key_rotation_json" \ --argjson recoveredDistribution "$recovered_distribution_delta_json" \ --argjson abruptRecoveredDistribution "$abrupt_recovered_distribution_delta_json" \ - '{schemaVersion: 5, result: "pass", evidenceBoundary: "disposable loopback kind metadata-only content-distinct image rollout and baseline rollback, versioned immutable inbound-server TLS Secret rotation and identity rollback, planned worker loss, and operator-remediated abrupt worker-container loss; not an ingress-controller, automatic infrastructure-failure detection, application-layer release compatibility, registry/source binding, external certificate-authority, client trust-distribution, or deployment-capacity proof", + '{schemaVersion: 6, result: "pass", evidenceBoundary: "disposable loopback kind metadata-only content-distinct image rollout and baseline rollback, versioned immutable inbound-server TLS Secret rotation and identity rollback, bounded two-key API credential overlap/commit/rollback, planned worker loss, and operator-remediated abrupt worker-container loss; not dynamic Secret reload, not an ingress-controller, and not automatic infrastructure-failure detection, application-layer release compatibility, registry/source binding, external certificate-authority, client trust-distribution, external secret-manager, or deployment-capacity proof", profileId: $profileId, repositoryRevision: $sourceRevision, images: {identityType: "local Docker content-addressed image ID", baseline: {contentId: $proxyImageId}, @@ -1357,14 +1666,18 @@ jq -n \ fixtureContentId: $fixtureImageId, applicationLayersIdentical: true}, topology: {workers: 2, zones: 2, initialProxyReplicas: 2, postRolloutProxyReplicas: 2, postRollbackProxyReplicas: 2, - postCertificateRotationProxyReplicas: 2, postCertificateRollbackProxyReplicas: 2, + postCertificateRotationProxyReplicas: 2, postCertificateRollbackProxyReplicas: 2, + postApiKeyOverlapProxyReplicas: 2, postApiKeyCommitProxyReplicas: 2, + postApiKeyRollbackOverlapProxyReplicas: 2, postApiKeyRollbackCommitProxyReplicas: 2, degradedProxyReplicas: 1, recoveredProxyReplicas: 2, abruptDegradedProxyReplicas: 1, abruptRecoveredProxyReplicas: 2}, traffic: {bothProxyReplicasServed: true, baseline: $baselineDistribution, rollout: "pass", postRollout: $postRolloutDistribution, rollback: "pass", postRollback: $postRollbackDistribution, certificateRotation: "pass", postCertificateRotation: $postCertificateRotationDistribution, - certificateRollback: "pass", postCertificateRollback: $postCertificateRollbackDistribution, + certificateRollback: "pass", postCertificateRollback: $postCertificateRollbackDistribution, + apiKeyOverlap: "pass", apiKeyCommit: "pass", + apiKeyRollbackOverlap: "pass", apiKeyRollbackCommit: "pass", drainTransition: "pass", degraded: "pass", recovered: $recoveredDistribution, abruptTransition: "pass", abruptDegraded: "pass", abruptRecovered: $abruptRecoveredDistribution}, @@ -1410,6 +1723,7 @@ jq -n \ rollbackSeconds: $certificateRollbackSeconds, continuitySamples: $certificateRollbackSamples, minimumReadyProxyPods: $certificateRollbackMinimumReadyPods, minimumReadyServiceEndpoints: $certificateRollbackMinimumReadyEndpoints}}, + apiKeyRotationExercise: $apiKeyRotation, workerExercise: {planned: {drainedAndStopped: $drainedWorker, recoverySeconds: $recoverySeconds}, abrupt: {stoppedWithoutDrain: $abruptWorker, remediation: "verified-down out-of-service:NoExecute taint plus forced API deletion", @@ -1419,4 +1733,11 @@ jq -n \ recoverySeconds: $abruptRecoverySeconds}}}' \ > "$output_dir/summary.json" -printf 'Kubernetes two-zone image rollout/rollback, immutable TLS identity rotation/rollback, planned-loss, and abrupt-loss proof passed; evidence: %s\n' "$output_dir" +for secret_file in "$api_key_file" "$candidate_api_key_file"; do + if grep -R -F -q -- "$(<"$secret_file")" "$output_dir"; then + echo "Kubernetes API-key value leaked into evidence" >&2 + exit 1 + fi +done + +printf 'Kubernetes two-zone image rollout/rollback, immutable TLS identity rotation/rollback, bounded API-key rotation/rollback, planned-loss, and abrupt-loss proof passed; evidence: %s\n' "$output_dir" diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/config/ApiKeyVerifier.java b/src/main/java/com/richmond423/loadbalancerpro/api/config/ApiKeyVerifier.java new file mode 100644 index 00000000..49fdd4bd --- /dev/null +++ b/src/main/java/com/richmond423/loadbalancerpro/api/config/ApiKeyVerifier.java @@ -0,0 +1,48 @@ +package com.richmond423.loadbalancerpro.api.config; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +/** + * Verifies the required primary API key and one optional, operator-bounded rotation key. + */ +public final class ApiKeyVerifier { + private final boolean configured; + private final byte[] primaryKeyDigest; + private final byte[] rotationKeyDigest; + + ApiKeyVerifier(String primaryKey, String rotationKey) { + byte[] normalizedPrimaryKey = normalizedBytes(primaryKey); + this.configured = normalizedPrimaryKey.length > 0; + this.primaryKeyDigest = sha256(normalizedPrimaryKey); + this.rotationKeyDigest = sha256(normalizedBytes(rotationKey)); + } + + public boolean isConfigured() { + return configured; + } + + public boolean matches(String presentedKey) { + if (presentedKey == null || presentedKey.isBlank()) { + return false; + } + byte[] presentedDigest = sha256(presentedKey.getBytes(StandardCharsets.UTF_8)); + boolean primaryMatch = MessageDigest.isEqual(primaryKeyDigest, presentedDigest); + boolean rotationMatch = MessageDigest.isEqual(rotationKeyDigest, presentedDigest); + return primaryMatch | rotationMatch; + } + + private static byte[] normalizedBytes(String value) { + String normalized = value == null ? "" : value.trim(); + return normalized.getBytes(StandardCharsets.UTF_8); + } + + private static byte[] sha256(byte[] value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value); + } catch (NoSuchAlgorithmException exception) { + throw new IllegalStateException("SHA-256 digest algorithm is unavailable", exception); + } + } +} diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthModeConfiguration.java b/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthModeConfiguration.java index 09c1e1e5..d5bdab74 100644 --- a/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthModeConfiguration.java +++ b/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthModeConfiguration.java @@ -12,16 +12,23 @@ public class AuthModeConfiguration { private static final Logger logger = LoggerFactory.getLogger(AuthModeConfiguration.class); + @Bean + ApiKeyVerifier apiKeyVerifier( + @Value("${loadbalancerpro.api.key:}") String apiKey, + @Value("${loadbalancerpro.api.rotation-key:}") String rotationKey) { + return new ApiKeyVerifier(apiKey, rotationKey); + } + @Bean AuthModeValidator authModeValidator( AuthProperties authProperties, - @Value("${loadbalancerpro.api.key:}") String apiKey) { - return new AuthModeValidator(authProperties, apiKey); + ApiKeyVerifier apiKeyVerifier) { + return new AuthModeValidator(authProperties, apiKeyVerifier); } static final class AuthModeValidator { - AuthModeValidator(AuthProperties authProperties, String apiKey) { - authProperties.validateApiKeyMode(apiKey); + AuthModeValidator(AuthProperties authProperties, ApiKeyVerifier apiKeyVerifier) { + authProperties.validateApiKeyMode(apiKeyVerifier.isConfigured()); authProperties.validateOAuth2Mode(); if (authProperties.isNoneMode()) { logger.warn("SECURITY WARNING: loadbalancerpro.auth.mode=none is active; authentication is disabled. " diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthProperties.java b/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthProperties.java index b9f3e1c6..c67deabe 100644 --- a/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthProperties.java +++ b/src/main/java/com/richmond423/loadbalancerpro/api/config/AuthProperties.java @@ -54,8 +54,8 @@ public boolean isNoneMode() { return mode == Mode.NONE; } - public void validateApiKeyMode(String apiKey) { - if (isApiKeyMode() && !StringUtils.hasText(apiKey)) { + public void validateApiKeyMode(boolean apiKeyConfigured) { + if (isApiKeyMode() && !apiKeyConfigured) { throw new IllegalStateException("Application refuses to start with loadbalancerpro.auth.mode=api-key " + "because loadbalancerpro.api.key is missing or blank; configure a key, select OAuth2, " + "or explicitly set loadbalancerpro.auth.mode=none for bounded local development"); diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/config/ProdApiKeyFilter.java b/src/main/java/com/richmond423/loadbalancerpro/api/config/ProdApiKeyFilter.java index e1224b1c..35e62971 100644 --- a/src/main/java/com/richmond423/loadbalancerpro/api/config/ProdApiKeyFilter.java +++ b/src/main/java/com/richmond423/loadbalancerpro/api/config/ProdApiKeyFilter.java @@ -2,8 +2,6 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; @@ -35,18 +33,15 @@ public class ProdApiKeyFilter extends OncePerRequestFilter { private static final String API_KEY_HEADER = "X-API-Key"; private final ObjectMapper objectMapper; - private final byte[] configuredApiKey; - private final boolean apiKeyConfigured; + private final ApiKeyVerifier apiKeyVerifier; private final boolean protectActuator; private final AtomicBoolean missingKeyWarningLogged = new AtomicBoolean(false); public ProdApiKeyFilter(ObjectMapper objectMapper, - @Value("${loadbalancerpro.api.key:}") String configuredApiKey, + ApiKeyVerifier apiKeyVerifier, @Value("${loadbalancerpro.auth.protect-actuator:false}") boolean protectActuator) { this.objectMapper = objectMapper; - String normalizedKey = configuredApiKey == null ? "" : configuredApiKey.trim(); - this.apiKeyConfigured = !normalizedKey.isEmpty(); - this.configuredApiKey = normalizedKey.getBytes(StandardCharsets.UTF_8); + this.apiKeyVerifier = apiKeyVerifier; this.protectActuator = protectActuator; } @@ -58,15 +53,14 @@ protected void doFilterInternal(HttpServletRequest request, HttpServletResponse return; } - if (!apiKeyConfigured) { + if (!apiKeyVerifier.isConfigured()) { logMissingKeyWarningOnce(); writeUnauthorized(request, response); return; } String presentedApiKey = request.getHeader(API_KEY_HEADER); - if (presentedApiKey == null || presentedApiKey.isBlank() - || !constantTimeEquals(configuredApiKey, presentedApiKey.getBytes(StandardCharsets.UTF_8))) { + if (!apiKeyVerifier.matches(presentedApiKey)) { writeUnauthorized(request, response); return; } @@ -111,18 +105,6 @@ private static boolean isActuatorRequest(HttpServletRequest request) { return "/actuator".equals(requestUri) || requestUri.startsWith("/actuator/"); } - private static boolean constantTimeEquals(byte[] expected, byte[] actual) { - return MessageDigest.isEqual(sha256(expected), sha256(actual)); - } - - private static byte[] sha256(byte[] value) { - try { - return MessageDigest.getInstance("SHA-256").digest(value); - } catch (NoSuchAlgorithmException exception) { - throw new IllegalStateException("SHA-256 digest algorithm is unavailable", exception); - } - } - private void logMissingKeyWarningOnce() { if (missingKeyWarningLogged.compareAndSet(false, true)) { logger.warn("API-key auth mode has no configured key; protected API requests will be rejected."); diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusController.java b/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusController.java index da0e9283..1cfe910f 100644 --- a/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusController.java +++ b/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusController.java @@ -2,9 +2,6 @@ import java.net.URI; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -13,10 +10,10 @@ import java.util.Objects; import com.richmond423.loadbalancerpro.api.ApiErrorResponse; +import com.richmond423.loadbalancerpro.api.config.ApiKeyVerifier; import com.richmond423.loadbalancerpro.api.explain.LiveRoutingDecisionExplanation; import com.richmond423.loadbalancerpro.api.explain.LiveRoutingExplanationService; import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; @@ -39,27 +36,25 @@ @RestController @RequestMapping("/api/proxy") public class ReverseProxyStatusController { - private static final String API_KEY_HEADER = "X-API-Key"; - private final ReverseProxyProperties properties; private final ReverseProxyMetrics metrics; private final ObjectProvider reverseProxyService; private final LiveRoutingExplanationService routingExplanationService; private final Environment environment; - private final String configuredApiKey; + private final ApiKeyVerifier apiKeyVerifier; public ReverseProxyStatusController(ReverseProxyProperties properties, ReverseProxyMetrics metrics, ObjectProvider reverseProxyService, LiveRoutingExplanationService routingExplanationService, Environment environment, - @Value("${loadbalancerpro.api.key:}") String configuredApiKey) { + ApiKeyVerifier apiKeyVerifier) { this.properties = properties; this.metrics = metrics; this.reverseProxyService = reverseProxyService; this.routingExplanationService = routingExplanationService; this.environment = environment; - this.configuredApiKey = configuredApiKey; + this.apiKeyVerifier = apiKeyVerifier; } @GetMapping("/status") @@ -264,7 +259,7 @@ private ReverseProxyStatusResponse decorate(ReverseProxyStatusResponse response) response.metrics(), ReverseProxyStatusSummaries.observability(response.proxyEnabled(), response.routes(), response.upstreams(), response.metrics()), - ReverseProxyStatusSummaries.securityBoundary(environment, configuredApiKey), + ReverseProxyStatusSummaries.securityBoundary(environment, apiKeyVerifier.isConfigured()), response.privateNetworkLiveValidation(), response.reload()); } @@ -473,14 +468,7 @@ private boolean oauth2Mode() { } private boolean validApiKey(HttpServletRequest request) { - String expected = configuredApiKey == null ? "" : configuredApiKey.trim(); - if (expected.isEmpty()) { - return false; - } - String presented = request.getHeader(API_KEY_HEADER); - return presented != null && !presented.isBlank() - && constantTimeEquals(expected.getBytes(StandardCharsets.UTF_8), - presented.getBytes(StandardCharsets.UTF_8)); + return apiKeyVerifier.matches(request.getHeader("X-API-Key")); } private ResponseEntity rejectAdminAuthentication(HttpServletRequest request) { @@ -501,18 +489,6 @@ private static ResponseEntity adminUnavailable(HttpServletRequ List.of())); } - private static boolean constantTimeEquals(byte[] expected, byte[] actual) { - return MessageDigest.isEqual(sha256(expected), sha256(actual)); - } - - private static byte[] sha256(byte[] value) { - try { - return MessageDigest.getInstance("SHA-256").digest(value); - } catch (NoSuchAlgorithmException exception) { - throw new IllegalStateException("SHA-256 digest algorithm is unavailable", exception); - } - } - private static ReverseProxyReloadResponse reloadRejected(String status, List errors) { return new ReverseProxyReloadResponse( false, diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusSummaries.java b/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusSummaries.java index 91c04f60..e7538e04 100644 --- a/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusSummaries.java +++ b/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyStatusSummaries.java @@ -5,7 +5,6 @@ import java.util.Locale; import org.springframework.core.env.Environment; -import org.springframework.util.StringUtils; final class ReverseProxyStatusSummaries { private ReverseProxyStatusSummaries() { @@ -50,7 +49,7 @@ static ReverseProxyStatusResponse.ObservabilitySummary observability( static ReverseProxyStatusResponse.SecurityBoundaryStatus securityBoundary( Environment environment, - String configuredApiKey) { + boolean apiKeyConfigured) { String authMode = normalizeAuthMode(environment == null ? "api-key" : environment.getProperty("loadbalancerpro.auth.mode", "api-key")); @@ -60,8 +59,6 @@ static ReverseProxyStatusResponse.SecurityBoundaryStatus securityBoundary( boolean oauth2Mode = "oauth2".equals(authMode); boolean apiKeyBoundary = "api-key".equals(authMode) && prodLikeProfile; boolean protectedBoundary = oauth2Mode || apiKeyBoundary; - boolean apiKeyConfigured = StringUtils.hasText(configuredApiKey); - return new ReverseProxyStatusResponse.SecurityBoundaryStatus( authMode, activeProfiles, diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/ApiKeyRotationProtectionTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/ApiKeyRotationProtectionTest.java new file mode 100644 index 00000000..dc237462 --- /dev/null +++ b/src/test/java/com/richmond423/loadbalancerpro/api/ApiKeyRotationProtectionTest.java @@ -0,0 +1,66 @@ +package com.richmond423.loadbalancerpro.api; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; + +@SpringBootTest(properties = { + "spring.profiles.active=prod", + "loadbalancerpro.auth.mode=api-key", + "loadbalancerpro.api.key=TEST_ROTATION_PRIMARY_KEY", + "loadbalancerpro.api.rotation-key=TEST_ROTATION_OVERLAP_KEY" +}) +@AutoConfigureMockMvc +class ApiKeyRotationProtectionTest { + private static final String PRIMARY_KEY = "TEST_ROTATION_PRIMARY_KEY"; + private static final String ROTATION_KEY = "TEST_ROTATION_OVERLAP_KEY"; + + @Autowired + private MockMvc mockMvc; + + @Test + void boundedOverlapAcceptsPrimaryAndRotationKeysWithoutExposingEither() throws Exception { + for (String apiKey : new String[] {PRIMARY_KEY, ROTATION_KEY}) { + mockMvc.perform(get("/api/proxy/status").header("X-API-Key", apiKey)) + .andExpect(status().isOk()) + .andExpect(content().string(not(containsString(PRIMARY_KEY)))) + .andExpect(content().string(not(containsString(ROTATION_KEY)))) + .andExpect(jsonPath("$.securityBoundary.apiKeyConfigured", is(true))); + } + } + + @Test + void rotationKeyAlsoPassesTheIndependentProxyAdministrationCheck() throws Exception { + mockMvc.perform(post("/api/proxy/reload") + .header("X-API-Key", ROTATION_KEY) + .contentType(MediaType.APPLICATION_JSON) + .content("{}")) + .andExpect(status().isConflict()) + .andExpect(jsonPath("$.success", is(false))) + .andExpect(jsonPath("$.status", is("unsupported"))) + .andExpect(content().string(not(containsString(PRIMARY_KEY)))) + .andExpect(content().string(not(containsString(ROTATION_KEY)))); + } + + @Test + void missingUnknownAndWhitespaceKeysRemainDenied() throws Exception { + for (String apiKey : new String[] {"", " ", "TEST_UNKNOWN_API_KEY"}) { + mockMvc.perform(get("/api/proxy/status").header("X-API-Key", apiKey)) + .andExpect(status().isUnauthorized()); + } + mockMvc.perform(get("/api/proxy/status")) + .andExpect(status().isUnauthorized()); + } +} diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index fe100c71..d2453710 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -41,6 +41,8 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "drop: [\"ALL\"]", "kind: PodDisruptionBudget", "kind: NetworkPolicy", + "secretName: loadbalancerpro-api-key-a", + "path: loadbalancerpro.api.rotation-key", "secretName: loadbalancerpro-server-tls-a")) { assertTrue(workload.contains(invariant), "missing restricted workload invariant: " + invariant); } @@ -64,7 +66,7 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { "kindest/node:v1.34.3@sha256:08497ee19eace7b4b5348db5c6a1591d7752b164530a36f855cb0f2bdcbadd48")); JsonNode profile = new ObjectMapper().readTree(read(PROFILE)); - assertEquals(5, profile.path("schemaVersion").asInt()); + assertEquals(6, profile.path("schemaVersion").asInt()); assertEquals("example", profile.path("review").path("status").asText()); assertEquals("v1.34.3", profile.path("cluster").path("kubectlVersion").asText()); assertEquals(2, profile.path("cluster").path("workers").asInt()); @@ -88,11 +90,21 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { assertTrue(profile.path("objectives").path("minimumPostCertificateRotationSuccessRatio").asDouble() >= 0.95); assertTrue(profile.path("objectives").path("minimumCertificateRollbackSuccessRatio").asDouble() >= 0.95); assertTrue(profile.path("objectives").path("minimumPostCertificateRollbackSuccessRatio").asDouble() >= 0.95); + assertTrue(profile.path("workload").path("apiKeyTransitionSeconds").asInt() + >= profile.path("objectives").path("maximumApiKeyTransitionSeconds").asInt() + 5); + assertTrue(profile.path("objectives").path("minimumApiKeyTransitionSuccessRatio").asDouble() >= 0.95); + assertTrue(profile.path("objectives").path("minimumPostApiKeyTransitionSuccessRatio").asDouble() >= 0.95); assertEquals("lbp-kubernetes.local", profile.path("tlsRotation").path("hostname").asText()); assertEquals("loadbalancerpro-server-tls-a", profile.path("tlsRotation").path("baselineSecret").asText()); assertEquals("loadbalancerpro-server-tls-b", profile.path("tlsRotation").path("candidateSecret").asText()); + assertEquals("loadbalancerpro-api-key-a", + profile.path("apiKeyRotation").path("baselineSecret").asText()); + assertEquals("loadbalancerpro-api-key-a-b", + profile.path("apiKeyRotation").path("overlapSecret").asText()); + assertEquals("loadbalancerpro-api-key-b", + profile.path("apiKeyRotation").path("candidateSecret").asText()); assertTrue(profile.path("workload").path("abruptTransitionSeconds").asInt() >= profile.path("objectives").path("maximumAbruptEndpointWithdrawalSeconds").asInt() + 5); assertTrue(profile.path("objectives").path("minimumAbruptTransitionSuccessRatio").asDouble() >= 0.90); @@ -160,6 +172,21 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "baselineOnlyRejectedAfterRotation: true", "candidateOnlyRejectedAfterRollback: true", "not an ingress-controller", + "create_immutable_api_key_secret", + "api-key-secret-metadata.json", + "loadbalancerpro.io/qualification-api-key-overlap", + "loadbalancerpro.io/qualification-api-key-commit", + "loadbalancerpro.io/qualification-api-key-rollback-overlap", + "loadbalancerpro.io/qualification-api-key-rollback-commit", + "${phase}-continuity.csv", + "Candidate API key before overlap", + "Retired baseline API key after commit", + "Retired candidate API key after rollback", + "bothApiKeysAcceptedDuringOverlap", + "candidateApiKeyRetiredAfterRollback", + "required primary plus at most one operator-bounded rotation key", + "Kubernetes API-key value leaked into evidence", + "not dynamic Secret reload", "priorPodUids: $priorPodUids", "candidatePodUids: $candidatePodUids", "restoredPodUids: $restoredPodUids", @@ -186,7 +213,7 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "bothProxyReplicasServed: true")) { assertTrue(runner.contains(behavior), "missing live Kubernetes proof behavior: " + behavior); } - assertTrue(read(CONTRACT).contains("rejected 52 unsafe profiles without creating a cluster")); + assertTrue(read(CONTRACT).contains("rejected 64 unsafe profiles without creating a cluster")); assertFalse(runner.contains("--insecure")); assertFalse(runner.contains("--validate=false")); } diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/ProdApiKeyMissingConfigurationTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/ProdApiKeyMissingConfigurationTest.java index b447403e..48011b65 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/ProdApiKeyMissingConfigurationTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/ProdApiKeyMissingConfigurationTest.java @@ -26,6 +26,22 @@ void prodProfileRefusesStartupWhenApiKeyIsBlank() { }); } + @Test + void rotationKeyCannotReplaceTheRequiredPrimaryKey() { + contextRunner.withPropertyValues( + "spring.profiles.active=prod", + "loadbalancerpro.auth.mode=api-key", + "loadbalancerpro.api.key=", + "loadbalancerpro.api.rotation-key=TEST_ROTATION_ONLY_KEY") + .run(context -> { + assertThat(context).hasFailed(); + assertThat(rootCause(context.getStartupFailure())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("loadbalancerpro.api.key") + .hasMessageContaining("refuses to start"); + }); + } + private static Throwable rootCause(Throwable throwable) { Throwable cause = throwable; while (cause.getCause() != null) { From 3b282098707101a92bf7deeb86e5b9521450e340 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 18:07:00 -0700 Subject: [PATCH 02/11] fix: package api key verifier --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 0018d658..57903189 100644 --- a/pom.xml +++ b/pom.xml @@ -259,6 +259,7 @@ com/richmond423/loadbalancerpro/api/ApiErrorResponse*.class com/richmond423/loadbalancerpro/api/RestExceptionHandler*.class com/richmond423/loadbalancerpro/api/config/ApiRateLimit*.class + com/richmond423/loadbalancerpro/api/config/ApiKeyVerifier*.class com/richmond423/loadbalancerpro/api/config/ApiSecurityConfiguration*.class com/richmond423/loadbalancerpro/api/config/AuthModeConfiguration*.class com/richmond423/loadbalancerpro/api/config/AuthProperties*.class From f9a17f4d111f1ca77ec7389d42d92129568a86f2 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 18:47:48 -0700 Subject: [PATCH 03/11] fix: gate kubernetes readiness on proxy traffic --- deploy/kubernetes/qualification.yaml | 2 +- scripts/bench/kubernetes-topology-profile.example.json | 4 ++-- scripts/bench/proxy-kubernetes-topology.sh | 1 + .../api/KubernetesLiveTopologyContractTest.java | 1 + 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index 4fb7f588..fa241d93 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -277,7 +277,7 @@ spec: curl --fail --silent --show-error --cacert /run/tls/ca.pem --header "X-API-Key: $(cat /run/secrets/loadbalancerpro.api.key)" --resolve "${LBP_TLS_HOSTNAME}:8080:127.0.0.1" - "https://${LBP_TLS_HOSTNAME}:8080/actuator/health" > /dev/null + "https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology" > /dev/null periodSeconds: 3 timeoutSeconds: 2 failureThreshold: 3 diff --git a/scripts/bench/kubernetes-topology-profile.example.json b/scripts/bench/kubernetes-topology-profile.example.json index 37cba791..a6bec16b 100644 --- a/scripts/bench/kubernetes-topology-profile.example.json +++ b/scripts/bench/kubernetes-topology-profile.example.json @@ -27,7 +27,7 @@ "postCertificateRotationSeconds": 8, "certificateRollbackSeconds": 90, "postCertificateRollbackSeconds": 8, - "apiKeyTransitionSeconds": 50, + "apiKeyTransitionSeconds": 65, "postApiKeyTransitionSeconds": 6, "transitionSeconds": 20, "degradedSeconds": 8, @@ -60,7 +60,7 @@ "maximumRollbackSeconds": 75, "maximumCertificateRotationSeconds": 75, "maximumCertificateRollbackSeconds": 75, - "maximumApiKeyTransitionSeconds": 45, + "maximumApiKeyTransitionSeconds": 60, "maximumRecoverySeconds": 180, "maximumAbruptEndpointWithdrawalSeconds": 15, "maximumAbruptRecoverySeconds": 180 diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index ccab91a8..6558680f 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -124,6 +124,7 @@ for invariant in \ 'kind: NetworkPolicy' \ 'secretName: loadbalancerpro-server-tls-a' \ 'secretName: loadbalancerpro-api-key-a' \ + 'https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology' \ 'path: loadbalancerpro.api.rotation-key'; do grep -Fq "$invariant" "$workload_manifest" || { echo "Kubernetes workload is missing: $invariant" >&2; exit 2; } done diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index d2453710..6e527099 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -43,6 +43,7 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "kind: NetworkPolicy", "secretName: loadbalancerpro-api-key-a", "path: loadbalancerpro.api.rotation-key", + "https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology", "secretName: loadbalancerpro-server-tls-a")) { assertTrue(workload.contains(invariant), "missing restricted workload invariant: " + invariant); } From 541435c1f2489057d5e79fe62ccec3ba52f0e5e0 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 19:29:36 -0700 Subject: [PATCH 04/11] fix: bound kubernetes qualification retries --- deploy/kubernetes/qualification.yaml | 8 ++++++++ docs/DEPLOYMENT.md | 8 ++++++++ scripts/bench/proxy-kubernetes-topology.sh | 4 ++++ .../resources/application-proxy-prod.properties | 8 ++++++++ .../api/KubernetesLiveTopologyContractTest.java | 4 ++++ .../api/ProxyProdProfileConfigurationTest.java | 16 ++++++++++++++-- 6 files changed, 46 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index fa241d93..d4e2d615 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -29,6 +29,14 @@ data: LBP_HEALTH_CHECK_INTERVAL: 1s LBP_HEALTHY_THRESHOLD: "1" LBP_UNHEALTHY_THRESHOLD: "2" + LBP_RETRY_ENABLED: "true" + LBP_RETRY_MAX_ATTEMPTS: "2" + LBP_RETRY_BUDGET_PERCENT: "100" + LBP_RETRY_BACKOFF_BASE: 10ms + LBP_RETRY_BACKOFF_MAX: 50ms + LBP_RETRY_NON_IDEMPOTENT: "false" + LBP_RETRY_METHODS: GET,HEAD + LBP_RETRY_STATUSES: 502,503,504 --- apiVersion: apps/v1 kind: Deployment diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 209a201d..fb147919 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -105,6 +105,14 @@ Hostname verification remains mandatory; `tls.verify=false` is rejected. Server | `LBP_COOLDOWN_DURATION` | `loadbalancerpro.proxy.cooldown.duration` | `30s` | | `LBP_DRAIN_TIMEOUT` | `loadbalancerpro.proxy.reload.drain-timeout` | `30s` | | `LBP_SLOW_START_DURATION` | `loadbalancerpro.proxy.slow-start.duration` | `5s` | +| `LBP_RETRY_ENABLED` | `loadbalancerpro.proxy.retry.enabled` | `false` | +| `LBP_RETRY_MAX_ATTEMPTS` | `loadbalancerpro.proxy.retry.max-attempts` | `2` | +| `LBP_RETRY_BUDGET_PERCENT` | `loadbalancerpro.proxy.retry.budget-percent` | `20` | +| `LBP_RETRY_BACKOFF_BASE` | `loadbalancerpro.proxy.retry.backoff.base` | `50ms` | +| `LBP_RETRY_BACKOFF_MAX` | `loadbalancerpro.proxy.retry.backoff.max` | `1s` | +| `LBP_RETRY_NON_IDEMPOTENT` | `loadbalancerpro.proxy.retry.retry-non-idempotent` | `false` | +| `LBP_RETRY_METHODS` | `loadbalancerpro.proxy.retry.methods` | `GET,HEAD` | +| `LBP_RETRY_STATUSES` | `loadbalancerpro.proxy.retry.retry-statuses` | `502,503,504` | | `LBP_BACKEND_TRUST_BUNDLE` | `loadbalancerpro.proxy.backend-tls.truststore` | blank | | `LBP_UPSTREAM_0_CLIENT_CERT_BUNDLE` | `loadbalancerpro.proxy.upstreams[0].tls.client-cert` | blank | diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index 6558680f..452d30bc 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -125,6 +125,10 @@ for invariant in \ 'secretName: loadbalancerpro-server-tls-a' \ 'secretName: loadbalancerpro-api-key-a' \ 'https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology' \ + 'LBP_RETRY_ENABLED: "true"' \ + 'LBP_RETRY_MAX_ATTEMPTS: "2"' \ + 'LBP_RETRY_BUDGET_PERCENT: "100"' \ + 'LBP_RETRY_NON_IDEMPOTENT: "false"' \ 'path: loadbalancerpro.api.rotation-key'; do grep -Fq "$invariant" "$workload_manifest" || { echo "Kubernetes workload is missing: $invariant" >&2; exit 2; } done diff --git a/src/main/resources/application-proxy-prod.properties b/src/main/resources/application-proxy-prod.properties index b660d54c..2b9cf018 100644 --- a/src/main/resources/application-proxy-prod.properties +++ b/src/main/resources/application-proxy-prod.properties @@ -35,6 +35,14 @@ loadbalancerpro.proxy.limits.max-in-flight=${LBP_MAX_IN_FLIGHT:100} loadbalancerpro.proxy.limits.adaptive=false loadbalancerpro.proxy.reload.drain-timeout=${LBP_DRAIN_TIMEOUT:30s} loadbalancerpro.proxy.slow-start.duration=${LBP_SLOW_START_DURATION:5s} +loadbalancerpro.proxy.retry.enabled=${LBP_RETRY_ENABLED:false} +loadbalancerpro.proxy.retry.max-attempts=${LBP_RETRY_MAX_ATTEMPTS:2} +loadbalancerpro.proxy.retry.budget-percent=${LBP_RETRY_BUDGET_PERCENT:20} +loadbalancerpro.proxy.retry.backoff.base=${LBP_RETRY_BACKOFF_BASE:50ms} +loadbalancerpro.proxy.retry.backoff.max=${LBP_RETRY_BACKOFF_MAX:1s} +loadbalancerpro.proxy.retry.retry-non-idempotent=${LBP_RETRY_NON_IDEMPOTENT:false} +loadbalancerpro.proxy.retry.methods=${LBP_RETRY_METHODS:GET,HEAD} +loadbalancerpro.proxy.retry.retry-statuses=${LBP_RETRY_STATUSES:502,503,504} loadbalancerpro.proxy.upstreams[0].id=${LBP_UPSTREAM_0_ID:backend-a} loadbalancerpro.proxy.upstreams[0].url=${LBP_UPSTREAM_0_URL} diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 6e527099..55684469 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -44,6 +44,10 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "secretName: loadbalancerpro-api-key-a", "path: loadbalancerpro.api.rotation-key", "https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology", + "LBP_RETRY_ENABLED: \"true\"", + "LBP_RETRY_MAX_ATTEMPTS: \"2\"", + "LBP_RETRY_BUDGET_PERCENT: \"100\"", + "LBP_RETRY_NON_IDEMPOTENT: \"false\"", "secretName: loadbalancerpro-server-tls-a")) { assertTrue(workload.contains(invariant), "missing restricted workload invariant: " + invariant); } diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java index ae2d0dea..e6d5c38d 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java @@ -21,7 +21,15 @@ "spring.profiles.active=prod,proxy-prod", "loadbalancerpro.api.key=TEST_PROXY_PROD_API_KEY", "LBP_UPSTREAM_0_URL=http://127.0.0.1:18081", - "LBP_UPSTREAM_1_URL=http://127.0.0.1:18082" + "LBP_UPSTREAM_1_URL=http://127.0.0.1:18082", + "LBP_RETRY_ENABLED=true", + "LBP_RETRY_MAX_ATTEMPTS=2", + "LBP_RETRY_BUDGET_PERCENT=100", + "LBP_RETRY_BACKOFF_BASE=10ms", + "LBP_RETRY_BACKOFF_MAX=50ms", + "LBP_RETRY_NON_IDEMPOTENT=false", + "LBP_RETRY_METHODS=GET,HEAD", + "LBP_RETRY_STATUSES=502,503,504" }) @AutoConfigureMockMvc class ProxyProdProfileConfigurationTest { @@ -63,7 +71,11 @@ void profileKeepsHealthPublicButProtectsProxyStatusAndActuator() throws Exceptio .andExpect(status().isUnauthorized()); mockMvc.perform(get("/api/proxy/status").header("X-API-Key", API_KEY)) .andExpect(status().isOk()) - .andExpect(jsonPath("$.proxyEnabled", is(true))); + .andExpect(jsonPath("$.proxyEnabled", is(true))) + .andExpect(jsonPath("$.retry.enabled", is(true))) + .andExpect(jsonPath("$.retry.maxAttempts", is(2))) + .andExpect(jsonPath("$.retry.budgetPercent", is(100))) + .andExpect(jsonPath("$.retry.retryNonIdempotent", is(false))); double proxyRequestsBefore = meterRegistry.find("lbp.proxy.requests").counters().stream() .mapToDouble(io.micrometer.core.instrument.Counter::count) From 6b69f2b1c71b71d34499bbb6d7ea12d874b9de84 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 20:06:39 -0700 Subject: [PATCH 05/11] fix: serialize planned kubernetes drain --- scripts/bench/proxy-kubernetes-topology.sh | 30 +++++++++++++++++++ .../KubernetesLiveTopologyContractTest.java | 7 +++++ 2 files changed, 37 insertions(+) diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index 452d30bc..17b2d509 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -426,6 +426,14 @@ ready_endpoint_count() { | select(.conditions.ready == true and .conditions.terminating != true)] | length' } +ready_backend_endpoint_count() { + local service_name="$1" + kubectl get endpointslice --namespace "$namespace" \ + -l kubernetes.io/service-name="$service_name" -o json \ + | jq '[.items[].endpoints[] + | select(.conditions.ready == true and .conditions.terminating != true)] | length' +} + abrupt_source_pod_count() { kubectl get pod --namespace "$namespace" -o json \ | jq --argjson names "$abrupt_forced_pod_names_json" \ @@ -444,6 +452,18 @@ wait_for_count() { return 1 } +wait_for_backend_endpoint_count() { + local service_name="$1" expected="$2" timeout_seconds="$3" + local deadline=$((SECONDS + timeout_seconds)) actual=unknown + while (( SECONDS < deadline )); do + actual="$(ready_backend_endpoint_count "$service_name")" + [[ "$actual" == "$expected" ]] && return 0 + sleep 1 + done + echo "$service_name ready endpoint count did not become $expected; last observed $actual" >&2 + return 1 +} + sample_transition_continuity() { local phase="$1" local output="$output_dir/${phase}-continuity.csv" @@ -1421,6 +1441,16 @@ vegeta attack -duration="${transition_seconds}s" -rate="${rate}/s" -timeout=5s - -root-certs="$tls_trust_bundle" -targets="$targets" > "$work_dir/transition.bin" & attack_pid=$! sleep 3 +kubectl cordon "$failed_node" +kubectl drain "$failed_node" --ignore-daemonsets --delete-emptydir-data --timeout=120s \ + --pod-selector='loadbalancerpro.io/backend=backend-a' +wait_for_backend_endpoint_count backend-a 1 30 +kubectl drain "$failed_node" --ignore-daemonsets --delete-emptydir-data --timeout=120s \ + --pod-selector='loadbalancerpro.io/backend=backend-b' +wait_for_backend_endpoint_count backend-b 1 30 +kubectl drain "$failed_node" --ignore-daemonsets --delete-emptydir-data --timeout=120s \ + --pod-selector='app.kubernetes.io/name=loadbalancerpro' +wait_for_count 'ready Service endpoints during planned drain' 1 ready_endpoint_count 30 kubectl drain "$failed_node" --ignore-daemonsets --delete-emptydir-data --timeout=120s docker stop "$failed_node" >/dev/null stopped_node="$failed_node" diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 55684469..991feaa0 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -197,7 +197,14 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "restoredPodUids: $restoredPodUids", "contentDistinctRuntimeImageId: true", "restoredInitialRuntimeImageId: true", + "kubectl cordon", "kubectl drain", + "--pod-selector='loadbalancerpro.io/backend=backend-a'", + "wait_for_backend_endpoint_count backend-a 1 30", + "--pod-selector='loadbalancerpro.io/backend=backend-b'", + "wait_for_backend_endpoint_count backend-b 1 30", + "--pod-selector='app.kubernetes.io/name=loadbalancerpro'", + "ready Service endpoints during planned drain", "docker stop", "ready Service endpoints while one worker is stopped", "docker start", From 6a06ff9f636d1fa4efde9e536e28e8ecba9c082a Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 20:50:03 -0700 Subject: [PATCH 06/11] fix: align kubernetes service health ownership --- deploy/kubernetes/qualification.yaml | 2 ++ docs/DEPLOYMENT.md | 7 ++++++- scripts/bench/kubernetes-topology-profile.example.json | 2 +- scripts/bench/proxy-kubernetes-topology.sh | 4 +++- src/main/resources/application-proxy-prod.properties | 4 ++-- .../api/DeploymentPackagingContractTest.java | 4 ++-- .../api/KubernetesLiveTopologyContractTest.java | 3 +++ .../api/ProxyProdProfileConfigurationTest.java | 8 ++++++-- 8 files changed, 25 insertions(+), 9 deletions(-) diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index d4e2d615..ed906b3a 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -26,9 +26,11 @@ data: LBP_UPSTREAM_1_URL: http://backend-b:8080 LBP_TLS_HOSTNAME: lbp-kubernetes.local LBP_PROXY_STRATEGY: ROUND_ROBIN + LBP_HEALTH_CHECK_ENABLED: "false" LBP_HEALTH_CHECK_INTERVAL: 1s LBP_HEALTHY_THRESHOLD: "1" LBP_UNHEALTHY_THRESHOLD: "2" + LBP_COOLDOWN_ENABLED: "false" LBP_RETRY_ENABLED: "true" LBP_RETRY_MAX_ATTEMPTS: "2" LBP_RETRY_BUDGET_PERCENT: "100" diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index fb147919..b96d22bc 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -100,8 +100,10 @@ Hostname verification remains mandatory; `tls.verify=false` is rejected. Server | `LBP_MAX_REQUEST_BYTES` | `loadbalancerpro.proxy.max-request-bytes` | `65536` | | `LBP_MAX_RESPONSE_BYTES` | `loadbalancerpro.proxy.max-response-bytes` | `0` (streaming/unbounded) | | `LBP_MAX_IN_FLIGHT` | `loadbalancerpro.proxy.limits.max-in-flight` | `100` | +| `LBP_HEALTH_CHECK_ENABLED` | `loadbalancerpro.proxy.health-check.enabled` | `true` | | `LBP_HEALTH_CHECK_PATH` | `loadbalancerpro.proxy.health-check.path` | `/health` | | `LBP_HEALTH_CHECK_INTERVAL` | `loadbalancerpro.proxy.health-check.interval` | `5s` | +| `LBP_COOLDOWN_ENABLED` | `loadbalancerpro.proxy.cooldown.enabled` | `true` | | `LBP_COOLDOWN_DURATION` | `loadbalancerpro.proxy.cooldown.duration` | `30s` | | `LBP_DRAIN_TIMEOUT` | `loadbalancerpro.proxy.reload.drain-timeout` | `30s` | | `LBP_SLOW_START_DURATION` | `loadbalancerpro.proxy.slow-start.duration` | `5s` | @@ -133,7 +135,10 @@ system. The lane next proves bounded API-key rotation through immutable A-only, sequence for rollback. Both keys are accepted only in the overlap phases; the retired key must return 401 after each commit, while zero-unavailable endpoint continuity, fresh pod UIDs, fixed runtime image identity, and traffic through both replicas and backends remain required. This is startup configuration rollout proof, not dynamic Secret reload or -external secret-manager proof. The lane then proves two-zone Service distribution, planned worker removal, and +external secret-manager proof. Because each configured upstream is a Kubernetes Service rather than a pod, the local +qualification lane uses EndpointSlice readiness as the pod-health authority and disables process-local active health +checks and cooldown; bounded idempotent retry across the two Services remains enabled. The lane then proves two-zone +Service distribution, planned worker removal, and operator-remediated no-drain worker loss and recovery. The abrupt-loss exercise forcibly stops the kind worker, confirms its container is down, applies the out-of-service `NoExecute` taint, and force-removes the three exact stateless qualification pods from the API. The disposable cluster also pins immediate EndpointSlice-triggered iptables updates diff --git a/scripts/bench/kubernetes-topology-profile.example.json b/scripts/bench/kubernetes-topology-profile.example.json index a6bec16b..7ddf8105 100644 --- a/scripts/bench/kubernetes-topology-profile.example.json +++ b/scripts/bench/kubernetes-topology-profile.example.json @@ -29,7 +29,7 @@ "postCertificateRollbackSeconds": 8, "apiKeyTransitionSeconds": 65, "postApiKeyTransitionSeconds": 6, - "transitionSeconds": 20, + "transitionSeconds": 45, "degradedSeconds": 8, "recoveredSeconds": 8, "abruptTransitionSeconds": 45, diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index 17b2d509..15622e92 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -50,7 +50,7 @@ jq -e ' and (.workload.postCertificateRollbackSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.apiKeyTransitionSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.workload.postApiKeyTransitionSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) - and (.workload.transitionSeconds | type == "number" and . >= 15 and . <= 120 and floor == .) + and (.workload.transitionSeconds | type == "number" and . >= 40 and . <= 120 and floor == .) and (.workload.degradedSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.recoveredSeconds | type == "number" and . >= 5 and . <= 60 and floor == .) and (.workload.abruptTransitionSeconds | type == "number" and . >= 15 and . <= 120 and floor == .) @@ -125,6 +125,8 @@ for invariant in \ 'secretName: loadbalancerpro-server-tls-a' \ 'secretName: loadbalancerpro-api-key-a' \ 'https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology' \ + 'LBP_HEALTH_CHECK_ENABLED: "false"' \ + 'LBP_COOLDOWN_ENABLED: "false"' \ 'LBP_RETRY_ENABLED: "true"' \ 'LBP_RETRY_MAX_ATTEMPTS: "2"' \ 'LBP_RETRY_BUDGET_PERCENT: "100"' \ diff --git a/src/main/resources/application-proxy-prod.properties b/src/main/resources/application-proxy-prod.properties index 2b9cf018..9479fff2 100644 --- a/src/main/resources/application-proxy-prod.properties +++ b/src/main/resources/application-proxy-prod.properties @@ -21,13 +21,13 @@ loadbalancerpro.proxy.access-log.format=${LBP_ACCESS_LOG_FORMAT:JSON} loadbalancerpro.proxy.access-log.path=${LBP_ACCESS_LOG_PATH:/tmp/loadbalancerpro/proxy-access.log} loadbalancerpro.proxy.access-log.sample-rate=${LBP_ACCESS_LOG_SAMPLE_RATE:1.0} loadbalancerpro.proxy.forwarded.mode=${LBP_FORWARDED_MODE:strip-and-set} -loadbalancerpro.proxy.health-check.enabled=true +loadbalancerpro.proxy.health-check.enabled=${LBP_HEALTH_CHECK_ENABLED:true} loadbalancerpro.proxy.health-check.path=${LBP_HEALTH_CHECK_PATH:/health} loadbalancerpro.proxy.health-check.timeout=${LBP_HEALTH_CHECK_TIMEOUT:1s} loadbalancerpro.proxy.health-check.interval=${LBP_HEALTH_CHECK_INTERVAL:5s} loadbalancerpro.proxy.health-check.healthy-threshold=${LBP_HEALTHY_THRESHOLD:2} loadbalancerpro.proxy.health-check.unhealthy-threshold=${LBP_UNHEALTHY_THRESHOLD:3} -loadbalancerpro.proxy.cooldown.enabled=true +loadbalancerpro.proxy.cooldown.enabled=${LBP_COOLDOWN_ENABLED:true} loadbalancerpro.proxy.cooldown.consecutive-failure-threshold=${LBP_COOLDOWN_FAILURE_THRESHOLD:2} loadbalancerpro.proxy.cooldown.duration=${LBP_COOLDOWN_DURATION:30s} loadbalancerpro.proxy.cooldown.recover-on-successful-health-check=true diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java index 1aab9dcc..b8c3fa69 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java @@ -36,8 +36,8 @@ void proxyProdIsExplicitAndUsesRequiredExternalUpstreams() throws Exception { "loadbalancerpro.auth.mode=api-key", "loadbalancerpro.auth.protect-actuator=true", "loadbalancerpro.proxy.enabled=true", - "loadbalancerpro.proxy.health-check.enabled=true", - "loadbalancerpro.proxy.cooldown.enabled=true", + "loadbalancerpro.proxy.health-check.enabled=${LBP_HEALTH_CHECK_ENABLED:true}", + "loadbalancerpro.proxy.cooldown.enabled=${LBP_COOLDOWN_ENABLED:true}", "loadbalancerpro.proxy.limits.max-in-flight=${LBP_MAX_IN_FLIGHT:100}", "loadbalancerpro.proxy.max-request-bytes=${LBP_MAX_REQUEST_BYTES:65536}", "loadbalancerpro.proxy.upstreams[0].url=${LBP_UPSTREAM_0_URL}", diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 991feaa0..0a514224 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -44,6 +44,8 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "secretName: loadbalancerpro-api-key-a", "path: loadbalancerpro.api.rotation-key", "https://${LBP_TLS_HOSTNAME}:8080/proxy/kubernetes/topology", + "LBP_HEALTH_CHECK_ENABLED: \"false\"", + "LBP_COOLDOWN_ENABLED: \"false\"", "LBP_RETRY_ENABLED: \"true\"", "LBP_RETRY_MAX_ATTEMPTS: \"2\"", "LBP_RETRY_BUDGET_PERCENT: \"100\"", @@ -79,6 +81,7 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { assertEquals("lbp-kubernetes-smoke", profile.path("cluster").path("namespace").asText()); assertEquals(30443, profile.path("cluster").path("nodePort").asInt()); assertEquals("close-per-request", profile.path("workload").path("connectionMode").asText()); + assertTrue(profile.path("workload").path("transitionSeconds").asInt() >= 40); assertTrue(profile.path("workload").path("rolloutSeconds").asInt() >= profile.path("objectives").path("maximumRolloutSeconds").asInt() + 5); assertTrue(profile.path("objectives").path("minimumRolloutSuccessRatio").asDouble() >= 0.95); diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java index e6d5c38d..c81e4291 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/ProxyProdProfileConfigurationTest.java @@ -22,6 +22,8 @@ "loadbalancerpro.api.key=TEST_PROXY_PROD_API_KEY", "LBP_UPSTREAM_0_URL=http://127.0.0.1:18081", "LBP_UPSTREAM_1_URL=http://127.0.0.1:18082", + "LBP_HEALTH_CHECK_ENABLED=false", + "LBP_COOLDOWN_ENABLED=false", "LBP_RETRY_ENABLED=true", "LBP_RETRY_MAX_ATTEMPTS=2", "LBP_RETRY_BUDGET_PERCENT=100", @@ -51,8 +53,8 @@ void profileEnablesTheBoundedProxyDeploymentDefaults() { assertEquals("true", environment.getProperty("loadbalancerpro.proxy.enabled")); assertEquals("true", environment.getProperty("server.http2.enabled")); assertEquals("false", environment.getProperty("loadbalancerpro.proxy.websocket.enabled")); - assertEquals("true", environment.getProperty("loadbalancerpro.proxy.health-check.enabled")); - assertEquals("true", environment.getProperty("loadbalancerpro.proxy.cooldown.enabled")); + assertEquals("false", environment.getProperty("loadbalancerpro.proxy.health-check.enabled")); + assertEquals("false", environment.getProperty("loadbalancerpro.proxy.cooldown.enabled")); assertEquals("100", environment.getProperty("loadbalancerpro.proxy.limits.max-in-flight")); assertEquals("http://127.0.0.1:18081", environment.getProperty("loadbalancerpro.proxy.upstreams[0].url")); @@ -72,6 +74,8 @@ void profileKeepsHealthPublicButProtectsProxyStatusAndActuator() throws Exceptio mockMvc.perform(get("/api/proxy/status").header("X-API-Key", API_KEY)) .andExpect(status().isOk()) .andExpect(jsonPath("$.proxyEnabled", is(true))) + .andExpect(jsonPath("$.healthCheck.enabled", is(false))) + .andExpect(jsonPath("$.cooldown.enabled", is(false))) .andExpect(jsonPath("$.retry.enabled", is(true))) .andExpect(jsonPath("$.retry.maxAttempts", is(2))) .andExpect(jsonPath("$.retry.budgetPercent", is(100))) From b0b1c05ba438cfbc77e2a73443546be97f6ff97b Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 21:28:03 -0700 Subject: [PATCH 07/11] fix: bound kubernetes ci runtime --- .github/workflows/ci.yml | 2 +- .../loadbalancerpro/api/KubernetesLiveTopologyContractTest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c7a8d99..5cdca1b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: build: name: Build, Test, Package, Smoke runs-on: ubuntu-latest - timeout-minutes: 35 + timeout-minutes: 45 steps: - name: Check out repository diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 0a514224..4a1324de 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -236,6 +236,7 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() @Test void ciPinsKindAndRunsAndUploadsTheLiveProof() throws IOException { String ci = read(CI); + assertTrue(ci.contains("timeout-minutes: 45")); assertTrue(ci.contains("eb244cbafcc157dff60cf68693c14c9a75c4e6e6fedaf9cd71c58117cb93e3fa")); assertTrue(ci.contains("ab60ca5f0fd60c1eb81b52909e67060e3ba0bd27e55a8ac147cbc2172ff14212")); assertTrue(ci.contains("bash scripts/bench/kubernetes-topology-contract-test.sh")); From 7cdad6b691d4f63771c56bea214421f7ec74f57e Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 22:07:39 -0700 Subject: [PATCH 08/11] fix: bound abrupt kubernetes degradation --- docs/LOAD_BALANCER_BUILD_OUT.md | 4 +++- scripts/bench/README.md | 4 +++- scripts/bench/kubernetes-topology-contract-test.sh | 3 ++- scripts/bench/kubernetes-topology-profile.example.json | 3 ++- scripts/bench/proxy-kubernetes-topology.sh | 7 +++++-- .../api/KubernetesLiveTopologyContractTest.java | 5 +++-- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/LOAD_BALANCER_BUILD_OUT.md b/docs/LOAD_BALANCER_BUILD_OUT.md index 041ae091..5c49ac5d 100644 --- a/docs/LOAD_BALANCER_BUILD_OUT.md +++ b/docs/LOAD_BALANCER_BUILD_OUT.md @@ -177,7 +177,9 @@ out-of-service `NoExecute` remediation. It force-removes the three exact statele endpoint withdrawal, proves degraded traffic, rejects the failed pod identity after recovery, and requires both recovered replicas and backends to serve new traffic. The disposable cluster pins iptables-mode kube-proxy to immediate EndpointSlice-triggered updates and a one-second cleanup sync; deployment environments must review the -equivalent Service/ingress failure-detection and reconciliation behavior. The candidate has a distinct local image +equivalent Service/ingress failure-detection and reconciliation behavior. Its abrupt transition and degraded phases +bound stale conntrack impact at 90% and 95% success with 5.5-second p99 ceilings; recovered traffic must return to the +normal 99.9% success and 1.5-second p99 objectives. The candidate has a distinct local image content ID but preserves the baseline application layers, so it proves Kubernetes transition and rollback mechanics rather than compatibility between application releases. The reviewed deployment ingress, deployment-equivalent resources, registry diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 67f37f37..05fa1475 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -140,7 +140,9 @@ forcibly stops that recovered worker without a drain. After confirming the worke Kubernetes' out-of-service `NoExecute` remediation and force-removes the three exact stateless workload pods from the API, bounds endpoint withdrawal, proves degraded traffic, and requires fresh pod identity, two-zone placement, and traffic distribution after recovery. The lab cluster pins iptables kube-proxy to immediate EndpointSlice-triggered updates and a -one-second cleanup sync so the Service failover objective is executable and recorded: +one-second cleanup sync so the Service failover objective is executable and recorded. The abrupt transition and degraded +windows retain bounded 90% and 95% success floors with 5.5-second p99 ceilings for stale conntrack paths; recovered +traffic returns to the normal 99.9% success and 1.5-second p99 objectives: ```bash bash scripts/bench/proxy-kubernetes-topology.sh --mode validate diff --git a/scripts/bench/kubernetes-topology-contract-test.sh b/scripts/bench/kubernetes-topology-contract-test.sh index 07980357..42fc0438 100644 --- a/scripts/bench/kubernetes-topology-contract-test.sh +++ b/scripts/bench/kubernetes-topology-contract-test.sh @@ -81,6 +81,7 @@ assert_rejected weak-abrupt-transition-objective '.objectives.minimumAbruptTrans assert_rejected weak-abrupt-degraded-objective '.objectives.minimumAbruptDegradedSuccessRatio = 0.5' assert_rejected weak-abrupt-recovered-objective '.objectives.minimumAbruptRecoveredSuccessRatio = 0.5' assert_rejected long-abrupt-transition-p99 '.objectives.maximumAbruptTransitionP99Millis = 10000' +assert_rejected long-abrupt-degraded-p99 '.objectives.maximumAbruptDegradedP99Millis = 10000' assert_rejected long-abrupt-endpoint-withdrawal '.objectives.maximumAbruptEndpointWithdrawalSeconds = 60' assert_rejected long-recovery '.objectives.maximumRecoverySeconds = 600' assert_rejected long-abrupt-recovery '.objectives.maximumAbruptRecoverySeconds = 600' @@ -93,4 +94,4 @@ assert_rejected wrong-candidate-api-key-secret '.apiKeyRotation.candidateSecret assert_rejected unchanged-overlap-api-key-secret '.apiKeyRotation.overlapSecret = .apiKeyRotation.baselineSecret' assert_rejected unchanged-candidate-api-key-secret '.apiKeyRotation.candidateSecret = .apiKeyRotation.baselineSecret' -printf 'Kubernetes topology contract rejected 64 unsafe profiles without creating a cluster.\n' +printf 'Kubernetes topology contract rejected 65 unsafe profiles without creating a cluster.\n' diff --git a/scripts/bench/kubernetes-topology-profile.example.json b/scripts/bench/kubernetes-topology-profile.example.json index 7ddf8105..98967278 100644 --- a/scripts/bench/kubernetes-topology-profile.example.json +++ b/scripts/bench/kubernetes-topology-profile.example.json @@ -52,10 +52,11 @@ "minimumDegradedSuccessRatio": 0.999, "minimumRecoveredSuccessRatio": 0.999, "minimumAbruptTransitionSuccessRatio": 0.90, - "minimumAbruptDegradedSuccessRatio": 0.999, + "minimumAbruptDegradedSuccessRatio": 0.95, "minimumAbruptRecoveredSuccessRatio": 0.999, "maximumP99Millis": 1500, "maximumAbruptTransitionP99Millis": 5500, + "maximumAbruptDegradedP99Millis": 5500, "maximumRolloutSeconds": 75, "maximumRollbackSeconds": 75, "maximumCertificateRotationSeconds": 75, diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index 15622e92..e6a27886 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -75,6 +75,7 @@ jq -e ' and (.objectives.minimumAbruptRecoveredSuccessRatio | type == "number" and . >= 0.95 and . <= 1) and (.objectives.maximumP99Millis | type == "number" and . >= 100 and . <= 5000 and floor == .) and (.objectives.maximumAbruptTransitionP99Millis | type == "number" and . >= 1000 and . <= 6000 and floor == .) + and (.objectives.maximumAbruptDegradedP99Millis | type == "number" and . >= 1000 and . <= 6000 and floor == .) and (.objectives.maximumRolloutSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.objectives.maximumRollbackSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) and (.objectives.maximumCertificateRotationSeconds | type == "number" and . >= 20 and . <= 120 and floor == .) @@ -743,9 +744,10 @@ report_attack() { run_attack() { local name="$1" seconds="$2" minimum_success="$3" local attack_targets="${4:-$targets}" + local maximum_p99_millis="${5:-$(jq -r '.objectives.maximumP99Millis' "$profile")}" vegeta attack -duration="${seconds}s" -rate="${rate}/s" -timeout=5s -keepalive=false -http2=false \ -root-certs="$tls_trust_bundle" -targets="$attack_targets" > "$work_dir/${name}.bin" - report_attack "$name" "$minimum_success" + report_attack "$name" "$minimum_success" "$maximum_p99_millis" } run_api_key_secret_transition() { @@ -1568,7 +1570,8 @@ report_attack abrupt-transition \ "$(jq -r '.objectives.maximumAbruptTransitionP99Millis' "$profile")" capture_state abrupt-degraded run_attack abrupt-degraded "$(jq -r '.workload.abruptDegradedSeconds' "$profile")" \ - "$(jq -r '.objectives.minimumAbruptDegradedSuccessRatio' "$profile")" + "$(jq -r '.objectives.minimumAbruptDegradedSuccessRatio' "$profile")" "$targets" \ + "$(jq -r '.objectives.maximumAbruptDegradedP99Millis' "$profile")" abrupt_recovery_started_epoch="$(date +%s)" docker start "$stopped_node" >/dev/null diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 4a1324de..45725103 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -116,9 +116,10 @@ void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { assertTrue(profile.path("workload").path("abruptTransitionSeconds").asInt() >= profile.path("objectives").path("maximumAbruptEndpointWithdrawalSeconds").asInt() + 5); assertTrue(profile.path("objectives").path("minimumAbruptTransitionSuccessRatio").asDouble() >= 0.90); - assertTrue(profile.path("objectives").path("minimumAbruptDegradedSuccessRatio").asDouble() >= 0.95); + assertEquals(0.95, profile.path("objectives").path("minimumAbruptDegradedSuccessRatio").asDouble()); assertTrue(profile.path("objectives").path("minimumAbruptRecoveredSuccessRatio").asDouble() >= 0.95); assertTrue(profile.path("objectives").path("maximumAbruptTransitionP99Millis").asInt() <= 6000); + assertTrue(profile.path("objectives").path("maximumAbruptDegradedP99Millis").asInt() <= 6000); } @Test @@ -228,7 +229,7 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "bothProxyReplicasServed: true")) { assertTrue(runner.contains(behavior), "missing live Kubernetes proof behavior: " + behavior); } - assertTrue(read(CONTRACT).contains("rejected 64 unsafe profiles without creating a cluster")); + assertTrue(read(CONTRACT).contains("rejected 65 unsafe profiles without creating a cluster")); assertFalse(runner.contains("--insecure")); assertFalse(runner.contains("--validate=false")); } From 549c0823cfb6ee7c7b32ca44c717ef0acdcc06d2 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 22:41:14 -0700 Subject: [PATCH 09/11] fix: drain kubernetes endpoints before shutdown --- deploy/kubernetes-proxy-prod.yaml | 4 ++-- deploy/kubernetes/qualification.yaml | 4 ++-- docs/DEPLOYMENT.md | 8 +++++--- docs/LOAD_BALANCER_BUILD_OUT.md | 4 +++- scripts/bench/README.md | 4 +++- scripts/bench/kubernetes-staging-kubectl-fixture.py | 4 ++-- scripts/bench/kubernetes-topology-profile.example.json | 4 ++-- scripts/bench/proxy-kubernetes-topology.sh | 2 ++ .../api/DeploymentPackagingContractTest.java | 4 ++-- .../api/KubernetesLiveTopologyContractTest.java | 2 ++ 10 files changed, 25 insertions(+), 15 deletions(-) diff --git a/deploy/kubernetes-proxy-prod.yaml b/deploy/kubernetes-proxy-prod.yaml index 383269cc..d632585d 100644 --- a/deploy/kubernetes-proxy-prod.yaml +++ b/deploy/kubernetes-proxy-prod.yaml @@ -43,7 +43,7 @@ spec: serviceAccountName: loadbalancerpro automountServiceAccountToken: false enableServiceLinks: false - terminationGracePeriodSeconds: 40 + terminationGracePeriodSeconds: 45 topologySpreadConstraints: - maxSkew: 1 minDomains: 2 @@ -132,7 +132,7 @@ spec: lifecycle: preStop: exec: - command: ["sh", "-c", "sleep 5"] + command: ["sh", "-c", "sleep 10"] securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index ed906b3a..d223d449 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -227,7 +227,7 @@ spec: serviceAccountName: loadbalancerpro automountServiceAccountToken: false enableServiceLinks: false - terminationGracePeriodSeconds: 40 + terminationGracePeriodSeconds: 45 nodeSelector: loadbalancerpro.io/qualification-worker: "true" topologySpreadConstraints: @@ -307,7 +307,7 @@ spec: lifecycle: preStop: exec: - command: ["sh", "-c", "sleep 5"] + command: ["sh", "-c", "sleep 10"] securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index b96d22bc..dec35d72 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -120,9 +120,11 @@ Hostname verification remains mandatory; `tls.verify=false` is rejected. Server [`../deploy/kubernetes-proxy-prod.yaml`](../deploy/kubernetes-proxy-prod.yaml) is the canonical deployment base. It encodes two replicas, zero-unavailable rolling replacement, a two-domain zone-spread rule that permits a temporary -surge pod, preferred host spreading, a one-replica disruption budget, startup/readiness/liveness probes, a five-second -preStop delay, a 40-second termination window, a token-free service account, numeric non-root execution, and external -Secret/ConfigMap mounts. Its image remains a deliberately non-resolving digest placeholder. The disposable +surge pod, preferred host spreading, a one-replica disruption budget, startup/readiness/liveness probes, a ten-second +preStop delay, a 45-second termination window, a token-free service account, numeric non-root execution, and external +Secret/ConfigMap mounts. The drain delay exceeds the five-second qualification client timeout, while the termination +window contains the application's 30-second graceful-shutdown bound. Its image remains a deliberately non-resolving +digest placeholder. The disposable [`../scripts/bench/proxy-kubernetes-topology.sh`](../scripts/bench/proxy-kubernetes-topology.sh) lane applies the separate loopback qualification workload and proves a metadata-only content-distinct candidate rollout and baseline rollback under continuous traffic, complete pod-UID turnover in both directions, runtime-image identity transition and diff --git a/docs/LOAD_BALANCER_BUILD_OUT.md b/docs/LOAD_BALANCER_BUILD_OUT.md index 5c49ac5d..650340c0 100644 --- a/docs/LOAD_BALANCER_BUILD_OUT.md +++ b/docs/LOAD_BALANCER_BUILD_OUT.md @@ -179,7 +179,9 @@ recovered replicas and backends to serve new traffic. The disposable cluster pin to immediate EndpointSlice-triggered updates and a one-second cleanup sync; deployment environments must review the equivalent Service/ingress failure-detection and reconciliation behavior. Its abrupt transition and degraded phases bound stale conntrack impact at 90% and 95% success with 5.5-second p99 ceilings; recovered traffic must return to the -normal 99.9% success and 1.5-second p99 objectives. The candidate has a distinct local image +normal 99.9% success and 1.5-second p99 objectives. The ten-second endpoint drain exceeds the five-second qualification +client timeout, and the 45-second termination grace contains the 30-second application shutdown bound. The candidate +has a distinct local image content ID but preserves the baseline application layers, so it proves Kubernetes transition and rollback mechanics rather than compatibility between application releases. The reviewed deployment ingress, deployment-equivalent resources, registry diff --git a/scripts/bench/README.md b/scripts/bench/README.md index 05fa1475..b93ec63e 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -142,7 +142,9 @@ API, bounds endpoint withdrawal, proves degraded traffic, and requires fresh pod distribution after recovery. The lab cluster pins iptables kube-proxy to immediate EndpointSlice-triggered updates and a one-second cleanup sync so the Service failover objective is executable and recorded. The abrupt transition and degraded windows retain bounded 90% and 95% success floors with 5.5-second p99 ceilings for stale conntrack paths; recovered -traffic returns to the normal 99.9% success and 1.5-second p99 objectives: +traffic returns to the normal 99.9% success and 1.5-second p99 objectives. Rollouts drain endpoints for ten seconds, +longer than the five-second qualification client timeout, while the 45-second grace period contains the application's +30-second graceful-shutdown bound: ```bash bash scripts/bench/proxy-kubernetes-topology.sh --mode validate diff --git a/scripts/bench/kubernetes-staging-kubectl-fixture.py b/scripts/bench/kubernetes-staging-kubectl-fixture.py index cab3c784..bcb92800 100755 --- a/scripts/bench/kubernetes-staging-kubectl-fixture.py +++ b/scripts/bench/kubernetes-staging-kubectl-fixture.py @@ -63,7 +63,7 @@ def deployment(state: dict[str, Any], name: str) -> dict[str, Any]: }, }, "spec": { - "terminationGracePeriodSeconds": 40, + "terminationGracePeriodSeconds": 45, "containers": [{ "name": CONTAINER, "image": state["imageReference"], @@ -72,7 +72,7 @@ def deployment(state: dict[str, Any], name: str) -> dict[str, Any]: "requests": {"cpu": "100m", "memory": "256Mi"}, "limits": {"cpu": "1", "memory": "512Mi"}, }, - "lifecycle": {"preStop": {"exec": {"command": ["sh", "-c", "sleep 5"]}}}, + "lifecycle": {"preStop": {"exec": {"command": ["sh", "-c", "sleep 10"]}}}, "volumeMounts": [{"name": "server-tls", "mountPath": "/run/tls", "readOnly": True}], }], "volumes": [{"name": "server-tls", "secret": {"secretName": state["tlsSecret"]}}], diff --git a/scripts/bench/kubernetes-topology-profile.example.json b/scripts/bench/kubernetes-topology-profile.example.json index 98967278..f9f78b59 100644 --- a/scripts/bench/kubernetes-topology-profile.example.json +++ b/scripts/bench/kubernetes-topology-profile.example.json @@ -27,7 +27,7 @@ "postCertificateRotationSeconds": 8, "certificateRollbackSeconds": 90, "postCertificateRollbackSeconds": 8, - "apiKeyTransitionSeconds": 65, + "apiKeyTransitionSeconds": 80, "postApiKeyTransitionSeconds": 6, "transitionSeconds": 45, "degradedSeconds": 8, @@ -61,7 +61,7 @@ "maximumRollbackSeconds": 75, "maximumCertificateRotationSeconds": 75, "maximumCertificateRollbackSeconds": 75, - "maximumApiKeyTransitionSeconds": 60, + "maximumApiKeyTransitionSeconds": 75, "maximumRecoverySeconds": 180, "maximumAbruptEndpointWithdrawalSeconds": 15, "maximumAbruptRecoverySeconds": 180 diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index e6a27886..0770b13b 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -117,6 +117,8 @@ for invariant in \ 'automountServiceAccountToken: false' \ 'maxUnavailable: 0' \ 'maxSurge: 1' \ + 'terminationGracePeriodSeconds: 45' \ + 'command: ["sh", "-c", "sleep 10"]' \ 'minDomains: 2' \ 'runAsUser: 10001' \ 'readOnlyRootFilesystem: true' \ diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java index b8c3fa69..ee194e27 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/DeploymentPackagingContractTest.java @@ -120,8 +120,8 @@ void kubernetesBaseHasTheCanonicalLifecycleAndSecurityBoundary() throws Exceptio "readOnlyRootFilesystem: true", "allowPrivilegeEscalation: false", "drop: [\"ALL\"]", - "terminationGracePeriodSeconds: 40", - "sleep 5", + "terminationGracePeriodSeconds: 45", + "sleep 10", "startupProbe:", "https://${LBP_TLS_HOSTNAME}:8080/actuator/health", "X-API-Key: $(cat /run/secrets/loadbalancerpro.api.key)", diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 45725103..de413103 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -32,6 +32,8 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "automountServiceAccountToken: false", "maxUnavailable: 0", "maxSurge: 1", + "terminationGracePeriodSeconds: 45", + "command: [\"sh\", \"-c\", \"sleep 10\"]", "minDomains: 2", "runAsNonRoot: true", "runAsUser: 10001", From 007e0798c39d57d4fe40df70e8c3701b6474eb11 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Wed, 19 Aug 2026 23:33:07 -0700 Subject: [PATCH 10/11] fix: cycle bounded proxy retries --- deploy/kubernetes/qualification.yaml | 2 +- docs/DEPLOYMENT.md | 3 +- docs/REVERSE_PROXY_RESILIENCE.md | 4 ++ scripts/bench/README.md | 4 +- scripts/bench/proxy-kubernetes-topology.sh | 2 +- .../api/proxy/ReverseProxyService.java | 4 ++ .../KubernetesLiveTopologyContractTest.java | 2 +- .../ReverseProxyRetryBudgetSlowStartTest.java | 67 +++++++++++++++++-- 8 files changed, 77 insertions(+), 11 deletions(-) diff --git a/deploy/kubernetes/qualification.yaml b/deploy/kubernetes/qualification.yaml index d223d449..dc354c6d 100644 --- a/deploy/kubernetes/qualification.yaml +++ b/deploy/kubernetes/qualification.yaml @@ -32,7 +32,7 @@ data: LBP_UNHEALTHY_THRESHOLD: "2" LBP_COOLDOWN_ENABLED: "false" LBP_RETRY_ENABLED: "true" - LBP_RETRY_MAX_ATTEMPTS: "2" + LBP_RETRY_MAX_ATTEMPTS: "3" LBP_RETRY_BUDGET_PERCENT: "100" LBP_RETRY_BACKOFF_BASE: 10ms LBP_RETRY_BACKOFF_MAX: 50ms diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index dec35d72..8582c632 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -139,7 +139,8 @@ commit, while zero-unavailable endpoint continuity, fresh pod UIDs, fixed runtim both replicas and backends remain required. This is startup configuration rollout proof, not dynamic Secret reload or external secret-manager proof. Because each configured upstream is a Kubernetes Service rather than a pod, the local qualification lane uses EndpointSlice readiness as the pod-health authority and disables process-local active health -checks and cooldown; bounded idempotent retry across the two Services remains enabled. The lane then proves two-zone +checks and cooldown; up to three bounded attempts remain enabled for `GET`/`HEAD`, trying both Services before cycling +after stale pooled connections have been discarded. The lane then proves two-zone Service distribution, planned worker removal, and operator-remediated no-drain worker loss and recovery. The abrupt-loss exercise forcibly stops the kind worker, confirms its container is down, applies the out-of-service `NoExecute` taint, and force-removes the three exact stateless diff --git a/docs/REVERSE_PROXY_RESILIENCE.md b/docs/REVERSE_PROXY_RESILIENCE.md index 7fe8e848..ae8d8ed9 100644 --- a/docs/REVERSE_PROXY_RESILIENCE.md +++ b/docs/REVERSE_PROXY_RESILIENCE.md @@ -19,6 +19,10 @@ loadbalancerpro.proxy.retry.retry-statuses=502,503,504 When enabled, the first attempt is the original forward and the maximum attempt count remains a hard ceiling. Each admitted primary request adds `budget-percent` process-local credits, at most 100 credits are stored, and an additional attempt requires 100 credits. At the default `20`, sustained failures can therefore trigger at most one retry per five primary requests without banking a large healthy-period burst. `0` suppresses retries; `100` allows at most one retry per primary request. A granted retry waits for full jitter between zero and an exponential ceiling: `backoff.base` for the first retry, doubling for later retries, capped at `backoff.max`. The base and maximum may be `0ms` for a no-delay local test and may not exceed 60 seconds. By default, only `GET` and `HEAD` are retried. Enabling retries for `POST`, `PUT`, `PATCH`, or `DELETE` can duplicate upstream side effects and should only be used with upstream-specific idempotency controls. +Within one request, retries exhaust the currently eligible upstreams before cycling through them again. Cycling still +requires another retry-budget grant and never exceeds `max-attempts`; it lets a later attempt use an upstream again after +a failed pooled connection has been discarded. + ## Cooldown Configuration Cooldown is disabled by default. diff --git a/scripts/bench/README.md b/scripts/bench/README.md index b93ec63e..861b1754 100644 --- a/scripts/bench/README.md +++ b/scripts/bench/README.md @@ -134,7 +134,9 @@ API-key Secrets. Four zero-unavailable rollouts prove baseline-key traffic throu commit, the reverse rollback overlap, and final restoration of A-only. Positive and negative authentication checks prove both overlap windows and both key-retirement boundaries; every phase also requires fresh pod UIDs, two ready endpoints, an unchanged runtime image, and traffic through both replicas and backends. The runtime accepts only the required primary -plus one optional rotation key and does not dynamically reload Secret files. It then drains and stops one worker under load, +plus one optional rotation key and does not dynamically reload Secret files. Qualification permits up to three bounded +`GET`/`HEAD` attempts, exhausting both Service upstreams before cycling after a stale pooled connection. It then drains +and stops one worker under load, tests the one-replica degraded service, requires both recovered replicas and backends to serve new traffic, then forcibly stops that recovered worker without a drain. After confirming the worker container is down, it applies Kubernetes' out-of-service `NoExecute` remediation and force-removes the three exact stateless workload pods from the diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index 0770b13b..b55b61e9 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -131,7 +131,7 @@ for invariant in \ 'LBP_HEALTH_CHECK_ENABLED: "false"' \ 'LBP_COOLDOWN_ENABLED: "false"' \ 'LBP_RETRY_ENABLED: "true"' \ - 'LBP_RETRY_MAX_ATTEMPTS: "2"' \ + 'LBP_RETRY_MAX_ATTEMPTS: "3"' \ 'LBP_RETRY_BUDGET_PERCENT: "100"' \ 'LBP_RETRY_NON_IDEMPOTENT: "false"' \ 'path: loadbalancerpro.api.rotation-key'; do diff --git a/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyService.java b/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyService.java index 55ec8654..6999a1d0 100644 --- a/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyService.java +++ b/src/main/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyService.java @@ -617,6 +617,10 @@ private ReverseProxyResponse forwardAdmitted( List upstreams = configuredUpstreams( config, route, splitTargetIds, excludedIds); if (upstreams.isEmpty()) { + if (pendingAttempt != null && !attemptedUpstreamIds.isEmpty()) { + attemptedUpstreamIds.clear(); + continue; + } if (pendingAttempt != null) { ForwardAttemptResult finalAttempt = pendingAttempt; pendingAttempt = null; diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index de413103..9a6aab10 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -49,7 +49,7 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep "LBP_HEALTH_CHECK_ENABLED: \"false\"", "LBP_COOLDOWN_ENABLED: \"false\"", "LBP_RETRY_ENABLED: \"true\"", - "LBP_RETRY_MAX_ATTEMPTS: \"2\"", + "LBP_RETRY_MAX_ATTEMPTS: \"3\"", "LBP_RETRY_BUDGET_PERCENT: \"100\"", "LBP_RETRY_NON_IDEMPOTENT: \"false\"", "secretName: loadbalancerpro-server-tls-a")) { diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyRetryBudgetSlowStartTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyRetryBudgetSlowStartTest.java index 3d88ee04..40caed15 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyRetryBudgetSlowStartTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/proxy/ReverseProxyRetryBudgetSlowStartTest.java @@ -20,9 +20,14 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZoneOffset; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import com.richmond423.loadbalancerpro.core.RoutingStrategyRegistry; @@ -65,6 +70,48 @@ void brownoutRetriesAreCappedAtTwentyPercentOfPrimaryRequests() throws Exception .count()); } + @Test + void boundedRetriesCycleOnlyAfterEveryEligibleUpstreamWasAttempted() throws Exception { + AtomicInteger brownoutCalls = new AtomicInteger(); + List attemptedPorts = new ArrayList<>(); + CountDownLatch secondAttemptStarted = new CountDownLatch(1); + CountDownLatch releaseSecondAttempt = new CountDownLatch(1); + HttpClient client = mock(HttpClient.class); + when(client.send(any(HttpRequest.class), + org.mockito.ArgumentMatchers.>any())) + .thenAnswer(invocation -> { + HttpRequest outbound = invocation.getArgument(0); + if (outbound.uri().getPath().endsWith("/funding")) { + return response(200); + } + int call = brownoutCalls.incrementAndGet(); + attemptedPorts.add(outbound.uri().getPort()); + if (call == 2) { + secondAttemptStarted.countDown(); + assertTrue(releaseSecondAttempt.await(5, TimeUnit.SECONDS)); + } + return response(call == 3 ? 200 : 503); + }); + ReverseProxyProperties properties = properties("ROUND_ROBIN", List.of(alpha(), beta())); + configureRetry(properties, 100); + properties.getRetry().setMaxAttempts(3); + ReverseProxyService service = service(properties, client, new MutableClock(START)); + + CompletableFuture brownout = CompletableFuture.supplyAsync( + () -> service.forward(request("brownout"), new byte[0])); + assertTrue(secondAttemptStarted.await(5, TimeUnit.SECONDS)); + assertEquals(200, service.forward(request("funding"), new byte[0]).statusCode()); + releaseSecondAttempt.countDown(); + + assertEquals(200, brownout.get(5, TimeUnit.SECONDS).statusCode()); + assertEquals(3, brownoutCalls.get()); + assertEquals(2, Set.copyOf(attemptedPorts.subList(0, 2)).size()); + assertTrue(attemptedPorts.subList(0, 2).contains(attemptedPorts.get(2))); + assertEquals(2, service.statusSnapshot().metrics().totalRetryAttempts()); + assertEquals(2, service.statusSnapshot().retry().budgetGrantedRetries()); + assertEquals(0, service.statusSnapshot().retry().budgetRejectedRetries()); + } + @Test void newlyAddedUpstreamRampsLinearlyBeforeReceivingItsFullShare() throws Exception { AtomicInteger upstreamCalls = new AtomicInteger(); @@ -217,15 +264,19 @@ private static HttpClient clientReturning(int statusCode, AtomicInteger calls) t org.mockito.ArgumentMatchers.>any())) .thenAnswer(invocation -> { calls.incrementAndGet(); - HttpResponse response = mock(HttpResponse.class); - when(response.statusCode()).thenReturn(statusCode); - when(response.headers()).thenReturn(HttpHeaders.of(Map.of(), (name, value) -> true)); - when(response.body()).thenAnswer(ignored -> new ByteArrayInputStream(new byte[0])); - return response; + return response(statusCode); }); return client; } + private static HttpResponse response(int statusCode) { + HttpResponse response = mock(HttpResponse.class); + when(response.statusCode()).thenReturn(statusCode); + when(response.headers()).thenReturn(HttpHeaders.of(Map.of(), (name, value) -> true)); + when(response.body()).thenAnswer(ignored -> new ByteArrayInputStream(new byte[0])); + return response; + } + private static ReverseProxyProperties properties( String strategy, List upstreams) { ReverseProxyProperties properties = new ReverseProxyProperties(); @@ -275,9 +326,13 @@ private static ReverseProxyProperties.Upstream upstream(String id, String url) { } private static HttpServletRequest request() { + return request("brownout"); + } + + private static HttpServletRequest request(String path) { HttpServletRequest request = mock(HttpServletRequest.class); when(request.getContextPath()).thenReturn(""); - when(request.getRequestURI()).thenReturn("/proxy/api/brownout"); + when(request.getRequestURI()).thenReturn("/proxy/api/" + path); when(request.getMethod()).thenReturn("GET"); when(request.getHeaderNames()).thenReturn(Collections.emptyEnumeration()); when(request.getRemoteAddr()).thenReturn("198.51.100.10"); From d3293d2bd0b7fbfbd62989ecfeb2bcec3fce28e8 Mon Sep 17 00:00:00 2001 From: RicheyWorks <730richey730@gmail.com> Date: Thu, 20 Aug 2026 10:41:24 -0700 Subject: [PATCH 11/11] fix: bound kubernetes fixture memory --- deploy/fixture/Dockerfile | 2 +- deploy/fixture/FixtureBackend.java | 16 ++++++++++--- scripts/bench/proxy-kubernetes-topology.sh | 24 +++++++++++++++++++ .../KubernetesLiveTopologyContractTest.java | 22 +++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) diff --git a/deploy/fixture/Dockerfile b/deploy/fixture/Dockerfile index 18e248a4..ff23794e 100644 --- a/deploy/fixture/Dockerfile +++ b/deploy/fixture/Dockerfile @@ -11,4 +11,4 @@ RUN groupadd --gid 10001 fixture \ COPY --from=build --chown=fixture:fixture /output/FixtureBackend.class /app/FixtureBackend.class USER fixture:fixture EXPOSE 8080 -ENTRYPOINT ["java", "--add-modules", "jdk.httpserver", "-cp", "/app", "FixtureBackend"] +ENTRYPOINT ["java", "-Xmx32m", "-Xss256k", "--add-modules", "jdk.httpserver", "-cp", "/app", "FixtureBackend"] diff --git a/deploy/fixture/FixtureBackend.java b/deploy/fixture/FixtureBackend.java index d91c89f9..57a8dada 100644 --- a/deploy/fixture/FixtureBackend.java +++ b/deploy/fixture/FixtureBackend.java @@ -6,15 +6,18 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; public final class FixtureBackend { private static final int PORT = 8080; private static final int MAX_REQUEST_BYTES = 1_048_576; private static final int MAX_RESPONSE_BYTES = 1_048_576; private static final long MAX_DELAY_MILLIS = 10_000; + private static final int MAX_REQUEST_THREADS = 32; + private static final int MAX_PENDING_REQUESTS = 256; private FixtureBackend() { } @@ -22,7 +25,14 @@ private FixtureBackend() { public static void main(String[] args) throws Exception { String backendId = requiredEnvironment("FIXTURE_ID"); HttpServer server = HttpServer.create(new InetSocketAddress("0.0.0.0", PORT), 0); - ExecutorService executor = Executors.newCachedThreadPool(); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + MAX_REQUEST_THREADS, + MAX_REQUEST_THREADS, + 30, + TimeUnit.SECONDS, + new ArrayBlockingQueue<>(MAX_PENDING_REQUESTS), + new ThreadPoolExecutor.CallerRunsPolicy()); + executor.allowCoreThreadTimeOut(true); CountDownLatch stopped = new CountDownLatch(1); server.createContext("/", exchange -> handle(exchange, backendId)); server.setExecutor(executor); diff --git a/scripts/bench/proxy-kubernetes-topology.sh b/scripts/bench/proxy-kubernetes-topology.sh index b55b61e9..e19c2016 100644 --- a/scripts/bench/proxy-kubernetes-topology.sh +++ b/scripts/bench/proxy-kubernetes-topology.sh @@ -642,6 +642,26 @@ initial_proxy_runtime_image_ids_json="$(jq '[.[].status.containerStatuses[]? <<< "$initial_ready_proxy_pods_json")" == true ]] || { echo "Initial proxy pods do not reference the immutable baseline TLS Secret" >&2; exit 1; } + +assert_no_container_restarts() { + local phase="$1" restart_evidence + restart_evidence="$(kubectl get pod --namespace "$namespace" -o json | jq '[.items[] + | select(.metadata.deletionTimestamp == null) + | .metadata.name as $pod + | .status.containerStatuses[]? + | select(.restartCount > 0) + | {pod: $pod, container: .name, restartCount: .restartCount, + lastTerminationReason: (.lastState.terminated.reason // "unknown"), + lastExitCode: (.lastState.terminated.exitCode // null)}]')" + if [[ "$(jq 'length' <<< "$restart_evidence")" != 0 ]]; then + jq -n --arg phase "$phase" --argjson containers "$restart_evidence" \ + '{phase: $phase, containers: $containers}' \ + > "$output_dir/${phase}-container-restarts.json" + echo "Kubernetes workload containers restarted before completing $phase" >&2 + jq -c '.[]' <<< "$restart_evidence" >&2 + return 1 + fi +} [[ "$(jq --arg secret "$baseline_api_key_secret" '[.[] | any(.spec.volumes[]?; .name == "api-key" and .secret.secretName == $secret)] | all' \ <<< "$initial_ready_proxy_pods_json")" == true ]] || { @@ -1438,6 +1458,8 @@ jq -n \ traffic: $rollbackCommitDistribution[0]}}' \ > "$output_dir/api-key-rotation.json" +assert_no_container_restarts pre-planned-drain + failed_node="$(jq -r '.[0].spec.nodeName' <<< "$api_key_rollback_commit_pods_json")" [[ "$failed_node" == "${cluster_name}-worker" || "$failed_node" == "${cluster_name}-worker2" ]] || { echo "Refusing to drain unexpected node $failed_node" >&2; exit 1; @@ -1466,6 +1488,7 @@ report_attack transition "$(jq -r '.objectives.minimumTransitionSuccessRatio' "$ wait_for_count 'ready proxy replicas while one worker is stopped' 1 ready_proxy_count 90 wait_for_count 'ready Service endpoints while one worker is stopped' 1 ready_endpoint_count 90 capture_state degraded +assert_no_container_restarts degraded run_attack degraded "$(jq -r '.workload.degradedSeconds' "$profile")" \ "$(jq -r '.objectives.minimumDegradedSuccessRatio' "$profile")" @@ -1483,6 +1506,7 @@ wait_for_count 'recovered Service endpoints' 2 ready_endpoint_count "$maximum_re recovery_seconds=$(( $(date +%s) - recovery_started_epoch )) (( recovery_seconds <= maximum_recovery_seconds )) || { echo "Worker recovery exceeded the objective" >&2; exit 1; } capture_state recovered +assert_no_container_restarts recovered collect_distribution recovered-before false run_attack recovered "$(jq -r '.workload.recoveredSeconds' "$profile")" \ "$(jq -r '.objectives.minimumRecoveredSuccessRatio' "$profile")" diff --git a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java index 9a6aab10..68500323 100644 --- a/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java +++ b/src/test/java/com/richmond423/loadbalancerpro/api/KubernetesLiveTopologyContractTest.java @@ -15,6 +15,8 @@ class KubernetesLiveTopologyContractTest { private static final Path DOCKERFILE = Path.of("Dockerfile"); + private static final Path FIXTURE_DOCKERFILE = Path.of("deploy/fixture/Dockerfile"); + private static final Path FIXTURE_SOURCE = Path.of("deploy/fixture/FixtureBackend.java"); private static final Path CLUSTER = Path.of("deploy/kubernetes/kind-cluster.yaml"); private static final Path WORKLOAD = Path.of("deploy/kubernetes/qualification.yaml"); private static final Path CANDIDATE = Path.of("deploy/topology/RolloutCandidate.Dockerfile"); @@ -59,6 +61,22 @@ void imageAndWorkloadUseAKubernetesVerifiableRestrictedIdentity() throws IOExcep assertFalse(workload.contains("stringData:")); } + @Test + void qualificationFixtureBoundsNativeThreadAndHeapMemory() throws IOException { + String fixtureSource = read(FIXTURE_SOURCE); + assertFalse(fixtureSource.contains("newCachedThreadPool")); + assertTrue(fixtureSource.contains("MAX_REQUEST_THREADS = 32")); + assertTrue(fixtureSource.contains("MAX_PENDING_REQUESTS = 256")); + assertTrue(fixtureSource.contains("new ArrayBlockingQueue<>(MAX_PENDING_REQUESTS)")); + assertTrue(fixtureSource.contains("new ThreadPoolExecutor.CallerRunsPolicy()")); + assertTrue(fixtureSource.contains("executor.allowCoreThreadTimeOut(true)")); + + String fixtureDockerfile = read(FIXTURE_DOCKERFILE); + assertTrue(fixtureDockerfile.contains("\"-Xmx32m\"")); + assertTrue(fixtureDockerfile.contains("\"-Xss256k\"")); + assertEquals(2, count(read(WORKLOAD), "memory: 128Mi")); + } + @Test void clusterAndProfilePinTheDisposableTwoZoneBoundary() throws IOException { String cluster = read(CLUSTER); @@ -198,6 +216,10 @@ void runnerExecutesLiveImageTransitionRollbackPlannedAndAbruptWorkerLossChecks() "required primary plus at most one operator-bounded rotation key", "Kubernetes API-key value leaked into evidence", "not dynamic Secret reload", + "assert_no_container_restarts pre-planned-drain", + "assert_no_container_restarts degraded", + "assert_no_container_restarts recovered", + "lastTerminationReason", "priorPodUids: $priorPodUids", "candidatePodUids: $candidatePodUids", "restoredPodUids: $restoredPodUids",