Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions ci/kind/backing-services.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ spec:
metadata:
labels: {app.kubernetes.io/name: mysql}
spec:
# Pod Security Standard `restricted` (setup.sh labels the kind namespace enforce=restricted, as on GKE).
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 999
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: mysql
image: mysql:5.7
Expand All @@ -30,11 +38,18 @@ spec:
exec:
command: ["sh", "-c", "mysql -uroot -p\"$MYSQL_ROOT_PASSWORD\" -e 'select 1' enginedb"]
periodSeconds: 5
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
volumeMounts:
- {name: initdb, mountPath: /docker-entrypoint-initdb.d, readOnly: true}
- {name: data, mountPath: /var/lib/mysql}
volumes:
- name: initdb
configMap: {name: mysql-initdb}
- name: data
emptyDir: {}
---
apiVersion: v1
kind: Service
Expand All @@ -59,6 +74,14 @@ spec:
metadata:
labels: {app.kubernetes.io/name: rabbitmq}
spec:
# Pod Security Standard `restricted` (setup.sh labels the kind namespace enforce=restricted, as on GKE).
securityContext:
runAsNonRoot: true
runAsUser: 999
runAsGroup: 999
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: rabbitmq
image: rabbitmq:3.7-management
Expand All @@ -75,6 +98,15 @@ spec:
command: ["rabbitmq-diagnostics", "-q", "ping"]
periodSeconds: 10
timeoutSeconds: 10
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
volumeMounts:
- {name: data, mountPath: /var/lib/rabbitmq}
volumes:
- name: data
emptyDir: {}
---
apiVersion: v1
kind: Service
Expand Down
34 changes: 33 additions & 1 deletion ci/kind/setup.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Sourced by verify-axon-routing.sh, verify-helm-smoke.sh and verify-deploy-rollback.sh (expects CLUSTER, NS, k and fail from the caller).
# Sourced by verify-axon-routing.sh, verify-helm-smoke.sh, verify-deploy-rollback.sh and verify-security-baseline.sh (expects CLUSTER, NS, k and fail from the caller).
# kind_setup: kind cluster, local images, throwaway Secrets, the MySQL/RabbitMQ stand-ins (ci/kind/kustomization.yaml),
# then `helm upgrade --install` of deploy/helm/axon-trader with ci/kind/values-kind.yaml. With ESARCH_KIND_GATEWAY=1
# it also installs Envoy Gateway and renders the chart's Gateway + HTTPRoute (ci/kind/values-kind-gateway.yaml).
Expand All @@ -24,6 +24,9 @@ kind_cluster() {
fi
done
kubectl --context "kind-$CLUSTER" create namespace "$NS" --dry-run=client -o yaml | kubectl --context "kind-$CLUSTER" apply -f - >/dev/null
# Same Pod Security Standard as infra/k8s/platform/namespace.yaml: anything not `restricted` is rejected at admission.
kubectl --context "kind-$CLUSTER" label namespace "$NS" --overwrite pod-security.kubernetes.io/enforce=restricted \
pod-security.kubernetes.io/enforce-version=v1.33 pod-security.kubernetes.io/warn=restricted >/dev/null
local mysql_password rabbitmq_password app
mysql_password=$(openssl rand -hex 16)
rabbitmq_password=$(openssl rand -hex 16)
Expand Down Expand Up @@ -75,6 +78,35 @@ YAML

kind_setup() { kind_cluster; kind_install; }

# kind_probe <name> <app label|-> <host> <port>: curl from a throwaway pod (restricted securityContext) carrying that
# app label. Prints one of:
# blocked curl timed out (exit 28) without ever connecting (time_connect 0): a NetworkPolicy dropped the SYN;
# open(...) TCP connected: HTTP answer (0), a non-HTTP greeting from MySQL/AMQP (1, "HTTP/0.9 not allowed"),
# empty reply / recv error (52/56), or a response timeout after the connect (28, time_connect > 0);
# error(...) anything else, e.g. connection refused (7) or DNS failure (6): neither proves a flow open nor blocked.
kind_probe() {
local labels="esarch-probe=1"
[ "$2" != - ] && labels="$labels,app.kubernetes.io/name=$2"
k run "np-$1" --image=curlimages/curl:8.16.0 --image-pull-policy=IfNotPresent --restart=Never --labels="$labels" \
--dry-run=client -o json \
--command -- sh -c "t=\$(curl -s -o /dev/null -w '%{time_connect}' --connect-timeout 5 -m 8 http://$3:$4/); echo rc=\$? connect=\$t" |
jq '.spec.automountServiceAccountToken = false
| .spec.securityContext = {runAsNonRoot: true, runAsUser: 100, runAsGroup: 101, seccompProfile: {type: "RuntimeDefault"}}
| .spec.containers[0].securityContext = {allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, capabilities: {drop: ["ALL"]}}' |
k apply -f - >/dev/null
k wait --for=jsonpath='{.status.phase}'=Succeeded "pod/np-$1" --timeout=120s >/dev/null || true
local out rc connect
out=$(k logs "np-$1" | grep '^rc=' || true)
rc=$(sed -n 's/^rc=\([0-9]*\).*/\1/p' <<<"$out"); connect=$(sed -n 's/.*connect=//p' <<<"$out")
k delete pod "np-$1" --wait=false >/dev/null
case "$rc,${connect:-0}" in
28,0.000000) echo blocked ;;
0,* | 1,* | 52,* | 56,*) echo "open(rc=$rc connect=${connect}s)" ;;
28,*) echo "open(rc=28 connect=${connect}s)" ;;
*) echo "error(rc=${rc:-none} connect=${connect:-none}s)" ;;
esac
}

# Eureka helpers (need EUREKA_URL, jq and fail()).
registry() {
curl -sf -H 'Accept: application/json' "$EUREKA_URL/eureka/apps" | jq -r '
Expand Down
2 changes: 2 additions & 0 deletions ci/kind/values-kind-gateway.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ route:
parentRefs:
- name: axon-trader
sectionName: http
# kind has no certificate: the stand-in Gateway is HTTP-only, so no HSTS (GKE serves the app on HTTPS only).
hsts: ""
gateway:
create: true
className: envoy-gateway
Expand Down
19 changes: 2 additions & 17 deletions ci/kind/verify-helm-smoke.sh
Original file line number Diff line number Diff line change
Expand Up @@ -65,27 +65,12 @@ appURL=http://localhost:$APP_PORT engineURL=http://localhost:$ENGINE_PORT proxyU
bash ci/local/verify-local.sh

echo "== 3. NetworkPolicies"
# probe <name> <app label|-> <host> <port>: curl from a throwaway pod carrying that label. Blocked means curl timed
# out (exit 28) without ever connecting (time_connect 0), i.e. the policy dropped the SYN; anything else (HTTP status,
# a protocol error from MySQL/AMQP, or a response timeout after the connect) means the TCP connection was accepted.
# Hosts are ClusterIPs so the result does not depend on DNS egress.
# Probe pods borrow the apps' labels (kind_probe in setup.sh). Hosts are ClusterIPs so the result does not depend on DNS egress.
ip() { k get svc "$1" -o jsonpath='{.spec.clusterIP}'; }
probe() {
local labels="esarch-probe=1"
[ "$2" != - ] && labels="$labels,app.kubernetes.io/name=$2"
k run "np-$1" --image=curlimages/curl:8.16.0 --image-pull-policy=IfNotPresent --restart=Never --labels="$labels" \
--command -- sh -c "t=\$(curl -s -o /dev/null -w '%{time_connect}' --connect-timeout 5 -m 8 http://$3:$4/); echo rc=\$? connect=\$t" >/dev/null
k wait --for=jsonpath='{.status.phase}'=Succeeded "pod/np-$1" --timeout=120s >/dev/null || true
local out rc connect
out=$(k logs "np-$1" | grep '^rc=' || true)
rc=$(sed -n 's/^rc=\([0-9]*\).*/\1/p' <<<"$out"); connect=$(sed -n 's/.*connect=//p' <<<"$out")
k delete pod "np-$1" --wait=false >/dev/null
if [ "$rc" = 28 ] && [ "${connect:-0}" = 0.000000 ]; then echo blocked; else echo "open(rc=$rc)"; fi
}
status=0
src() { if [ "$1" = - ]; then echo 'unlabelled pod'; else echo "$1 pod"; fi; }
expect() { # want name label service port
local got; got=$(probe "$2" "$3" "$(ip "$4")" "$5")
local got; got=$(kind_probe "$2" "$3" "$(ip "$4")" "$5")
if [ "${got%%(*}" = "$1" ]; then echo "OK $2: $(src "$3") -> $4:$5 $got"
else echo "FAIL $2: $(src "$3") -> $4:$5 $got (want $1)" >&2; status=1; fi
}
Expand Down
Loading
Loading