diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..12d1a43 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,27 @@ +# Keeps the operator's Go modules, digest-pinned base images and SHA-pinned actions current. +version: 2 +updates: + - package-ecosystem: gomod + directory: /operator + schedule: + interval: weekly + cooldown: + default-days: 7 + groups: + # k8s.io and controller-runtime only build against matching versions. + kubernetes: + patterns: + - k8s.io/* + - sigs.k8s.io/* + - package-ecosystem: docker + directories: ["/operator"] + schedule: + interval: weekly + cooldown: + default-days: 7 + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + cooldown: + default-days: 7 diff --git a/.github/workflows/operator-ci.yml b/.github/workflows/operator-ci.yml new file mode 100644 index 0000000..d3a1dfd --- /dev/null +++ b/.github/workflows/operator-ci.yml @@ -0,0 +1,44 @@ +name: Operator CI + +on: + pull_request: + paths: + - operator/** + - .github/workflows/operator-ci.yml + # On master and on release this runs from the publish workflows, before they push. + workflow_call: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: operator + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: operator/go.mod + cache-dependency-path: operator/go.sum + # Regenerates CRDs/RBAC/deepcopy first; a diff means they were not committed. + - name: Generated files are up to date + run: make manifests generate && git diff --exit-code + - name: Controller and spawn API tests (envtest) + run: make test + + image: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Build notebook operator image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: operator + platforms: linux/amd64 + tags: mip-notebook-operator:ci + cache-from: type=gha,scope=operator + cache-to: type=gha,scope=operator,mode=max diff --git a/.github/workflows/operator-container-scan.yml b/.github/workflows/operator-container-scan.yml new file mode 100644 index 0000000..cab56e8 --- /dev/null +++ b/.github/workflows/operator-container-scan.yml @@ -0,0 +1,123 @@ +name: Operator container scanning + +on: + pull_request: + paths: + - operator/** + - ci/** + - .github/workflows/operator-container-scan.yml + workflow_dispatch: + workflow_call: {} + schedule: + - cron: '0 2 * * 1' # weekly scan, every Monday 02:00 UTC + +jobs: + container-scan: + name: Container Vulnerability Scanning (${{ matrix.image }}) + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write # required for uploading results to github security + strategy: + fail-fast: false + matrix: + include: + - image: mip-notebook-operator + context: operator + dockerfile: operator/Dockerfile + cache-scope: operator + env: + IMAGE_NAME: ${{ matrix.image }}:testing + DOCKERFILE: ${{ matrix.dockerfile }} + CONTAINER_SCAN_MERGED_SARIF_OUTPUT: container-scan-${{ matrix.image }}-merged.sarif + + # SCA / CVE + TRIVY_IGNOREFILE: ci/suppress_trivy.yaml + OSV_IGNOREFILE: ci/suppress_osv_scanner.toml + TRIVY_SCA_SARIF_OUTPUT: sca-trivy-container.sarif + OSV_SCA_SARIF_OUTPUT: sca-osv-container.sarif + + # SAST / Linting + SEMGREP_CONFIG_RULESETS: >- + semgrep-rules/dockerfile auto + OPENGREP_SAST_SARIF_OUTPUT: sast-opengrep-dockerfile.sarif + HADOLINT_SAST_SARIF_OUTPUT: sast-hadolint-dockerfile.sarif + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 #v7.1.0 + with: + python-version: '3.15.1' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 #v4.4.1 + + # Reads the same GHA layer cache as operator-ci.yml, so unchanged layers are not rebuilt. + - name: Build Docker image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc #v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + tags: ${{ env.IMAGE_NAME }} + load: true + cache-from: type=gha,scope=${{ matrix.cache-scope }} + + - name: Setup tools + run: | + bash ci/setup-tools.sh \ + --install-tool trivy,osv-scanner,opengrep,hadolint,semgrep-rules + + - name: Run SAST scanning + run: python ci/container_scan.py --scan-type sast + + - name: Run SCA scanning + if: ${{ !cancelled() }} + run: python ci/container_scan.py --scan-type sca --image ${{ env.IMAGE_NAME }} + + - name: Merge all SARIF reports + if: ${{ !cancelled() }} + run: | + python ci/container_scan.py \ + --merge-sarif "${{ env.TRIVY_SCA_SARIF_OUTPUT }}" "${{ env.OSV_SCA_SARIF_OUTPUT }}" "${{ env.OPENGREP_SAST_SARIF_OUTPUT }}" "${{ env.HADOLINT_SAST_SARIF_OUTPUT }}" \ + --merge-output "${{ env.CONTAINER_SCAN_MERGED_SARIF_OUTPUT }}" + + - name: Upload merged SARIF to GitHub Security tab + if: ${{ !cancelled() && hashFiles(env.CONTAINER_SCAN_MERGED_SARIF_OUTPUT) != '' }} + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + sarif_file: ${{ env.CONTAINER_SCAN_MERGED_SARIF_OUTPUT }} + category: container-scan-${{ matrix.image }} + + - name: Encrypt SARIF report + id: encrypt + if: ${{ !cancelled() && hashFiles(env.CONTAINER_SCAN_MERGED_SARIF_OUTPUT) != '' }} + env: + PW: ${{ secrets.ARTIFACT_PASSWORD }} + OUT: ${{ env.CONTAINER_SCAN_MERGED_SARIF_OUTPUT }}.gpg + run: | + if [ -z "$PW" ]; then + echo "::warning title=SARIF report not uploaded::ARTIFACT_PASSWORD is not available in this run (Dependabot, Renovate or fork PR). Skipping encryption and upload." + rm -f -- "$CONTAINER_SCAN_MERGED_SARIF_OUTPUT" + exit 0 + fi + printf '%s' "$PW" | gpg --symmetric --batch --yes --quiet \ + --pinentry-mode loopback --passphrase-fd 0 \ + --no-symkey-cache \ + --cipher-algo AES256 \ + --s2k-mode 3 --s2k-digest-algo SHA512 --s2k-count 65011712 \ + --output "$OUT" -- "$CONTAINER_SCAN_MERGED_SARIF_OUTPUT" + rm -f -- "$CONTAINER_SCAN_MERGED_SARIF_OUTPUT" + echo "encrypted=$OUT" >> "$GITHUB_OUTPUT" + + - name: Upload encrypted SARIF artifact + if: ${{ !cancelled() && steps.encrypt.outputs.encrypted != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: container-scan-sarif-report-${{ matrix.image }} + path: ${{ steps.encrypt.outputs.encrypted }} + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/operator-dev.yml b/.github/workflows/operator-dev.yml new file mode 100644 index 0000000..c3f9e87 --- /dev/null +++ b/.github/workflows/operator-dev.yml @@ -0,0 +1,49 @@ +# The dev image: every operator change on master, tagged dev and sha-. Pin it by +# digest, dev moves. A MIP release goes through operator-publish.yml instead. +name: Publish operator dev image + +on: + push: + branches: [master] + paths: + - operator/** + - .github/workflows/operator-dev.yml + +concurrency: operator-dev + +permissions: + contents: read + +jobs: + test: + uses: ./.github/workflows/operator-ci.yml + + build: + needs: test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Log in to Docker Hub + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + with: + images: hbpmip/mip-notebook-operator + tags: | + type=raw,value=dev + type=sha + - name: Build and push + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: operator + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=operator + cache-to: type=gha,scope=operator,mode=max diff --git a/.github/workflows/operator-publish.yml b/.github/workflows/operator-publish.yml new file mode 100644 index 0000000..c8be955 --- /dev/null +++ b/.github/workflows/operator-publish.yml @@ -0,0 +1,118 @@ +name: Publish operator image + +on: + release: + types: [published] + +env: + HARBOR_REGISTRY: docker-registry.ebrains.eu + DOCKER_BUILDKIT: 1 + DOCKER_CLIENT_TIMEOUT: 300 + COMPOSE_HTTP_TIMEOUT: 300 + RETRY_ATTEMPTS: "4" + RETRY_SLEEP: "10" + +# Registries use their own credentials; GITHUB_TOKEN only reads. +permissions: + contents: read + # The called scan workflows upload their SARIF to the Security tab. + security-events: write + +jobs: + test: + uses: ./.github/workflows/operator-ci.yml + + sca: + name: Operator SCA + uses: ./.github/workflows/operator-sca.yml + secrets: + ARTIFACT_PASSWORD: ${{ secrets.ARTIFACT_PASSWORD }} + + container-scan: + name: Operator container scanning + uses: ./.github/workflows/operator-container-scan.yml + secrets: + ARTIFACT_PASSWORD: ${{ secrets.ARTIFACT_PASSWORD }} + + sast: + name: Operator SAST + uses: ./.github/workflows/operator-sast.yml + secrets: + ARTIFACT_PASSWORD: ${{ secrets.ARTIFACT_PASSWORD }} + + secrets-scan: + name: Operator secret scanning + uses: ./.github/workflows/operator-secrets-scan.yml + + build_and_push: + name: Build ${{ matrix.image }} image and push to registries + needs: [test, sca, container-scan, sast, secrets-scan] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - image: mip-notebook-operator + context: operator + dockerfile: operator/Dockerfile + cache-scope: operator + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #v7.0.1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 #v4.4.1 + + - name: Harbor connectivity probe + run: | + set -x + getent hosts ${{ env.HARBOR_REGISTRY }} || true + curl -vI --connect-timeout 10 https://${{ env.HARBOR_REGISTRY }}/v2/ || true + + - name: Disable IPv6 (job-scoped) + run: | + sudo sysctl -w net.ipv6.conf.all.disable_ipv6=1 + sudo sysctl -w net.ipv6.conf.default.disable_ipv6=1 + + - name: Log in to Docker Hub + uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 #v4.7.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to EBRAINS HARBOR (retry) + env: + HARBOR_USERNAME: ${{ secrets.HARBOR_USERNAME }} + HARBOR_PASSWORD: ${{ secrets.HARBOR_PASSWORD }} + run: | + for i in $(seq 1 ${{ env.RETRY_ATTEMPTS }}); do + printf '%s' "$HARBOR_PASSWORD" | \ + docker login ${{ env.HARBOR_REGISTRY }} \ + -u "$HARBOR_USERNAME" --password-stdin && exit 0 + echo "Harbor login attempt $i failed; retrying in ${{ env.RETRY_SLEEP }}s..." + sleep ${{ env.RETRY_SLEEP }} + done + echo "Harbor login failed after ${{ env.RETRY_ATTEMPTS }} attempts" >&2 + exit 1 + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 #v6.2.0 + with: + images: | + hbpmip/${{ matrix.image }} + ${{ env.HARBOR_REGISTRY }}/medical-informatics-platform/${{ matrix.image }} + + # Shares the GHA layer cache with operator-ci.yml and operator-container-scan.yml. + - name: Build ${{ matrix.image }} docker image + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc #v7.4.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.cache-scope }} + cache-to: type=gha,scope=${{ matrix.cache-scope }},mode=max diff --git a/.github/workflows/operator-sast.yml b/.github/workflows/operator-sast.yml new file mode 100644 index 0000000..86c2933 --- /dev/null +++ b/.github/workflows/operator-sast.yml @@ -0,0 +1,79 @@ +name: Operator SAST + +on: + pull_request: + paths: + - operator/** + - ci/** + - .github/workflows/operator-sast.yml + workflow_dispatch: + workflow_call: {} + schedule: + - cron: '0 2 * * 1' # weekly scan, every Monday 02:00 UTC + +jobs: + sast: + name: Static Application Security Testing + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write # required for uploading SAST results to github security + env: + SEMGREP_CONFIG_RULESETS: >- + semgrep-rules/generic semgrep-rules/problem-based-packs semgrep-rules/bash semgrep-rules/python semgrep-rules/go auto + semgrep-rules/yaml semgrep-rules/package_managers p/default + OPENGREP_EXCLUDE: >- + *.sarif ci/ Dockerfile* deployment/ documentation/ README.md + OPENGREP_SARIF_OUTPUT: sast-semgrep-app.sarif + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 #v7.1.0 + with: + python-version: '3.15.1' + + - name: Setup tools + run: bash ci/setup-tools.sh --install-tool opengrep,semgrep-rules + + - name: Run SAST scanning + run: python ci/sast_scan.py + + - name: Upload SARIF to GitHub Security tab + if: ${{ !cancelled() && hashFiles(env.OPENGREP_SARIF_OUTPUT) != '' }} + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + sarif_file: ${{ env.OPENGREP_SARIF_OUTPUT }} + category: semgrep-app + + - name: Encrypt SARIF report + id: encrypt + if: ${{ !cancelled() && hashFiles(env.OPENGREP_SARIF_OUTPUT) != '' }} + env: + PW: ${{ secrets.ARTIFACT_PASSWORD }} + OUT: ${{ env.OPENGREP_SARIF_OUTPUT }}.gpg + run: | + if [ -z "$PW" ]; then + echo "::warning title=SARIF report not uploaded::ARTIFACT_PASSWORD is not available in this run (Dependabot, Renovate or fork PR). Skipping encryption and upload." + rm -f -- "$OPENGREP_SARIF_OUTPUT" + exit 0 + fi + printf '%s' "$PW" | gpg --symmetric --batch --yes --quiet \ + --pinentry-mode loopback --passphrase-fd 0 \ + --no-symkey-cache \ + --cipher-algo AES256 \ + --s2k-mode 3 --s2k-digest-algo SHA512 --s2k-count 65011712 \ + --output "$OUT" -- "$OPENGREP_SARIF_OUTPUT" + rm -f -- "$OPENGREP_SARIF_OUTPUT" + echo "encrypted=$OUT" >> "$GITHUB_OUTPUT" + + - name: Upload encrypted SARIF artifact + if: ${{ !cancelled() && steps.encrypt.outputs.encrypted != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sast-scan-sarif-report + path: ${{ steps.encrypt.outputs.encrypted }} + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/operator-sca.yml b/.github/workflows/operator-sca.yml new file mode 100644 index 0000000..b3e4534 --- /dev/null +++ b/.github/workflows/operator-sca.yml @@ -0,0 +1,83 @@ +name: Operator SCA + +on: + pull_request: + paths: + - operator/** + - ci/** + - .github/workflows/operator-sca.yml + workflow_dispatch: + workflow_call: {} + schedule: + - cron: '0 2 * * 1' # weekly SCA scan, every Monday 02:00 UTC + +jobs: + sca: + name: Software Composition Analysis + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write # required for uploading SCA results to github security + env: + SCA_TARGET: operator + TRIVY_IGNOREFILE: ci/suppress_trivy.yaml + OSV_IGNOREFILE: ci/suppress_osv_scanner.toml + TRIVY_SARIF_OUTPUT: trivy-mip-notebook-operator.sarif + OSV_SARIF_OUTPUT: osv-scanner-mip-notebook-operator.sarif + SCA_MERGED_SARIF_OUTPUT: SCA-mip-notebook-operator-merged.sarif + # CVSS gate: fail at >= FAIL, warn between WARN and FAIL. + GATE_FAIL_THRESHOLD: "8.0" + GATE_WARN_THRESHOLD: "5.0" + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 #v7.1.0 + with: + python-version: '3.15.1' + + # Scans the committed lockfiles directly (operator/go.mod); no SBOM step. + - name: Setup tools + run: bash ci/setup-tools.sh --install-tool trivy,osv-scanner + + - name: Run SCA tools + run: python ci/sca_scan.py + + - name: Upload merged SARIF to GitHub Security tab + if: ${{ !cancelled() && hashFiles(env.SCA_MERGED_SARIF_OUTPUT) != '' }} + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + sarif_file: ${{ env.SCA_MERGED_SARIF_OUTPUT }} + category: sca-app + + - name: Encrypt SARIF report + id: encrypt + if: ${{ !cancelled() && hashFiles(env.SCA_MERGED_SARIF_OUTPUT) != '' }} + env: + PW: ${{ secrets.ARTIFACT_PASSWORD }} + OUT: ${{ env.SCA_MERGED_SARIF_OUTPUT }}.gpg + run: | + if [ -z "$PW" ]; then + echo "::warning title=SARIF report not uploaded::ARTIFACT_PASSWORD is not available in this run (Dependabot, Renovate or fork PR). Skipping encryption and upload." + rm -f -- "$SCA_MERGED_SARIF_OUTPUT" + exit 0 + fi + printf '%s' "$PW" | gpg --symmetric --batch --yes --quiet \ + --pinentry-mode loopback --passphrase-fd 0 \ + --no-symkey-cache \ + --cipher-algo AES256 \ + --s2k-mode 3 --s2k-digest-algo SHA512 --s2k-count 65011712 \ + --output "$OUT" -- "$SCA_MERGED_SARIF_OUTPUT" + rm -f -- "$SCA_MERGED_SARIF_OUTPUT" + echo "encrypted=$OUT" >> "$GITHUB_OUTPUT" + + - name: Upload encrypted SARIF artifact + if: ${{ !cancelled() && steps.encrypt.outputs.encrypted != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: sca-scan-sarif-report + path: ${{ steps.encrypt.outputs.encrypted }} + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/operator-secrets-scan.yml b/.github/workflows/operator-secrets-scan.yml new file mode 100644 index 0000000..84b9d98 --- /dev/null +++ b/.github/workflows/operator-secrets-scan.yml @@ -0,0 +1,46 @@ +name: Operator secret scanning (gitleaks) + +on: + pull_request: + paths: + - operator/** + - ci/** + - .github/workflows/operator-secrets-scan.yml + push: + branches: + - master + paths: + - operator/** + workflow_dispatch: + workflow_call: {} + +permissions: + contents: read + security-events: write + +jobs: + gitleaks: + runs-on: ubuntu-latest + env: + GITLEAKS_SARIF_OUTPUT: gitleaks.sarif + + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Install gitleaks + run: bash ci/setup-tools.sh --install-tool gitleaks + + - name: Install and run gitleaks + run: > + gitleaks dir operator + --redact + --report-format sarif + --report-path "$GITLEAKS_SARIF_OUTPUT" + + - name: Upload SARIF to code scanning + if: always() + uses: github/codeql-action/upload-sarif@9b3254085494b55a0f7c4c2b3d2d52cc712ce370 # v4.39.1 + with: + sarif_file: ${{ env.GITLEAKS_SARIF_OUTPUT }} + category: gitleaks diff --git a/ci/container_scan.py b/ci/container_scan.py new file mode 100644 index 0000000..bc57194 --- /dev/null +++ b/ci/container_scan.py @@ -0,0 +1,297 @@ +import subprocess +import os +import sys +import logging +import json +import argparse +from parse_sarif import evaluate, drop_suppressed, print_failures, GATE_FAIL_THRESHOLD, GATE_WARN_THRESHOLD + + +GREEN = '\033[92m' +RED = '\033[91m' +RESET = '\033[0m' +BOLD = '\033[1m' +YELLOW = '\033[93m' + +logging.basicConfig( + level=logging.INFO, + format='%(message)s' # Clean format to prevent double-timestamps in CI logs +) +logger = logging.getLogger("container-scan-orchestrator") + +# --- Configurable values +IMAGE_NAME = os.getenv("IMAGE_NAME", "mip-jupyter:local") +DOCKERFILE = os.getenv("DOCKERFILE", "docker/singleuser/Dockerfile") + +# --- SCA / CVE (Trivy + OSV) --- +TRIVY_IGNOREFILE = os.getenv("TRIVY_IGNOREFILE", "ci/suppress_trivy.yaml") +OSV_IGNOREFILE = os.getenv("OSV_IGNOREFILE", "ci/suppress_osv_scanner.toml") +TRIVY_SCA_SARIF_OUTPUT = os.getenv("TRIVY_SCA_SARIF_OUTPUT", "sca-trivy-container.sarif") +OSV_SCA_SARIF_OUTPUT = os.getenv("OSV_SCA_SARIF_OUTPUT", "sca-osv-container.sarif") + +# --- SAST/Code Linting (OpenGrep + Hadolint) --- +SEMGREP_CONFIG_RULESETS = os.getenv( + "SEMGREP_CONFIG_RULESETS", + "semgrep-rules/dockerfile" +).split() +OPENGREP_SAST_SARIF_OUTPUT = os.getenv("OPENGREP_SAST_SARIF_OUTPUT", "sast-opengrep-dockerfile.sarif") +HADOLINT_SAST_SARIF_OUTPUT = os.getenv("HADOLINT_SAST_SARIF_OUTPUT", "sast-hadolint-dockerfile.sarif") + +# --- Functions to run each SCA tool and handle their outputs +def run_trivy(): + cmd = [ + "trivy", "image", + IMAGE_NAME, + "--format", "sarif", + "--ignore-unfixed", + "--ignorefile", TRIVY_IGNOREFILE, + "--output", TRIVY_SCA_SARIF_OUTPUT + ] + return subprocess.run(cmd).returncode + +def run_osv_scanner(): + cmd = [ + "osv-scanner", "scan", "image", + IMAGE_NAME, + "--config", OSV_IGNOREFILE, + "--format", "sarif", + "--output-file", OSV_SCA_SARIF_OUTPUT + ] + exit_code = subprocess.run(cmd).returncode + if exit_code == 1: + exit_code = 0 # OSV Scanner returns 1 if vulnerabilities are found, but we want to continue the pipeline + if exit_code == 0: + exit_code = drop_osv_unfixed() + return exit_code + +def drop_osv_unfixed(): + # OSV-Scanner has no --ignore-unfixed; mirror Trivy's by dropping results + # whose advisory has no fixed version in the scanned package's own + # ecosystem (an Ubuntu Pro-only fix does not count). + json_cmd = [ + "osv-scanner", "scan", "image", IMAGE_NAME, + "--config", OSV_IGNOREFILE, "--format", "json", + ] + scan = subprocess.run(json_cmd, capture_output=True, text=True) + # 0: nothing to filter. Not 1: no data, so keep every result rather than pass the gate. + if scan.returncode == 0: + return 0 + if scan.returncode != 1: + logger.error(f"{RED}[!] osv-scanner JSON scan exit code {scan.returncode}: {scan.stderr}{RESET}") + return scan.returncode + fixed = set() + for result in json.loads(scan.stdout).get("results", []): + for pkg in result.get("packages", []): + name = pkg["package"]["name"] + ecosystem = pkg["package"].get("ecosystem") + for vuln in pkg.get("vulnerabilities", []): + for affected in vuln.get("affected", []): + if affected.get("package", {}).get("name") != name: + continue + if affected["package"].get("ecosystem") != ecosystem: + continue + if any("fixed" in e for r in affected.get("ranges", []) for e in r.get("events", [])): + for vid in [vuln["id"], *vuln.get("aliases", [])]: + fixed.add((vid, name)) + with open(OSV_SCA_SARIF_OUTPUT) as f: + sarif = json.load(f) + for run in sarif.get("runs", []): + # A message that does not parse keeps its result: unknown is not unfixed. + run["results"] = [r for r in run.get("results", []) + if osv_package(r) is None or (r.get("ruleId"), osv_package(r)) in fixed] + with open(OSV_SCA_SARIF_OUTPUT, "w") as f: + json.dump(sarif, f) + return 0 + +def osv_package(result): + # Message reads "Package 'name@version' is vulnerable to ...". + text = result.get("message", {}).get("text", "") + return text.split("'", 2)[1].rsplit("@", 1)[0] if text.count("'") >= 2 else None + +def handle_sca(): + + tools = {"trivy": run_trivy, "osv-scanner": run_osv_scanner} + sarif_files = {"trivy": TRIVY_SCA_SARIF_OUTPUT, "osv-scanner": OSV_SCA_SARIF_OUTPUT} + tool_status = {} # "PASSED" | "WARNING" | "FAILED" | "ERROR" + gate_failed = False + + # Run each SCA tool and collect their exit codes + for name, tool_fn in tools.items(): + exit_code = tool_fn() + logger.info("-" * 40) + + path = sarif_files[name] + if exit_code != 0 and os.path.exists(path): + logger.error(f"{RED}[!] {name} exit code {exit_code} but wrote {path}{RESET}") + tool_status[name] = "ERROR" + gate_failed = True + + # Evaluate each SARIF file for gate decision + for name, path in sarif_files.items(): + if name in tool_status: + continue # already flagged ERROR above, don't overwrite it + + if not os.path.exists(path): + logger.error(f"{RED}[!] {name} SARIF missing: {path},tool failed to run (not a vulnerability){RESET}") + tool_status[name] = "ERROR" + gate_failed = True + continue + + eval_result = evaluate(path) + + if eval_result.gate_failed: + tool_status[name] = "FAILED" + gate_failed = True + print_failures(path) + elif eval_result.gate_warn: + tool_status[name] = "WARNING" + else: + tool_status[name] = "PASSED" + + # Print summary of results + logger.info(f"\n{BOLD}========== SCA PIPELINE SUMMARY =========={RESET}") + for name, status in tool_status.items(): + if status == "PASSED": + logger.info(f"[{name}]: {GREEN}PASSED{RESET}") + elif status == "WARNING": + logger.warning(f"[{name}]: {YELLOW}WARNING (findings between {GATE_WARN_THRESHOLD} and {GATE_FAIL_THRESHOLD}){RESET}") + elif status == "ERROR": + logger.error(f"[{name}]: {RED}ERROR (tool did not run correctly){RESET}") + else: + logger.error(f"[{name}]: {RED}FAILED (CVSS >= {GATE_FAIL_THRESHOLD} found){RESET}") + + # Exit with non-zero code if any tool failed the gate + if gate_failed: + logger.error(f"{RED}One or more SCA tools failed the gate check.{RESET}") + sys.exit(1) + + +# --- Functions to run each SAST tool and handle their outputs +def run_hadolint(): + cmd = [ + "hadolint", DOCKERFILE, + "--failure-threshold", "error", + "--format", "sarif", + ] + with open(HADOLINT_SAST_SARIF_OUTPUT, "w") as f: + result = subprocess.run(cmd, stdout=f) + + return result.returncode + +def run_opengrep(): + base_cmd = ["opengrep", "scan", "-q"] + \ + [f"--config {config}" for config in SEMGREP_CONFIG_RULESETS] + + report_cmd = (base_cmd + ["--sarif", "--output", OPENGREP_SAST_SARIF_OUTPUT, DOCKERFILE]) + report_cmd = " ".join(report_cmd).split() + logger.info(f"{BOLD}Running (report):{RESET} {' '.join(report_cmd)}") + subprocess.run(report_cmd) + drop_suppressed(OPENGREP_SAST_SARIF_OUTPUT) + + gate_cmd = (base_cmd + ["--severity=ERROR", "--error", DOCKERFILE]) + gate_cmd = " ".join(gate_cmd).split() + logger.info(f"{BOLD}Running (gate):{RESET} {' '.join(gate_cmd)}") + return subprocess.run(gate_cmd).returncode + +def handle_sast(): + tools = { + "hadolint": run_hadolint, + "opengrep": run_opengrep, + } + + tool_status = {} + exit_codes = {} + + for name, run_fn in tools.items(): + exit_code = run_fn() + exit_codes[name] = exit_code + if exit_code == 0: + tool_status[name] = "PASSED" + elif exit_code == 1: + tool_status[name] = "FAILED" + else: + tool_status[name] = "ERROR" + + logger.info(f"\n{BOLD}========== SAST PIPELINE SUMMARY =========={RESET}") + for name, status in tool_status.items(): + if status == "PASSED": + logger.info(f"[{name}]: {GREEN}PASSED (exit code 0){RESET}") + elif status == "FAILED": + logger.error(f"[{name}]: {RED}FAILED (exit code 1 - error-severity findings){RESET}") + else: + logger.error(f"[{name}]: {RED}ERROR (exit code {exit_codes[name]}, tool did not run correctly){RESET}") + logger.info(f"{BOLD}==========================================={RESET}\n") + + if any(status != "PASSED" for status in tool_status.values()): + sys.exit(1) + +def merge_sarifs(sarif_paths, output_path): + merged = { + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [], + } + + for path in sarif_paths: + if not os.path.exists(path): + logger.warning(f"{path} not found, skipping in merge") + continue + with open(path) as f: + sarif = json.load(f, strict=False) + merged["runs"].extend(sarif.get("runs", [])) + + with open(output_path, "w") as f: + json.dump(merged, f) + + logger.info(f"SARIF files merged successfully into {output_path}") + +def main(): + parser = argparse.ArgumentParser( + prog="sec-orchestrator", + description="Agnostic DevSecOps Container scanning Pipeline Orchestrator" + ) + + # The primary router + parser.add_argument( + "-s", "--scan-type", + choices=["sast", "sca"], + help="Specify the security methodology to execute (e.g., sast, sca)" + ) + + # Image flag specifically SCA approach + parser.add_argument( + "-i", "--image", + help="Target Docker image reference" + ) + + parser.add_argument( + "--merge-sarif", + nargs="+", + metavar="SARIF_FILE", + help="List of SARIF files to merge into one report" + ) + + parser.add_argument( + "--merge-output", + default="merged-container-scan.sarif", + help="Output path for the merged SARIF file" + ) + args = parser.parse_args() + + if args.scan_type == "sast": + logger.info(f"{BOLD}Initiating SAST pipeline: {RESET}") + handle_sast() + # Execution + elif args.scan_type == "sca": + logger.info(f"{BOLD}Initiating SCA pipeline on {args.image}{RESET}") + global IMAGE_NAME + if args.image: # Override the global IMAGE_NAME + IMAGE_NAME = args.image + handle_sca() + + if args.merge_sarif: + merge_sarifs(args.merge_sarif, args.merge_output) + return + +if __name__ == "__main__": + main() diff --git a/ci/parse_sarif.py b/ci/parse_sarif.py new file mode 100644 index 0000000..ae1ed5b --- /dev/null +++ b/ci/parse_sarif.py @@ -0,0 +1,94 @@ +import json +import os +from dataclasses import dataclass + +GATE_FAIL_THRESHOLD = float(os.getenv("GATE_FAIL_THRESHOLD", "8.0")) +GATE_WARN_THRESHOLD = float(os.getenv("GATE_WARN_THRESHOLD", "5.0")) + +def drop_suppressed(path): + # opengrep keeps `# nosemgrep` findings in SARIF (suppressions: inSource), + # and the GitHub Security tab still opens alerts for them. + if not os.path.exists(path): + return + with open(path) as f: + sarif = json.load(f) + for run in sarif.get("runs", []): + run["results"] = [r for r in run.get("results", []) if not r.get("suppressions")] + with open(path, "w") as f: + json.dump(sarif, f) + +@dataclass +class EvaluationResult: + gate_failed: bool + gate_warn: bool + +def evaluate(sarif_paths): + if isinstance(sarif_paths, str): + sarif_paths = [sarif_paths] + + max_score = 0.0 + unscored = 0 + for path in sarif_paths: + try: + with open(path) as f: + sarif = json.load(f, strict=False) + except FileNotFoundError: + raise FileNotFoundError(f"[parse_sarif] SARIF file not found: {path}") + except json.JSONDecodeError as e: + raise ValueError(f"[parse_sarif] SARIF file at {path} is not valid JSON: {e}") + + for run in sarif.get("runs", []): + try: + rules = run["tool"]["driver"].get("rules", []) + except KeyError as e: + raise KeyError(f"[parse_sarif] SARIF file at {path} has a run missing expected field {e}") + + severities = {r["id"]: r.get("properties", {}).get("security-severity") for r in rules} + + for result in run.get("results", []): + score = severities.get(result["ruleId"]) + if score is not None: + max_score = max(max_score, float(score)) + else: + unscored += 1 + + # The gate cannot judge these, so they must not read as a clean pass. + if unscored: + print(f"::warning title=Unscored findings::{unscored} result(s) in {', '.join(sarif_paths)} have no security-severity; the CVSS gate did not evaluate them.") + + return EvaluationResult( + gate_failed=max_score >= GATE_FAIL_THRESHOLD, + gate_warn=unscored > 0 or GATE_WARN_THRESHOLD <= max_score < GATE_FAIL_THRESHOLD, + ) + +def print_failures(sarif_path, threshold=None): + """Print ruleId, severity and location of findings at or above the fail + threshold, so a red job log is enough to act on without a local re-run.""" + if threshold is None: + threshold = GATE_FAIL_THRESHOLD + try: + with open(sarif_path) as f: + sarif = json.load(f, strict=False) + except (FileNotFoundError, json.JSONDecodeError): + return + for run in sarif.get("runs", []): + rules = run.get("tool", {}).get("driver", {}).get("rules", []) + severities = {r["id"]: r.get("properties", {}).get("security-severity") for r in rules} + for result in run.get("results", []): + score = severities.get(result.get("ruleId")) + try: + at_or_above = score is not None and float(score) >= threshold + except (TypeError, ValueError): + at_or_above = False + if not at_or_above: + continue + location = "" + phys = (result.get("locations") or [{}])[0].get("physicalLocation", {}) + art = phys.get("artifactLocation", {}).get("uri", "") + if art: + location = f" in {art}" + region = phys.get("region", {}).get("startLine") + if region: + location += f":{region}" + msg = (result.get("message", {}).get("text") or "").splitlines()[0] if result.get("message", {}).get("text") else "" + print(f"::error title={result.get('ruleId')} (severity {score})::{msg}{location}") diff --git a/ci/sast_scan.py b/ci/sast_scan.py new file mode 100644 index 0000000..ebd5d95 --- /dev/null +++ b/ci/sast_scan.py @@ -0,0 +1,80 @@ +import subprocess +import os +import sys +import logging +from parse_sarif import drop_suppressed + +GREEN = '\033[92m' +RED = '\033[91m' +RESET = '\033[0m' +BOLD = '\033[1m' +YELLOW = '\033[93m' + +logging.basicConfig( + level=logging.INFO, + format='%(message)s' # Clean format to prevent double-timestamps in CI logs +) +logger = logging.getLogger("sast-orchestrator") + +# --- Configurable values ----------------------------------------------- +SEMGREP_CONFIG_RULESETS = os.getenv( + "SEMGREP_CONFIG_RULESETS", + " semgrep-rules/generic semgrep-rules/problem-based-packs semgrep-rules/bash " + " semgrep-rules/python semgrep-rules/go auto semgrep-rules/yaml semgrep-rules/package_managers p/default" +).split() +OPENGREP_EXCLUDE = os.getenv( + "OPENGREP_EXCLUDE", + "*.sarif ci/ Dockerfile* .pre-commit-config.yaml docs/** README.md AGENTS.md" +).split() +OPENGREP_SARIF_OUTPUT = os.getenv("OPENGREP_SARIF_OUTPUT", "sast-opengrep-app.sarif") + +def run_opengrep(): + base_cmd = ["opengrep", "scan"] + \ + [f"--config {config}" for config in SEMGREP_CONFIG_RULESETS] + \ + [f"--exclude={pattern}" for pattern in OPENGREP_EXCLUDE] + + report_cmd = (base_cmd + ["--sarif", "--output", OPENGREP_SARIF_OUTPUT]) + report_cmd = " ".join(report_cmd).split() + logger.info(f"{BOLD}Running (report):{RESET} {' '.join(report_cmd)}") + subprocess.run(report_cmd) + drop_suppressed(OPENGREP_SARIF_OUTPUT) + + gate_cmd = (base_cmd + ["--severity=ERROR", "--error"]) + gate_cmd = " ".join(gate_cmd).split() + logger.info(f"{BOLD}Running (gate):{RESET} {' '.join(gate_cmd)}") + return subprocess.run(gate_cmd).returncode + +def main(): + + exit_code = run_opengrep() + logger.info("-" * 40) + + if exit_code == 0: + status = "PASSED" + elif exit_code == 1: + status = "FAILED" + else: + status = "ERROR" + + if not os.path.exists(OPENGREP_SARIF_OUTPUT): + logger.error(f"{RED}[!] opengrep SARIF missing: {OPENGREP_SARIF_OUTPUT}, tool failed to run{RESET}") + status = "ERROR" + + logger.info(f"\n{BOLD}========== SAST PIPELINE SUMMARY =========={RESET}") + if status == "PASSED": + logger.info(f"[opengrep]: {GREEN}PASSED (exit code 0){RESET}") + elif status == "FAILED": + logger.error(f"[opengrep]: {RED}FAILED (exit code 1 - error-severity findings){RESET}") + else: + logger.error(f"[opengrep]: {RED}ERROR (exit code {exit_code}, tool did not run correctly){RESET}") + logger.info(f"{BOLD}==========================================={RESET}\n") + + if status == "ERROR": + sys.exit(1) + + if status == "FAILED": + logger.error(f"{RED}SAST gate failed: blocking-severity findings present.{RESET}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/ci/sca_scan.py b/ci/sca_scan.py new file mode 100644 index 0000000..80cef8e --- /dev/null +++ b/ci/sca_scan.py @@ -0,0 +1,132 @@ +import subprocess +import os +import sys +import logging +import json +from parse_sarif import evaluate, print_failures, GATE_FAIL_THRESHOLD, GATE_WARN_THRESHOLD + +GREEN = '\033[92m' +RED = '\033[91m' +RESET = '\033[0m' +BOLD = '\033[1m' +YELLOW = '\033[93m' + +logging.basicConfig( + level=logging.INFO, + format='%(message)s' # Clean format to prevent double-timestamps in CI logs +) +logger = logging.getLogger("sca-orchestrator") + +# Configurable values +# Directory scanned for lockfiles (uv.lock, go.mod/go.sum). +SCA_TARGET = os.getenv("SCA_TARGET", ".") +TRIVY_IGNOREFILE = os.getenv("TRIVY_IGNOREFILE", "ci/suppress_trivy.yaml") +OSV_IGNOREFILE = os.getenv("OSV_IGNOREFILE", "ci/suppress_osv_scanner.toml") +TRIVY_SARIF_OUTPUT = os.getenv("TRIVY_SARIF_OUTPUT", "trivy-mip-jupyter.sarif") +OSV_SARIF_OUTPUT = os.getenv("OSV_SARIF_OUTPUT", "osv-scanner-mip-jupyter.sarif") +SCA_MERGED_SARIF_OUTPUT = os.getenv("SCA_MERGED_SARIF_OUTPUT", "SCA-mip-jupyter-merged.sarif") + +def run_trivy(): + cmd = [ + "trivy", "fs", SCA_TARGET, + "--scanners", "vuln", + "--format", "sarif", + "--ignorefile", TRIVY_IGNOREFILE, + "--output", TRIVY_SARIF_OUTPUT + ] + return subprocess.run(cmd).returncode + +def run_osv_scanner(): + cmd = [ + "osv-scanner", "scan", "source", + "--recursive", SCA_TARGET, + "--config", OSV_IGNOREFILE, + "--format", "sarif", + "--output-file", OSV_SARIF_OUTPUT + ] + exit_code = subprocess.run(cmd).returncode + if exit_code == 1: + return 0 # OSV Scanner returns 1 if vulnerabilities are found, but we want to continue the pipeline + return exit_code + +def merge_sarifs(): + merged = { + "$schema": "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/main/sarif-2.1/schema/sarif-schema-2.1.0.json", + "version": "2.1.0", + "runs": [], + } + + for path in (TRIVY_SARIF_OUTPUT, OSV_SARIF_OUTPUT): + if not os.path.exists(path): + logger.warning(f"{path} not found, skipping in merge") + continue + with open(path) as f: + sarif = json.load(f, strict=False) + merged["runs"].extend(sarif.get("runs", [])) + + with open(SCA_MERGED_SARIF_OUTPUT, "w") as f: + json.dump(merged, f) + + logger.info("SARIF files merged successfully.") + +def main(): + + tools = {"trivy": run_trivy, "osv-scanner": run_osv_scanner} + sarif_files = {"trivy": TRIVY_SARIF_OUTPUT, "osv-scanner": OSV_SARIF_OUTPUT} + tool_status = {} # "PASSED" | "WARNING" | "FAILED" | "ERROR" + gate_failed = False + + # Run each SCA tool and collect their exit codes + for name, tool_fn in tools.items(): + exit_code = tool_fn() + logger.info("-" * 40) + + path = sarif_files[name] + if exit_code != 0 and os.path.exists(path): + logger.error(f"{RED}[!] {name} exit code {exit_code} but wrote {path}{RESET}") + tool_status[name] = "ERROR" + gate_failed = True + + merge_sarifs() # combined artifact only, not used for the gate decision + + # Evaluate each SARIF file for gate decision + for name, path in sarif_files.items(): + if name in tool_status: + continue # already flagged ERROR above, don't overwrite it + + if not os.path.exists(path): + logger.error(f"{RED}[!] {name} SARIF missing: {path},tool failed to run (not a vulnerability){RESET}") + tool_status[name] = "ERROR" + gate_failed = True + continue + + eval_result = evaluate(path) + + if eval_result.gate_failed: + tool_status[name] = "FAILED" # this tool found CVSS >= 8.0 + gate_failed = True + print_failures(path) + elif eval_result.gate_warn: + tool_status[name] = "WARNING" # this tool found 5.0 <= CVSS < 8.0 + else: + tool_status[name] = "PASSED" # this tool found nothing >= 5.0 + + # Print summary of results + logger.info(f"\n{BOLD}========== SCA PIPELINE SUMMARY =========={RESET}") + for name, status in tool_status.items(): + if status == "PASSED": + logger.info(f"[{name}]: {GREEN}PASSED{RESET}") + elif status == "WARNING": + logger.warning(f"[{name}]: {YELLOW}WARNING (findings between {GATE_WARN_THRESHOLD} and {GATE_FAIL_THRESHOLD}){RESET}") + elif status == "ERROR": + logger.error(f"[{name}]: {RED}ERROR (tool did not run correctly){RESET}") + else: + logger.error(f"[{name}]: {RED}FAILED (CVSS >= {GATE_FAIL_THRESHOLD} found){RESET}") + + # Exit with non-zero code if any tool failed the gate + if gate_failed: + logger.error(f"{RED}One or more SCA tools failed the gate check.{RESET}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/ci/setup-tools.sh b/ci/setup-tools.sh new file mode 100755 index 0000000..2dda818 --- /dev/null +++ b/ci/setup-tools.sh @@ -0,0 +1,159 @@ +#!/bin/bash +set -e # stop the pipeline if any command fails +set -o pipefail # Prevents silent pipeline successes if the curl download drops +set -u # treat unset variables as an error + +trap 'echo "[setup-tools] ERROR: command failed (exit $?) at line $LINENO: $BASH_COMMAND" >&2' ERR + +# Tool versions and the SHA256 of the release asset we download. Bumped by hand: +# no update bot reads this file. When overriding a *_VERSION via env, the matching +# *_SHA256 must be overridden as well or verification will fail. + +TRIVY_VERSION="${TRIVY_VERSION:-v0.76.1}" +TRIVY_SHA256="${TRIVY_SHA256:-c7cbe3137fb8a48a17574bede3dd6beb16742d1e9772929940fc90082c39d9c0}" + +OSV_SCANNER_VERSION="${OSV_SCANNER_VERSION:-v2.7.0}" +OSV_SCANNER_SHA256="${OSV_SCANNER_SHA256:-160c6253e2922e9e0a6a9771c5c4c40a4b3c6090b34a3cdd5e80c96cc2e4218b}" + +OPENGREP_VERSION="${OPENGREP_VERSION:-v1.32.0}" +OPENGREP_SHA256="${OPENGREP_SHA256:-7360d60174880bd9214f2dfbe439b7bce82b7a3d44cfe945ba833c85cc8bda75}" + +SEMGREP_RULES_REF="${SEMGREP_RULES_REF:-16b9e6d45f10649f6a531ec4838ed9924372c3bc}" +SEMGREP_RULES_DIR="semgrep-rules" + +HADOLINT_VERSION="${HADOLINT_VERSION:-v2.15.1}" +HADOLINT_SHA256="${HADOLINT_SHA256:-c7187db94eeeeca956519a6af171adc31453941a1e777961f6e680f697c8c507}" + +GITLEAKS_VERSION="${GITLEAKS_VERSION:-v8.30.1}" +GITLEAKS_SHA256="${GITLEAKS_SHA256:-551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb}" + +CYCLONEDX_NPM_VERSION="${CYCLONEDX_NPM_VERSION:-6.0.1}" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +# --- Flag parsing ----------------------------------------------------- +INSTALL_TOOL="none" +SBOM_ECOSYSTEM="none" + +while [[ $# -gt 0 ]]; do + case "$1" in + --install-tool) + [[ $# -ge 2 ]] || { echo "[setup-tools] --install-tool requires a value (e.g. trivy,osv-scanner|all)" >&2; exit 1; } + INSTALL_TOOL="$2" + shift 2 + ;; + --sbom-ecosystem) + [[ $# -ge 2 ]] || { echo "[setup-tools] --sbom-ecosystem requires a value (e.g. maven|npm|none)" >&2; exit 1; } + SBOM_ECOSYSTEM="$2" + shift 2 + ;; + *) + echo "[setup-tools] Unknown flag: $1" >&2 + exit 1 + ;; + esac +done + +should_install() { + [[ "$INSTALL_TOOL" == "all" || ",$INSTALL_TOOL," == *",$1,"* ]] +} + +# Download a file and refuse to proceed unless its SHA256 matches the pinned one +download_and_verify() { + local url="$1" dest="$2" sha256="$3" + curl -fsSL --retry 3 "${url}" -o "${dest}" + echo "${sha256} ${dest}" | sha256sum -c - +} + +# --- Trivy -------------------------------------------------------------- +if should_install "trivy"; then + echo "[setup-tools] Installing Trivy ${TRIVY_VERSION}" + TRIVY_TARBALL="trivy_${TRIVY_VERSION#v}_Linux-64bit.tar.gz" + download_and_verify \ + "https://github.com/aquasecurity/trivy/releases/download/${TRIVY_VERSION}/${TRIVY_TARBALL}" \ + "${TMP_DIR}/${TRIVY_TARBALL}" \ + "${TRIVY_SHA256}" + sudo tar -xzf "${TMP_DIR}/${TRIVY_TARBALL}" -C /usr/local/bin trivy + trivy --version + echo "Trivy installed OK" +fi + +# --- OSV Scanner ---------------------------------------------------------- +if should_install "osv-scanner"; then + echo "[setup-tools] Installing OSV Scanner ${OSV_SCANNER_VERSION}" + download_and_verify \ + "https://github.com/google/osv-scanner/releases/download/${OSV_SCANNER_VERSION}/osv-scanner_linux_amd64" \ + "${TMP_DIR}/osv-scanner" \ + "${OSV_SCANNER_SHA256}" + sudo install -m 0755 "${TMP_DIR}/osv-scanner" /usr/local/bin/osv-scanner + osv-scanner --version + echo "OSV Scanner installed OK" +fi + +# --- OpenGrep ------------------------------------------------------------- +if should_install "opengrep"; then + echo "[setup-tools] Installing OpenGrep ${OPENGREP_VERSION}" + download_and_verify \ + "https://github.com/opengrep/opengrep/releases/download/${OPENGREP_VERSION}/opengrep_manylinux_x86" \ + "${TMP_DIR}/opengrep" \ + "${OPENGREP_SHA256}" + sudo install -m 0755 "${TMP_DIR}/opengrep" /usr/local/bin/opengrep + opengrep --version + echo "OpenGrep installed OK" +fi + +# --- Semgrep community ruleset (cloned, not registry) --------- +if should_install "semgrep-rules"; then + echo "[setup-tools] Cloning semgrep-rules @ ${SEMGREP_RULES_REF}" + rm -rf "${SEMGREP_RULES_DIR}" + git clone --quiet https://github.com/semgrep/semgrep-rules.git "${SEMGREP_RULES_DIR}" + git -C "${SEMGREP_RULES_DIR}" checkout --quiet "${SEMGREP_RULES_REF}" + echo "semgrep-rules ready at ${SEMGREP_RULES_DIR} (ref: ${SEMGREP_RULES_REF})" +fi + +# --- Hadolint --------------------------------------------------------- +if should_install "hadolint"; then + echo "[setup-tools] Installing Hadolint ${HADOLINT_VERSION}" + download_and_verify \ + "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VERSION}/hadolint-linux-x86_64" \ + "${TMP_DIR}/hadolint" \ + "${HADOLINT_SHA256}" + sudo install -m 0755 "${TMP_DIR}/hadolint" /usr/local/bin/hadolint + hadolint --version + echo "Hadolint installed OK" +fi + +# --- Gitleaks ----------------------------------------------------------- +if should_install "gitleaks"; then + echo "[setup-tools] Installing Gitleaks ${GITLEAKS_VERSION}" + GITLEAKS_TARBALL="gitleaks_${GITLEAKS_VERSION#v}_linux_x64.tar.gz" + download_and_verify \ + "https://github.com/gitleaks/gitleaks/releases/download/${GITLEAKS_VERSION}/${GITLEAKS_TARBALL}" \ + "${TMP_DIR}/${GITLEAKS_TARBALL}" \ + "${GITLEAKS_SHA256}" + sudo tar -xzf "${TMP_DIR}/${GITLEAKS_TARBALL}" -C /usr/local/bin gitleaks + gitleaks version + echo "Gitleaks installed OK" +fi + +# --- SBOM generation ---------------------------------------------------- +case "$SBOM_ECOSYSTEM" in + maven) + echo "Generating SBOM for Maven project" + mvn -B -ntp dependency:resolve -q + mvn -B -ntp org.cyclonedx:cyclonedx-maven-plugin:makeAggregateBom -q + ;; + npm) + echo "Generating SBOM for NPM project" + npm ci + npx --yes "@cyclonedx/cyclonedx-npm@${CYCLONEDX_NPM_VERSION}" --output-file target/bom.json + ;; + none) + echo "No SBOM generation needed" + ;; + *) + echo "Unknown SBOM_ECOSYSTEM: $SBOM_ECOSYSTEM" >&2 + exit 1 + ;; +esac diff --git a/ci/suppress_osv_scanner.toml b/ci/suppress_osv_scanner.toml new file mode 100644 index 0000000..0deaf2d --- /dev/null +++ b/ci/suppress_osv_scanner.toml @@ -0,0 +1 @@ +# No active suppressions. Add entries here only with an id, reason and an ignoreUntil date. diff --git a/ci/suppress_trivy.yaml b/ci/suppress_trivy.yaml new file mode 100644 index 0000000..2be407b --- /dev/null +++ b/ci/suppress_trivy.yaml @@ -0,0 +1,2 @@ +# Add entries here only with a CVE id, a reason, and a removal condition. +vulnerabilities: [] diff --git a/deployment/dev/.env.example b/deployment/dev/.env.example index dc4097e..7c37b4c 100644 --- a/deployment/dev/.env.example +++ b/deployment/dev/.env.example @@ -25,3 +25,14 @@ KEYCLOAK_REALM=MIP KEYCLOAK_CLIENT_ID=mippublic KEYCLOAK_CLIENT_SECRET=replace-me KEYCLOAK_SSL_REQUIRED=none +# Public Keycloak client for the mip Python client on users' own computers +# (account-page tokens and `mip login`). Empty: not offered. +KEYCLOAK_CLI_CLIENT_ID= +# Bearer tokens must also name this client in aud. Set together with +# KEYCLOAK_CLI_CLIENT_ID, once all this deployment's clients issue such tokens. +KEYCLOAK_REQUIRED_AUDIENCE= +# Require DPoP-bound tokens for the laptop client. Turn on only together with +# DPoP on the laptop client in Keycloak and a DPoP-capable mip-client. +KEYCLOAK_CLI_REQUIRE_DPOP=false +# Origin the UI's Content-Security-Policy allows for Keycloak, e.g. https://iam.ebrains.eu. +KEYCLOAK_ORIGIN= diff --git a/deployment/dev/docker-compose.yml b/deployment/dev/docker-compose.yml index 0061de9..beb31f2 100644 --- a/deployment/dev/docker-compose.yml +++ b/deployment/dev/docker-compose.yml @@ -167,6 +167,9 @@ services: KEYCLOAK_REALM: "${KEYCLOAK_REALM}" KEYCLOAK_CLIENT_ID: "${KEYCLOAK_CLIENT_ID}" KEYCLOAK_CLIENT_SECRET: "${KEYCLOAK_CLIENT_SECRET}" + KEYCLOAK_CLI_CLIENT_ID: "${KEYCLOAK_CLI_CLIENT_ID:-}" + KEYCLOAK_REQUIRED_AUDIENCE: "${KEYCLOAK_REQUIRED_AUDIENCE:-}" + KEYCLOAK_CLI_REQUIRE_DPOP: "${KEYCLOAK_CLI_REQUIRE_DPOP:-false}" KEYCLOAK_SSL_REQUIRED: "${KEYCLOAK_SSL_REQUIRED}" volumes: - ./config:/opt/platform/api @@ -181,6 +184,7 @@ services: PLATFORM_BACKEND_CONTEXT: services NOTEBOOK_ENABLED: 0 MIP_VERSION: "${MIP}" + KEYCLOAK_ORIGIN: "${KEYCLOAK_ORIGIN:-}" depends_on: - platform-backend restart: unless-stopped diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md index e9a1b5e..fa2c206 100644 --- a/deployment/kubernetes/README.md +++ b/deployment/kubernetes/README.md @@ -84,7 +84,7 @@ Prior to deploying it (on a microk8s K8s cluster of one or more nodes), there ar * `jupyterhub`: JupyterHub and single-user Jupyter images, hub resources, ingress, storage, crypt key, and notebook resource requests/limits. Rendered only when `platform-ui.notebook.enabled` is true, which also requires `keycloak.enabled`. Notebook traffic for the portal iframe should use platform-ui's `/notebook/` proxy; keep `jupyterhub.ingress.enabled: false` unless you need a separate admin-only host. Notebook examples are mounted only in operator mode on MIP infra clusters (mip-infra `common/examples`). KubeSpawner installs start without them. -* `keycloak`: toggles the connection parameters to the external Keycloak instance (`enabled`, `host`, `protocol`, `realm`). +* `keycloak`: toggles the connection parameters to the external Keycloak instance (`enabled`, `host`, `protocol`, `realm`), and `cliClientId` for using MIP from users' own computers (below). Copy `values.yaml` to a new file (for example `my-values.yaml`) and edit it in-place. A few important knobs: @@ -123,6 +123,8 @@ Register a Keycloak redirect URI for the Hub OAuth client: For Kubernetes, make sure the images set in `jupyterhub.image` and `jupyterhub.singleuser.image` are pushed to the configured registry or loaded onto every node that may run the pods. +`jupyterhub.spawner: operator-api` removes the hub's Kubernetes access entirely. The hub creates and deletes notebooks through the notebook operator's HTTPS spawn API (`jupyterhub.operatorApi.url`) instead of the Kubernetes API, and the chart renders no Role or RoleBinding, whatever `jupyterhub.rbac.create` says. The hub pod gets no ServiceAccount token at the standard path; it authenticates with a projected token bound to the audience `mip-notebook-operator`, which the kubelet rotates and the API server rejects, and trusts the spawn API's CA from the `mip-notebooks-ca` ConfigMap (`jupyterhub.operatorApi.caConfigMap`), which mip-infra copies into each federation. The hub picks up a rotated token or CA without a restart. This mode needs a hub image with the operator-api spawner and the spawn API deployed by mip-infra. + The reachability diagram from the legacy profiles is still valid as a reference for deciding the correct public URL: ![MIP Reachability Scheme](../docs/MIP_Configuration.png) @@ -138,6 +140,52 @@ If you deploy behind a reverse proxy or load balancer, forward the standard `X-F After tailoring `values.yaml` you can deploy the UI Helm chart. We still recommend deploying the engine Helm charts first, before installing the UI components. +### Using MIP from users' own computers + +The `mip` Python client signs in through a **public** Keycloak client of its own, one per federation, e.g. `mip-cli-`. Users create a token on the portal's account page, or run `mip login`, and the client renews access with Keycloak by itself. Set the client's ID in `keycloak.cliClientId`; leave it empty to keep the feature off (the account page then hides the guide). + +The client needs, in Keycloak: client authentication off; standard flow on with PKCE `S256` **enforced** (not an empty challenge method), redirect URI exactly `https:///account` (no wildcard) and web origin `https://`; the OAuth 2.0 Device Authorization Grant on; implicit flow, direct access grants and service accounts **off**; *Full scope allowed* **off**, with the platform client's data roles (or one composite role holding them all) added to the laptop client's dedicated scope instead — whenever a new dataset role is created on the platform client, add it there too, or laptop users silently lose that dataset; the `roles` client scope set to *Optional* (so other clients' and realm roles stay out of its tokens); a copy of the federation client's *User Client Role* mapper (client `KEYCLOAK_CLIENT_ID`, claim `authorities`); an *Audience* mapper for that client; the `offline_access` scope; a *Client Offline Session Idle* of 30 days and a *Client Offline Session Max* of 180 days. Keycloak applies offline maximums only when the realm's *Offline Session Max Limited* is on, and caps the client's at the realm's *Offline Session Max*; to leave the realm's other clients unaffected, set that realm maximum high (e.g. 3650 days). Without it, only the offline idle timeout ever signs a computer out. Also ask EBRAINS for an authentication flow override on the laptop client without the Cookie execution (every sign-in then needs the password, whatever the request asks) and for *Consent required* with a consent text naming the client. Users and roles stay on the federation's client. + +### Ending a laptop sign-in + +Revoke a computer's access through Keycloak; the next `mip` call then fails when it tries to renew. + +*As the user:* open the Keycloak account console at `https:///realms//account`, go to *Applications*, find `mip-cli-` and choose *Remove access*; this ends the offline session. On the laptop, `mip logout` clears the stored token. + +*As an administrator:* in *Users*, open the user, then *Sessions* to sign out the offline sessions and *Consents* to revoke the client's consent. Disabling the user also stops renewal. + +*Lost laptop:* revoke as above, have the user change their password if the laptop may have been unlocked, and tell the federation operator. After revocation, an access token already issued still works for up to 5 minutes. + +For the administrator steps at EBRAINS, contact the realm administrators; record the contact and expected response time here once confirmed. When a new dataset role is created on the platform client, add it to the laptop client's scope as well (see the settings above), or laptop users silently lose that dataset. + +Before turning the feature on (setting `keycloak.cliClientId`), check off: + +- [ ] The Keycloak client settings above are verified: the sign-in settings, the federation-limited scope, and the re-authentication override and consent (ask EBRAINS). +- [ ] `keycloak.requiredAudience` is set on the backend (step 6 under *Security settings for each federation*). +- [ ] A platform-ui image with `prompt=login` on the account page is deployed (name the minimum version here once released). +- [ ] The revocation runbook above (*Ending a laptop sign-in*) is in place. + + +### Security settings for each federation + +Do these in order; each step needs the one before it. + +1. **Deploy a backend with the token checks everywhere first.** Older backends accept any token of the realm, so one federation's token reads another's data. Laptop sign-in stays off until this is done. +2. **Give each federation its own platform client.** Federations sharing a client share tokens: their backends accept each other's. This needs a new client in Keycloak, with users and roles assigned to it. +3. **Set the platform client's *Backchannel Logout URL*** to `https:///services/logout/connect/back-channel/keycloak`. Without it, a disabled user keeps a portal session for up to 8 hours. +4. **Keep notebook pods away from Exaflow and the backend database.** Exaflow has no user authentication, and notebook pods run users' code. This chart ships no NetworkPolicy: `mip-infra` limits the notebooks' egress to the hub API, the backend API and DNS (`common/security/federation/templates/notebook-network-policy.yaml`). Clusters without it need an equivalent policy, on a CNI that enforces NetworkPolicies. A policy on Exaflow's pods does not help if another policy opens them to the namespace. From a notebook terminal, check that each of these times out; a name resolution error proves nothing: + ```bash + curl -m 5 http://exaflow-controller-service:5000/healthcheck + python -c "import socket; socket.create_connection(('exaflow-workers-service', 5672), 5)" + python -c "import socket; socket.create_connection(('exaflow-aggregation-service', 50051), 5)" + ``` +5. **Create the secret `exaflow-api-token`** (key `token`) in the namespace, then restart the backend, the Exaflow controller and the Exaflow workers; all of them read it. The controller then requires the token on its HTTP API, except on `/livez` and `/healthcheck`. It does not protect the workers or the aggregation server. +6. **Set `keycloak.requiredAudience`** to the platform client once its portal, notebook and laptop tokens all carry it in `aud`. Keycloak does not add it by default: add an *Audience* mapper for the platform client to each client that issues these tokens. Decode one of each first — for a laptop token, check `azp` (the laptop client), `aud` (contains the platform client) and `typ` (`Bearer`) — a wrong value refuses every API call. Laptop sign-in (`keycloak.cliClientId`) stays off until this is set. + +Upgrade notes: + +- Users without a role on the platform client can no longer open notebooks, and users without a shared experiment's datasets can no longer read it. +- The chart does not change image tags. `EXAFLOW_API_TOKEN`, `KEYCLOAK_REQUIRED_AUDIENCE` and `KEYCLOAK_ORIGIN` do nothing until you deploy images that read them. ### Microk8s installation On a running Ubuntu (we recommend 22.04) distribution, install microk8s (we **HIGHLY** recommend to **NOT** install Docker on your Kubernetes cluster!): diff --git a/deployment/kubernetes/templates/jupyterhub.yaml b/deployment/kubernetes/templates/jupyterhub.yaml index 69bdeda..68d3fb0 100644 --- a/deployment/kubernetes/templates/jupyterhub.yaml +++ b/deployment/kubernetes/templates/jupyterhub.yaml @@ -12,7 +12,14 @@ {{- $keycloakRealm := default "MIP" .Values.keycloak.realm -}} {{- $keycloakAuthUrl := include "mip.keycloakAuthUrl" . -}} {{- $publicHost := .Values.global.publicHost -}} -{{- $useOperator := eq (default "kubespawner" $jupyterHub.spawner) "operator" -}} +{{- $spawner := default "kubespawner" $jupyterHub.spawner -}} +{{- if not (has $spawner (list "kubespawner" "operator" "operator-api")) -}} +{{- fail (printf "jupyterhub.spawner must be kubespawner, operator or operator-api, not %q" $spawner) -}} +{{- end -}} +{{- /* Both operator modes render the NotebookProfile; operator-api also drops the hub's Kubernetes access. */ -}} +{{- $useOperator := has $spawner (list "operator" "operator-api") -}} +{{- $useOperatorApi := eq $spawner "operator-api" -}} +{{- $operatorApi := default (dict) $jupyterHub.operatorApi -}} {{- if $jupyterHub.cryptKey }} --- # Without jupyterhub.cryptKey, create the secret out-of-band (see README) so the @@ -50,7 +57,8 @@ metadata: name: jupyterhub namespace: {{ $namespace }} -{{- if ne (toString $jupyterHub.rbac.create) "false" }} +{{- /* operator-api: the hub has no Kubernetes permissions at all. */ -}} +{{- if and (ne (toString $jupyterHub.rbac.create) "false") (not $useOperatorApi) }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -64,7 +72,7 @@ rules: # the per-server API token Secret, owned by its Notebook. - apiGroups: ["notebooks.mip.ebrains.eu"] resources: ["notebooks"] - verbs: ["get", "list", "watch", "create", "delete"] + verbs: ["get", "create", "delete"] - apiGroups: [""] resources: ["secrets"] verbs: ["create"] @@ -157,6 +165,11 @@ spec: component: hub spec: serviceAccountName: jupyterhub +{{- if $useOperatorApi }} + # No Kubernetes credentials: the hub only holds the audience-bound token + # in notebook-api below, which the API server rejects. + automountServiceAccountToken: false +{{- end }} {{- if eq (toString .Values.cluster.managed) "false" }} nodeSelector: master: "true" @@ -165,6 +178,23 @@ spec: - name: jupyterhub-claim0 persistentVolumeClaim: claimName: jupyterhub-claim0 +{{- if $useOperatorApi }} + # Token for the notebook operator's spawn API. Its audience is not the + # API server's, so the API server rejects it; the kubelet rotates it. + # The spawn API accepts only this audience (a constant there). + - name: notebook-api + projected: + sources: + - serviceAccountToken: + path: token + audience: mip-notebook-operator + expirationSeconds: {{ $operatorApi.tokenExpirationSeconds }} + - configMap: + name: {{ $operatorApi.caConfigMap | quote }} + items: + - key: ca.crt + path: ca.crt +{{- end }} containers: - name: jupyterhub image: {{ $hubImage | quote }} @@ -181,7 +211,15 @@ spec: fieldRef: fieldPath: metadata.namespace - name: JUPYTERHUB_SPAWNER - value: {{ ternary "operator" "kubespawner" $useOperator | quote }} + value: {{ $spawner | quote }} +{{- if $useOperatorApi }} + - name: NOTEBOOK_API_URL + value: {{ $operatorApi.url | quote }} + - name: NOTEBOOK_API_TOKEN_FILE + value: /var/run/secrets/mip-notebook-operator/token + - name: NOTEBOOK_API_CA_FILE + value: /var/run/secrets/mip-notebook-operator/ca.crt +{{- end }} - name: JUPYTER_SINGLEUSER_IMAGE value: {{ $singleuserImage | quote }} {{- if eq (toString .Values.cluster.managed) "false" }} @@ -247,10 +285,15 @@ spec: volumeMounts: - name: jupyterhub-claim0 mountPath: /srv/jupyterhub +{{- if $useOperatorApi }} + - name: notebook-api + mountPath: /var/run/secrets/mip-notebook-operator + readOnly: true +{{- end }} {{- if $useOperator }} --- -# Pod template for the MIP notebook operator (mip-jupyter/operator). The hub +# Pod template for the MIP notebook operator (operator/ in this repository). The hub # cannot read or change it; the operator applies the security context itself. apiVersion: notebooks.mip.ebrains.eu/v1alpha1 kind: NotebookProfile diff --git a/deployment/kubernetes/templates/platform-backend.yaml b/deployment/kubernetes/templates/platform-backend.yaml index e3b5fbd..b9f3eda 100644 --- a/deployment/kubernetes/templates/platform-backend.yaml +++ b/deployment/kubernetes/templates/platform-backend.yaml @@ -1,3 +1,6 @@ +{{- if and (default "" .Values.keycloak.cliClientId) (not (default "" .Values.keycloak.requiredAudience)) -}} +{{- fail "keycloak.cliClientId requires keycloak.requiredAudience: laptop tokens would be accepted without the audience check. See the README turn-on checklist and rollout step 6." -}} +{{- end -}} {{- $namespace := default "default" .Release.Namespace -}} {{- $platformBackend := index .Values "platform-backend" -}} {{- $authEnabled := ne (toString .Values.keycloak.enabled) "false" -}} @@ -201,6 +204,18 @@ spec: value: {{ $keycloakAuthUrl | quote }} - name: KEYCLOAK_REALM value: {{ $keycloakRealm | quote }} + - name: KEYCLOAK_CLI_CLIENT_ID + value: {{ default "" .Values.keycloak.cliClientId | quote }} + - name: KEYCLOAK_CLI_REQUIRE_DPOP + value: {{ ternary "true" "false" (eq (toString .Values.keycloak.cliRequireDpop) "true") | quote }} + - name: EXAFLOW_API_TOKEN + valueFrom: + secretKeyRef: + name: exaflow-api-token + key: token + optional: true + - name: KEYCLOAK_REQUIRED_AUDIENCE + value: {{ default "" .Values.keycloak.requiredAudience | quote }} - name: KEYCLOAK_CLIENT_ID valueFrom: secretKeyRef: diff --git a/deployment/kubernetes/templates/platform-ui.yaml b/deployment/kubernetes/templates/platform-ui.yaml index 2d69062..dae60b9 100644 --- a/deployment/kubernetes/templates/platform-ui.yaml +++ b/deployment/kubernetes/templates/platform-ui.yaml @@ -50,6 +50,10 @@ spec: value: {{ printf "%s:%v" $backendHost $backendPort | quote }} - name: PLATFORM_BACKEND_CONTEXT value: {{ $backendContext | quote }} +{{- end }} +{{- if .Values.keycloak.host }} + - name: KEYCLOAK_ORIGIN + value: {{ printf "%s://%s" (default "https" .Values.keycloak.protocol) .Values.keycloak.host | quote }} {{- end }} - name: NOTEBOOK_ENABLED value: {{ ternary "1" "0" $notebookEnabled | quote }} diff --git a/deployment/kubernetes/values.yaml b/deployment/kubernetes/values.yaml index 6cbbc59..a8062ae 100644 --- a/deployment/kubernetes/values.yaml +++ b/deployment/kubernetes/values.yaml @@ -52,6 +52,18 @@ platformBackendDatabase: keycloak: enabled: true host: iam.ebrains.eu + # Public Keycloak client for the mip Python client on users' own computers + # (account-page tokens and `mip login`). Empty: not offered. See README. + # Turns laptop sign-in on. Set only after the checklist in README + # *Using MIP from users' own computers*. + cliClientId: "" + # Require DPoP-bound tokens for the laptop client (RFC 9449). Turn on only + # together with DPoP on the laptop client in Keycloak and a DPoP-capable + # mip-client; platform and notebook tokens keep working over Bearer. + cliRequireDpop: false + # When set, Bearer tokens must also name this in aud. Set it once all this federation's + # clients issue such tokens. Empty: aud is not checked (azp always is). + requiredAudience: "" jupyterhub: # Leave empty and create the jupyterhub-crypt secret out-of-band (see README), @@ -70,14 +82,26 @@ jupyterhub: ingress: enabled: false # kubespawner: the hub creates notebook pods itself (Role on pods/PVCs). - # operator: the MIP notebook operator does (mip-jupyter/operator); the hub + # operator: the MIP notebook operator does (operator/ in this repository); the hub # only creates Notebook resources. Needs the operator and its CRDs installed. + # operator-api: as operator, but the hub asks the operator's spawn API over + # HTTPS (see operatorApi) and holds no Kubernetes credentials or Role. spawner: kubespawner rbac: # false: an admin applies the hub Role/RoleBinding (e.g. with # mip-infra-staging's out-of-band RBAC), so the GitOps tool needs no - # RBAC write. The rules must match the spawner mode. + # RBAC write. The rules must match the spawner mode. Ignored with + # operator-api: no Role or RoleBinding is rendered. create: true + # operator-api only. Needs a hub image with the operator-api spawner and the + # spawn API deployed (mip-infra). The hub authenticates with a ServiceAccount + # token bound to the spawn API's audience: the kubelet rotates it, and the + # API server rejects it. caConfigMap holds the spawn API's CA under ca.crt + # (mip-infra copies it into each federation). + operatorApi: + url: https://mip-notebook-spawn-api.mip-notebooks-system.svc.cluster.local + caConfigMap: mip-notebooks-ca + tokenExpirationSeconds: 3600 # OpenAI-compatible inference server for the Jupyter AI Codex persona. # baseUrl ends in /v1; empty leaves Codex unconfigured. Point it at a proxy # (e.g. mip-infra-staging's athena-proxy) to keep the real server URL out of diff --git a/documentation/notebook-operator.md b/documentation/notebook-operator.md new file mode 100644 index 0000000..9679fdc --- /dev/null +++ b/documentation/notebook-operator.md @@ -0,0 +1,321 @@ +# Notebook operator + +JupyterHub no longer creates pods or PVCs. It creates a `Notebook` custom +resource; the MIP notebook operator (`operator/`, Operator SDK `go/v4`) turns it +into a pod and a PVC from a `NotebookProfile` the hub cannot edit. + +## Why + +With KubeSpawner the hub needed a Role with `pods`/`persistentvolumeclaims` +create/delete in every `federation-*` namespace. When Argo CD creates that +Role, RBAC escalation prevention forces the Argo application controller to hold +`pods: create` (and RBAC write) cluster-wide. A compromised hub could run any image in a +namespace that also holds the platform databases and Exareme workers. + +After the switch: + +| Component | Can create pods | How | +|---|---|---| +| JupyterHub, `operator` mode | no | `notebooks` create/get/list/watch/delete, `secrets` create (Role applied out-of-band by an admin) | +| JupyterHub, `operator-api` mode | no | no Kubernetes credential at all; calls the spawn API | +| Spawn API | no | `notebooks` get/create/delete, `secrets` create per federation, `tokenreviews` create | +| Argo CD controller | no | no RBAC write either; only syncs `NotebookProfile` | +| Notebook operator | yes, only in namespaces an admin bound | ClusterRole + per-namespace RoleBinding; no access to Secrets | + +`operator-api` is the target: a compromised hub then holds nothing the +Kubernetes API server accepts. + +## Resources + +API group `notebooks.mip.ebrains.eu/v1alpha1`, both namespaced. + +### NotebookProfile + +Shipped by the chart in this repository (`deployment/kubernetes`), synced by Argo CD. The hub has no access. + +```yaml +spec: + image: hbpmip/mip-jupyter:0.0.1_candidate + imagePullPolicy: IfNotPresent + resources: {requests: {cpu: 500m, memory: 1G}, limits: {cpu: "1", memory: 4G}} + storage: + storageClassName: ceph-corbo-cephfs + size: 2Gi + mountPath: /home/jovyan + fixOwnership: true # root init container chowns the volume to 1000:100 (provisioners that ignore fsGroup) + nodeSelector: {} + env: # static, non-secret + - {name: PLATFORM_BACKEND_URL, value: http://platform-backend-service:8080/services} +``` + +The operator, not the profile, sets the security context: `runAsUser: 1000`, +`runAsGroup: 100`, `fsGroup: 100`, `runAsNonRoot`, seccomp `RuntimeDefault`, +all capabilities dropped, no privilege escalation, no service account token. +The notebook listens on port 8888. + +### Notebook + +Created per server by the hub, or by the spawn API on its behalf. + +```yaml +metadata: {name: jupyter-{user_server}} # KubeSpawner's pod name ("safe" slugs) +spec: + profile: default + user: + serverName: "" + env: {JUPYTERHUB_SERVICE_PREFIX: /notebook/user//, ...} +status: + phase: Pending | Running | Failed + url: http://:8888 + message: +``` + +CRD validation (CEL, enforced by the API server): + +- `spec` is immutable; a restart is delete + create. +- `env` keys match `^(JUPYTERHUB|JPY)_[A-Z0-9_]+$` and exclude + `JUPYTERHUB_API_TOKEN` / `JPY_API_TOKEN`. +- Names start with `jupyter-` and are at most 57 characters. + +The hub (or, in `operator-api` mode, the spawn API) puts the hub API token in +Secret `--token` (key `token`, owned by the +`Notebook`). The API server assigns the UID, so no +Secret that existed before the `Notebook` is ever mounted: not +`keycloak-credentials`, and not a token left by an earlier `Notebook` of the +same name. The hub can create Secrets but not overwrite them. The +ValidatingAdmissionPolicy `mip-jupyterhub-secrets` (mip-infra, test copy in +`operator/testdata/token-secrets-policy.yaml`) +admits only Opaque Secrets named and owned that way from the hub and from the +spawn API (both, so hubs can move one at a time), so neither can +create a `kubernetes.io/service-account-token` Secret, which the API server +would fill with another ServiceAccount's token, for a pod to mount. The +operator never reads Secrets. + +## Operator + +One Deployment in `mip-notebooks-system`, leader election on, watching only the +namespaces in `WATCH_NAMESPACES` (comma separated). Adding a federation means +adding it to that list and binding the ClusterRole there. + +Shared examples (optional): with `EXAMPLES_CLAIM` set (mip-infra uses +`mip-examples`), every new notebook pod mounts that claim read-only at +`work/.mip-examples` under the profile's `storage.mountPath`, where the +single-user entrypoint links `examples/` to it (every pod gets `MIP_WORK_DIR` = +`/work`, so the entrypoint looks there). The mount is added only when the +claim in the notebook's namespace is `Bound` (and not being deleted) and +the profile sets `fixOwnership: true` (its init container creates `work/`, so the +runtime does not create it as root for the mount point). A missing or unbound +claim is skipped and the notebook starts without examples; other API +errors retry the reconcile. A claim pruned between that check and pod creation +leaves the pod `Pending` until the notebook is recreated. The decision is in the +"created notebook pod" log line. Rollback: unset `EXAMPLES_CLAIM`, and new +notebooks start without the mount; running ones keep it until they restart. + +Reconcile of a `Notebook`: + +1. Deleted: nothing to do. Pod and token Secret are owned by the `Notebook` and + garbage collected. +2. No pod yet: load the profile. Missing: `Failed`, retried every 30s. A running + pod does not need its profile, so deleting one does not stop servers. +3. Ensure PVC `claim-{user_server}`, KubeSpawner's name for the user's default + server, so homes KubeSpawner created are reused. Never owned, updated or + deleted, so home directories survive restarts. +4. Ensure pod (bare pod, owned by the `Notebook`). A pod that ends is not + recreated; the hub sees `Failed` and the user starts again. Until the hub + has created the token Secret the kubelet holds the pod in + `CreateContainerConfigError`, reported as the message. If the API server + rejects the PVC or pod (quota, Pod Security), `Pending` with that error, + retried with backoff. +5. Status from the pod: `Running` + `url` when Ready, `Pending` with the waiting + reason (`ImagePullBackOff`, `Unschedulable`, ...), `Failed` with the + termination message. A pod of that name the `Notebook` does not own, such as + one KubeSpawner left, is `Failed`: delete it before starting again. + +Profile edits apply from the next start; running pods are not touched. + +Operator RBAC (ClusterRole `mip-notebook-operator`, generated in +`operator/config/rbac/role.yaml`, bound per namespace): `pods` and +`persistentvolumeclaims` get/list/watch/create, `notebooks` and +`notebookprofiles` get/list/watch, `notebooks/status` get/update/patch. No +Secrets, no delete: garbage collection removes pods. + +## Spawn API + +The same image with `--mode=spawn-api`: a separate Deployment in +`mip-notebooks-system`, ServiceAccount `mip-notebook-spawn-api`, 2 replicas, +no leader election (stateless), no informer cache (a direct client; RBAC +limits it to the namespaces it is bound in). Served at +`https://mip-notebook-spawn-api.mip-notebooks-system.svc.cluster.local` +(Service 443 to container 8443). + +```text +--mode=spawn-api +--spawn-api-bind-address=:8443 +--tls-cert-dir=/etc/mip-notebook-spawn-api/tls # tls.crt, tls.key; reloaded on change +--health-probe-bind-address=:8081 # /healthz, /readyz +--metrics-bind-address=0 # off by default +--spawn-api-rate-limit=5 --spawn-api-rate-burst=20 # per namespace +``` + +TLS 1.2 minimum. The certificate (cert-manager in mip-infra) must name the +Service DNS name above: the hub verifies it against its CA file. + +### Authentication and authorization + +`Authorization: Bearer `, checked with a TokenReview for audience +`mip-notebook-operator`. The request passes only when the review says +`authenticated` and lists that audience in `status.audiences`. Every request +makes its own TokenReview, uncached: a pod-bound token stops working as soon as +its hub pod is deleted. Hub traffic (one PUT per spawn, then status polls) keeps +that load small. + +The user must be exactly `system:serviceaccount:{ns}:jupyterhub`, where `{ns}` +is the namespace in the path and matches `^federation-[a-z0-9-]+$`. Anything +else is 403. Then a token bucket per namespace (429 with `Retry-After`). +Each denial is logged with namespace, remote address, reason and user name; +tokens and bodies are never logged. Metrics: `mip_spawn_api_requests_total` +(method, code) and `mip_spawn_api_denied_total` (reason). + +### Endpoints + +JSON; errors are `{"error": ""}`. 401 unauthenticated, 403 +forbidden, 429 rate limited, 503 when the TokenReview itself fails. + +| Request | Result | +|---|---| +| `PUT /v1/namespaces/{ns}/servers/{name}` | 201 created; 200 same spec again; 409 other spec or terminating; 400 invalid; 413 body over 64 KiB | +| `GET /v1/namespaces/{ns}/servers/{name}` | 200 `{"phase", "url", "message", "deleting"}`; 404 gone | +| `DELETE /v1/namespaces/{ns}/servers/{name}` | 202 while the `Notebook` exists (foreground delete); 404 gone | + +PUT body: `{"profile", "user", "serverName", "env", "apiToken"}`, unknown +fields rejected. `name` and `env` follow the CRD rules (`jupyter-*`, at most +57 characters, DNS label; `JUPYTERHUB_*`/`JPY_*` keys, no token keys); +`apiToken` is required. The spawn API creates the `Notebook` with annotation +`notebooks.mip.ebrains.eu/requested-by: ` (the API server's +audit log now shows only the spawn API), then the token Secret (Opaque, one +controller owner reference to the `Notebook`). If the Secret fails it deletes +the `Notebook` and answers 500. + +The spawn API cannot read Secrets, so a PUT for an existing `Notebook` with the +same spec creates the Secret to find out whether it exists: an `AlreadyExists` +means it does, a success completes a request that stopped between the two. +It cannot tell whether an existing Secret holds the same `apiToken`; the hub +always deletes and waits for 404 before a PUT, so that only arises for a retry +of the same start. + +### Why a separate account + +The controller can create pods in every federation, which means it can read +every Secret there by mounting it. The spawn API faces the hubs over the +network. Kept apart: + +- the controller has no listener the hubs can reach, and never gets + `secrets: create`; +- the spawn API has no pods, PVCs, Secret reads, list or watch. Compromised, it + can start notebooks from the profiles the hubs already use and write token + Secrets the admission policy admits, in the federations it is bound in. + +RBAC, deployed from mip-infra +`base/mip-infrastructure/notebook-operator/rbac.yaml`. The copy the tests apply, +`operator/testdata/spawn-api-role.yaml`, is hand-maintained (not kubebuilder +markers, which would merge into `manager-role`); change both together: + +- ClusterRole `mip-notebook-spawn-api`, bound per federation namespace: + `notebooks` get/create/delete, `secrets` create. +- ClusterRole `mip-notebook-spawn-api-tokenreview`, bound cluster-wide: + `tokenreviews` create. + +## Hub spawner + +`JUPYTERHUB_SPAWNER` selects one (default `kubespawner` until the operator is +installed): + +- `operator`: mip-jupyter `docker/hub/notebook_spawner.py`, talks to the Kubernetes API. +- `operator-api`: mip-jupyter `docker/hub/notebook_api_spawner.py`, talks to the spawn API + with tornado's `AsyncHTTPClient`. Imports no Kubernetes client. + +Both: + +- `start()`: take `JUPYTERHUB_API_TOKEN` / `JPY_API_TOKEN` out of + `get_env()`, create the `Notebook` (replacing a leftover one), create the + token Secret `--token` owned by it, wait for `Running`, return + `status.url`. On timeout delete the `Notebook`. +- `progress()`: yields `status.message` changes. +- `poll()`: `None` while `Pending`/`Running`, `1` when `Failed` or gone. +- `stop()`: delete the `Notebook` (foreground) and wait until it is gone. + +`operator-api` maps these onto the endpoints above: `start()` is DELETE and +wait for 404 (60 s), PUT, then GET every second until `Running` with a `url` +or `Failed` (then DELETE and wait). It retries 429 (after `Retry-After`), 502, +503, 504 and no response, up to 6 attempts. Environment (chart in this repository): + +| Variable | Value | +|---|---| +| `NOTEBOOK_API_URL` | `https://mip-notebook-spawn-api.mip-notebooks-system.svc.cluster.local` | +| `NOTEBOOK_API_TOKEN_FILE` | `/var/run/secrets/mip-notebook-operator/token`: projected, audience `mip-notebook-operator`, 3600 s; read on every request | +| `NOTEBOOK_API_CA_FILE` | `/var/run/secrets/mip-notebook-operator/ca.crt`: passed as `ca_certs`, loaded per connection | +| `POD_NAMESPACE` | downward API | + +The hub pod sets `automountServiceAccountToken: false`. + +`MIP_TOKEN` is no longer put in the pod env (the `env` rule rejects it). The +`mip` client fetches the first token from the hub `/api/platform-token` +endpoint, as it already did for refreshes. + +## Rollout + +1. `mip`: operator image; `mip-jupyter`: hub image with both spawners, client change. +2. `mip-infra`: install CRDs + operator, the hub Secret policy (Kubernetes 1.30+), + out-of-band operator and hub RBAC, whitelist `NotebookProfile` in the federation AppProject, Argo + controller gets `notebookprofiles` write. +3. `mip` chart: `jupyterhub.spawner: operator` renders the profile; + `jupyterhub.rbac.create: false` leaves the hub Role to the admin. +4. `mip-infra`: drop `pods: create` and RBAC write from the Argo + controller. + +In `operator` mode the operator and hub images must agree on the token Secret +name and the CRD. Upgrade the CRD and operator first, then the hub, with no +server running in operator mode in between. + +Moving to `operator-api`, per federation: + +1. `mip-infra`: the spawn API Deployment, Service and certificate; the + updated Secret policy (admits both accounts); RoleBinding to + `mip-notebook-spawn-api` in the federation, ClusterRoleBinding to + `mip-notebook-spawn-api-tokenreview` once. +2. `mip` chart: `operator-api` mode renders the env above, the + projected token and CA volume, and `automountServiceAccountToken: false`. + Running servers survive: both modes use the same `Notebook` names. +3. Drop the hub Role and RoleBinding. Once every federation has moved, drop + the `jupyterhub` match from the Secret policy. + +## Contracts with other repositories + +The operator is built here, the hub and single-user images in mip-jupyter, and +the cluster installation is in mip-infra. A change to any of these needs both sides: + +| Between | Contract | +|---|---| +| Hub spawner, spawn API | `/v1` endpoints, audience `mip-notebook-operator`, name rule (`jupyter-*`, at most 57 characters), `env` key rule | +| Hub `operator` spawner, operator | `Notebook` CRD, token Secret name `--token`. Goes with that spawner once every federation runs `operator-api` | +| Single-user image, operator | Port 8888, uid `1000:100`, `MIP_WORK_DIR`, `work/.mip-examples` | +| mip-infra, operator | Flags and env above, ClusterRoles, policy `mip-jupyterhub-secrets`, CRDs from `operator/config/crd/bases` at the release tag | + +## Testing + +- Operator: `cd operator && make test` (envtest: CRD validation rules, hub + Secret policy, profile missing, pod build, status mapping, KubeSpawner PVC + names, shared examples mount). +- Spawn API: `cd operator && make test` also runs `internal/spawnapi` + (envtest: TokenReview audiences, authorization, PUT/GET/DELETE semantics, + Secret rollback, rate limit, TLS reload, and a spawn as the spawn API + account with only its RBAC through the Secret policy, mounted by the + controller). +- Spawners: mip-jupyter `docker/hub/test_notebook_spawner.py` and + `test_notebook_api_spawner.py`, run inside the hub image (CI there does this + after building it). +- Staging: spawn, stop, restart the hub with a server running, and confirm + `kubectl auth can-i create pods --as=system:serviceaccount::jupyterhub` + returns `no`. In `operator-api` mode also confirm the hub pod has no + `/var/run/secrets/kubernetes.io/serviceaccount`, and that its projected + token is refused by the API server (401). diff --git a/operator/.dockerignore b/operator/.dockerignore new file mode 100644 index 0000000..a3aab7a --- /dev/null +++ b/operator/.dockerignore @@ -0,0 +1,3 @@ +# More info: https://docs.docker.com/engine/reference/builder/#dockerignore-file +# Ignore build and test binaries. +bin/ diff --git a/operator/.gitignore b/operator/.gitignore new file mode 100644 index 0000000..869fe9f --- /dev/null +++ b/operator/.gitignore @@ -0,0 +1,2 @@ +bin/ +cover.out diff --git a/operator/Dockerfile b/operator/Dockerfile new file mode 100644 index 0000000..0f63440 --- /dev/null +++ b/operator/Dockerfile @@ -0,0 +1,18 @@ +# Base images are pinned by digest; bump the digest to pick up base fixes. +FROM golang:1.27@sha256:f20d83a83e18dbd2a4714b8ffd4b30b287b6243962dd75608e86abcc63ba4efc AS builder +ARG TARGETOS +ARG TARGETARCH +WORKDIR /workspace +# Dependencies first, so source changes reuse the module download layer. +COPY go.mod go.sum ./ +RUN go mod download +COPY cmd/main.go cmd/main.go +COPY api/ api/ +COPY internal/ internal/ +RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o manager cmd/main.go + +FROM gcr.io/distroless/static:nonroot@sha256:18f4cc26c9ffa7f20a359298be7deadac5c607c8e03a02b8119f1a5fc2f82c99 +WORKDIR / +COPY --from=builder /workspace/manager . +USER 65532:65532 +ENTRYPOINT ["/manager"] diff --git a/operator/Makefile b/operator/Makefile new file mode 100644 index 0000000..fa944d0 --- /dev/null +++ b/operator/Makefile @@ -0,0 +1,29 @@ +LOCALBIN ?= $(shell pwd)/bin +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest +CONTROLLER_TOOLS_VERSION ?= v0.23.0 +ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') +ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') + +.PHONY: manifests +manifests: $(CONTROLLER_GEN) ## CRDs (config/crd/bases) and RBAC (config/rbac/role.yaml). + $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: generate +generate: $(CONTROLLER_GEN) ## DeepCopy methods. + $(CONTROLLER_GEN) object paths="./..." + +.PHONY: test +test: manifests generate $(ENVTEST) ## envtest controller tests. + go fmt ./... + go vet ./... + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out + +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +$(CONTROLLER_GEN): | $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +$(ENVTEST): | $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@$(ENVTEST_VERSION) diff --git a/operator/README.md b/operator/README.md new file mode 100644 index 0000000..5f0b296 --- /dev/null +++ b/operator/README.md @@ -0,0 +1,26 @@ +# mip-notebook-operator + +Builds JupyterHub notebook pods from `Notebook` / `NotebookProfile` resources so +the hub needs no permission on pods or PVCs. With `--mode=spawn-api` the same +image serves the hubs' HTTPS spawn API, so they need no Kubernetes credential +at all. Design, RBAC and rollout: +[documentation/notebook-operator.md](../documentation/notebook-operator.md). + +Scaffolded with Operator SDK (`go/v4` plugin). + +```bash +make test # envtest controller and spawn API tests +make manifests generate # after editing api/ or RBAC markers; commit the output +docker build -t hbpmip/mip-notebook-operator: . +``` + +CRDs: `config/crd/bases/`. Operator RBAC: `config/rbac/role.yaml` (generated). +`testdata/` holds the spawn API RBAC and the policy on the token Secrets that the +tests apply: hand-maintained copies of mip-infra's +`base/mip-infrastructure/notebook-operator/rbac.yaml`, not deployed from here. +The deployments live in `mip-infra`. The controller (`--mode=controller`, +the default) needs `WATCH_NAMESPACES` (comma separated) and leader election +(`--leader-elect`). The spawn API (`--mode=spawn-api`) needs neither; it takes +`--spawn-api-bind-address`, `--tls-cert-dir`, `--health-probe-bind-address`, +and optionally `--metrics-bind-address` and `--spawn-api-rate-limit` / +`--spawn-api-rate-burst`. diff --git a/operator/api/v1alpha1/groupversion_info.go b/operator/api/v1alpha1/groupversion_info.go new file mode 100644 index 0000000..a5f22ae --- /dev/null +++ b/operator/api/v1alpha1/groupversion_info.go @@ -0,0 +1,20 @@ +// Package v1alpha1 contains API Schema definitions for the notebooks v1alpha1 API group. +// +kubebuilder:object:generate=true +// +groupName=notebooks.mip.ebrains.eu +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/scheme" +) + +var ( + // GroupVersion is group version used to register these objects. + GroupVersion = schema.GroupVersion{Group: "notebooks.mip.ebrains.eu", Version: "v1alpha1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme. + SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/operator/api/v1alpha1/notebook_types.go b/operator/api/v1alpha1/notebook_types.go new file mode 100644 index 0000000..2e93703 --- /dev/null +++ b/operator/api/v1alpha1/notebook_types.go @@ -0,0 +1,94 @@ +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Notebook phases reported in status.phase. +const ( + PhasePending = "Pending" + PhaseRunning = "Running" + PhaseFailed = "Failed" +) + +// NotebookSpec is what JupyterHub asks for. Everything that decides what runs +// (image, resources, volumes, security) comes from the referenced profile. +// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="spec is immutable; delete and recreate the Notebook" +type NotebookSpec struct { + // Profile is the name of a NotebookProfile in the same namespace. + // +kubebuilder:validation:MinLength=1 + Profile string `json:"profile"` + + // User is the JupyterHub user name. It names the home PVC. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=253 + User string `json:"user"` + + // ServerName is the JupyterHub named server, empty for the default server. + // +optional + ServerName string `json:"serverName,omitempty"` + + // Env holds the per-spawn JupyterHub variables. Only JUPYTERHUB_* and JPY_* + // names are accepted; the API token comes from the Notebook's token Secret. + // +optional + // +kubebuilder:validation:XValidation:rule="self.all(k, k.matches('^(JUPYTERHUB|JPY)_[A-Z0-9_]+$'))",message="env keys must match ^(JUPYTERHUB|JPY)_[A-Z0-9_]+$" + // +kubebuilder:validation:XValidation:rule="!('JUPYTERHUB_API_TOKEN' in self) && !('JPY_API_TOKEN' in self)",message="the API token comes from the Notebook's token Secret" + Env map[string]string `json:"env,omitempty"` +} + +// NotebookStatus is the observed state of the notebook pod. +type NotebookStatus struct { + // +optional + // +kubebuilder:validation:Enum=Pending;Running;Failed + Phase string `json:"phase,omitempty"` + + // URL is where the hub proxy reaches the notebook server, set when Running. + // +optional + URL string `json:"url,omitempty"` + + // Message is the latest human readable reason, shown on the spawn page. + // +optional + Message string `json:"message,omitempty"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="User",type=string,JSONPath=`.spec.user` +// +kubebuilder:printcolumn:name="Phase",type=string,JSONPath=`.status.phase` +// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.message`,priority=1 +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` + +// Notebook is one JupyterHub single-user server. The hub creates Secret +// --token (key "token", owned by the Notebook) with its API token. +// The API server assigns the UID, so no Secret that existed before the +// Notebook, such as one left by an earlier Notebook of the same name, is ever +// mounted. Names are jupyter-* and at most 57 characters, as the hub makes them. +// +kubebuilder:validation:XValidation:rule="self.metadata.name.startsWith('jupyter-') && size(self.metadata.name) <= 57",message="name must start with jupyter- and be at most 57 characters" +type Notebook struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NotebookSpec `json:"spec"` + Status NotebookStatus `json:"status,omitempty"` +} + +// TokenSecretKey is the key of the API token in the token Secret. +const TokenSecretKey = "token" + +// TokenSecretName is the name of the Notebook's token Secret. +func (nb *Notebook) TokenSecretName() string { + return nb.Name + "-" + string(nb.UID) + "-token" +} + +// +kubebuilder:object:root=true + +// NotebookList contains a list of Notebook. +type NotebookList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []Notebook `json:"items"` +} + +func init() { + SchemeBuilder.Register(&Notebook{}, &NotebookList{}) +} diff --git a/operator/api/v1alpha1/notebookprofile_types.go b/operator/api/v1alpha1/notebookprofile_types.go new file mode 100644 index 0000000..5a00e3c --- /dev/null +++ b/operator/api/v1alpha1/notebookprofile_types.go @@ -0,0 +1,71 @@ +package v1alpha1 + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NotebookProfileSpec is the pod template for notebooks. It is shipped with the +// chart; the hub cannot read or change it. The security context is not part of +// it on purpose: the operator always applies the restricted one. +type NotebookProfileSpec struct { + // +kubebuilder:validation:MinLength=1 + Image string `json:"image"` + + // +optional + // +kubebuilder:default=IfNotPresent + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + Storage NotebookStorage `json:"storage"` + + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Env is static, non-secret configuration for every notebook. + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` +} + +// NotebookStorage describes the per-user home volume. +type NotebookStorage struct { + // +optional + StorageClassName string `json:"storageClassName,omitempty"` + + Size resource.Quantity `json:"size"` + + // +optional + // +kubebuilder:default=/home/jovyan + MountPath string `json:"mountPath,omitempty"` + + // FixOwnership adds a root init container that chowns the volume to + // 1000:100, for provisioners that ignore fsGroup (local-path, hostPath). + // +optional + FixOwnership bool `json:"fixOwnership,omitempty"` +} + +// +kubebuilder:object:root=true + +// NotebookProfile is a pod template that Notebooks reference by name. +type NotebookProfile struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec NotebookProfileSpec `json:"spec"` +} + +// +kubebuilder:object:root=true + +// NotebookProfileList contains a list of NotebookProfile. +type NotebookProfileList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NotebookProfile `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NotebookProfile{}, &NotebookProfileList{}) +} diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go new file mode 100644 index 0000000..8aecc97 --- /dev/null +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -0,0 +1,211 @@ +//go:build !ignore_autogenerated + +// Code generated by controller-gen. DO NOT EDIT. + +package v1alpha1 + +import ( + "k8s.io/api/core/v1" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Notebook) DeepCopyInto(out *Notebook) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Notebook. +func (in *Notebook) DeepCopy() *Notebook { + if in == nil { + return nil + } + out := new(Notebook) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Notebook) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookList) DeepCopyInto(out *NotebookList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Notebook, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookList. +func (in *NotebookList) DeepCopy() *NotebookList { + if in == nil { + return nil + } + out := new(NotebookList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NotebookList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookProfile) DeepCopyInto(out *NotebookProfile) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookProfile. +func (in *NotebookProfile) DeepCopy() *NotebookProfile { + if in == nil { + return nil + } + out := new(NotebookProfile) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NotebookProfile) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookProfileList) DeepCopyInto(out *NotebookProfileList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NotebookProfile, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookProfileList. +func (in *NotebookProfileList) DeepCopy() *NotebookProfileList { + if in == nil { + return nil + } + out := new(NotebookProfileList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NotebookProfileList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookProfileSpec) DeepCopyInto(out *NotebookProfileSpec) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + in.Storage.DeepCopyInto(&out.Storage) + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookProfileSpec. +func (in *NotebookProfileSpec) DeepCopy() *NotebookProfileSpec { + if in == nil { + return nil + } + out := new(NotebookProfileSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookSpec) DeepCopyInto(out *NotebookSpec) { + *out = *in + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookSpec. +func (in *NotebookSpec) DeepCopy() *NotebookSpec { + if in == nil { + return nil + } + out := new(NotebookSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookStatus) DeepCopyInto(out *NotebookStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookStatus. +func (in *NotebookStatus) DeepCopy() *NotebookStatus { + if in == nil { + return nil + } + out := new(NotebookStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NotebookStorage) DeepCopyInto(out *NotebookStorage) { + *out = *in + out.Size = in.Size.DeepCopy() +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NotebookStorage. +func (in *NotebookStorage) DeepCopy() *NotebookStorage { + if in == nil { + return nil + } + out := new(NotebookStorage) + in.DeepCopyInto(out) + return out +} diff --git a/operator/cmd/main.go b/operator/cmd/main.go new file mode 100644 index 0000000..3ad489b --- /dev/null +++ b/operator/cmd/main.go @@ -0,0 +1,147 @@ +package main + +import ( + "flag" + "os" + "path/filepath" + "strings" + + "github.com/go-logr/logr" + "golang.org/x/time/rate" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/certwatcher" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" + "github.com/Medical-Informatics-Platform/mip/operator/internal/controller" + "github.com/Medical-Informatics-Platform/mip/operator/internal/spawnapi" +) + +func main() { + var mode, probeAddr string + var leaderElect bool + var spawnAPIAddr, tlsCertDir, metricsAddr string + var rateLimit float64 + var rateBurst int + flag.StringVar(&mode, "mode", "controller", "controller (reconcile Notebooks) or spawn-api (the hubs' HTTPS API).") + flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") + flag.BoolVar(&leaderElect, "leader-elect", false, "Enable leader election (a Lease in the pod namespace).") + flag.StringVar(&spawnAPIAddr, "spawn-api-bind-address", ":8443", "spawn-api: the address the HTTPS API binds to.") + flag.StringVar(&tlsCertDir, "tls-cert-dir", "/etc/mip-notebook-spawn-api/tls", + "spawn-api: directory with tls.crt and tls.key, reloaded when they change.") + flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "spawn-api: the address the metrics endpoint binds to; 0 disables it.") + flag.Float64Var(&rateLimit, "spawn-api-rate-limit", 5, "spawn-api: requests per second per namespace.") + flag.IntVar(&rateBurst, "spawn-api-rate-burst", 20, "spawn-api: request burst per namespace.") + opts := zap.Options{} + opts.BindFlags(flag.CommandLine) + flag.Parse() + ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) + setupLog := ctrl.Log.WithName("setup") + + scheme := runtime.NewScheme() + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + utilruntime.Must(notebooksv1alpha1.AddToScheme(scheme)) + + switch mode { + case "controller": + runController(setupLog, scheme, probeAddr, leaderElect) + case "spawn-api": + runSpawnAPI(setupLog, scheme, probeAddr, metricsAddr, spawnAPIAddr, tlsCertDir, rate.Limit(rateLimit), rateBurst) + default: + setupLog.Error(nil, "--mode must be controller or spawn-api", "mode", mode) + os.Exit(1) + } +} + +func runController(setupLog logr.Logger, scheme *runtime.Scheme, probeAddr string, leaderElect bool) { + // The operator is bound per namespace (RoleBinding to its ClusterRole), so + // its cache must be limited to exactly those namespaces. + watchNamespaces := map[string]cache.Config{} + for _, ns := range strings.Split(os.Getenv("WATCH_NAMESPACES"), ",") { + if ns = strings.TrimSpace(ns); ns != "" { + watchNamespaces[ns] = cache.Config{} + } + } + if len(watchNamespaces) == 0 { + setupLog.Error(nil, "WATCH_NAMESPACES must list at least one namespace") + os.Exit(1) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Cache: cache.Options{DefaultNamespaces: watchNamespaces}, + Metrics: metricsserver.Options{BindAddress: "0"}, + HealthProbeBindAddress: probeAddr, + LeaderElection: leaderElect, + LeaderElectionID: "mip-notebook-operator.mip.ebrains.eu", + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + if err := (&controller.NotebookReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ExamplesClaim: os.Getenv("EXAMPLES_CLAIM"), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "Notebook") + os.Exit(1) + } + utilruntime.Must(mgr.AddHealthzCheck("healthz", healthz.Ping)) + utilruntime.Must(mgr.AddReadyzCheck("readyz", healthz.Ping)) + + setupLog.Info("starting manager") + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +// runSpawnAPI serves the hubs' API under its own ServiceAccount: Notebooks +// get/create/delete and Secrets create in the federations it is bound in, +// tokenreviews create. Stateless, so no leader election, and no cache: the +// account may not list or watch. +func runSpawnAPI(setupLog logr.Logger, scheme *runtime.Scheme, probeAddr, metricsAddr, addr, certDir string, + limit rate.Limit, burst int) { + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + Scheme: scheme, + Metrics: metricsserver.Options{BindAddress: metricsAddr}, + HealthProbeBindAddress: probeAddr, + }) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + c, err := client.New(mgr.GetConfig(), client.Options{Scheme: scheme, Mapper: mgr.GetRESTMapper()}) + if err != nil { + setupLog.Error(err, "unable to create client") + os.Exit(1) + } + certs, err := certwatcher.New(filepath.Join(certDir, "tls.crt"), filepath.Join(certDir, "tls.key")) + if err != nil { + setupLog.Error(err, "unable to load the TLS certificate", "dir", certDir) + os.Exit(1) + } + utilruntime.Must(mgr.Add(certs)) + + auth := &spawnapi.TokenReviewAuthenticator{Client: c} + srv := spawnapi.NewServer(c, auth, limit, burst, ctrl.Log.WithName("spawn-api")) + listener := &spawnapi.Listener{Addr: addr, Handler: srv.Handler(), GetCertificate: certs.GetCertificate} + utilruntime.Must(mgr.Add(listener)) + utilruntime.Must(mgr.AddHealthzCheck("healthz", healthz.Ping)) + utilruntime.Must(mgr.AddReadyzCheck("readyz", listener.ReadyCheck)) + + setupLog.Info("starting spawn API", "address", addr) + if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + setupLog.Error(err, "problem running spawn API") + os.Exit(1) + } +} diff --git a/operator/config/crd/bases/notebooks.mip.ebrains.eu_notebookprofiles.yaml b/operator/config/crd/bases/notebooks.mip.ebrains.eu_notebookprofiles.yaml new file mode 100644 index 0000000..33351b9 --- /dev/null +++ b/operator/config/crd/bases/notebooks.mip.ebrains.eu_notebookprofiles.yaml @@ -0,0 +1,306 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.23.0 + name: notebookprofiles.notebooks.mip.ebrains.eu +spec: + group: notebooks.mip.ebrains.eu + names: + kind: NotebookProfile + listKind: NotebookProfileList + plural: notebookprofiles + singular: notebookprofile + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NotebookProfile is a pod template that Notebooks reference by + name. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NotebookProfileSpec is the pod template for notebooks. It is shipped with the + chart; the hub cannot read or change it. The security context is not part of + it on purpose: the operator always applies the restricted one. + properties: + env: + description: Env is static, non-secret configuration for every notebook. + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: |- + Name of the environment variable. + May consist of any printable ASCII characters except '='. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. Cannot + be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: |- + The key to select from the ConfigMap's Data field. + Keys in the BinaryData field are not currently propagated to container env vars. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath is + written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified + API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + description: |- + FileKeyRef selects a key of the env file. + Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: |- + The key within the env file. An invalid key will prevent the pod from starting. + The keys defined within a source may consist of any printable ASCII characters except '='. + During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + default: false + description: |- + Specify whether the file or its key must be defined. If the file or key + does not exist, then the env var is not published. + If optional is set to true and the specified key does not exist, + the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, + an error will be returned during Pod creation. + type: boolean + path: + description: |- + The path within the volume from which to select the file. + Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing + the env file. + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the exposed + resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + image: + minLength: 1 + type: string + imagePullPolicy: + default: IfNotPresent + description: PullPolicy describes a policy for if/when to pull a container + image + type: string + nodeSelector: + additionalProperties: + type: string + type: object + resources: + description: ResourceRequirements describes the compute resource requirements. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + storage: + description: NotebookStorage describes the per-user home volume. + properties: + fixOwnership: + description: |- + FixOwnership adds a root init container that chowns the volume to + 1000:100, for provisioners that ignore fsGroup (local-path, hostPath). + type: boolean + mountPath: + default: /home/jovyan + type: string + size: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + storageClassName: + type: string + required: + - size + type: object + required: + - image + - storage + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/operator/config/crd/bases/notebooks.mip.ebrains.eu_notebooks.yaml b/operator/config/crd/bases/notebooks.mip.ebrains.eu_notebooks.yaml new file mode 100644 index 0000000..1e0cd76 --- /dev/null +++ b/operator/config/crd/bases/notebooks.mip.ebrains.eu_notebooks.yaml @@ -0,0 +1,125 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.23.0 + name: notebooks.notebooks.mip.ebrains.eu +spec: + group: notebooks.mip.ebrains.eu + names: + kind: Notebook + listKind: NotebookList + plural: notebooks + singular: notebook + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.user + name: User + type: string + - jsonPath: .status.phase + name: Phase + type: string + - jsonPath: .status.message + name: Message + priority: 1 + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Notebook is one JupyterHub single-user server. The hub creates Secret + --token (key "token", owned by the Notebook) with its API token. + The API server assigns the UID, so no Secret that existed before the + Notebook, such as one left by an earlier Notebook of the same name, is ever + mounted. Names are jupyter-* and at most 57 characters, as the hub makes them. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NotebookSpec is what JupyterHub asks for. Everything that decides what runs + (image, resources, volumes, security) comes from the referenced profile. + properties: + env: + additionalProperties: + type: string + description: |- + Env holds the per-spawn JupyterHub variables. Only JUPYTERHUB_* and JPY_* + names are accepted; the API token comes from the Notebook's token Secret. + type: object + x-kubernetes-validations: + - message: env keys must match ^(JUPYTERHUB|JPY)_[A-Z0-9_]+$ + rule: self.all(k, k.matches('^(JUPYTERHUB|JPY)_[A-Z0-9_]+$')) + - message: the API token comes from the Notebook's token Secret + rule: '!(''JUPYTERHUB_API_TOKEN'' in self) && !(''JPY_API_TOKEN'' + in self)' + profile: + description: Profile is the name of a NotebookProfile in the same + namespace. + minLength: 1 + type: string + serverName: + description: ServerName is the JupyterHub named server, empty for + the default server. + type: string + user: + description: User is the JupyterHub user name. It names the home PVC. + maxLength: 253 + minLength: 1 + type: string + required: + - profile + - user + type: object + x-kubernetes-validations: + - message: spec is immutable; delete and recreate the Notebook + rule: self == oldSelf + status: + description: NotebookStatus is the observed state of the notebook pod. + properties: + message: + description: Message is the latest human readable reason, shown on + the spawn page. + type: string + phase: + enum: + - Pending + - Running + - Failed + type: string + url: + description: URL is where the hub proxy reaches the notebook server, + set when Running. + type: string + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: name must start with jupyter- and be at most 57 characters + rule: self.metadata.name.startsWith('jupyter-') && size(self.metadata.name) + <= 57 + served: true + storage: true + subresources: + status: {} diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml new file mode 100644 index 0000000..d55b1d2 --- /dev/null +++ b/operator/config/rbac/role.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - persistentvolumeclaims + - pods + verbs: + - create + - get + - list + - watch +- apiGroups: + - notebooks.mip.ebrains.eu + resources: + - notebookprofiles + - notebooks + verbs: + - get + - list + - watch +- apiGroups: + - notebooks.mip.ebrains.eu + resources: + - notebooks/status + verbs: + - get + - patch + - update diff --git a/operator/go.mod b/operator/go.mod new file mode 100644 index 0000000..a59b273 --- /dev/null +++ b/operator/go.mod @@ -0,0 +1,81 @@ +module github.com/Medical-Informatics-Platform/mip/operator + +go 1.27.0 + +require ( + github.com/go-logr/logr v1.4.4 + github.com/onsi/ginkgo/v2 v2.33.1 + github.com/onsi/gomega v1.44.0 + github.com/prometheus/client_golang v1.24.0 + golang.org/x/time v0.15.0 + k8s.io/api v0.37.1 + k8s.io/apimachinery v0.37.1 + k8s.io/client-go v0.37.1 + k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 + sigs.k8s.io/controller-runtime v0.25.2 +) + +require ( + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.1 // indirect + github.com/go-logr/zapr v1.3.0 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/jsonreference v1.0.0 // indirect + github.com/go-openapi/swag v0.28.0 // indirect + github.com/go-openapi/swag/cmdutils v0.28.0 // indirect + github.com/go-openapi/swag/conv v0.28.0 // indirect + github.com/go-openapi/swag/fileutils v0.28.0 // indirect + github.com/go-openapi/swag/jsonutils v0.28.0 // indirect + github.com/go-openapi/swag/loading v0.28.0 // indirect + github.com/go-openapi/swag/mangling v0.28.0 // indirect + github.com/go-openapi/swag/netutils v0.28.0 // indirect + github.com/go-openapi/swag/pools v0.28.0 // indirect + github.com/go-openapi/swag/stringutils v0.28.0 // indirect + github.com/go-openapi/swag/typeutils v0.28.0 // indirect + github.com/go-openapi/swag/yamlutils v0.28.0 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.0 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/testify v1.12.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect + go.yaml.in/yaml/v2 v2.4.4 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/mod v0.41.0 // indirect + golang.org/x/net v0.62.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect + golang.org/x/tools v0.51.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/protobuf v1.36.12 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/apiextensions-apiserver v0.37.0 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect +) diff --git a/operator/go.sum b/operator/go.sum new file mode 100644 index 0000000..ef61a1e --- /dev/null +++ b/operator/go.sum @@ -0,0 +1,202 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= +github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= +github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= +github.com/go-openapi/swag v0.28.0 h1:xkgbOSKj6DZziNpyqRRAOt3GJGtgjgsd2RoyT30VWuw= +github.com/go-openapi/swag v0.28.0/go.mod h1:4qYnT3Cqr1p1VknOdPo70evN4rgQnAg6jwApHyxSGIg= +github.com/go-openapi/swag/cmdutils v0.28.0 h1:7TOeNtkYru1SG8Y34tDh9WBbLsMqGnptuxWiHREPZ4Q= +github.com/go-openapi/swag/cmdutils v0.28.0/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.28.0 h1:GtqqbyFe7vR5Y7ehxG9W6/OvrSFdf1OLeTGp40TqxH8= +github.com/go-openapi/swag/conv v0.28.0/go.mod h1:mbUE+mzctnhxi864m0Q07SpN8OowD9JhxmxuYvZZD/k= +github.com/go-openapi/swag/fileutils v0.28.0 h1:Z04XWQD7R8Eq+7GnOrjovBxPPmZzsS4gt2H2GPGIViU= +github.com/go-openapi/swag/fileutils v0.28.0/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.28.0 h1:YIch6FwO7RXzeAnbO8Tu7dWBZeUEH+4nA0HXltVTnv4= +github.com/go-openapi/swag/jsonutils v0.28.0/go.mod h1:CYM3WlTUcagR2ZoHdz54di/cbBqt82tuxuXgAjxw+mg= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0 h1:qV+VVUAx5Oro8WjVWpZeql7YReTKhT4smR4zhcOQZr0= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.28.0/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.28.0 h1:td8QZdZC9MIYGGSnSPKShKiK22I2tU5UQvuUhIBPRLU= +github.com/go-openapi/swag/loading v0.28.0/go.mod h1:rXB0QiQX5mMveXEA7ouM4KiiM9jVJe4K6BVbwhD1M4k= +github.com/go-openapi/swag/mangling v0.28.0 h1:pH8eyeNO9SLYsTMWJrurnNfKmDa28XrlA+HePVD53VM= +github.com/go-openapi/swag/mangling v0.28.0/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.28.0 h1:YXN6TALEi2pzts8/8GNm6T61HTAZsieukGZidap989k= +github.com/go-openapi/swag/netutils v0.28.0/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.28.0 h1:HPMZWSAfce3rdVTFcjFiCIBtDg9h4x2QlRrHipwhxeU= +github.com/go-openapi/swag/pools v0.28.0/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.28.0 h1:ixsc9iYgDPubHL/8nSkbnryEHpD2VRlBMLKpQyPXcDU= +github.com/go-openapi/swag/stringutils v0.28.0/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.28.0 h1:nRBKSBXjDgf01VDPB3fWeD9nQuhCOVeIYAkUx2tbkyY= +github.com/go-openapi/swag/typeutils v0.28.0/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.28.0 h1:TV3JXH6DS46KUroDtMLAYHGkdWf5VDq3wVWFirmzROY= +github.com/go-openapi/swag/yamlutils v0.28.0/go.mod h1:x0q/yndZHEgk9Rx3DyDqzFUmHy55KTvIZldvF2dTJXs= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oXJCETMO23COyaKGP6fHVpkpWpg= +github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/onsi/ginkgo/v2 v2.33.1 h1:62XfWJbEIjPSCSDYrsCPP/SHcmH7O0M8ne8zfqQ2iz4= +github.com/onsi/ginkgo/v2 v2.33.1/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= +github.com/onsi/gomega v1.44.0 h1:eAiGl3Pw5jz5GQdDff0BcxYpAX1JxW8xD7mFUuwNfZQ= +github.com/onsi/gomega v1.44.0/go.mod h1:e/C2HwaZ1DhvjzXXuFhcR7hY7Sh9pl7MmoWKEjzwcdA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.24.0 h1:5XStIklKuAtJSNpdD3s8XJj/Yv78IQmE1kbNk87JrAI= +github.com/prometheus/client_golang v1.24.0/go.mod h1:QcsNdotprC2nS4BTM2ucbcqxd2CeXTEa9jW7zHO9iDE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.0 h1:bcpru3tWPVnxGnETLgOV5jbp/JRXgYEyv65CuBLAMMI= +github.com/prometheus/common v0.70.0/go.mod h1:S/SFasQmgGiYH6C81LKCtYa8QACgthGg5zxL2udV7SY= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.37.1 h1:l6N77U7tjwB5L056bgrBTJIEdevac/naBZ3iSvDNfpM= +k8s.io/api v0.37.1/go.mod h1:zSlbB1YpJ1YQlFVQy20UYll81UJSJJUMLhkhvg6Z78M= +k8s.io/apiextensions-apiserver v0.37.0 h1:zRMQ3+/LIE5oZ0tVvXwYHC+dIkSP5cjNWju7AZU1LOI= +k8s.io/apiextensions-apiserver v0.37.0/go.mod h1:HU0PfSBwchHL5iDau6jjt9zU6ryWkDDlaVUiq91NK80= +k8s.io/apimachinery v0.37.1 h1:hGCYyvKHCwtwMitj2vU4vYx0Z16N9GyZk9BBnz0wDAE= +k8s.io/apimachinery v0.37.1/go.mod h1:jF84AyUi/IRIXRot5f+lm6MpxoWI+F1XgjaMmwCdTFw= +k8s.io/client-go v0.37.1 h1:QTv/5ha4jAHtW9qxxVBkQVFBRDb4jHfFopQqqMdc+wM= +k8s.io/client-go v0.37.1/go.mod h1:dnAPtTnCNY38Ho04D2KdY1F4IKausa9UbqaAZKl60SY= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3 h1:jVkFFVfXdXP74B/zbO3hM3hpSFD0xvhQ5U686DPurkE= +k8s.io/utils v0.0.0-20260707023825-cf1189d6abe3/go.mod h1:M2s5JB1lIYP3jzZdorPLHXIPJzt9vv2muW5a6L9DtNM= +sigs.k8s.io/controller-runtime v0.25.2 h1:bEkK3PVOIVK9X8QWLGVhJgmFc++47vfT6wakSzAOLsQ= +sigs.k8s.io/controller-runtime v0.25.2/go.mod h1:4QqLdT6z/L6Olj8JJCtvztid4/fnIiYsfaTFScegctc= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/operator/internal/controller/notebook_controller.go b/operator/internal/controller/notebook_controller.go new file mode 100644 index 0000000..1a31f60 --- /dev/null +++ b/operator/internal/controller/notebook_controller.go @@ -0,0 +1,419 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "maps" + "net" + "path" + "regexp" + "slices" + "strconv" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" +) + +const ( + notebookPort = 8888 + homeVolume = "home" + profileRetryWait = 30 * time.Second +) + +// NotebookReconciler turns Notebooks into a home PVC and a notebook pod. +type NotebookReconciler struct { + client.Client + Scheme *runtime.Scheme + // ExamplesClaim, when set and bound in the notebook's namespace, is mounted read-only + // at work/.mip-examples in the home of every new notebook. + ExamplesClaim string +} + +// +kubebuilder:rbac:groups=notebooks.mip.ebrains.eu,resources=notebooks,verbs=get;list;watch +// +kubebuilder:rbac:groups=notebooks.mip.ebrains.eu,resources=notebooks/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=notebooks.mip.ebrains.eu,resources=notebookprofiles,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=pods;persistentvolumeclaims,verbs=get;list;watch;create + +func (r *NotebookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + nb := ¬ebooksv1alpha1.Notebook{} + if err := r.Get(ctx, req.NamespacedName, nb); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + // Pod and token Secret are owned by the Notebook; garbage collection + // removes them, so deletion needs no finalizer. + if !nb.DeletionTimestamp.IsZero() { + return ctrl.Result{}, nil + } + + pod := &corev1.Pod{} + err := r.Get(ctx, client.ObjectKeyFromObject(nb), pod) + if apierrors.IsNotFound(err) { + return r.start(ctx, nb) + } + if err != nil { + return ctrl.Result{}, err + } + // Same name, other owner: a KubeSpawner pod, or an earlier Notebook's. Its IP + // would send the hub to a server with another API token. + if !metav1.IsControlledBy(pod, nb) { + msg := fmt.Sprintf("pod %s exists and does not belong to this Notebook", pod.Name) + return ctrl.Result{}, r.setStatus(ctx, nb, notebooksv1alpha1.PhaseFailed, "", msg) + } + + phase, url, msg := podStatus(pod) + return ctrl.Result{}, r.setStatus(ctx, nb, phase, url, msg) +} + +// start creates the home and the pod. Only here is the profile needed: a running +// pod does not depend on it, and "Failed" makes the hub stop the server. +func (r *NotebookReconciler) start(ctx context.Context, nb *notebooksv1alpha1.Notebook) (ctrl.Result, error) { + profile := ¬ebooksv1alpha1.NotebookProfile{} + err := r.Get(ctx, types.NamespacedName{Namespace: nb.Namespace, Name: nb.Spec.Profile}, profile) + if apierrors.IsNotFound(err) { + msg := fmt.Sprintf("NotebookProfile %q not found", nb.Spec.Profile) + return ctrl.Result{RequeueAfter: profileRetryWait}, r.setStatus(ctx, nb, notebooksv1alpha1.PhaseFailed, "", msg) + } + if err != nil { + return ctrl.Result{}, err + } + if err := r.createPod(ctx, nb, profile); err != nil { + // Pending, not Failed: a quota can free up. The message reaches the spawn + // page, which otherwise shows nothing until the hub's timeout. + return ctrl.Result{}, errors.Join(err, r.setStatus(ctx, nb, notebooksv1alpha1.PhasePending, "", "cannot create pod: "+err.Error())) + } + return ctrl.Result{}, r.setStatus(ctx, nb, notebooksv1alpha1.PhasePending, "", "pod created") +} + +func (r *NotebookReconciler) createPod(ctx context.Context, nb *notebooksv1alpha1.Notebook, profile *notebooksv1alpha1.NotebookProfile) error { + if err := r.ensurePVC(ctx, nb, profile); err != nil { + return err + } + pod := buildPod(nb, profile) + bound, err := r.examplesBound(ctx, nb.Namespace) + if err != nil { + return err + } + examples := "not bound" + switch { + case bound && !profile.Spec.Storage.FixOwnership: + // Without fix-home-perm the runtime creates work/ root-owned for the nested mount. + examples = "skipped: fixOwnership is false" + case bound: + addExamples(pod, r.ExamplesClaim, profile.Spec.Storage.MountPath) + examples = "mounted" + } + if err := controllerutil.SetControllerReference(nb, pod, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, pod); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + logf.FromContext(ctx).Info("created notebook pod", "pod", pod.Name, "examplesClaim", r.ExamplesClaim, "examples", examples) + return nil +} + +// ensurePVC creates the user's home volume. It is never owned, updated or +// deleted by the operator, so homes survive server restarts. +func (r *NotebookReconciler) ensurePVC(ctx context.Context, nb *notebooksv1alpha1.Notebook, profile *notebooksv1alpha1.NotebookProfile) error { + pvc := &corev1.PersistentVolumeClaim{} + key := types.NamespacedName{Namespace: nb.Namespace, Name: pvcName(nb.Spec.User)} + err := r.Get(ctx, key, pvc) + if !apierrors.IsNotFound(err) { + return err + } + pvc = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: key.Name, + Namespace: key.Namespace, + Labels: notebookLabels, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: profile.Spec.Storage.Size}, + }, + }, + } + if sc := profile.Spec.Storage.StorageClassName; sc != "" { + pvc.Spec.StorageClassName = &sc + } + if err := r.Create(ctx, pvc); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +func buildPod(nb *notebooksv1alpha1.Notebook, profile *notebooksv1alpha1.NotebookProfile) *corev1.Pod { + ps := profile.Spec + + // The entrypoint's work dir, on the home volume: fix-home-perm creates it and the + // examples mount sits in it. First, so profile or notebook env can still override it. + env := append([]corev1.EnvVar{{Name: "MIP_WORK_DIR", Value: path.Join(ps.Storage.MountPath, "work")}}, ps.Env...) + for _, k := range slices.Sorted(maps.Keys(nb.Spec.Env)) { + env = append(env, corev1.EnvVar{Name: k, Value: nb.Spec.Env[k]}) + } + for _, name := range []string{"JUPYTERHUB_API_TOKEN", "JPY_API_TOKEN"} { + env = append(env, corev1.EnvVar{Name: name, ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: &corev1.SecretKeySelector{ + // Never read by the operator: until the hub has created it the + // pod waits in CreateContainerConfigError, shown as the message. + // The UID keeps out a Secret left by an earlier Notebook of the + // same name. + LocalObjectReference: corev1.LocalObjectReference{Name: nb.TokenSecretName()}, + Key: notebooksv1alpha1.TokenSecretKey, + }, + }}) + } + + restricted := &corev1.SecurityContext{ + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + } + + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: nb.Name, + Namespace: nb.Namespace, + Labels: notebookLabels, + }, + Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, + AutomountServiceAccountToken: ptr.To(false), + EnableServiceLinks: ptr.To(false), + NodeSelector: ps.NodeSelector, + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: ptr.To[int64](1000), + RunAsGroup: ptr.To[int64](100), + FSGroup: ptr.To[int64](100), + FSGroupChangePolicy: ptr.To(corev1.FSGroupChangeOnRootMismatch), + RunAsNonRoot: ptr.To(true), + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{ + Name: "notebook", + Image: ps.Image, + ImagePullPolicy: ps.ImagePullPolicy, + Resources: ps.Resources, + Env: env, + Ports: []corev1.ContainerPort{{Name: "notebook", ContainerPort: notebookPort}}, + VolumeMounts: []corev1.VolumeMount{{Name: homeVolume, MountPath: ps.Storage.MountPath}}, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{TCPSocket: &corev1.TCPSocketAction{Port: intstr.FromInt32(notebookPort)}}, + PeriodSeconds: 2, + }, + SecurityContext: restricted, + }}, + Volumes: []corev1.Volume{{ + Name: homeVolume, + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: pvcName(nb.Spec.User), + }}, + }}, + }, + } + if ps.Storage.FixOwnership { + // Runs as root, so a profile using it cannot run under PSS "restricted". + pod.Spec.InitContainers = []corev1.Container{{ + Name: "fix-home-perm", + Image: ps.Image, + Command: []string{"sh", "-c", + `[ "$(stat -c %u:%g /mnt/home)" = 1000:100 ] || chown -R 1000:100 /mnt/home; ` + + "mkdir -p /mnt/home/work && chown 1000:100 /mnt/home/work"}, + VolumeMounts: []corev1.VolumeMount{{Name: homeVolume, MountPath: "/mnt/home"}}, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: ptr.To[int64](0), + RunAsGroup: ptr.To[int64](0), + RunAsNonRoot: ptr.To(false), + // Root only for chown/mkdir on the home volume; no escalation and no + // other capabilities, so the init cannot do more than fix ownership. + AllowPrivilegeEscalation: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + Add: []corev1.Capability{"CHOWN", "FOWNER", "DAC_OVERRIDE"}, + }, + }, + }} + } + return pod +} + +// examplesBound reports whether ns holds the shared examples claim, bound and not being +// deleted. A missing or unbound claim is false, not an error. +func (r *NotebookReconciler) examplesBound(ctx context.Context, ns string) (bool, error) { + if r.ExamplesClaim == "" { + return false, nil + } + pvc := &corev1.PersistentVolumeClaim{} + if err := r.Get(ctx, types.NamespacedName{Namespace: ns, Name: r.ExamplesClaim}, pvc); err != nil { + return false, client.IgnoreNotFound(err) + } + return pvc.DeletionTimestamp.IsZero() && pvc.Status.Phase == corev1.ClaimBound, nil +} + +// addExamples mounts the claim at work/.mip-examples inside the home volume, where the +// single-user entrypoint links examples/ to it. The runtime creates the mount point, as +// root, in the parent directory, so work/ must already exist: fix-home-perm creates it. +func addExamples(pod *corev1.Pod, claim, home string) { + pod.Spec.Volumes = append(pod.Spec.Volumes, corev1.Volume{ + Name: "examples", + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: claim, + ReadOnly: true, + }}, + }) + c := &pod.Spec.Containers[0] + // A path in the Linux container, not on the operator's OS, so never filepath.Join. + // nosemgrep: semgrep-rules.go.lang.correctness.use-filepath-join + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{Name: "examples", MountPath: path.Join(home, "work/.mip-examples"), ReadOnly: true}) +} + +// podStatus maps a pod to the Notebook phase, proxy URL and spawn message. +func podStatus(pod *corev1.Pod) (string, string, string) { + switch pod.Status.Phase { + case corev1.PodSucceeded, corev1.PodFailed: + return notebooksv1alpha1.PhaseFailed, "", terminationMessage(pod) + } + if !pod.DeletionTimestamp.IsZero() { + return notebooksv1alpha1.PhaseFailed, "", "pod is being deleted" + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue && pod.Status.PodIP != "" { + // JoinHostPort brackets IPv6 addresses; a bare Sprintf does not. + return notebooksv1alpha1.PhaseRunning, "http://" + net.JoinHostPort(pod.Status.PodIP, strconv.Itoa(notebookPort)), "server ready" + } + } + return notebooksv1alpha1.PhasePending, "", waitingMessage(pod) +} + +func waitingMessage(pod *corev1.Pod) string { + statuses := append(append([]corev1.ContainerStatus{}, pod.Status.InitContainerStatuses...), pod.Status.ContainerStatuses...) + for _, cs := range statuses { + if w := cs.State.Waiting; w != nil && w.Reason != "" { + return strings.TrimSpace(w.Reason + ": " + w.Message) + } + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse { + return strings.TrimSpace(c.Reason + ": " + c.Message) + } + } + for _, cs := range pod.Status.ContainerStatuses { + if cs.State.Running != nil { + return "starting notebook server" + } + } + return "pod " + strings.ToLower(string(pod.Status.Phase)) +} + +func terminationMessage(pod *corev1.Pod) string { + for _, cs := range pod.Status.InitContainerStatuses { + if t := cs.State.Terminated; t != nil && t.ExitCode != 0 { + return strings.TrimSpace(fmt.Sprintf("init container %s exited with code %d: %s %s", cs.Name, t.ExitCode, t.Reason, t.Message)) + } + } + for _, cs := range pod.Status.ContainerStatuses { + if t := cs.State.Terminated; t != nil { + return strings.TrimSpace(fmt.Sprintf("notebook exited with code %d: %s %s", t.ExitCode, t.Reason, t.Message)) + } + } + if pod.Status.Message != "" { + return pod.Status.Message + } + return "pod " + strings.ToLower(string(pod.Status.Phase)) +} + +func (r *NotebookReconciler) setStatus(ctx context.Context, nb *notebooksv1alpha1.Notebook, phase, url, msg string) error { + want := notebooksv1alpha1.NotebookStatus{Phase: phase, URL: url, Message: msg} + if nb.Status == want { + return nil + } + // Merge patch: no resourceVersion, so a stale cached copy cannot conflict. + base := nb.DeepCopy() + nb.Status = want + return r.Status().Patch(ctx, nb, client.MergeFrom(base)) +} + +// component=singleuser-server is what the federation NetworkPolicies select. +var notebookLabels = map[string]string{ + "app.kubernetes.io/managed-by": "mip-notebook-operator", + "component": "singleuser-server", +} + +// pvcName is KubeSpawner's claim-{user_server} for a user's default server, +// so homes KubeSpawner created are found again. The slug functions below port +// its "safe" scheme (kubespawner/slugs.py, 7.1): a name kept as-is never +// contains "--", so it cannot equal an escaped one ending in "---". +func pvcName(user string) string { + slug := safeSlug(user) + if len(slug)+2 > slugMaxLength { + slug = multiSlug(user, "") + } + return "claim-" + slug +} + +const slugMaxLength = 48 + +var ( + nonAlnum = regexp.MustCompile(`[^a-z0-9]+`) + objectName = regexp.MustCompile(`^[a-z]([a-z0-9-]*[a-z0-9])?$`) +) + +func safeSlug(name string) string { + if !strings.Contains(name, "--") && len(name) <= slugMaxLength && objectName.MatchString(name) { + return name + } + return safeName(name, slugMaxLength-11) + "---" + shortHash(name) +} + +// multiSlug hashes all names together; "\xff" never occurs in UTF-8, so +// different splits of the same characters hash differently. +func multiSlug(names ...string) string { + limit := (slugMaxLength-9)/len(names) - 2 + slugs := make([]string, len(names)) + for i, n := range names { + slugs[i] = safeName(n, limit) + } + return strings.Join(slugs, "--") + "---" + shortHash(strings.Join(names, "\xff")) +} + +func safeName(name string, limit int) string { + s := strings.TrimLeft(nonAlnum.ReplaceAllString(strings.ToLower(name), "-"), "-") + s = strings.TrimRight(s[:min(len(s), limit)], "-") + if s != "" && (s[0] < 'a' || s[0] > 'z') { + s = "x-" + s[:min(len(s), limit-2)] + } + if s == "" { + s = "x" + } + return s +} + +func shortHash(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:8] +} + +// SetupWithManager sets up the controller with the Manager. +func (r *NotebookReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(¬ebooksv1alpha1.Notebook{}). + Owns(&corev1.Pod{}). + Named("notebook"). + Complete(r) +} diff --git a/operator/internal/controller/notebook_controller_test.go b/operator/internal/controller/notebook_controller_test.go new file mode 100644 index 0000000..9926322 --- /dev/null +++ b/operator/internal/controller/notebook_controller_test.go @@ -0,0 +1,397 @@ +package controller + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" +) + +// rejectPods fails pod creation as an admission plugin would. +type rejectPods struct{ client.Client } + +func (c rejectPods) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*corev1.Pod); ok { + return apierrors.NewForbidden(corev1.Resource("pods"), obj.GetName(), errors.New("exceeded quota")) + } + return c.Client.Create(ctx, obj, opts...) +} + +var _ = Describe("Notebook controller", func() { + var ( + ns string + reconciler *NotebookReconciler + counter int + ) + + newNotebook := func(name string, env map[string]string) *notebooksv1alpha1.Notebook { + return ¬ebooksv1alpha1.Notebook{ + ObjectMeta: metav1.ObjectMeta{Name: "jupyter-" + name, Namespace: ns}, + Spec: notebooksv1alpha1.NotebookSpec{ + Profile: "default", + User: "Alice@Example.org", + Env: env, + }, + } + } + newProfile := func() *notebooksv1alpha1.NotebookProfile { + return ¬ebooksv1alpha1.NotebookProfile{ + ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: ns}, + Spec: notebooksv1alpha1.NotebookProfileSpec{ + Image: "hbpmip/mip-jupyter:test", + Storage: notebooksv1alpha1.NotebookStorage{Size: resource.MustParse("2Gi"), StorageClassName: "fast"}, + Env: []corev1.EnvVar{{Name: "PLATFORM_BACKEND_URL", Value: "http://backend"}}, + }, + } + } + reconcileNB := func(nb *notebooksv1alpha1.Notebook) reconcile.Result { + res, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(nb)}) + Expect(err).NotTo(HaveOccurred()) + return res + } + fetch := func(nb *notebooksv1alpha1.Notebook) *notebooksv1alpha1.Notebook { + out := ¬ebooksv1alpha1.Notebook{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(nb), out)).To(Succeed()) + return out + } + + BeforeEach(func() { + counter++ + ns = fmt.Sprintf("federation-%d", counter) + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})).To(Succeed()) + reconciler = &NotebookReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + }) + + Context("CRD validation", func() { + It("rejects env names outside JUPYTERHUB_* / JPY_*", func() { + err := k8sClient.Create(ctx, newNotebook("bad-env", map[string]string{"LD_PRELOAD": "/x.so"})) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "got %v", err) + }) + It("rejects the API token in env", func() { + err := k8sClient.Create(ctx, newNotebook("token-env", map[string]string{"JUPYTERHUB_API_TOKEN": "x"})) + Expect(apierrors.IsInvalid(err)).To(BeTrue(), "got %v", err) + }) + It("accepts only jupyter-* names of at most 57 characters", func() { + nb := newNotebook("x", nil) + nb.Name = "keycloak-credentials" + Expect(apierrors.IsInvalid(k8sClient.Create(ctx, nb))).To(BeTrue()) + Expect(apierrors.IsInvalid(k8sClient.Create(ctx, newNotebook(strings.Repeat("a", 50), nil)))).To(BeTrue()) + Expect(k8sClient.Create(ctx, newNotebook(strings.Repeat("a", 49), nil))).To(Succeed()) + }) + It("rejects spec changes", func() { + nb := newNotebook("immutable", nil) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + nb.Spec.Profile = "other" + Expect(apierrors.IsInvalid(k8sClient.Update(ctx, nb))).To(BeTrue()) + }) + }) + + It("fails when the profile is missing", func() { + nb := newNotebook("no-profile", nil) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + Expect(reconcileNB(nb).RequeueAfter).To(Equal(profileRetryWait)) + got := fetch(nb) + Expect(got.Status.Phase).To(Equal(notebooksv1alpha1.PhaseFailed)) + Expect(got.Status.Message).To(ContainSubstring(`"default" not found`)) + }) + + It("creates the PVC and a restricted pod, then reports Running", func() { + Expect(k8sClient.Create(ctx, newProfile())).To(Succeed()) + nb := newNotebook("happy", map[string]string{"JUPYTERHUB_SERVICE_PREFIX": "/notebook/user/alice/"}) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + reconcileNB(nb) + + pvc := &corev1.PersistentVolumeClaim{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName(nb.Spec.User)}, pvc)).To(Succeed()) + Expect(pvc.OwnerReferences).To(BeEmpty()) + Expect(*pvc.Spec.StorageClassName).To(Equal("fast")) + + pod := &corev1.Pod{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(nb), pod)).To(Succeed()) + Expect(pod.OwnerReferences).To(HaveLen(1)) + Expect(pod.Labels).To(HaveKeyWithValue("component", "singleuser-server")) + Expect(*pod.Spec.AutomountServiceAccountToken).To(BeFalse()) + Expect(*pod.Spec.SecurityContext.RunAsNonRoot).To(BeTrue()) + Expect(pod.Spec.InitContainers).To(BeEmpty()) + c := pod.Spec.Containers[0] + Expect(c.Image).To(Equal("hbpmip/mip-jupyter:test")) + // Defaulted by the CRD; buildPod has no fallback of its own. + Expect(c.VolumeMounts[0].MountPath).To(Equal("/home/jovyan")) + Expect(*c.SecurityContext.AllowPrivilegeEscalation).To(BeFalse()) + env := map[string]corev1.EnvVar{} + for _, e := range c.Env { + env[e.Name] = e + } + Expect(env["PLATFORM_BACKEND_URL"].Value).To(Equal("http://backend")) + Expect(env["MIP_WORK_DIR"].Value).To(Equal("/home/jovyan/work")) + Expect(env["JUPYTERHUB_SERVICE_PREFIX"].Value).To(Equal("/notebook/user/alice/")) + Expect(env["JUPYTERHUB_API_TOKEN"].ValueFrom.SecretKeyRef.Name).To(Equal(nb.Name + "-" + string(nb.UID) + "-token")) + Expect(env["JUPYTERHUB_API_TOKEN"].Value).To(BeEmpty()) + Expect(fetch(nb).Status.Phase).To(Equal(notebooksv1alpha1.PhasePending)) + + pod.Status.PodIP = "10.0.0.7" + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + Expect(k8sClient.Status().Update(ctx, pod)).To(Succeed()) + reconcileNB(nb) + got := fetch(nb) + Expect(got.Status.Phase).To(Equal(notebooksv1alpha1.PhaseRunning)) + Expect(got.Status.URL).To(Equal("http://10.0.0.7:8888")) + }) + + It("reports Failed when the notebook exits", func() { + pod := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + ContainerStatuses: []corev1.ContainerStatus{{State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 137, Reason: "OOMKilled"}, + }}}, + }} + phase, url, msg := podStatus(pod) + Expect(phase).To(Equal(notebooksv1alpha1.PhaseFailed)) + Expect(url).To(BeEmpty()) + Expect(msg).To(ContainSubstring("code 137: OOMKilled")) + }) + + It("reports why the init container failed", func() { + pod := &corev1.Pod{Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + InitContainerStatuses: []corev1.ContainerStatus{{Name: "fix-home-perm", State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ExitCode: 1, Reason: "Error"}, + }}}, + }} + _, _, msg := podStatus(pod) + Expect(msg).To(Equal("init container fix-home-perm exited with code 1: Error")) + }) + + It("keeps a running notebook Running when its profile goes away", func() { + profile := newProfile() + Expect(k8sClient.Create(ctx, profile)).To(Succeed()) + nb := newNotebook("profile-gone", nil) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + reconcileNB(nb) + pod := &corev1.Pod{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(nb), pod)).To(Succeed()) + pod.Status.PodIP = "10.0.0.8" + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + Expect(k8sClient.Status().Update(ctx, pod)).To(Succeed()) + + Expect(k8sClient.Delete(ctx, profile)).To(Succeed()) + reconcileNB(nb) + got := fetch(nb) + Expect(got.Status.Phase).To(Equal(notebooksv1alpha1.PhaseRunning)) + Expect(got.Status.URL).To(Equal("http://10.0.0.8:8888")) + }) + + It("fails on a pod of the same name that it does not own", func() { + Expect(k8sClient.Create(ctx, newProfile())).To(Succeed()) + nb := newNotebook("foreign", nil) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: nb.Name, Namespace: ns}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "notebook", Image: "kubespawner"}}}, + } + Expect(k8sClient.Create(ctx, pod)).To(Succeed()) + pod.Status.PodIP = "10.0.0.9" + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + Expect(k8sClient.Status().Update(ctx, pod)).To(Succeed()) + + reconcileNB(nb) + got := fetch(nb) + Expect(got.Status.Phase).To(Equal(notebooksv1alpha1.PhaseFailed)) + Expect(got.Status.URL).To(BeEmpty()) + Expect(got.Status.Message).To(ContainSubstring("does not belong")) + }) + + It("shows why the pod cannot be created, and retries", func() { + Expect(k8sClient.Create(ctx, newProfile())).To(Succeed()) + nb := newNotebook("rejected", nil) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + reconciler.Client = rejectPods{k8sClient} + _, err := reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(nb)}) + Expect(err).To(HaveOccurred()) + got := fetch(nb) + Expect(got.Status.Phase).To(Equal(notebooksv1alpha1.PhasePending)) + Expect(got.Status.Message).To(ContainSubstring("exceeded quota")) + }) + + It("adds the root init container only when asked", func() { + profile := newProfile() + profile.Spec.Storage.FixOwnership = true + pod := buildPod(newNotebook("x", nil), profile) + Expect(pod.Spec.InitContainers).To(HaveLen(1)) + Expect(pod.Spec.InitContainers[0].SecurityContext.RunAsUser).To(Equal(ptr.To[int64](0))) + }) + + Context("shared examples", func() { + BeforeEach(func() { + reconciler.ExamplesClaim = "mip-examples" + }) + + // No PV controller runs under envtest, so the test binds the claim itself. + examplesClaim := func(bound bool) { + pvc := &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: "mip-examples", Namespace: ns}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + } + Expect(k8sClient.Create(ctx, pvc)).To(Succeed()) + if bound { + pvc.Status.Phase = corev1.ClaimBound + Expect(k8sClient.Status().Update(ctx, pvc)).To(Succeed()) + } + } + // home "" keeps the CRD default, /home/jovyan. + spawn := func(fixOwnership bool, home string) (*notebooksv1alpha1.Notebook, *corev1.Pod) { + profile := newProfile() + profile.Spec.Storage.FixOwnership = fixOwnership + profile.Spec.Storage.MountPath = home + Expect(k8sClient.Create(ctx, profile)).To(Succeed()) + nb := newNotebook("examples", nil) + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + reconcileNB(nb) + pod := &corev1.Pod{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(nb), pod)).To(Succeed()) + return nb, pod + } + + It("mounts a bound claim read-only", func() { + examplesClaim(true) + _, pod := spawn(true, "") + Expect(pod.Spec.Volumes).To(ContainElement(corev1.Volume{ + Name: "examples", + VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: "mip-examples", + ReadOnly: true, + }}, + })) + Expect(pod.Spec.Containers[0].VolumeMounts).To(ContainElement( + corev1.VolumeMount{Name: "examples", MountPath: "/home/jovyan/work/.mip-examples", ReadOnly: true})) + }) + It("mounts inside the profile's home", func() { + examplesClaim(true) + _, pod := spawn(true, "/home/user") + Expect(pod.Spec.Containers[0].VolumeMounts).To(ContainElement( + corev1.VolumeMount{Name: "examples", MountPath: "/home/user/work/.mip-examples", ReadOnly: true})) + // The entrypoint looks for the mount under MIP_WORK_DIR. + Expect(pod.Spec.Containers[0].Env).To(ContainElement(corev1.EnvVar{Name: "MIP_WORK_DIR", Value: "/home/user/work"})) + }) + It("starts without the mount when there is no claim", func() { + nb, pod := spawn(true, "") + Expect(pod.Spec.Volumes).To(HaveLen(1)) + Expect(pod.Spec.Volumes[0].Name).To(Equal(homeVolume)) + Expect(fetch(nb).Status.Phase).To(Equal(notebooksv1alpha1.PhasePending)) + }) + It("ignores a claim that is not bound", func() { + examplesClaim(false) + _, pod := spawn(true, "") + Expect(pod.Spec.Volumes).To(HaveLen(1)) + }) + It("does nothing without EXAMPLES_CLAIM", func() { + reconciler.ExamplesClaim = "" + examplesClaim(true) + _, pod := spawn(true, "") + Expect(pod.Spec.Volumes).To(HaveLen(1)) + }) + It("needs fixOwnership, so work/ exists before the mount", func() { + examplesClaim(true) + _, pod := spawn(false, "") + Expect(pod.Spec.Volumes).To(HaveLen(1)) + }) + }) + + It("lets the hub create only Opaque --token Secrets owned by the Notebook", func() { + policy, err := os.ReadFile(filepath.Join("..", "..", "testdata", "token-secrets-policy.yaml")) + Expect(err).NotTo(HaveOccurred()) + for _, doc := range strings.Split(string(policy), "\n---\n") { + obj := &unstructured.Unstructured{} + Expect(utilyaml.Unmarshal([]byte(doc), &obj.Object)).To(Succeed()) + Expect(k8sClient.Create(ctx, obj)).To(Succeed()) + } + // The hub Role from the madgik/mip chart, bound to the hub ServiceAccount. + Expect(k8sClient.Create(ctx, &rbacv1.Role{ + ObjectMeta: metav1.ObjectMeta{Name: "jupyterhub", Namespace: ns}, + Rules: []rbacv1.PolicyRule{ + {APIGroups: []string{notebooksv1alpha1.GroupVersion.Group}, Resources: []string{"notebooks"}, Verbs: []string{"create"}}, + {APIGroups: []string{""}, Resources: []string{"secrets"}, Verbs: []string{"create"}}, + }, + })).To(Succeed()) + Expect(k8sClient.Create(ctx, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "jupyterhub", Namespace: ns}, + Subjects: []rbacv1.Subject{{Kind: rbacv1.ServiceAccountKind, Name: "jupyterhub", Namespace: ns}}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "Role", Name: "jupyterhub"}, + })).To(Succeed()) + hubCfg := rest.CopyConfig(cfg) + hubCfg.Impersonate.UserName = "system:serviceaccount:" + ns + ":jupyterhub" + hub, err := client.New(hubCfg, client.Options{Scheme: k8sClient.Scheme()}) + Expect(err).NotTo(HaveOccurred()) + + nb := newNotebook("policy", nil) + Expect(hub.Create(ctx, nb)).To(Succeed()) + secret := func(name string, typ corev1.SecretType) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, OwnerReferences: []metav1.OwnerReference{{ + APIVersion: notebooksv1alpha1.GroupVersion.String(), Kind: "Notebook", Name: nb.Name, UID: nb.UID, + }}}, + Type: typ, + } + } + denied := MatchError(ContainSubstring("mip-jupyterhub-secrets")) + // The policy takes effect a moment after it is created. + Eventually(func() error { + return hub.Create(ctx, secret("keycloak-credentials", corev1.SecretTypeOpaque)) + }).WithTimeout(10 * time.Second).Should(denied) + own := nb.Name + "-" + string(nb.UID) + "-token" + // The API server would fill this with the named ServiceAccount's token. + saToken := secret(own, corev1.SecretTypeServiceAccountToken) + saToken.Annotations = map[string]string{corev1.ServiceAccountNameKey: "default"} + Expect(hub.Create(ctx, saToken)).To(denied) + Expect(hub.Create(ctx, secret(own, corev1.SecretTypeOpaque))).To(Succeed()) + }) +}) + +var _ = Describe("names", func() { + It("matches KubeSpawner's claim-{user_server}", func() { + // From KubeSpawner 7.1 (slug_scheme "safe") in the hub image. + for user, want := range map[string]string{ + "alice": "claim-alice", + "john.doe": "claim-john-doe---30f69670", + "john-doe-30f69670": "claim-john-doe-30f69670", // must not equal john.doe's + "Alice": "claim-alice---3bc51062", + "alice--x": "claim-alice-x---2bbc534b", + "Alice@Example.org": "claim-alice-example-org---cfd44c4f", + "3b64fbf7-4e7b-4455-a61e-c19e64905f06": "claim-x-3b64fbf7-4e7b-4--x---07e56cda", + strings.Repeat("a", 46): "claim-" + strings.Repeat("a", 46), + strings.Repeat("a", 47): "claim-aaaaaaaaaaaaaaaaa--x---b191b19d", + strings.Repeat("x.y", 20): "claim-x-yx-yx-yx-yx-yx--x---9a32d384", + "user-": "claim-user---b5d4a457", + "_": "claim-x---d2e2adf7", + "1": "claim-x-1---6b86b273", + } { + Expect(pvcName(user)).To(Equal(want), user) + } + }) +}) diff --git a/operator/internal/controller/suite_test.go b/operator/internal/controller/suite_test.go new file mode 100644 index 0000000..34f67e2 --- /dev/null +++ b/operator/internal/controller/suite_test.go @@ -0,0 +1,71 @@ +package controller + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.Client +) + +func TestControllers(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Controller Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + var err error + err = notebooksv1alpha1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/operator/internal/spawnapi/auth.go b/operator/internal/spawnapi/auth.go new file mode 100644 index 0000000..7889a9f --- /dev/null +++ b/operator/internal/spawnapi/auth.go @@ -0,0 +1,46 @@ +package spawnapi + +import ( + "context" + "errors" + "fmt" + "slices" + + authenticationv1 "k8s.io/api/authentication/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Audience of the hub's projected token. The API server does not accept it, so +// the token is useless anywhere but here. +const Audience = "mip-notebook-operator" + +// ErrUnauthenticated means the token is not valid for Audience. +var ErrUnauthenticated = errors.New("token is not valid for audience " + Audience) + +// Authenticator maps a bearer token to a Kubernetes user name. +type Authenticator interface { + // Authenticate returns ErrUnauthenticated for a token that is not valid, + // another error when it cannot tell. + Authenticate(ctx context.Context, token string) (string, error) +} + +// TokenReviewAuthenticator asks the API server (tokenreviews create). +type TokenReviewAuthenticator struct { + Client client.Client +} + +func (a *TokenReviewAuthenticator) Authenticate(ctx context.Context, token string) (string, error) { + tr := &authenticationv1.TokenReview{Spec: authenticationv1.TokenReviewSpec{ + Token: token, + Audiences: []string{Audience}, + }} + if err := a.Client.Create(ctx, tr); err != nil { + return "", fmt.Errorf("token review: %w", err) + } + // An authenticator without audience support may ignore spec.audiences; + // status.audiences is what was actually checked. + if !tr.Status.Authenticated || !slices.Contains(tr.Status.Audiences, Audience) { + return "", ErrUnauthenticated + } + return tr.Status.User.Username, nil +} diff --git a/operator/internal/spawnapi/auth_test.go b/operator/internal/spawnapi/auth_test.go new file mode 100644 index 0000000..3e4a11d --- /dev/null +++ b/operator/internal/spawnapi/auth_test.go @@ -0,0 +1,71 @@ +package spawnapi + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +var _ = Describe("TokenReviewAuthenticator", func() { + var ( + ns string + counter int + auth *TokenReviewAuthenticator + ) + token := func(audience string) string { + clientset, err := kubernetes.NewForConfig(cfg) + Expect(err).NotTo(HaveOccurred()) + tr, err := clientset.CoreV1().ServiceAccounts(ns).CreateToken(ctx, "jupyterhub", &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{Audiences: []string{audience}, ExpirationSeconds: ptr.To[int64](600)}, + }, metav1.CreateOptions{}) + Expect(err).NotTo(HaveOccurred()) + return tr.Status.Token + } + + BeforeEach(func() { + counter++ + ns = fmt.Sprintf("federation-auth-%d", counter) + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})).To(Succeed()) + Expect(k8sClient.Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "jupyterhub", Namespace: ns}})).To(Succeed()) + auth = &TokenReviewAuthenticator{Client: k8sClient} + }) + + It("accepts a token for the mip-notebook-operator audience", func() { + username, err := auth.Authenticate(ctx, token(Audience)) + Expect(err).NotTo(HaveOccurred()) + Expect(username).To(Equal("system:serviceaccount:" + ns + ":jupyterhub")) + }) + It("rejects a token for another audience", func() { + _, err := auth.Authenticate(ctx, token("https://kubernetes.default.svc")) + Expect(err).To(MatchError(ErrUnauthenticated)) + }) + It("rejects a token that is not one", func() { + _, err := auth.Authenticate(ctx, "not-a-token") + Expect(err).To(MatchError(ErrUnauthenticated)) + }) + It("rejects an answer that does not confirm the audience", func() { + auth.Client = interceptor.NewClient(k8sClient, interceptor.Funcs{ + Create: func(_ context.Context, _ client.WithWatch, obj client.Object, _ ...client.CreateOption) error { + tr := obj.(*authenticationv1.TokenReview) + tr.Status = authenticationv1.TokenReviewStatus{ + Authenticated: true, + User: authenticationv1.UserInfo{Username: "system:serviceaccount:" + ns + ":jupyterhub"}, + Audiences: []string{"https://kubernetes.default.svc"}, + } + return nil + }, + }) + _, err := auth.Authenticate(ctx, "anything") + Expect(err).To(MatchError(ErrUnauthenticated)) + }) +}) diff --git a/operator/internal/spawnapi/listener.go b/operator/internal/spawnapi/listener.go new file mode 100644 index 0000000..2a4c522 --- /dev/null +++ b/operator/internal/spawnapi/listener.go @@ -0,0 +1,112 @@ +package spawnapi + +import ( + "context" + "crypto/tls" + "errors" + "net" + "net/http" + "strconv" + "sync/atomic" + "time" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + requestsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "mip_spawn_api_requests_total", + Help: "Spawn API requests by method and response code.", + }, []string{"method", "code"}) + deniedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "mip_spawn_api_denied_total", + Help: "Spawn API requests denied, by reason.", + }, []string{"reason"}) +) + +func init() { + metrics.Registry.MustRegister(requestsTotal, deniedTotal) +} + +type statusRecorder struct { + http.ResponseWriter + code int +} + +func (s *statusRecorder) WriteHeader(code int) { + s.code = code + s.ResponseWriter.WriteHeader(code) +} + +// instrument counts requests; labels carry no names, so cardinality is fixed. +func instrument(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec := &statusRecorder{ResponseWriter: w, code: http.StatusOK} + next.ServeHTTP(rec, r) + method := r.Method + switch method { + case http.MethodGet, http.MethodPut, http.MethodDelete: + default: + method = "other" + } + requestsTotal.WithLabelValues(method, strconv.Itoa(rec.code)).Inc() + }) +} + +// Listener serves a handler over TLS as a manager Runnable. +type Listener struct { + Addr string + Handler http.Handler + // GetCertificate comes from a certwatcher, so a renewed certificate is + // picked up without a restart. + GetCertificate func(*tls.ClientHelloInfo) (*tls.Certificate, error) + + ready atomic.Bool +} + +func (l *Listener) Start(ctx context.Context) error { + srv := &http.Server{ + Handler: l.Handler, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: l.GetCertificate, + }, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 2 * time.Minute, + MaxHeaderBytes: 32 << 10, + } + ln, err := net.Listen("tcp", l.Addr) + if err != nil { + return err + } + l.ready.Store(true) + done := make(chan struct{}) + go func() { + defer close(done) + <-ctx.Done() + l.ready.Store(false) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = srv.Shutdown(shutdownCtx) + }() + err = srv.ServeTLS(ln, "", "") + if errors.Is(err, http.ErrServerClosed) { + <-done + return nil + } + return err +} + +// NeedLeaderElection is false: every replica serves. +func (l *Listener) NeedLeaderElection() bool { return false } + +// ReadyCheck is a readyz check: ready while the listener is open. +func (l *Listener) ReadyCheck(_ *http.Request) error { + if !l.ready.Load() { + return errors.New("spawn API listener is not serving") + } + return nil +} diff --git a/operator/internal/spawnapi/listener_test.go b/operator/internal/spawnapi/listener_test.go new file mode 100644 index 0000000..67d404d --- /dev/null +++ b/operator/internal/spawnapi/listener_test.go @@ -0,0 +1,101 @@ +package spawnapi + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "sigs.k8s.io/controller-runtime/pkg/certwatcher" +) + +// writeCert writes a self-signed tls.crt / tls.key for 127.0.0.1 into dir. +func writeCert(dir, cn string) *x509.Certificate { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(time.Now().UnixNano()), + Subject: pkix.Name{CommonName: cn}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Minute), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + Expect(err).NotTo(HaveOccurred()) + keyDER, err := x509.MarshalECPrivateKey(key) + Expect(err).NotTo(HaveOccurred()) + // Key first: the watcher reloads on the cert write and needs a matching key. + Expect(os.WriteFile(filepath.Join(dir, "tls.key"), pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}), 0o600)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(dir, "tls.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600)).To(Succeed()) + cert, err := x509.ParseCertificate(der) + Expect(err).NotTo(HaveOccurred()) + return cert +} + +var _ = Describe("Listener", func() { + It("serves TLS 1.2+ and picks up a renewed certificate", func() { + dir := GinkgoT().TempDir() + first := writeCert(dir, "first") + certs, err := certwatcher.New(filepath.Join(dir, "tls.crt"), filepath.Join(dir, "tls.key")) + Expect(err).NotTo(HaveOccurred()) + certs = certs.WithWatchInterval(100 * time.Millisecond) + + ln, err := net.Listen("tcp", "127.0.0.1:0") + Expect(err).NotTo(HaveOccurred()) + addr := ln.Addr().String() + Expect(ln.Close()).To(Succeed()) + + srv := NewServer(k8sClient, fakeAuth{}, 5, 20, GinkgoLogr) + l := &Listener{Addr: addr, Handler: srv.Handler(), GetCertificate: certs.GetCertificate} + runCtx, stop := context.WithCancel(ctx) + defer stop() + go func() { _ = certs.Start(runCtx) }() + go func() { _ = l.Start(runCtx) }() + Eventually(func() error { return l.ReadyCheck(nil) }).Should(Succeed()) + + trusting := func(cert *x509.Certificate, version uint16) *tls.Config { + pool := x509.NewCertPool() + pool.AddCert(cert) + return &tls.Config{RootCAs: pool, MinVersion: version, MaxVersion: version} + } + served := func(cfg *tls.Config) (string, error) { + conn, err := tls.Dial("tcp", addr, cfg) + if err != nil { + return "", err + } + defer conn.Close() + return conn.ConnectionState().PeerCertificates[0].Subject.CommonName, nil + } + Expect(served(trusting(first, tls.VersionTLS12))).To(Equal("first")) + _, err = served(trusting(first, tls.VersionTLS11)) + Expect(err).To(HaveOccurred()) + + second := writeCert(dir, "second") + Eventually(func() (string, error) { return served(trusting(second, tls.VersionTLS13)) }). + WithTimeout(5 * time.Second).Should(Equal("second")) + + hc := &http.Client{Transport: &http.Transport{TLSClientConfig: trusting(second, tls.VersionTLS13)}} + resp, err := hc.Get("https://" + addr + "/v1/namespaces/federation-x/servers/jupyter-alice") + Expect(err).NotTo(HaveOccurred()) + resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized)) + + stop() + Eventually(func() error { return l.ReadyCheck(nil) }).ShouldNot(Succeed()) + }) +}) diff --git a/operator/internal/spawnapi/ratelimit.go b/operator/internal/spawnapi/ratelimit.go new file mode 100644 index 0000000..8939f7d --- /dev/null +++ b/operator/internal/spawnapi/ratelimit.go @@ -0,0 +1,44 @@ +package spawnapi + +import ( + "sync" + "time" + + "golang.org/x/time/rate" +) + +// namespaceLimiter is a token bucket per federation namespace. It is consulted +// only after authorization, so its keys are bounded by the namespaces whose hub +// holds a valid token. +type namespaceLimiter struct { + limit rate.Limit + burst int + + mu sync.Mutex + limiters map[string]*rate.Limiter +} + +func newNamespaceLimiter(limit rate.Limit, burst int) *namespaceLimiter { + return &namespaceLimiter{limit: limit, burst: burst, limiters: map[string]*rate.Limiter{}} +} + +// allow reports whether a request may go ahead, and if not, when to retry. +func (l *namespaceLimiter) allow(ns string, now time.Time) (bool, time.Duration) { + l.mu.Lock() + lim, ok := l.limiters[ns] + if !ok { + lim = rate.NewLimiter(l.limit, l.burst) + l.limiters[ns] = lim + } + l.mu.Unlock() + + r := lim.ReserveN(now, 1) + if !r.OK() { + return false, time.Second + } + if d := r.DelayFrom(now); d > 0 { + r.CancelAt(now) + return false, d + } + return true, 0 +} diff --git a/operator/internal/spawnapi/server.go b/operator/internal/spawnapi/server.go new file mode 100644 index 0000000..a24a115 --- /dev/null +++ b/operator/internal/spawnapi/server.go @@ -0,0 +1,349 @@ +// Package spawnapi is the HTTPS API the federation hubs use to start and stop +// notebook servers, so they hold no Kubernetes credential. It runs as its own +// process (--mode=spawn-api) and ServiceAccount, never inside the controller: +// the controller can create pods, so it must not listen where hubs reach it. +// See documentation/notebook-operator.md. +package spawnapi + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "math" + "net/http" + "regexp" + "strings" + "time" + + "github.com/go-logr/logr" + "golang.org/x/time/rate" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" +) + +const ( + // RequestedByAnnotation keeps the hub account on the Notebook: the API + // server only sees the spawn API's. + RequestedByAnnotation = "notebooks.mip.ebrains.eu/requested-by" + maxBodyBytes = 64 << 10 + maxNameLength = 57 + cleanupTimeout = 10 * time.Second +) + +var ( + federationNamespace = regexp.MustCompile(`^federation-[a-z0-9-]+$`) + // Same rules as the CRD on Notebook.spec.env. + allowedEnv = regexp.MustCompile(`^(JUPYTERHUB|JPY)_[A-Z0-9_]+$`) + tokenEnv = []string{"JUPYTERHUB_API_TOKEN", "JPY_API_TOKEN"} +) + +// Server serves /v1/namespaces/{ns}/servers/{name}. +type Server struct { + // Client must be uncached: the spawn API may not list or watch. + Client client.Client + Auth Authenticator + Log logr.Logger + limiter *namespaceLimiter + now func() time.Time +} + +func NewServer(c client.Client, auth Authenticator, limit rate.Limit, burst int, log logr.Logger) *Server { + return &Server{Client: c, Auth: auth, Log: log, limiter: newNamespaceLimiter(limit, burst), now: time.Now} +} + +// ServerRequest is the PUT body. +type ServerRequest struct { + Profile string `json:"profile"` + User string `json:"user"` + ServerName string `json:"serverName"` + Env map[string]string `json:"env"` + APIToken string `json:"apiToken"` +} + +// ServerStatus is what GET (and PUT) answer, from Notebook.status. +type ServerStatus struct { + Phase string `json:"phase"` + URL string `json:"url"` + Message string `json:"message"` + Deleting bool `json:"deleting"` +} + +type handlerFunc func(w http.ResponseWriter, r *http.Request, ns, name, username string) + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + const path = "/v1/namespaces/{ns}/servers/{name}" + mux.Handle("PUT "+path, s.guard(s.put)) + mux.Handle("GET "+path, s.guard(s.get)) + mux.Handle("DELETE "+path, s.guard(s.delete)) + return instrument(mux) +} + +// guard authenticates, authorizes and rate limits, in that order. +func (s *Server) guard(next handlerFunc) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ns := r.PathValue("ns") + token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if !ok || token == "" { + w.Header().Set("WWW-Authenticate", "Bearer") + s.deny(w, r, ns, "", http.StatusUnauthorized, "missing_token", "bearer token required") + return + } + username, err := s.Auth.Authenticate(r.Context(), token) + if errors.Is(err, ErrUnauthenticated) { + w.Header().Set("WWW-Authenticate", "Bearer") + s.deny(w, r, ns, "", http.StatusUnauthorized, "unauthenticated", "invalid token") + return + } + if err != nil { + s.Log.Error(err, "cannot authenticate request", "namespace", ns, "remoteAddr", r.RemoteAddr) + writeError(w, http.StatusServiceUnavailable, "cannot verify the token; retry") + return + } + if !federationNamespace.MatchString(ns) { + s.deny(w, r, ns, username, http.StatusForbidden, "namespace", "not a federation namespace") + return + } + // Each federation's hub manages its own namespace only. + if username != "system:serviceaccount:"+ns+":jupyterhub" { + s.deny(w, r, ns, username, http.StatusForbidden, "account", "account may not manage servers in this namespace") + return + } + if ok, wait := s.limiter.allow(ns, s.now()); !ok { + w.Header().Set("Retry-After", fmt.Sprint(int(math.Ceil(wait.Seconds())))) + s.deny(w, r, ns, username, http.StatusTooManyRequests, "rate_limited", "too many requests") + return + } + name := r.PathValue("name") + if err := validName(name); err != nil { + writeError(w, http.StatusBadRequest, err.Error()) + return + } + next(w, r, ns, name, username) + }) +} + +func (s *Server) deny(w http.ResponseWriter, r *http.Request, ns, username string, code int, reason, msg string) { + deniedTotal.WithLabelValues(reason).Inc() + s.Log.Info("denied request", "namespace", ns, "remoteAddr", r.RemoteAddr, "reason", reason, "username", username) + writeError(w, code, msg) +} + +func (s *Server) put(w http.ResponseWriter, r *http.Request, ns, name, username string) { + req, code, err := decodeRequest(w, r) + if err != nil { + writeError(w, code, err.Error()) + return + } + ctx := r.Context() + log := s.Log.WithValues("namespace", ns, "notebook", name, "requestedBy", username) + nb := ¬ebooksv1alpha1.Notebook{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: ns, + Annotations: map[string]string{RequestedByAnnotation: username}, + }, + Spec: notebooksv1alpha1.NotebookSpec{ + Profile: req.Profile, User: req.User, ServerName: req.ServerName, Env: req.Env, + }, + } + err = s.Client.Create(ctx, nb) + switch { + case err == nil: + if err := s.createSecret(ctx, nb, req.APIToken); err != nil { + log.Error(err, "cannot create token Secret; deleting the Notebook") + s.rollback(ctx, log, nb) + writeError(w, http.StatusInternalServerError, "cannot create the token Secret") + return + } + log.Info("created notebook", "uid", nb.UID) + writeJSON(w, http.StatusCreated, statusOf(nb)) + case apierrors.IsAlreadyExists(err): + s.retry(w, r, log, nb, req.APIToken) + case apierrors.IsInvalid(err): + writeError(w, http.StatusBadRequest, err.Error()) + default: + log.Error(err, "cannot create notebook") + writeError(w, http.StatusInternalServerError, "cannot create the Notebook") + } +} + +// retry answers a PUT for a Notebook that exists: 200 when it is the same +// request again, 409 otherwise. The token is not compared: an existing Secret +// keeps the first request's. The hub deletes the old server before every PUT, +// so a same-spec PUT with a new token does not happen. +func (s *Server) retry(w http.ResponseWriter, r *http.Request, log logr.Logger, want *notebooksv1alpha1.Notebook, token string) { + ctx := r.Context() + nb := ¬ebooksv1alpha1.Notebook{} + err := s.Client.Get(ctx, client.ObjectKeyFromObject(want), nb) + switch { + case apierrors.IsNotFound(err): + writeError(w, http.StatusConflict, "the Notebook is being replaced; retry") + return + case err != nil: + log.Error(err, "cannot get notebook") + writeError(w, http.StatusInternalServerError, "cannot get the Notebook") + return + case nb.DeletionTimestamp != nil: + writeError(w, http.StatusConflict, "the Notebook is terminating") + return + case !sameSpec(nb.Spec, want.Spec): + writeError(w, http.StatusConflict, "a Notebook with a different spec exists") + return + } + // No secrets get: creating it tells whether it exists, and completes a + // request that stopped between the Notebook and its Secret. + if err := s.createSecret(ctx, nb, token); err != nil { + log.Error(err, "cannot create token Secret") + writeError(w, http.StatusInternalServerError, "cannot create the token Secret") + return + } + writeJSON(w, http.StatusOK, statusOf(nb)) +} + +// createSecret creates --token, owned by nb. One that exists can +// only be from an earlier request for this Notebook (the UID is in the name). +func (s *Server) createSecret(ctx context.Context, nb *notebooksv1alpha1.Notebook, token string) error { + // Outlives a client that hangs up: a Notebook without its Secret is useless. + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: nb.TokenSecretName(), + Namespace: nb.Namespace, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: notebooksv1alpha1.GroupVersion.String(), + Kind: "Notebook", + Name: nb.Name, + UID: nb.UID, + Controller: ptr.To(true), + }}, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{notebooksv1alpha1.TokenSecretKey: []byte(token)}, + } + if err := s.Client.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { + return err + } + return nil +} + +func (s *Server) rollback(ctx context.Context, log logr.Logger, nb *notebooksv1alpha1.Notebook) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + // The UID precondition keeps a Notebook created since then. + err := s.Client.Delete(ctx, nb, client.PropagationPolicy(metav1.DeletePropagationForeground), + client.Preconditions{UID: &nb.UID}) + if err != nil && !apierrors.IsNotFound(err) { + log.Error(err, "cannot delete notebook after failed start") + } +} + +func (s *Server) get(w http.ResponseWriter, r *http.Request, ns, name, _ string) { + nb := ¬ebooksv1alpha1.Notebook{} + err := s.Client.Get(r.Context(), client.ObjectKey{Namespace: ns, Name: name}, nb) + switch { + case apierrors.IsNotFound(err): + writeError(w, http.StatusNotFound, "no such server") + case err != nil: + s.Log.Error(err, "cannot get notebook", "namespace", ns, "notebook", name) + writeError(w, http.StatusInternalServerError, "cannot get the Notebook") + default: + writeJSON(w, http.StatusOK, statusOf(nb)) + } +} + +func (s *Server) delete(w http.ResponseWriter, r *http.Request, ns, name, username string) { + nb := ¬ebooksv1alpha1.Notebook{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}} + // Foreground: the Notebook goes after its pod, so the hub's wait for 404 + // also waits for the pod. + err := s.Client.Delete(r.Context(), nb, client.PropagationPolicy(metav1.DeletePropagationForeground)) + switch { + case apierrors.IsNotFound(err): + writeError(w, http.StatusNotFound, "no such server") + case err != nil: + s.Log.Error(err, "cannot delete notebook", "namespace", ns, "notebook", name) + writeError(w, http.StatusInternalServerError, "cannot delete the Notebook") + default: + s.Log.Info("deleting notebook", "namespace", ns, "notebook", name, "requestedBy", username) + writeJSON(w, http.StatusAccepted, ServerStatus{Deleting: true}) + } +} + +func decodeRequest(w http.ResponseWriter, r *http.Request) (*ServerRequest, int, error) { + dec := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodyBytes)) + dec.DisallowUnknownFields() + req := &ServerRequest{} + err := dec.Decode(req) + if err == nil && dec.Decode(&struct{}{}) != io.EOF { + err = errors.New("trailing data after the JSON object") + } + if err != nil { + if maxErr := (*http.MaxBytesError)(nil); errors.As(err, &maxErr) { + return nil, http.StatusRequestEntityTooLarge, fmt.Errorf("body exceeds %d bytes", maxBodyBytes) + } + return nil, http.StatusBadRequest, fmt.Errorf("invalid body: %w", err) + } + switch { + case req.Profile == "": + return nil, http.StatusBadRequest, errors.New("profile is required") + case req.User == "": + return nil, http.StatusBadRequest, errors.New("user is required") + case req.APIToken == "": + return nil, http.StatusBadRequest, errors.New("apiToken is required") + } + for k := range req.Env { + if !allowedEnv.MatchString(k) { + return nil, http.StatusBadRequest, fmt.Errorf("env key %q must match %s", k, allowedEnv) + } + for _, t := range tokenEnv { + if k == t { + return nil, http.StatusBadRequest, fmt.Errorf("env may not set %s; send apiToken", k) + } + } + } + return req, 0, nil +} + +// validName applies the CRD's name rule; hub names are DNS labels. +func validName(name string) error { + if !strings.HasPrefix(name, "jupyter-") || len(name) > maxNameLength { + return fmt.Errorf("name must start with jupyter- and be at most %d characters", maxNameLength) + } + if errs := validation.IsDNS1123Label(name); len(errs) > 0 { + return fmt.Errorf("name: %s", strings.Join(errs, "; ")) + } + return nil +} + +func sameSpec(a, b notebooksv1alpha1.NotebookSpec) bool { + return a.Profile == b.Profile && a.User == b.User && a.ServerName == b.ServerName && maps.Equal(a.Env, b.Env) +} + +func statusOf(nb *notebooksv1alpha1.Notebook) ServerStatus { + return ServerStatus{ + Phase: nb.Status.Phase, + URL: nb.Status.URL, + Message: nb.Status.Message, + Deleting: nb.DeletionTimestamp != nil, + } +} + +func writeJSON(w http.ResponseWriter, code int, v any) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(v) +} + +func writeError(w http.ResponseWriter, code int, msg string) { + writeJSON(w, code, map[string]string{"error": msg}) +} diff --git a/operator/internal/spawnapi/server_test.go b/operator/internal/spawnapi/server_test.go new file mode 100644 index 0000000..16fadaf --- /dev/null +++ b/operator/internal/spawnapi/server_test.go @@ -0,0 +1,370 @@ +package spawnapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "time" + + "github.com/go-logr/logr" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/time/rate" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + utilyaml "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" + "github.com/Medical-Informatics-Platform/mip/operator/internal/controller" +) + +const spawnAPIUser = "system:serviceaccount:mip-notebooks-system:mip-notebook-spawn-api" + +// fakeAuth maps tokens to user names; "broken" stands for an API server that +// cannot be reached. +type fakeAuth map[string]string + +func (f fakeAuth) Authenticate(_ context.Context, token string) (string, error) { + if token == "broken" { + return "", errors.New("connection refused") + } + if u, ok := f[token]; ok { + return u, nil + } + return "", ErrUnauthenticated +} + +// applyFile creates every object in a YAML file, tolerating ones that exist +// (cluster-scoped objects are shared by the specs). +func applyFile(path ...string) { + data, err := os.ReadFile(filepath.Join(path...)) + Expect(err).NotTo(HaveOccurred()) + for _, doc := range strings.Split(string(data), "\n---\n") { + obj := &unstructured.Unstructured{} + Expect(utilyaml.Unmarshal([]byte(doc), &obj.Object)).To(Succeed()) + if err := k8sClient.Create(ctx, obj); !apierrors.IsAlreadyExists(err) { + Expect(err).NotTo(HaveOccurred()) + } + } +} + +var _ = Describe("Spawn API", func() { + var ( + ns string + srv *Server + counter int + ) + + path := func(name string) string { return "/v1/namespaces/" + ns + "/servers/" + name } + do := func(method, path, token string, body any) *httptest.ResponseRecorder { + var buf bytes.Buffer + switch b := body.(type) { + case nil: + case string: + buf.WriteString(b) + default: + Expect(json.NewEncoder(&buf).Encode(b)).To(Succeed()) + } + req := httptest.NewRequest(method, path, &buf) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + rec := httptest.NewRecorder() + srv.Handler().ServeHTTP(rec, req) + return rec + } + status := func(rec *httptest.ResponseRecorder) ServerStatus { + out := ServerStatus{} + Expect(json.Unmarshal(rec.Body.Bytes(), &out)).To(Succeed()) + return out + } + request := func() ServerRequest { + return ServerRequest{ + Profile: "default", User: "alice", Env: map[string]string{"JUPYTERHUB_USER": "alice"}, + APIToken: "hub-api-token", + } + } + fetch := func(name string) (*notebooksv1alpha1.Notebook, error) { + nb := ¬ebooksv1alpha1.Notebook{} + return nb, k8sClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: name}, nb) + } + newServer := func(c client.Client, limit rate.Limit, burst int) *Server { + return NewServer(c, fakeAuth{ + "hub": "system:serviceaccount:" + ns + ":jupyterhub", + "other-hub": "system:serviceaccount:federation-other:jupyterhub", + "other-account": "system:serviceaccount:" + ns + ":default", + "kube": "system:serviceaccount:kube-system:jupyterhub", + }, limit, burst, logr.Discard()) + } + + BeforeEach(func() { + counter++ + ns = fmt.Sprintf("federation-%d", counter) + Expect(k8sClient.Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}})).To(Succeed()) + srv = newServer(k8sClient, rate.Inf, 0) + }) + + Context("authentication and authorization", func() { + It("answers 401 without a bearer token", func() { + rec := do(http.MethodGet, path("jupyter-alice"), "", nil) + Expect(rec.Code).To(Equal(http.StatusUnauthorized)) + Expect(rec.Header().Get("WWW-Authenticate")).To(Equal("Bearer")) + Expect(rec.Body.String()).To(ContainSubstring(`"error"`)) + }) + It("answers 401 for a token the TokenReview rejects", func() { + Expect(do(http.MethodGet, path("jupyter-alice"), "nope", nil).Code).To(Equal(http.StatusUnauthorized)) + }) + It("answers 503 when the token cannot be checked", func() { + Expect(do(http.MethodGet, path("jupyter-alice"), "broken", nil).Code).To(Equal(http.StatusServiceUnavailable)) + }) + It("answers 403 to another federation's hub and creates nothing", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "other-hub", request()).Code).To(Equal(http.StatusForbidden)) + _, err := fetch("jupyter-alice") + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + It("answers 403 to another account of the namespace", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "other-account", request()).Code).To(Equal(http.StatusForbidden)) + }) + It("answers 403 outside federation-* namespaces", func() { + rec := do(http.MethodGet, "/v1/namespaces/kube-system/servers/jupyter-alice", "kube", nil) + Expect(rec.Code).To(Equal(http.StatusForbidden)) + }) + }) + + Context("PUT", func() { + It("creates the Notebook, then its token Secret owned by it", func() { + rec := do(http.MethodPut, path("jupyter-alice"), "hub", request()) + Expect(rec.Code).To(Equal(http.StatusCreated), rec.Body.String()) + + nb, err := fetch("jupyter-alice") + Expect(err).NotTo(HaveOccurred()) + Expect(nb.Spec).To(Equal(notebooksv1alpha1.NotebookSpec{ + Profile: "default", User: "alice", Env: map[string]string{"JUPYTERHUB_USER": "alice"}, + })) + Expect(nb.Annotations).To(HaveKeyWithValue(RequestedByAnnotation, "system:serviceaccount:"+ns+":jupyterhub")) + + secret := &corev1.Secret{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: "jupyter-alice-" + string(nb.UID) + "-token"}, secret)).To(Succeed()) + Expect(secret.Type).To(Equal(corev1.SecretTypeOpaque)) + Expect(secret.Data).To(Equal(map[string][]byte{"token": []byte("hub-api-token")})) + Expect(secret.OwnerReferences).To(HaveLen(1)) + owner := secret.OwnerReferences[0] + Expect(owner.Kind).To(Equal("Notebook")) + Expect(owner.UID).To(Equal(nb.UID)) + Expect(*owner.Controller).To(BeTrue()) + }) + It("answers 200 to the same request again", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusCreated)) + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusOK)) + }) + It("completes a Notebook whose Secret was never created", func() { + nb := ¬ebooksv1alpha1.Notebook{ + ObjectMeta: metav1.ObjectMeta{Name: "jupyter-alice", Namespace: ns}, + Spec: notebooksv1alpha1.NotebookSpec{Profile: "default", User: "alice", Env: map[string]string{"JUPYTERHUB_USER": "alice"}}, + } + Expect(k8sClient.Create(ctx, nb)).To(Succeed()) + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusOK)) + secret := &corev1.Secret{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: "jupyter-alice-" + string(nb.UID) + "-token"}, secret)).To(Succeed()) + }) + It("answers 409 when a Notebook with another spec exists", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusCreated)) + other := request() + other.Profile = "other" + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", other).Code).To(Equal(http.StatusConflict)) + other = request() + other.Env = nil + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", other).Code).To(Equal(http.StatusConflict)) + }) + It("answers 409 while the Notebook is terminating", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusCreated)) + // envtest has no garbage collector: a foreground delete stays terminating. + Expect(do(http.MethodDelete, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusAccepted)) + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusConflict)) + }) + It("rejects invalid requests before creating anything", func() { + for name, want := range map[string]int{ + "alice": http.StatusBadRequest, + "jupyter-Alice": http.StatusBadRequest, + "jupyter-a.b": http.StatusBadRequest, + "jupyter-" + strings.Repeat("a", 50): http.StatusBadRequest, + } { + Expect(do(http.MethodPut, path(name), "hub", request()).Code).To(Equal(want), name) + } + bad := func(edit func(r *ServerRequest)) ServerRequest { + r := request() + edit(&r) + return r + } + for i, body := range []any{ + bad(func(r *ServerRequest) { r.Env["LD_PRELOAD"] = "/x.so" }), + bad(func(r *ServerRequest) { r.Env["JUPYTERHUB_API_TOKEN"] = "x" }), + bad(func(r *ServerRequest) { r.Env["JPY_API_TOKEN"] = "x" }), + bad(func(r *ServerRequest) { r.APIToken = "" }), + bad(func(r *ServerRequest) { r.Profile = "" }), + `{"profile": "default", "user": "alice", "apiToken": "t", "image": "evil"}`, + `{"profile": "default", "user": "alice", "apiToken": "t"} {}`, + `not json`, + } { + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", body).Code).To(Equal(http.StatusBadRequest), "body %d", i) + } + huge := bad(func(r *ServerRequest) { r.Env["JUPYTERHUB_X"] = strings.Repeat("x", 64<<10) }) + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", huge).Code).To(Equal(http.StatusRequestEntityTooLarge)) + _, err := fetch("jupyter-alice") + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + }) + It("deletes the Notebook when its Secret cannot be created", func() { + srv = newServer(interceptor.NewClient(k8sClient, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*corev1.Secret); ok { + return apierrors.NewInternalError(errors.New("etcd is down")) + } + return c.Create(ctx, obj, opts...) + }, + }), rate.Inf, 0) + rec := do(http.MethodPut, path("jupyter-alice"), "hub", request()) + Expect(rec.Code).To(Equal(http.StatusInternalServerError)) + Expect(rec.Body.String()).NotTo(ContainSubstring("etcd")) + nb, err := fetch("jupyter-alice") + if err == nil { + Expect(nb.DeletionTimestamp).NotTo(BeNil(), "Notebook left without its Secret") + } else { + Expect(apierrors.IsNotFound(err)).To(BeTrue()) + } + }) + }) + + Context("GET and DELETE", func() { + It("reports the Notebook status", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusCreated)) + nb, err := fetch("jupyter-alice") + Expect(err).NotTo(HaveOccurred()) + nb.Status = notebooksv1alpha1.NotebookStatus{Phase: "Running", URL: "http://10.0.0.7:8888", Message: "server ready"} + Expect(k8sClient.Status().Update(ctx, nb)).To(Succeed()) + + rec := do(http.MethodGet, path("jupyter-alice"), "hub", nil) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(status(rec)).To(Equal(ServerStatus{Phase: "Running", URL: "http://10.0.0.7:8888", Message: "server ready"})) + }) + It("answers 404 for a server that does not exist", func() { + Expect(do(http.MethodGet, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusNotFound)) + Expect(do(http.MethodDelete, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusNotFound)) + }) + It("answers 202 to DELETE until the Notebook is gone", func() { + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusCreated)) + Expect(do(http.MethodDelete, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusAccepted)) + + rec := do(http.MethodGet, path("jupyter-alice"), "hub", nil) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(status(rec).Deleting).To(BeTrue()) + Expect(do(http.MethodDelete, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusAccepted)) + + // What the garbage collector does once the pod is gone. + nb, err := fetch("jupyter-alice") + Expect(err).NotTo(HaveOccurred()) + Expect(nb.Finalizers).To(ContainElement(metav1.FinalizerDeleteDependents)) + nb.Finalizers = nil + Expect(k8sClient.Update(ctx, nb)).To(Succeed()) + Eventually(func() int { + return do(http.MethodGet, path("jupyter-alice"), "hub", nil).Code + }).Should(Equal(http.StatusNotFound)) + Expect(do(http.MethodDelete, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusNotFound)) + }) + }) + + It("rate limits each namespace on its own", func() { + srv = newServer(k8sClient, rate.Limit(0.1), 2) + for range 2 { + Expect(do(http.MethodGet, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusNotFound)) + } + rec := do(http.MethodGet, path("jupyter-alice"), "hub", nil) + Expect(rec.Code).To(Equal(http.StatusTooManyRequests)) + Expect(rec.Header().Get("Retry-After")).To(Equal("10")) + + rec = do(http.MethodGet, "/v1/namespaces/federation-other/servers/jupyter-alice", "other-hub", nil) + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) + + It("spawns as its own ServiceAccount, with only its RBAC, through the token Secret policy", func() { + applyFile("..", "..", "testdata", "token-secrets-policy.yaml") + applyFile("..", "..", "testdata", "spawn-api-role.yaml") + Expect(k8sClient.Create(ctx, &rbacv1.RoleBinding{ + ObjectMeta: metav1.ObjectMeta{Name: "mip-notebook-spawn-api", Namespace: ns}, + Subjects: []rbacv1.Subject{{ + Kind: rbacv1.ServiceAccountKind, Name: "mip-notebook-spawn-api", Namespace: "mip-notebooks-system", + }}, + RoleRef: rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: "mip-notebook-spawn-api"}, + })).To(Succeed()) + spawnCfg := rest.CopyConfig(cfg) + spawnCfg.Impersonate.UserName = spawnAPIUser + spawnClient, err := client.New(spawnCfg, client.Options{Scheme: k8sClient.Scheme()}) + Expect(err).NotTo(HaveOccurred()) + + // The policy takes effect a moment after it is created. + Eventually(func() error { + return spawnClient.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "keycloak-credentials", Namespace: ns}, + Type: corev1.SecretTypeOpaque, + }) + }).WithTimeout(10 * time.Second).Should(MatchError(ContainSubstring("mip-jupyterhub-secrets"))) + + srv = newServer(spawnClient, rate.Inf, 0) + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusCreated)) + Expect(do(http.MethodPut, path("jupyter-alice"), "hub", request()).Code).To(Equal(http.StatusOK)) + Expect(do(http.MethodGet, path("jupyter-alice"), "hub", nil).Code).To(Equal(http.StatusOK)) + + // Nothing beyond that. + forbidden := func(err error) { ExpectWithOffset(1, apierrors.IsForbidden(err)).To(BeTrue(), "got %v", err) } + forbidden(spawnClient.Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "x", Namespace: ns}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "x", Image: "x"}}}, + })) + forbidden(spawnClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: "keycloak-credentials"}, &corev1.Secret{})) + forbidden(spawnClient.List(ctx, ¬ebooksv1alpha1.NotebookList{}, client.InNamespace(ns))) + forbidden(spawnClient.Get(ctx, client.ObjectKey{Namespace: "default", Name: "jupyter-alice"}, ¬ebooksv1alpha1.Notebook{})) + + // The controller mounts the Secret the spawn API created. + Expect(k8sClient.Create(ctx, ¬ebooksv1alpha1.NotebookProfile{ + ObjectMeta: metav1.ObjectMeta{Name: "default", Namespace: ns}, + Spec: notebooksv1alpha1.NotebookProfileSpec{ + Image: "hbpmip/mip-jupyter:test", + Storage: notebooksv1alpha1.NotebookStorage{Size: resource.MustParse("2Gi"), StorageClassName: "fast"}, + }, + })).To(Succeed()) + nb, err := fetch("jupyter-alice") + Expect(err).NotTo(HaveOccurred()) + reconciler := &controller.NotebookReconciler{Client: k8sClient, Scheme: k8sClient.Scheme()} + _, err = reconciler.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(nb)}) + Expect(err).NotTo(HaveOccurred()) + pod := &corev1.Pod{} + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(nb), pod)).To(Succeed()) + var refs []string + for _, e := range pod.Spec.Containers[0].Env { + if e.ValueFrom != nil && e.ValueFrom.SecretKeyRef != nil { + refs = append(refs, e.ValueFrom.SecretKeyRef.Name+"/"+e.ValueFrom.SecretKeyRef.Key) + } + } + Expect(refs).To(ConsistOf( + "jupyter-alice-"+string(nb.UID)+"-token/token", + "jupyter-alice-"+string(nb.UID)+"-token/token", + )) + secret := &corev1.Secret{} + Expect(k8sClient.Get(ctx, client.ObjectKey{Namespace: ns, Name: "jupyter-alice-" + string(nb.UID) + "-token"}, secret)).To(Succeed()) + }) +}) diff --git a/operator/internal/spawnapi/suite_test.go b/operator/internal/spawnapi/suite_test.go new file mode 100644 index 0000000..441d673 --- /dev/null +++ b/operator/internal/spawnapi/suite_test.go @@ -0,0 +1,61 @@ +package spawnapi + +import ( + "context" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + notebooksv1alpha1 "github.com/Medical-Informatics-Platform/mip/operator/api/v1alpha1" +) + +var ( + ctx context.Context + cancel context.CancelFunc + testEnv *envtest.Environment + cfg *rest.Config + k8sClient client.WithWatch +) + +func TestSpawnAPI(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Spawn API Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + ctx, cancel = context.WithCancel(context.TODO()) + + Expect(notebooksv1alpha1.AddToScheme(scheme.Scheme)).To(Succeed()) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, + } + + var err error + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + k8sClient, err = client.NewWithWatch(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + cancel() + Expect(testEnv.Stop()).To(Succeed()) +}) diff --git a/operator/testdata/spawn-api-role.yaml b/operator/testdata/spawn-api-role.yaml new file mode 100644 index 0000000..b6502b8 --- /dev/null +++ b/operator/testdata/spawn-api-role.yaml @@ -0,0 +1,42 @@ +# Test fixture, not deployed: the deployed rules are in mip-infra +# base/mip-infrastructure/notebook-operator/rbac.yaml. Change both together. +# +# Hand-maintained, not generated: kubebuilder markers would merge these rules +# into manager-role. The spawn API (--mode=spawn-api) runs as +# system:serviceaccount:mip-notebooks-system:mip-notebook-spawn-api. +# +# Bound per federation namespace (RoleBinding). No pods, PVCs, Secret reads or +# list/watch: a compromised spawn API can start notebooks from the profiles the +# hubs already use, and nothing else. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mip-notebook-spawn-api +rules: +- apiGroups: + - notebooks.mip.ebrains.eu + resources: + - notebooks + verbs: + - get + - create + - delete +- apiGroups: + - "" + resources: + - secrets + verbs: + - create +--- +# Bound cluster-wide (ClusterRoleBinding): validates the hubs' tokens. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: mip-notebook-spawn-api-tokenreview +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create diff --git a/operator/testdata/token-secrets-policy.yaml b/operator/testdata/token-secrets-policy.yaml new file mode 100644 index 0000000..0772e34 --- /dev/null +++ b/operator/testdata/token-secrets-policy.yaml @@ -0,0 +1,55 @@ +# Test fixture, not deployed: the deployed policy is mip-jupyterhub-secrets in mip-infra +# base/mip-infrastructure/notebook-operator/rbac.yaml. Change both together. +# +# The hub (operator mode) and the spawn API (operator-api mode) need +# `secrets: create` for the token Secrets. Unchecked, that would let a +# compromised one create a kubernetes.io/service-account-token Secret, which the +# API server fills with any ServiceAccount's token, under the name the operator +# mounts. The hub match goes once every federation runs operator-api. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: mip-jupyterhub-secrets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: [''] + apiVersions: [v1] + operations: [CREATE] + resources: [secrets] + matchConditions: + - name: token-secret-creators + expression: >- + request.userInfo.username.matches('^system:serviceaccount:federation-[^:]+:jupyterhub$') || + request.userInfo.username == + 'system:serviceaccount:mip-notebooks-system:mip-notebook-spawn-api' + variables: + - name: owners + expression: 'has(object.metadata.ownerReferences) ? object.metadata.ownerReferences + : []' + validations: + - expression: "!has(object.type) || object.type == 'Opaque'" + message: only Opaque Secrets may be created + - expression: >- + variables.owners.size() == 1 && + variables.owners[0].apiVersion.startsWith('notebooks.mip.ebrains.eu/') && + variables.owners[0].kind == 'Notebook' + message: the token Secret must be owned by its Notebook + # The name the operator mounts for that Notebook and no other. + - expression: >- + has(object.metadata.name) && variables.owners.size() == 1 && + variables.owners[0].name.startsWith('jupyter-') && + object.metadata.name == variables.owners[0].name + '-' + variables.owners[0].uid + + '-token' + message: >- + only the token Secret --token of the jupyter-* Notebook that + owns it may be created +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: mip-jupyterhub-secrets +spec: + policyName: mip-jupyterhub-secrets + validationActions: [Deny]