diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index af25a7677a..df3003dc8a 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -55,7 +55,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Set up uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version: ${{ env.UV_VERSION }} enable-cache: true @@ -379,7 +379,15 @@ jobs: --no-index \ --find-links "$BUNDLE_ROOT/wheelhouse" \ "$cli_wheel[cli]" - build/cli-smoke-venv/bin/powercontext skill remote-sync --help + if [ -f scripts/ci_role_smoke.py ]; then + build/cli-smoke-venv/bin/python -I scripts/ci_role_smoke.py cli + else + # Requested historical refs may predate the role smoke harness. + build/cli-smoke-venv/bin/powercontext skill remote-sync --help + fi + if [ -f src/powercontext/cli/archive.py ]; then + build/cli-smoke-venv/bin/powercontext archive --help + fi plugin_root="$BUNDLE_ROOT/integrations/plugins/powercontext" ( @@ -402,7 +410,7 @@ jobs: > "build/output/${ARTIFACT_NAME}.tar.gz.sha256" - name: Upload customer artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ steps.metadata.outputs.artifact_name }} path: | diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 8e2e04c79c..59f588313c 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version: "0.10.12" enable-cache: true @@ -117,7 +117,7 @@ jobs: exit 1 - name: Upload Docker image archives - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: ${{ steps.metadata.outputs.artifact_name }} path: docker-images/*.tar diff --git a/.github/workflows/candidate-schema-replay.yml b/.github/workflows/candidate-schema-replay.yml new file mode 100644 index 0000000000..dc30de99ff --- /dev/null +++ b/.github/workflows/candidate-schema-replay.yml @@ -0,0 +1,70 @@ +name: Candidate schema replay + +on: + pull_request: + paths: + - src/powercontext/builtin/persistence/** + - src/powercontext/builtin/artifacts/** + - tests/builtin/persistence/** + - scripts/replay_candidate_migration.py + - .github/workflows/candidate-schema-replay.yml + - .github/actions/setup-upgrade-oceanbase/** + - pyproject.toml + - uv.lock + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: candidate-replay-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + replay: + name: Candidate replay (${{ matrix.backend }}) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + backend: [sqlite, seekdb, oceanbase] + env: + POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_candidate_replay?charset=utf8mb4 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Python + uses: ./.github/actions/setup-python-env + - name: Start isolated OceanBase + if: matrix.backend == 'oceanbase' + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Create the scratch database namespace + if: matrix.backend == 'oceanbase' + run: >- + docker exec upgrade-oceanbase + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e + -e 'CREATE DATABASE pc_migration_probe_candidate_replay' + - name: Replay and recover the frozen Candidate conversion without skips + run: >- + uv run --locked --extra seekdb python scripts/replay_candidate_migration.py + ${{ matrix.backend }} --output "$RUNNER_TEMP/candidate-replay" + - name: Preserve OceanBase diagnostics + if: always() && matrix.backend == 'oceanbase' + run: | + mkdir -p "$RUNNER_TEMP/candidate-replay" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/candidate-replay/oceanbase.log" 2>&1 || true + docker stop --timeout 30 upgrade-oceanbase || true + - name: Save generated inventories and comparisons + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: candidate-replay-${{ matrix.backend }}-${{ github.sha }} + path: ${{ runner.temp }}/candidate-replay/ + if-no-files-found: error + retention-days: 14 diff --git a/.github/workflows/capture-release-schema.yml b/.github/workflows/capture-release-schema.yml new file mode 100644 index 0000000000..ddcfda28c7 --- /dev/null +++ b/.github/workflows/capture-release-schema.yml @@ -0,0 +1,63 @@ +name: Capture released Server schema + +on: + pull_request: + paths: + - scripts/capture_release_schema.py + - scripts/upgrade_database_probe.py + - scripts/upgrade_process.py + - tests/release_upgrades/test_full_server.py + - .github/workflows/capture-release-schema.yml + - .github/actions/setup-upgrade-oceanbase/** + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-schema-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + capture: + name: Capture 1.4.0 (${{ matrix.backend }}, ${{ matrix.configuration }}) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + backend: [sqlite, seekdb, oceanbase] + configuration: [full-server-no-vector, full-server-code, full-server-vector, full-server-vector-code] + env: + PC_UPGRADE_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Set up Python + uses: ./.github/actions/setup-python-env + - name: Start isolated OceanBase + if: matrix.backend == 'oceanbase' + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Capture installed release without current runtime initialization + run: >- + .venv/bin/python scripts/upgrade_process.py + --parent-pid $$ --timeout 1800 --shutdown-timeout 60 -- + .venv/bin/python scripts/capture_release_schema.py + ${{ matrix.backend }} --configuration ${{ matrix.configuration }} + --output "$RUNNER_TEMP/release-schema" + - name: Preserve OceanBase diagnostics + if: always() && matrix.backend == 'oceanbase' + run: | + mkdir -p "$RUNNER_TEMP/release-schema" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/release-schema/oceanbase.log" 2>&1 || true + docker stop --timeout 30 upgrade-oceanbase || true + - name: Save source schema and provenance + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-1.4.0-${{ matrix.backend }}-${{ matrix.configuration }}-${{ github.sha }} + path: ${{ runner.temp }}/release-schema/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/ci-changes.yml b/.github/workflows/ci-changes.yml new file mode 100644 index 0000000000..3d98a54ee4 --- /dev/null +++ b/.github/workflows/ci-changes.yml @@ -0,0 +1,30 @@ +name: Detect CI scope + +on: + workflow_call: + outputs: + suites: + description: JSON object of suites selected for this event + value: ${{ jobs.detect.outputs.suites }} + +permissions: + contents: read + +jobs: + detect: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + suites: ${{ steps.select.outputs.suites }} + steps: + - name: Check out the complete PR history + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + - name: Select checks + id: select + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: python3 scripts/ci_changes.py diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index 59007214f3..ceedebdd01 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -35,7 +35,13 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up website tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache website dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: website + cache-dependency-glob: website/pnpm-lock.yaml - name: Build website run: make docs-build diff --git a/.github/workflows/desktop.yml b/.github/workflows/desktop.yml new file mode 100644 index 0000000000..8e47a5a5e7 --- /dev/null +++ b/.github/workflows/desktop.yml @@ -0,0 +1,263 @@ +name: Desktop validation + +on: + pull_request: + paths: ['desktop/**', 'src/powercontext/**', 'pyproject.toml', 'uv.lock', 'openapi/powercontext.yaml', 'tests/fixtures/transport_loopback_vectors.json', 'tests/test_desktop_fixture_qualification.py', 'website/assets/**', '.github/workflows/desktop.yml'] + push: + branches: [master, codex/desktop-preview] + paths: ['desktop/**', 'src/powercontext/**', 'pyproject.toml', 'uv.lock', 'openapi/powercontext.yaml', 'tests/fixtures/transport_loopback_vectors.json', 'tests/test_desktop_fixture_qualification.py', 'website/assets/**', '.github/workflows/desktop.yml'] + workflow_dispatch: + inputs: + installer_run: + description: 'Optional run ID from this repository to diagnose its exact existing installer' + required: false + type: string + default: '' + +permissions: + contents: read + +concurrency: + group: desktop-${{ github.event.pull_request.number || github.ref }}-${{ inputs.installer_run || 'build' }} + cancel-in-progress: true + +jobs: + windows: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.installer_run == '' }} + runs-on: windows-2025 + timeout-minutes: 40 + defaults: + run: + working-directory: desktop + shell: pwsh + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + fetch-depth: 0 + - name: Verify Server version baseline tag + run: | + git fetch --no-tags https://github.com/oceanbase/powercontext.git refs/tags/powercontext-v1.0.0:refs/tags/powercontext-v1.0.0 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ((git rev-parse refs/tags/powercontext-v1.0.0) -ne 'ec97efba5f26fc972e9d8bf5027d8233acab3ca1') { + throw 'Unexpected Server version baseline tag' + } + - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + working_directory: desktop + - name: Install pinned Windows Rust toolchain + run: rustup toolchain install 1.95.0-x86_64-pc-windows-msvc --profile minimal --component rustfmt --component clippy + - name: Cache desktop dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: desktop + cache-dependency-glob: desktop/pnpm-lock.yaml + - name: Cache Rust dependencies and compilation + uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: desktop/src-tauri -> target + key: ${{ hashFiles('desktop/rust-toolchain.toml') }} + cache-bin: false + - name: Install locked dependencies + run: pnpm install --frozen-lockfile + - run: pnpm lint + - run: pnpm typecheck + - run: pnpm test + - run: pnpm build + - run: cargo fmt --manifest-path src-tauri/Cargo.toml --check + - run: cargo clippy --locked --manifest-path src-tauri/Cargo.toml --all-targets -- -D warnings + - run: cargo test --locked --manifest-path src-tauri/Cargo.toml + - run: pnpm ipc:check + - name: Verify diagnostic process containment + run: cargo run --locked --manifest-path src-tauri/Cargo.toml --example diagnostic_process_probe + - name: Verify Windows vault across processes + run: cargo run --locked --manifest-path src-tauri/Cargo.toml --example credential_probe + - name: Set up Python fixture tooling + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: "0.10.12" + enable-cache: true + cache-suffix: desktop-${{ github.job }}-py3.12 + cache-dependency-glob: | + pyproject.toml + uv.lock + integrations/langchain/pyproject.toml + integrations/langgraph/pyproject.toml + integrations/opendal/pyproject.toml + - name: Install locked Server test dependencies + working-directory: . + run: uv sync --locked --python 3.12 + - name: Build Server wheel for isolated acceptance + working-directory: . + run: uv build --wheel --out-dir desktop/.artifacts/server-wheel + - name: Qualify the disposable checkout and built-wheel fixture + working-directory: . + run: uv run --no-sync python desktop/tests/fixture_qualification.py --server-commit $env:GITHUB_SHA + - name: Check installed UI acceptance harness + working-directory: . + run: | + $harnessFiles = @(Get-ChildItem -Path desktop/tests/installed_*.py -File | Sort-Object Name | ForEach-Object { $_.FullName }) + $harnessFiles += 'desktop/tests/fixture_qualification.py' + uv run --no-sync ruff check --no-fix @harnessFiles + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv run --no-sync ruff format --check @harnessFiles + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv run --no-sync ty check @harnessFiles + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv run --no-sync ty check --python-platform linux @harnessFiles + - name: Check fixture qualification regressions + working-directory: . + run: uv run --no-sync python -m pytest tests/test_desktop_fixture_qualification.py + - name: Build native API acceptance client + run: cargo build --locked --manifest-path src-tauri/Cargo.toml --example server_probe --example diagnostic_cli_probe + - name: Test native adapter with real SQLite Server + working-directory: . + run: uv run --no-sync python desktop/tests/real_server.py + - name: Test native diagnostics with isolated real CLI + working-directory: . + run: uv run --no-sync python desktop/tests/real_cli.py + - name: Build fixture-qualified unsigned internal Windows installer + run: pnpm desktop:build:windows:ci + - name: Prepare matching WebView2 driver + run: ./scripts/prepare-webdriver.ps1 + - name: Smoke test installed package + timeout-minutes: 8 + run: ./scripts/windows-smoke.ps1 + - name: Upload installer and acceptance evidence + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() + with: + name: desktop-windows-internal-unsigned + include-hidden-files: true + path: | + desktop/src-tauri/target/release/bundle/nsis/*-setup.exe + desktop/.artifacts/windows-smoke.json + desktop/.artifacts/ci-compatibility.json + desktop/.artifacts/real-server.json + desktop/.artifacts/real-server-*/ + desktop/.artifacts/real-cli.json + desktop/.artifacts/installed-ui.json + desktop/.artifacts/installed-ui.png + desktop/.artifacts/installed-ui-driver.log + desktop/.artifacts/installed-ui-app.log + desktop/.artifacts/installed-ui-processes.json + desktop/.artifacts/installed-ui-window.png + desktop/.artifacts/installed-ui-server-*/ + # Early validation failures can leave no installer or acceptance evidence. + # Still reject missing artifacts when every preceding step succeeded. + if-no-files-found: ${{ job.status == 'success' && 'error' || 'warn' }} + + macos: + if: ${{ github.event_name != 'workflow_dispatch' || inputs.installer_run == '' }} + strategy: + fail-fast: false + matrix: + runner: [macos-15, macos-15-intel] + runs-on: ${{ matrix.runner }} + timeout-minutes: 40 + defaults: + run: + working-directory: desktop + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + with: + working_directory: desktop + - run: rustup toolchain install 1.95.0 --profile minimal --component rustfmt --component clippy + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + with: + workspaces: desktop/src-tauri -> target + key: macos-${{ matrix.runner }} + cache-bin: false + - run: pnpm install --frozen-lockfile + - run: pnpm lint + - run: pnpm test + - run: pnpm build + - run: cargo fmt --manifest-path src-tauri/Cargo.toml --check + - run: cargo clippy --locked --manifest-path src-tauri/Cargo.toml --all-targets -- -D warnings + - run: cargo test --locked --manifest-path src-tauri/Cargo.toml + - run: pnpm ipc:check + - name: Verify native Keychain across processes + run: cargo run --locked --manifest-path src-tauri/Cargo.toml --example credential_probe + - name: Build unsigned macOS app and DMG + run: pnpm desktop:build + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: desktop-macos-${{ matrix.runner }}-internal-unsigned + path: desktop/src-tauri/target/release/bundle/dmg/*.dmg + if-no-files-found: error + + installed-package: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.installer_run != '' }} + runs-on: windows-2025 + timeout-minutes: 15 + permissions: + contents: read + actions: read + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + fetch-depth: 0 + - name: Verify Server version baseline tag + run: | + git fetch --no-tags https://github.com/oceanbase/powercontext.git refs/tags/powercontext-v1.0.0:refs/tags/powercontext-v1.0.0 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ((git rev-parse refs/tags/powercontext-v1.0.0) -ne 'ec97efba5f26fc972e9d8bf5027d8233acab3ca1') { throw 'Unexpected Server version baseline tag' } + - uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 + with: + version: "0.10.12" + enable-cache: true + cache-suffix: desktop-${{ github.job }}-py3.12 + cache-dependency-glob: | + pyproject.toml + uv.lock + integrations/langchain/pyproject.toml + integrations/langgraph/pyproject.toml + integrations/opendal/pyproject.toml + - name: Prepare isolated Server fixture + run: | + uv sync --locked --python 3.12 + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv build --wheel --out-dir desktop/.artifacts/server-wheel + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv run --no-sync python desktop/tests/fixture_qualification.py --server-commit $env:GITHUB_SHA + - name: Retrieve and verify exact installer + env: + GH_TOKEN: ${{ github.token }} + INSTALLER_RUN: ${{ inputs.installer_run }} + run: | + if ($env:INSTALLER_RUN -notmatch '^\d+$') { throw 'Expected numeric workflow run ID.' } + $run = gh api "repos/$env:GITHUB_REPOSITORY/actions/runs/$env:INSTALLER_RUN" | ConvertFrom-Json + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if ($run.path -ne '.github/workflows/desktop.yml' -or $run.head_branch -ne 'codex/desktop-preview') { throw 'Expected Desktop preview workflow artifact.' } + gh run download $env:INSTALLER_RUN --repo $env:GITHUB_REPOSITORY --name desktop-windows-internal-unsigned --dir desktop/.artifacts/source-installer + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $packages = @(Get-ChildItem desktop/.artifacts/source-installer -Recurse -Filter '*-setup.exe') + $reports = @(Get-ChildItem desktop/.artifacts/source-installer -Recurse -Filter windows-smoke.json) + if ($packages.Count -ne 1 -or $reports.Count -ne 1) { throw 'Expected exactly one package and package report.' } + $report = Get-Content -LiteralPath $reports[0].FullName -Raw | ConvertFrom-Json + if ($report.commit -ne $run.head_sha -or (Get-FileHash -LiteralPath $packages[0].FullName).Hash -ne $report.installerSha256 -or $packages[0].Length -ne $report.installerBytes) { throw 'Installer identity mismatch.' } + $bundle = 'desktop/src-tauri/target/release/bundle/nsis' + New-Item -ItemType Directory -Path $bundle -Force | Out-Null + Copy-Item -LiteralPath $packages[0].FullName -Destination $bundle + "DESKTOP_INSTALLER_COMMIT=$($run.head_sha)" >> $env:GITHUB_ENV + - name: Prepare matching WebView2 driver + run: ./desktop/scripts/prepare-webdriver.ps1 + - name: Exercise existing installed package + timeout-minutes: 8 + run: ./desktop/scripts/windows-smoke.ps1 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: always() + with: + name: desktop-installed-diagnostics + include-hidden-files: true + path: | + desktop/.artifacts/windows-smoke.json + desktop/.artifacts/installed-ui* + if-no-files-found: warn diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index 537aed3ff4..ccd0afeb32 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -7,6 +7,14 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] workflow_dispatch: + inputs: + suite: + description: Validation suite to run + type: choice + default: all + options: + - all + - recall-effort permissions: contents: read @@ -16,11 +24,18 @@ env: UV_VERSION: "0.10.12" concurrency: - group: e2e-harness-${{ github.event.pull_request.number || github.ref }}-${{ github.sha }} - cancel-in-progress: false + group: e2e-harness-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: + changes: + uses: ./.github/workflows/ci-changes.yml + validate: + needs: changes + if: >- + ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && + (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').harness) }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -31,7 +46,7 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version: ${{ env.UV_VERSION }} @@ -48,10 +63,401 @@ jobs: make harness-check make harness-compose-check + recall-effort: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + name: Recall effort (OceanBase) + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 + POWERCONTEXT_TEST_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/powercontext_test?charset=utf8mb4 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python and project dependencies + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + + - name: Start the isolated OceanBase test instance + run: | + docker run --detach --name recall-effort-oceanbase \ + --publish 127.0.0.1:2881:2881 \ + --ulimit nofile=65536:65536 \ + --env MODE=slim \ + --env OB_DATABASE=powercontext_test \ + --env OB_TENANT_PASSWORD=powercontext-e2e \ + "$OCEANBASE_IMAGE" + for attempt in $(seq 1 120); do + if docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext_test -e 'SELECT 1' >/dev/null 2>&1; then + # The slim image's default 1 GB maximum is too small for the + # indexes created across the full Server recovery scenarios. + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/recall-effort-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/recall-effort-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/recall-effort-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext_test \ + -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW GLOBAL VARIABLES LIKE 'recyclebin';" + exit 0 + fi + if [ "$(docker inspect --format '{{.State.Running}}' recall-effort-oceanbase)" != true ]; then + echo 'OceanBase exited before becoming ready.' >&2 + exit 1 + fi + sleep 5 + done + echo 'OceanBase did not become ready within 10 minutes.' >&2 + exit 1 + + - name: Resolve Python version for hook cache + id: python + run: echo "version=$(python -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT" + + - name: Cache prek environments + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/prek + key: prek-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.python.outputs.version }}-${{ hashFiles('.pre-commit-config.yaml') }} + + - name: Check code and types on Linux + run: | + uv lock --check + uv run --locked prek run -a + + - name: Run OceanBase contracts and Jev integration without skips + run: | + mkdir -p .powercontext/recall-effort-ci + uv run --locked pytest \ + tests/builtin/persistence/test_oceanbase_profile.py::test_live_oceanbase_profile_smoke \ + 'tests/builtin/persistence/test_recall_effort_statistics.py::test_recall_effort_schema_upgrade_and_atomic_daily_scope_accounting[oceanbase]' \ + tests/builtin/persistence/test_oceanbase_profile.py::test_live_oceanbase_profile_rejects_old_schema_collation \ + tests/e2e/test_jev_memory_reranking.py \ + --junitxml=.powercontext/recall-effort-ci/oceanbase.xml -q -rs + uv run --locked python - <<'PY' + from pathlib import Path + from xml.etree import ElementTree + + suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/oceanbase.xml")).getroot() + cases = list(suites.iter("testcase")) + if len(cases) != 9 or any(case.find("skipped") is not None for case in cases): + raise SystemExit("All nine database contracts must execute; skipped tests are not acceptance.") + PY + + - name: Run the real OceanBase recovery upgrade without skips + run: | + uv run --locked pytest tests/e2e/test_topic_memory_recovery_migration.py -k oceanbase \ + --junitxml=.powercontext/recall-effort-ci/topic-recovery.xml -q -rs + uv run --locked python - <<'PY' + from pathlib import Path + from xml.etree import ElementTree + + suites = ElementTree.parse(Path(".powercontext/recall-effort-ci/topic-recovery.xml")).getroot() + cases = list(suites.iter("testcase")) + if len(cases) != 2 or any(case.find("skipped") is not None for case in cases): + raise SystemExit("Both live OceanBase recovery upgrades must execute; skipped tests are not acceptance.") + PY + + - name: Run recall, statistics and prepared-context regressions + run: | + uv run --locked pytest \ + tests/builtin/runtime/test_recall_sufficiency.py \ + tests/e2e/test_recall_sufficiency_gate.py \ + tests/builtin/runtime/test_statistics.py \ + tests/builtin/runtime/test_model_usage_recorder.py \ + tests/builtin/runtime/test_prepared_context.py \ + tests/e2e/test_context_text_assembly.py \ + tests/builtin/persistence/test_recall_effort_statistics.py \ + tests/builtin/persistence/test_mysql_schema.py \ + tests/builtin/persistence/test_sqlite_profile.py \ + tests/builtin/persistence/test_database_snapshots.py \ + --junitxml=.powercontext/recall-effort-ci/regressions.xml -q -rs + + - name: Recover interrupted Server initialization on real OceanBase without skips + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 + run: | + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext_test -e 'CREATE DATABASE pc_migration_probe_server_recovery' + timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_resumes_interrupted_initialization_before_manifest_seed \ + -o faulthandler_timeout=180 --durations=0 \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-initialization.xml + + # Each group keeps a hard deadline. The six independent recovery scenarios + # must not consume a single shared deadline before client-exit recovery runs. + - name: Recover committed Server DDL and domain checkpoints on real OceanBase without skips + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 + run: | + timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_full_server_interruption_preserves_history_and_original_maintenance \ + -o faulthandler_timeout=180 --durations=0 \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-checkpoints.xml + + - name: Recover an exited Server migration client on real OceanBase without skips + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 + run: | + timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_oceanbase_full_server_client_exit_requires_acknowledgement_and_preserves_recovery \ + -o faulthandler_timeout=180 --durations=0 \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-client-exit.xml + + - name: Maintain the registered Server chain on real OceanBase without skips + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_server_recovery?charset=utf8mb4 + run: | + timeout --signal=TERM --kill-after=10s 600s uv run --locked pytest -vv -s -rs \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_processing_mode_switch_preserves_revision_and_history \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_atomic_compatibility_command_uses_shared_server_upgrade \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_registered_projection_repair_preserves_authority_and_revision \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_topic_bundle_refuses_managed_server_storage \ + tests/builtin/persistence/test_mysql_migration_runner.py::test_mysql_managed_identity_initialization_requires_no_ddl \ + -o faulthandler_timeout=180 --durations=0 \ + -k oceanbase --junitxml=.powercontext/recall-effort-ci/full-server-maintenance.xml + uv run --locked python - <<'PY' + from pathlib import Path + from xml.etree import ElementTree + + reports = ( + Path(".powercontext/recall-effort-ci/full-server-initialization.xml"), + Path(".powercontext/recall-effort-ci/full-server-checkpoints.xml"), + Path(".powercontext/recall-effort-ci/full-server-client-exit.xml"), + Path(".powercontext/recall-effort-ci/full-server-maintenance.xml"), + ) + cases = [case for report in reports for case in ElementTree.parse(report).iter("testcase")] + if len(cases) != 11 or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("All eleven native full Server recovery and maintenance scenarios must pass without skips.") + PY + + - name: Collect the database log + if: always() + run: | + mkdir -p .powercontext/recall-effort-ci + docker logs recall-effort-oceanbase > .powercontext/recall-effort-ci/oceanbase.log 2>&1 || true + for logfile in observer.log observer.log.wf; do + docker exec recall-effort-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ + > ".powercontext/recall-effort-ci/$logfile" 2>&1 || true + done + df -h / > .powercontext/recall-effort-ci/runner-disk.txt 2>&1 || true + docker exec recall-effort-oceanbase df -h /root/demo \ + > .powercontext/recall-effort-ci/container-disk.txt 2>&1 || true + docker exec recall-effort-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next'); SELECT data_disk_allocated, data_disk_capacity, data_disk_in_use FROM oceanbase.GV\$OB_SERVERS;" \ + > .powercontext/recall-effort-ci/database-disk.txt 2>&1 || true + + - name: Save validation reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: recall-effort-validation-${{ github.sha }} + path: .powercontext/recall-effort-ci/ + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + + - name: Stop and remove the test instance + if: always() + run: docker rm --force recall-effort-oceanbase || true + + atomic-memory-clean-oceanbase: + name: Atomic Memory clean migration (OceanBase) + needs: changes + if: >- + ${{ !cancelled() && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && + (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + runs-on: ubuntu-latest + timeout-minutes: 145 + env: + OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.3.5.6-106000012026040916 + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: >- + mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/powercontext_test?charset=utf8mb4 + PYTHONUNBUFFERED: "1" + MIGRATION_REPORTS: ${{ github.workspace }}/.powercontext/atomic-memory-clean-oceanbase + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + + - name: Start the isolated OceanBase test instance + run: | + docker run --detach --name atomic-memory-clean-oceanbase \ + --publish 127.0.0.1:2881:2881 \ + --ulimit nofile=65536:65536 \ + --env MODE=slim \ + --env OB_DATABASE=powercontext_test \ + --env OB_TENANT_PASSWORD=powercontext-e2e \ + "$OCEANBASE_IMAGE" + for attempt in $(seq 1 120); do + if docker exec atomic-memory-clean-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext_test -e 'SELECT 1' >/dev/null 2>&1; then + # The slim image's default 1 GB maximum is too small for the + # indexes created across the clean migration acceptance cases. + docker exec atomic-memory-clean-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + docker exec atomic-memory-clean-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/atomic-memory-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/atomic-memory-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/atomic-memory-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi + docker exec atomic-memory-clean-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext_test \ + -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" + exit 0 + fi + if [ "$(docker inspect --format '{{.State.Running}}' atomic-memory-clean-oceanbase)" != true ]; then + echo 'OceanBase exited before becoming ready.' >&2 + exit 1 + fi + sleep 5 + done + echo 'OceanBase did not become ready within 10 minutes.' >&2 + exit 1 + + - name: Test interrupted Scope search migrations without skips + timeout-minutes: 10 + run: | + uv run --locked python scripts/run_atomic_memory_clean_migration.py \ + oceanbase --output "$MIGRATION_REPORTS/scope-search" --tests \ + tests/e2e/test_scope_search_migration.py + + - name: Test background listener lifecycle without skips + timeout-minutes: 5 + run: | + uv run --locked --extra server python scripts/run_atomic_memory_clean_migration.py \ + oceanbase --output "$MIGRATION_REPORTS/background-metrics" --tests \ + tests/e2e/test_background_metrics.py + + - name: Run clean migration evidence, grants and vectors without skips + timeout-minutes: 60 + run: | + uv run --locked python scripts/run_atomic_memory_clean_migration.py \ + oceanbase --output "$MIGRATION_REPORTS/clean-migration" + + - name: Test concurrent managed startup without skips + # Collection and three isolated cases retain their 180s process deadlines. + timeout-minutes: 15 + run: | + uv run --locked --extra server python scripts/run_atomic_memory_clean_migration.py \ + oceanbase --output "$MIGRATION_REPORTS/startup-snapshot" --tests \ + tests/e2e/test_managed_startup_read_consistency.py + test "$(wc -l < "$MIGRATION_REPORTS/startup-snapshot/nodes.txt")" -eq 3 + + - name: Test long-query retrieval and admission without skips + # Eight isolated cases include full Server setup and retain 180s deadlines. + timeout-minutes: 30 + run: | + uv run --locked python scripts/run_atomic_memory_clean_migration.py \ + oceanbase --output "$MIGRATION_REPORTS/long-query" --tests \ + tests/e2e/test_atomic_artifact_search_http.py \ + tests/e2e/test_atomic_memory_long_fts_query.py + + - name: Collect the database log + if: always() + run: | + mkdir -p "$MIGRATION_REPORTS" + docker logs atomic-memory-clean-oceanbase > "$MIGRATION_REPORTS/oceanbase.log" 2>&1 || true + for logfile in observer.log observer.log.wf; do + docker exec atomic-memory-clean-oceanbase tail -n 20000 "/root/demo/log/$logfile" \ + > "$MIGRATION_REPORTS/$logfile" 2>&1 || true + done + docker inspect --format 'MemoryBytes={{.HostConfig.Memory}} NanoCpus={{.HostConfig.NanoCpus}} CpuQuota={{.HostConfig.CpuQuota}}' \ + atomic-memory-clean-oceanbase > "$MIGRATION_REPORTS/container-resources.txt" 2>&1 || true + docker stats --no-stream atomic-memory-clean-oceanbase \ + > "$MIGRATION_REPORTS/container-stats.txt" 2>&1 || true + docker exec atomic-memory-clean-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next', 'memory_limit', 'system_memory', 'cpu_count');" \ + > "$MIGRATION_REPORTS/database-parameters.txt" 2>&1 || true + + - name: Save complete observer logs on failure + if: failure() + run: | + mkdir -p "$MIGRATION_REPORTS/observer" + docker cp atomic-memory-clean-oceanbase:/root/demo/log/. "$MIGRATION_REPORTS/observer/" || true + + - name: Save clean migration reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: atomic-memory-clean-oceanbase-${{ github.sha }} + path: ${{ env.MIGRATION_REPORTS }}/ + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + + - name: Stop and remove the test instance + if: always() + run: docker rm --force atomic-memory-clean-oceanbase || true + acceptance: needs: validate runs-on: ubuntu-latest - timeout-minutes: 40 + # OceanBase regressions can take 32 minutes before the separate replay scenarios. + timeout-minutes: ${{ matrix.database == 'oceanbase' && 60 || 40 }} strategy: fail-fast: false matrix: @@ -65,11 +471,97 @@ jobs: persist-credentials: false - name: Install uv - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0 with: version: ${{ env.UV_VERSION }} + - name: Verify OceanBase Atomic Memory and HTTP regressions + timeout-minutes: 40 + if: matrix.database == 'oceanbase' + env: + COMPOSE_PROJECT_NAME: powercontext-e2e-oceanbase + POWERCONTEXT_E2E_OUTPUT: ${{ github.workspace }}/.powercontext-e2e/oceanbase/acceptance + run: | + mkdir -p "$POWERCONTEXT_E2E_OUTPUT" + compose=(docker compose -f e2e/bub/compose.yaml -f e2e/bub/compose.oceanbase.yaml) + "${compose[@]}" up --detach --wait --wait-timeout 660 oceanbase + container_id=$("${compose[@]}" ps -q oceanbase) + # The slim image's default 1 GB maximum is too small for the + # indexes created across isolated acceptance databases. + docker exec "$container_id" \ + obclient -h127.0.0.1 -P2881 -uroot@sys \ + -e "ALTER SYSTEM SET datafile_maxsize='8G'; ALTER SYSTEM SET datafile_next='512M'; ALTER SYSTEM SET datafile_size='4G';" + datafile_ready=false + for config_attempt in $(seq 1 30); do + docker exec "$container_id" \ + obclient -h127.0.0.1 -P2881 -uroot@sys --batch --skip-column-names \ + -e "SHOW PARAMETERS WHERE name IN ('datafile_size', 'datafile_maxsize', 'datafile_next');" \ + > "$RUNNER_TEMP/acceptance-datafile.tsv" + if awk ' + $5 == "datafile_size" && $7 == "4G" { size = 1 } + $5 == "datafile_maxsize" && $7 == "8G" { maximum = 1 } + $5 == "datafile_next" && $7 == "512M" { increment = 1 } + END { exit !(size && maximum && increment) } + ' "$RUNNER_TEMP/acceptance-datafile.tsv"; then + datafile_ready=true + break + fi + sleep 2 + done + cat "$RUNNER_TEMP/acceptance-datafile.tsv" + if [ "$datafile_ready" != true ]; then + echo 'OceanBase datafile configuration did not become effective within one minute.' >&2 + exit 1 + fi + docker exec "$container_id" \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -Dpowercontext \ + -e "SET GLOBAL recyclebin=OFF; SELECT VERSION(); SHOW VARIABLES LIKE 'ob_compatibility_mode'; SHOW VARIABLES LIKE 'recyclebin';" + oceanbase_ip=$(docker inspect --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container_id") + test -n "$oceanbase_ip" + export POWERCONTEXT_TEST_OCEANBASE_URL="mysql+aoceanbase://root%40test:powercontext-e2e@${oceanbase_ip}:2881/powercontext?charset=utf8mb4" + export POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL="$POWERCONTEXT_TEST_OCEANBASE_URL" + uv run --locked --extra server python - <<'PY' + import asyncio + import os + + from pydantic import SecretStr + + from powercontext.builtin.persistence.oceanbase import OceanBaseConfig, OceanBaseProfile + + async def version(): + config = OceanBaseConfig(url=SecretStr(os.environ["POWERCONTEXT_TEST_OCEANBASE_URL"])) + async with OceanBaseProfile.open(config, tables=()) as profile: + async with profile.database.transaction() as connection: + result = await connection.exec_driver_sql("SELECT VERSION()") + print("OceanBase engine:", result.scalar_one()) + + asyncio.run(version()) + PY + uv run --locked --extra server python -m pytest -vv -ra \ + tests/e2e/test_atomic_memory_search_authorization_snapshot.py \ + tests/e2e/test_atomic_memory_read_snapshot.py \ + tests/e2e/test_server_metrics_oceanbase.py \ + tests/e2e/test_runtime_server.py -k oceanbase \ + --junitxml="$RUNNER_TEMP/atomic-memory-oceanbase-snapshots.xml" + uv run --locked --extra server python - <<'PY' + import json + import os + import xml.etree.ElementTree as ET + from pathlib import Path + + suites = ET.parse(Path(os.environ["RUNNER_TEMP"]) / "atomic-memory-oceanbase-snapshots.xml").iter("testsuite") + counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")} + for suite in suites: + for key in counts: + counts[key] += int(suite.get(key, "0")) + print("OceanBase Atomic Memory and HTTP results:", json.dumps(counts)) + if not counts["tests"] or any(counts[key] for key in ("failures", "errors", "skipped")): + raise SystemExit("OceanBase Atomic Memory and HTTP regressions must execute without failures, errors, or skips") + PY + - name: Run deterministic scenarios + timeout-minutes: ${{ matrix.database == 'oceanbase' && 15 || 40 }} id: acceptance_scenarios env: POWERCONTEXT_E2E_DATABASE: ${{ matrix.database }} @@ -91,7 +583,7 @@ jobs: always() && (steps.evidence_scan.outcome != 'success' || steps.acceptance_scenarios.outcome != 'success') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: e2e-acceptance-diagnostics-${{ matrix.database }}-${{ github.sha }} path: ${{ runner.temp }}/powercontext-e2e-diagnostics-${{ matrix.database }}/summary.txt @@ -100,7 +592,7 @@ jobs: - name: Upload replay evidence if: always() && steps.evidence_scan.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: e2e-acceptance-${{ matrix.database }}-${{ github.sha }} path: .powercontext-e2e/${{ matrix.database }}/acceptance diff --git a/.github/workflows/kimi-code-acceptance.yml b/.github/workflows/kimi-code-acceptance.yml new file mode 100644 index 0000000000..8d70880bf8 --- /dev/null +++ b/.github/workflows/kimi-code-acceptance.yml @@ -0,0 +1,149 @@ +name: Kimi Code acceptance + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [master] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: kimi-code-acceptance-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + # Upgrade only after qualifying the new CLI with the controlled host acceptance. + KIMI_VERSION: '2.1.1' + CI: 'true' + +jobs: + changes: + uses: ./.github/workflows/ci-changes.yml + + controlled-cli: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').kimi) }} + name: controlled-cli (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + defaults: + run: + shell: bash + steps: + - name: Check out PowerContext + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up Python + uses: ./.github/actions/setup-python-env + with: + python-version: '3.12' + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: '24.21.0' + + - name: Install the pinned Kimi CLI + run: | + npm install --prefix "${{ runner.temp }}/kimi-host" \ + --registry https://registry.npmjs.org --ignore-scripts --no-audit --no-fund \ + "@moonshot-ai/kimi-code@$KIMI_VERSION" + python - <<'PY' + import os + from pathlib import Path + + cli = Path(os.environ["RUNNER_TEMP"]) / "kimi-host/node_modules/@moonshot-ai/kimi-code/dist/main.mjs" + with open(os.environ["GITHUB_ENV"], "a", encoding="utf-8") as environment: + environment.write(f"KIMI_CLI_BIN={cli}\n") + PY + + - name: Verify the real CLI and record provenance + run: | + python - <<'PY' + import hashlib + import json + import os + import platform + import subprocess + from pathlib import Path + + cli = Path(os.environ["KIMI_CLI_BIN"]) + if not cli.is_file(): + raise SystemExit("Pinned Kimi CLI is missing; host acceptance must not skip") + version = subprocess.check_output(["node", str(cli), "--version"], text=True).strip() + if version != os.environ["KIMI_VERSION"]: + raise SystemExit(f"Unexpected Kimi CLI version: {version}") + reports = Path(os.environ["RUNNER_TEMP"]) / "kimi-ci-evidence" + reports.mkdir() + provenance = { + "repository_commit": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "host_package": "@moonshot-ai/kimi-code", + "host_version": version, + "host_bundle_sha256": hashlib.sha256(cli.read_bytes()).hexdigest(), + "node_version": subprocess.check_output(["node", "--version"], text=True).strip(), + "python_version": platform.python_version(), + "platform": platform.system(), + "inference": "controlled", + "server": "real-sqlite", + } + (reports / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n", encoding="utf-8") + print(json.dumps(provenance, indent=2)) + PY + + - name: Run plugin Hook behavior tests + run: node --test integrations/kimi-code/plugins/powercontext/tests/plugin.test.mjs + + - name: Check generated MCP profiles + run: uv run --locked --no-sync python scripts/generate_kimi_profiles.py --check + + - name: Run real-host acceptance without skips + run: | + uv run --locked --no-sync pytest tests/e2e/test_kimi_code_host.py tests/e2e/test_kimi_code_workflows.py \ + -q -p no:cacheprovider \ + --basetemp="${{ runner.temp }}/kimi-acceptance" \ + --junitxml="${{ runner.temp }}/kimi-ci-evidence/junit.xml" + python - <<'PY' + import os + import xml.etree.ElementTree as ET + from pathlib import Path + + report = Path(os.environ["RUNNER_TEMP"]) / "kimi-ci-evidence/junit.xml" + cases = list(ET.parse(report).iter("testcase")) + if not cases or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("Kimi host acceptance must execute and pass without skips") + PY + + - name: Collect acceptance summaries + if: always() + run: | + python - <<'PY' + import os + import shutil + from pathlib import Path + + temporary = Path(os.environ["RUNNER_TEMP"]) + reports = temporary / "kimi-ci-evidence" + reports.mkdir(exist_ok=True) + for evidence in (temporary / "kimi-acceptance").glob("*/evidence.json"): + shutil.copy2(evidence, reports / (evidence.parent.name + ".json")) + for evidence in (temporary / "kimi-acceptance").glob("*/server-startup.json"): + shutil.copy2(evidence, reports / (evidence.parent.name + "-server-startup.json")) + PY + + - name: Upload controlled acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: kimi-controlled-${{ matrix.os }}-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/kimi-ci-evidence/ + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 00e2b4ab02..1c9a36b651 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -6,6 +6,16 @@ on: - master pull_request: types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: + inputs: + upgrade_selection: + description: Upgrade cases (seekDB Code configurations are temporarily manual) + type: choice + default: automatic + options: + - automatic + - seekdb-code + - full permissions: contents: read @@ -15,22 +25,163 @@ concurrency: cancel-in-progress: true jobs: - quality: + seekdb-managed-startup: + name: SeekDB managed startup regression (priority) timeout-minutes: 20 runs-on: ubuntu-latest + env: + POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" + PYTHONUNBUFFERED: "1" + MIGRATION_REPORTS: ${{ github.workspace }}/.powercontext/managed-startup-seekdb steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + - name: Set up the environment + uses: ./.github/actions/setup-python-env with: - path: ~/.cache/prek - key: prek-${{ hashFiles('.pre-commit-config.yaml') }} + python-version: "3.12" + - name: Test concurrent managed startup without skips + # Collection and three isolated cases retain their 180s process deadlines. + timeout-minutes: 15 + env: + TMPDIR: ${{ runner.temp }}/am-seekdb + run: | + mkdir -p "$TMPDIR" + uv run --locked --extra seekdb --extra server python scripts/run_atomic_memory_clean_migration.py \ + seekdb --output "$MIGRATION_REPORTS" --tests \ + tests/e2e/test_managed_startup_read_consistency.py + test "$(wc -l < "$MIGRATION_REPORTS/nodes.txt")" -eq 3 + + - name: Collect native diagnostics + if: always() + env: + TMPDIR: ${{ runner.temp }}/am-seekdb + run: | + mkdir -p "$MIGRATION_REPORTS" + df -h . /tmp > "$MIGRATION_REPORTS/disk.txt" + if [ -d "$TMPDIR" ]; then + find "$TMPDIR" -type f \( -name '*.log' -o -name '*.log.wf' \) \ + -print -exec tail -c 8388608 {} \; > "$MIGRATION_REPORTS/seekdb-native.log" + fi + + - name: Save managed startup reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: managed-startup-seekdb-${{ github.sha }} + path: ${{ env.MIGRATION_REPORTS }}/ + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + + changes: + uses: ./.github/workflows/ci-changes.yml + + upgrade-acceptance: + needs: seekdb-managed-startup + # Collect selected upgrade cases even when the priority regression fails. + if: ${{ !cancelled() }} + uses: ./.github/workflows/upgrade-release.yml + with: + selection: ${{ github.event_name == 'workflow_dispatch' && inputs.upgrade_selection || 'automatic' }} + + upgrade-release-gate: + name: Upgrade release gate + needs: upgrade-acceptance + if: always() + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require the selected parallel acceptance workflow + env: + UPGRADE_RESULT: ${{ needs.upgrade-acceptance.result }} + run: test "$UPGRADE_RESULT" = success + + oceanbase-engine-recovery: + name: OceanBase engine crash recovery + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} + runs-on: ubuntu-24.04 + timeout-minutes: 20 + env: + POWERCONTEXT_TEST_MIGRATION_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_migration_probe_engine_crash?charset=utf8mb4 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Set up the environment uses: ./.github/actions/setup-python-env + - name: Start an isolated OceanBase engine + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Bind the destructive probe to this job's owned container + run: | + docker exec upgrade-oceanbase \ + obclient -h127.0.0.1 -P2881 -uroot@test -ppowercontext-e2e \ + -e 'CREATE DATABASE pc_migration_probe_engine_crash;' + container_id=$(docker inspect --format '{{.Id}}' upgrade-oceanbase) + [[ "$container_id" =~ ^[0-9a-f]{64}$ ]] + echo "POWERCONTEXT_TEST_MIGRATION_OCEANBASE_ENGINE_CRASH_ID=$container_id" >> "$GITHUB_ENV" + - name: Verify migration recovery after observer SIGKILL + run: | + mkdir -p "$RUNNER_TEMP/oceanbase-engine-recovery" + uv run pytest -q tests/native/test_oceanbase_engine_recovery.py \ + --junitxml "$RUNNER_TEMP/oceanbase-engine-recovery/junit.xml" \ + --basetemp "$RUNNER_TEMP/oceanbase-engine-recovery/cases" + uv run python - <<'PY' + import os + from pathlib import Path + from xml.etree import ElementTree + path = Path(os.environ["RUNNER_TEMP"]) / "oceanbase-engine-recovery/junit.xml" + suites = list(ElementTree.parse(path).getroot().iter("testsuite")) + assert sum(int(s.get("tests", 0)) for s in suites) == 1 + assert all(int(s.get(key, 0)) == 0 for s in suites for key in ("failures", "errors", "skipped")) + PY + - name: Report recovery failures + if: failure() + run: >- + uv run python .github/scripts/report_pytest_failures.py + "$RUNNER_TEMP/oceanbase-engine-recovery/junit.xml" + - name: Preserve diagnostics and stop the owned engine + if: always() + run: | + mkdir -p "$RUNNER_TEMP/oceanbase-engine-recovery" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/oceanbase-engine-recovery/container.log" 2>&1 || true + docker stop --timeout 30 upgrade-oceanbase || true + - name: Save engine recovery evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: oceanbase-engine-recovery-${{ github.sha }} + path: ${{ runner.temp }}/oceanbase-engine-recovery/ + if-no-files-found: error + retention-days: 7 + + quality: + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + + - name: Resolve Python version for hook cache + id: python + run: echo "version=$(python -c 'import platform; print(platform.python_version())')" >> "$GITHUB_OUTPUT" + + - name: Cache prek environments + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ~/.cache/prek + key: prek-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.python.outputs.version }}-${{ hashFiles('.pre-commit-config.yaml') }} - name: Run checks run: make check @@ -39,11 +190,14 @@ jobs: run: make contract-test tests: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} timeout-minutes: 40 runs-on: ubuntu-latest strategy: matrix: python-version: ["3.11", "3.12", "3.13", "3.14"] + suite: [unit, e2e] fail-fast: false defaults: run: @@ -59,15 +213,214 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Set up project tools + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache DSH configuration dependencies + if: matrix.suite == 'unit' + uses: ./.github/actions/cache-pnpm + with: + working-directory: integrations/dsh/plugins/powercontext/tests/config-runtime + cache-dependency-glob: integrations/dsh/plugins/powercontext/tests/config-runtime/pnpm-lock.yaml + + - name: Install and resolve native DSH configuration test APIs + if: matrix.suite == 'unit' + run: | + pnpm --dir integrations/dsh/plugins/powercontext/tests/config-runtime install --frozen-lockfile + boot="$(node --input-type=module -e "import { createRequire } from 'node:module'; import { resolve } from 'node:path'; const require = createRequire(resolve('integrations/dsh/plugins/powercontext/tests/config-runtime/package.json')); process.stdout.write(require.resolve('@deepseek-ai/dsh-app-boot'))")" + test -n "$boot" && test -f "$boot" || { echo "DSH configuration runtime unresolved"; exit 1; } + echo "DSH_TEST_CONFIG_BOOT=$boot" >> "$GITHUB_ENV" + - name: Run unit tests + if: matrix.suite == 'unit' + env: + PYTEST_ADDOPTS: --require-dsh-config-runtime run: make unit-test - name: Run end-to-end tests + if: matrix.suite == 'e2e' run: make e2e-test - code-seekdb: + isolated-installations: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} timeout-minutes: 20 runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.14"] + extras: ["builtin", "cli", "builtin,cli", "server,cli"] + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up uv + uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0 + with: + version: "0.10.12" + + - name: Build and install only the requested wheel extras + env: + ROLE_EXTRAS: ${{ matrix.extras }} + run: | + uv build --wheel --out-dir "$RUNNER_TEMP/role-wheel" + uv venv --python "$(command -v python)" "$RUNNER_TEMP/role-venv" + wheel="$(find "$RUNNER_TEMP/role-wheel" -name '*.whl' -print -quit)" + uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" "$wheel[$ROLE_EXTRAS]" + + - name: Verify installed role behavior without checkout imports + env: + ROLE_EXTRAS: ${{ matrix.extras }} + run: | + "$RUNNER_TEMP/role-venv/bin/python" -I scripts/ci_role_smoke.py "$ROLE_EXTRAS" + + - name: Verify unrelated packages do not enable optional CLI roles + if: matrix.extras == 'cli' + run: | + uv pip install --python "$RUNNER_TEMP/role-venv/bin/python" sqlalchemy fastapi + "$RUNNER_TEMP/role-venv/bin/python" -I scripts/ci_role_smoke.py cli + + docs-tests: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').docs_tests) }} + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + + - name: Test documentation contracts + run: | + uv run --locked python -m pytest -q -ra \ + tests/test_release_version.py \ + tests/test_full_capability_docs.py \ + tests/test_integration_guidance_evaluation.py \ + tests/test_docs_remote_access_contract.py \ + tests/e2e/test_mcp_full_profile.py + + worker-observability: + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + + - name: Test worker traces and metrics without skips + run: | + uv run --locked --extra tracing-otlp python -m pytest -q -ra \ + tests/e2e/test_worker_otlp.py \ + tests/e2e/test_worker_model_usage.py \ + tests/test_observability_health.py \ + --junitxml="$RUNNER_TEMP/worker-observability.xml" + uv run --locked --extra tracing-otlp python - <<'PY' + import os + import xml.etree.ElementTree as ET + from pathlib import Path + + report = Path(os.environ["RUNNER_TEMP"]) / "worker-observability.xml" + cases = list(ET.parse(report).iter("testcase")) + if not cases or any(case.find(tag) is not None for case in cases for tag in ("skipped", "failure", "error")): + raise SystemExit("Worker traces and metrics must execute and pass without skips") + print(f"Worker observability: {len(cases)} cases passed without skips.") + PY + + - name: Save worker observability results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: worker-observability + path: ${{ runner.temp }}/worker-observability.xml + if-no-files-found: error + + evaluation-tests: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').evaluation) }} + timeout-minutes: 20 + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + cache-dependency-glob: | + pyproject.toml + uv.lock + integrations/langchain/pyproject.toml + integrations/langgraph/pyproject.toml + integrations/opendal/pyproject.toml + evaluation/pyproject.toml + evaluation/uv.lock + + # Evaluation has its own uv project. Collect each suite's unit tests through + # its dedicated path; the root project's testpaths do not include them. + - name: Run the evaluation project unit tests + run: | + set -o pipefail + make evaluation-unit-test 2>&1 | tee evaluation/pytest.log + + # A pull request from a fork cannot read this run's raw job log — the log + # endpoint answers `Must have admin rights to Repository` — so a red check + # is otherwise an exit code and nothing else. Both the step summary and an + # annotation are readable by the contributor, and the annotation is the one + # that still says something when the failure happened before the tests ran. + - name: Report the failing cases + if: failure() + run: | + { + echo "### evaluation unit tests" + echo + echo '```' + uv --version || true + python3 -V || true + echo '```' + echo + if grep -qE '^(FAILED|ERROR) ' evaluation/pytest.log; then + echo '```' + grep -E '^(FAILED|ERROR) ' evaluation/pytest.log + echo '```' + else + echo "No failing case was reported, so the run failed before the tests ran:" + echo + echo '```' + tail -n 40 evaluation/pytest.log + echo '```' + fi + } >> "$GITHUB_STEP_SUMMARY" + + detail=$(tail -n 25 evaluation/pytest.log | awk '{gsub(/%/, "%25"); gsub(/\r/, "%0D"); printf "%s%%0A", $0}') + echo "::error title=evaluation unit tests failed::${detail}" + + code-seekdb: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + timeout-minutes: 30 + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -86,7 +439,131 @@ jobs: - name: Test code indexing with embedded seekdb run: make code-seekdb-test + - name: Test unified migrations with embedded seekdb + # Full-Server cases run in separate processes and exceed fifteen minutes + # in aggregate; each runner keeps its 180-second deadline below. + timeout-minutes: 25 + env: + POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" + POWERCONTEXT_TEST_MIGRATION_SQL_LOG: "1" + PYTHONUNBUFFERED: "1" + TMPDIR: ${{ runner.temp }}/seekdb-migration + run: | + set -o pipefail + mkdir -p "$TMPDIR" + collect_diagnostics() { + df -h . /tmp + # A timed-out pytest process cannot run fixture cleanup. Preserve + # native startup/recovery evidence too: a tail alone can omit the + # transaction that acquired the lock blocking the final SQL query. + find "$TMPDIR" -type f \( -name '*.log' -o -name '*.log.wf' \) \ + -printf '%P\0' | tar -czf seekdb-native.tar.gz -C "$TMPDIR" --null -T - + } + trap collect_diagnostics EXIT + df -h . /tmp + uv run --locked --extra seekdb pytest -vv \ + -o faulthandler_timeout=120 --durations=20 \ + tests/builtin/persistence/test_database_migration_backends.py \ + tests/builtin/persistence/test_database_backup_providers.py \ + tests/builtin/persistence/test_database_migration_connections.py \ + tests/e2e/test_topic_memory_recovery_migration.py \ + tests/builtin/persistence/test_mysql_migration_checks.py \ + tests/builtin/persistence/test_mysql_migration_schema.py -k seekdb \ + 2>&1 | tee seekdb-migrations.log + uv run --locked --extra seekdb pytest --collect-only -q --color=no \ + tests/builtin/persistence/test_mysql_migration_runner.py -k seekdb \ + 2>&1 | tee seekdb-runner-collection.log | tee -a seekdb-migrations.log + mapfile -t seekdb_runner_nodes < <( + sed -n '/^tests\/builtin\/persistence\/test_mysql_migration_runner\.py::/p' \ + seekdb-runner-collection.log + ) + if [ "${#seekdb_runner_nodes[@]}" -eq 0 ]; then + echo "No seekdb migration runner tests were collected" >&2 + exit 1 + fi + for seekdb_runner_node in "${seekdb_runner_nodes[@]}"; do + timeout --signal=TERM --kill-after=10s 180s \ + uv run --locked --extra seekdb pytest -vv \ + -o faulthandler_timeout=120 --durations=20 --log-cli-level=INFO \ + "$seekdb_runner_node" -k seekdb \ + 2>&1 | tee -a seekdb-migrations.log + done + + - name: Upload seekdb migration diagnostics + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: seekdb-migrations-${{ matrix.python-version }} + path: | + seekdb-migrations.log + seekdb-native.tar.gz + if-no-files-found: ignore + + atomic-memory-clean-seekdb: + needs: [changes, seekdb-managed-startup] + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').database) }} + name: Atomic Memory clean migration (SeekDB) + timeout-minutes: 55 + runs-on: ubuntu-latest + env: + POWERCONTEXT_TEST_MIGRATION_SEEKDB: "1" + PYTHONUNBUFFERED: "1" + MIGRATION_REPORTS: ${{ github.workspace }}/.powercontext/atomic-memory-clean-seekdb + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + + - name: Run clean migration evidence, grants and vectors without skips + timeout-minutes: 35 + env: + TMPDIR: ${{ runner.temp }}/am-seekdb + run: | + mkdir -p "$TMPDIR" + uv run --locked --extra seekdb python scripts/run_atomic_memory_clean_migration.py \ + seekdb --output "$MIGRATION_REPORTS" + + - name: Test long-query retrieval and admission without skips + timeout-minutes: 10 + env: + TMPDIR: ${{ runner.temp }}/am-seekdb + run: | + uv run --locked --extra seekdb python scripts/run_atomic_memory_clean_migration.py \ + seekdb --output "$MIGRATION_REPORTS/long-query" --tests \ + tests/e2e/test_atomic_artifact_search_http.py \ + tests/e2e/test_atomic_memory_long_fts_query.py + + - name: Collect native diagnostics + if: always() + env: + TMPDIR: ${{ runner.temp }}/am-seekdb + run: | + mkdir -p "$MIGRATION_REPORTS" + df -h . /tmp > "$MIGRATION_REPORTS/disk.txt" + if [ -d "$TMPDIR" ]; then + find "$TMPDIR" -type f \( -name '*.log' -o -name '*.log.wf' \) \ + -print -exec tail -c 8388608 {} \; > "$MIGRATION_REPORTS/seekdb-native.log" + fi + + - name: Save clean migration reports + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: atomic-memory-clean-seekdb-${{ github.sha }} + path: ${{ env.MIGRATION_REPORTS }}/ + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + windows-unit-portability: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').tests) }} timeout-minutes: 20 runs-on: windows-latest steps: @@ -107,6 +584,15 @@ jobs: - name: Verify spawned worker lifecycle on Windows run: uv run --locked --no-sync pytest -q tests/builtin/runtime/test_artifact_processing.py + - name: Verify spawned worker logging on Windows + run: uv run --locked --no-sync pytest -q tests/e2e/test_worker_logging.py + + - name: Verify bootstrap receipt persistence and failure recovery on Windows + run: >- + uv run --locked --no-sync pytest -q + tests/claude_code_plugin/test_bootstrap.py + tests/codex_plugin/test_bootstrap.py + - name: Verify unsupported native indexing is skipped on Windows run: >- uv run --locked --no-sync pytest -q @@ -114,11 +600,48 @@ jobs: tests/builtin/test_native_code_multilanguage.py tests/e2e/test_native_code_multilanguage.py + - name: Verify Codex resolves and launches cmd shims on Windows + run: >- + uv run --locked --no-sync pytest -q + tests/test_system_cli.py::test_codex_json_runner_launches_the_resolved_cmd_shim + tests/test_system_cli.py::test_codex_mcp_list_runner_launches_the_resolved_cmd_shim + tests/test_system_cli.py::test_codex_diagnostics_run_through_the_resolved_cmd_shim + tests/test_system_cli.py::test_codex_runners_report_non_utf8_cmd_errors + tests/test_system_cli.py::test_codex_runners_preserve_json_output + tests/test_system_cli.py::test_codex_runners_report_a_missing_executable + tests/test_system_cli.py::test_codex_batch_install_rejects_unsafe_input_before_launch + tests/test_system_cli.py::test_codex_batch_alias_with_trailing_space_still_rejects_unsafe_arguments + tests/test_system_cli.py::test_codex_runners_prefer_path_over_the_current_directory + tests/test_system_cli.py::test_codex_runners_honor_an_explicit_dot_path_entry + tests/test_system_cli.py::test_codex_path_executable_prefers_earlier_pathext_entries + tests/test_system_cli.py::test_codex_path_executable_treats_an_empty_path_value_as_no_search + tests/test_system_cli.py::test_codex_path_executable_resolves_empty_path_entries_in_the_working_directory + tests/test_system_cli.py::test_codex_batch_install_preserves_supported_input + tests/test_system_cli.py::test_codex_batch_install_rejects_unsafe_returned_marketplace + tests/test_system_cli.py::test_codex_batch_runners_reject_unsafe_executable_path + tests/test_system_cli.py::test_codex_native_executable_preserves_arguments + + - name: Verify managed guidance and exact rollback bytes on Windows + run: >- + uv run --locked --no-sync pytest -q + tests/test_guidance_generation.py + tests/test_guidance_installation.py + tests/test_opencode_installation.py + tests/test_cli_zcode.py::test_zcode_upgrade_rejects_missing_packaged_reference_and_preserves_installation + tests/test_setup_rollback.py + - name: Exercise fixtures without Windows long-path support shell: pwsh run: | Set-ItemProperty -LiteralPath 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name LongPathsEnabled -Value 0 + - name: Verify portable archive restoration and SQLite URI targets on Windows + run: >- + uv run --locked --no-sync pytest -q + tests/builtin/portability/test_bundle.py + tests/test_archive_cli.py + tests/e2e/test_portable_archive.py + # Keep pytest's default temporary root: a short --basetemp would hide # regressions in the nested checkout fixtures covered by issue #1501. - name: Run Windows portability regressions @@ -133,9 +656,76 @@ jobs: tests/test_service.py tests/test_service_bootstrap.py tests/test_service_environment.py - tests/test_cli_workbuddy.py::test_setup_workbuddy_remote_checkout_refreshes_the_requested_ref + + dify-tools: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').dify) }} + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + with: + python-version: "3.12" + cache-dependency-glob: | + pyproject.toml + uv.lock + integrations/langchain/pyproject.toml + integrations/langgraph/pyproject.toml + integrations/opendal/pyproject.toml + integrations/dify/pyproject.toml + integrations/dify/uv.lock + + - name: Check out the supported Dify host helpers + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: langgenius/dify + ref: 8387590ace4a094de812b7847fc6a4c3a27cd52b # 1.17.1 + path: .artifacts/dify-host + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + api/core/entities/parameter_entities.py + api/core/plugin/entities/parameters.py + api/core/tools/__base/tool.py + api/core/tools/entities/common_entities.py + api/core/tools/entities/tool_entities.py + docker/docker-compose.yaml + web/app/components/workflow/nodes/tool/default.ts + web/app/components/workflow/nodes/tool/output-schema-utils.ts + web/app/components/workflow/nodes/_base/components/variable/utils.ts + + - name: Check out the default Dify plugin daemon entities + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: langgenius/dify-plugin-daemon + ref: 1310a18b2f6bc6f18768a0a6265484830891433c # 0.6.10 (Dify 1.17.1 compose default) + path: .artifacts/dify-daemon + persist-credentials: false + + - name: Set up Go for daemon entity serialization + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: .artifacts/dify-daemon/go.mod + cache-dependency-path: .artifacts/dify-daemon/go.sum + + - name: Set up project tools + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Check daemon serialization, host casting, Workflow selectors and SDK HTTP readback + env: + POWERCONTEXT_DIFY_SOURCE: ${{ github.workspace }}/.artifacts/dify-host + POWERCONTEXT_DIFY_DAEMON_SOURCE: ${{ github.workspace }}/.artifacts/dify-daemon + run: make dify-test hermes-skills: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').hermes) }} timeout-minutes: 15 runs-on: ubuntu-latest steps: @@ -161,6 +751,8 @@ jobs: run: uv run pytest tests/e2e/test_hermes_skills.py dsh-package: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').dsh) }} timeout-minutes: 30 runs-on: ubuntu-latest steps: @@ -173,7 +765,13 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up project tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache DSH dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: integrations/dsh/plugins/powercontext + cache-dependency-glob: integrations/dsh/plugins/powercontext/**/pnpm-lock.yaml - name: Test DSH package run: make js-test @@ -181,10 +779,27 @@ jobs: - name: Test built plugin in the real DSH runtime run: make dsh-runtime-test + - name: Install native DSH configuration test APIs + run: pnpm --dir integrations/dsh/plugins/powercontext/tests/config-runtime install --frozen-lockfile + + - name: Resolve required DSH test runtimes + run: | + bin="$(node --input-type=module -e "import { dshBin } from './integrations/dsh/plugins/powercontext/tests/runtime/fixture.mjs'; process.stdout.write(dshBin)")" + test -n "$bin" && test -f "$bin" || { echo "DSH test executable unresolved"; exit 1; } + boot="$(node --input-type=module -e "import { createRequire } from 'node:module'; import { resolve } from 'node:path'; const require = createRequire(resolve('integrations/dsh/plugins/powercontext/tests/config-runtime/package.json')); process.stdout.write(require.resolve('@deepseek-ai/dsh-app-boot'))")" + test -n "$boot" && test -f "$boot" || { echo "DSH configuration runtime unresolved"; exit 1; } + echo "DSH_TEST_EXECUTABLE=$bin" >> "$GITHUB_ENV" + echo "DSH_TEST_CONFIG_BOOT=$boot" >> "$GITHUB_ENV" + + - name: Verify DSH setup with customized profiles + run: uv run pytest tests/test_dsh_transport.py --require-dsh-runtime + - name: Verify DSH guidance adapter requests and results - run: uv run pytest "tests/test_integration_guidance_evaluation.py::test_native_adapter_handoff_uses_real_request_mapping_and_response_envelopes[dsh]" + run: uv run pytest tests/test_integration_guidance_evaluation.py::test_dsh_controlled_native_handoff_uses_only_mcp_tools_and_preserves_complete_carriers pi-package: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').pi) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -197,7 +812,13 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up project tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache Pi dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: integrations/pi/plugins/powercontext + cache-dependency-glob: integrations/pi/plugins/powercontext/pnpm-lock.yaml - name: Test Pi package run: make pi-test @@ -206,6 +827,8 @@ jobs: run: uv run pytest "tests/test_integration_guidance_evaluation.py::test_native_adapter_handoff_uses_real_request_mapping_and_response_envelopes[pi]" opencode-package: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').opencode) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -215,7 +838,12 @@ jobs: - name: Set up the Python environment uses: ./.github/actions/setup-python-env - - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + - name: Cache OpenCode dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: integrations/opencode/plugins/powercontext + cache-dependency-glob: integrations/opencode/plugins/powercontext/pnpm-lock.yaml - name: Test OpenCode package run: make opencode-test @@ -223,6 +851,8 @@ jobs: run: uv run pytest "tests/test_integration_guidance_evaluation.py::test_native_adapter_handoff_uses_real_request_mapping_and_response_envelopes[opencode]" openclaw-package: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').openclaw) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -232,9 +862,14 @@ jobs: - name: Set up the Python environment uses: ./.github/actions/setup-python-env - - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + - uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 # The pinned OpenClaw SDK requires the SQLite runtime shipped with Node 24.15+. - run: mise install node@24.15.0 + - name: Cache OpenClaw dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: integrations/openclaw/plugins/memory-powercontext + cache-dependency-glob: integrations/openclaw/plugins/memory-powercontext/pnpm-lock.yaml - name: Test OpenClaw package run: mise exec node@24.15.0 -- make openclaw-plugin-test @@ -242,6 +877,8 @@ jobs: run: mise exec node@24.15.0 -- uv run pytest tests/test_integration_guidance_evaluation.py::test_openclaw_handoff_adapter_preserves_generated_source_identity opendal-package: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').opendal) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -260,6 +897,8 @@ jobs: website-link-validation-windows: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').website) }} timeout-minutes: 20 runs-on: windows-latest steps: @@ -269,7 +908,13 @@ jobs: persist-credentials: false - name: Set up website tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache website dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: website + cache-dependency-glob: website/pnpm-lock.yaml - name: Install website dependencies run: pnpm --dir website install --frozen-lockfile @@ -278,6 +923,8 @@ jobs: run: pnpm --dir website test:links check-website: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').website) }} timeout-minutes: 20 runs-on: ubuntu-latest steps: @@ -290,7 +937,13 @@ jobs: uses: ./.github/actions/setup-python-env - name: Set up website tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache website dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: website + cache-dependency-glob: website/pnpm-lock.yaml - name: Check website run: make docs-test diff --git a/.github/workflows/native-service.yml b/.github/workflows/native-service.yml index 99c42b8b38..69e1055663 100644 --- a/.github/workflows/native-service.yml +++ b/.github/workflows/native-service.yml @@ -6,6 +6,7 @@ on: - master pull_request: types: [opened, synchronize, reopened, ready_for_review] + workflow_dispatch: permissions: contents: read @@ -15,7 +16,12 @@ concurrency: cancel-in-progress: true jobs: + changes: + uses: ./.github/workflows/ci-changes.yml + linux-systemd-user: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').native) }} runs-on: ubuntu-latest timeout-minutes: 30 env: @@ -44,6 +50,9 @@ jobs: wheel=$(find dist -name '*.whl' -print -quit) uv venv .native-venv uv pip install --python .native-venv/bin/python "powercontext[cli,server] @ file://$PWD/$wheel" pytest + uv venv .native-source-venv + uv pip install --python .native-source-venv/bin/python "powercontext[cli,server] @ https://files.pythonhosted.org/packages/30/bd/463f998f8794bc7f8ff588d47157ca0e62dd03a9f3ae466e2eb34fb66045/powercontext-1.3.0-py3-none-any.whl#sha256=8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5" + echo "POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON=$PWD/.native-source-venv/bin/python" >> "$GITHUB_ENV" - name: Exercise the real systemd user lifecycle run: >- @@ -67,13 +76,15 @@ jobs: - name: Upload Linux native-service logs after failure if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: linux-native-service-logs path: ${{ runner.temp }}/powercontext-native-tests/**/logs/* if-no-files-found: ignore macos-launchagent: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').native) }} runs-on: macos-latest timeout-minutes: 30 env: @@ -97,6 +108,9 @@ jobs: # Finish bytecode compilation during installation, before launchd # runs the cold-start lifecycle under its bounded startup deadline. uv pip install --compile-bytecode --python .native-venv/bin/python "powercontext[cli,server] @ file://$PWD/$wheel" pytest + uv venv .native-source-venv + uv pip install --compile-bytecode --python .native-source-venv/bin/python "powercontext[cli,server] @ https://files.pythonhosted.org/packages/30/bd/463f998f8794bc7f8ff588d47157ca0e62dd03a9f3ae466e2eb34fb66045/powercontext-1.3.0-py3-none-any.whl#sha256=8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5" + echo "POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON=$PWD/.native-source-venv/bin/python" >> "$GITHUB_ENV" - name: Exercise the real LaunchAgent lifecycle run: >- @@ -118,13 +132,15 @@ jobs: - name: Upload macOS native-service logs after failure if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: macos-native-service-logs path: ${{ runner.temp }}/powercontext-native-tests/**/logs/* if-no-files-found: ignore windows-task-scheduler: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').native) }} runs-on: windows-latest timeout-minutes: 30 env: @@ -147,6 +163,9 @@ jobs: $wheelUri = "file:///" + $wheel.Replace('\', '/') uv venv .native-venv uv pip install --python .native-venv\Scripts\python.exe "powercontext[cli,server] @ $wheelUri" pytest + uv venv .native-source-venv + uv pip install --python .native-source-venv\Scripts\python.exe "powercontext[cli,server] @ https://files.pythonhosted.org/packages/30/bd/463f998f8794bc7f8ff588d47157ca0e62dd03a9f3ae466e2eb34fb66045/powercontext-1.3.0-py3-none-any.whl#sha256=8da7983763f3a6ab80b413c770a4973a46c557a54e1a1fa675099e58d47305e5" + "POWERCONTEXT_NATIVE_UPGRADE_SOURCE_PYTHON=$PWD\.native-source-venv\Scripts\python.exe" >> $env:GITHUB_ENV - name: Verify binary cursor persistence and restart compatibility shell: pwsh @@ -195,7 +214,7 @@ jobs: - name: Upload Windows native-service logs after failure if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: windows-native-service-logs path: ${{ runner.temp }}/powercontext-native-tests/**/logs/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d43abdc638..3bb7a0d1d8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,79 +3,107 @@ name: Release on: release: types: [published] + workflow_dispatch: + inputs: + release_tag: + description: Existing published Release tag to build and publish + required: true + type: string + skip_upgrade_acceptance: + description: Publish for new databases only, without database upgrade acceptance + type: boolean + default: false permissions: contents: read concurrency: - group: release-${{ github.event.release.tag_name || github.ref }} + group: release-${{ inputs.release_tag || github.event.release.tag_name || github.ref }} cancel-in-progress: false jobs: - release-build: + resolve-release: runs-on: ubuntu-24.04 - timeout-minutes: 20 + timeout-minutes: 5 + outputs: + tag: ${{ steps.release.outputs.tag }} + source_sha: ${{ steps.release.outputs.source_sha }} steps: - - name: Check out - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Set up the environment - uses: ./.github/actions/setup-python-env - - - name: Verify package version from Release tag + - name: Resolve the published Release and pin its source + id: release env: - RELEASE_TAG: ${{ github.event.release.tag_name }} + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.release_tag || github.event.release.tag_name }} + SKIP_UPGRADE_ACCEPTANCE: ${{ inputs.skip_upgrade_acceptance || false }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | python - <<'PY' + import json import os import re - import subprocess - - release_tag = os.environ["RELEASE_TAG"] - if release_tag.startswith("powercontext-v"): - release_version = release_tag.removeprefix("powercontext-v") - else: - release_version = release_tag.removeprefix("v") - if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:(?:a|b|rc)[0-9]+)?", release_version): - raise SystemExit( - "Release tag must use X.Y.Z with optional aN, bN, or rcN suffix and " - "optional v or powercontext-v prefix (for example, powercontext-v1.1.0)" - ) - - package_version = subprocess.check_output( - ["uvx", "--from", "hatchling", "--with", "hatch-vcs", "hatchling", "version"], - text=True, - ).strip() - if package_version != release_version: - raise SystemExit( - f"Release tag version {release_version!r} does not match VCS version {package_version!r}" + import urllib.parse + import urllib.request + from pathlib import Path + + tag = os.environ["RELEASE_TAG"] + if not re.fullmatch(r"(?:powercontext-v|v)?\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", tag): + raise SystemExit("Invalid PowerContext Release tag") + manual = os.environ["GITHUB_EVENT_NAME"] == "workflow_dispatch" + skip = os.environ["SKIP_UPGRADE_ACCEPTANCE"] == "true" + if manual and os.environ["GITHUB_REF"] != "refs/heads/" + os.environ["DEFAULT_BRANCH"]: + raise SystemExit("Run manual publication from the default branch; select the target with release_tag") + if skip and not manual: + raise SystemExit("Skipping upgrade acceptance requires explicit manual dispatch") + + def get(path): + request = urllib.request.Request( + f"https://api.github.com/repos/{os.environ['GITHUB_REPOSITORY']}/{path}", + headers={"Authorization": f"Bearer {os.environ['GH_TOKEN']}", + "Accept": "application/vnd.github+json"}, ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + quoted_tag = urllib.parse.quote(tag, safe="") + release = get(f"releases/tags/{quoted_tag}") + if release["tag_name"] != tag or release["draft"] or not release["published_at"]: + raise SystemExit("Select an existing published GitHub Release") + notice = ( + "Upgrade acceptance was skipped for this release. " + "Use only new databases; upgrades of existing databases are not supported." + ) + if skip and notice not in (release["body"] or ""): + raise SystemExit("Add this exact notice to the GitHub Release before skipping upgrades: " + notice) + commit = get(f"commits/{urllib.parse.quote('refs/tags/' + tag, safe='')}")["sha"] + if not re.fullmatch(r"[0-9a-f]{40}", commit): + raise SystemExit("Release tag did not resolve to a commit") + if not manual and commit != os.environ["GITHUB_SHA"]: + raise SystemExit("Release tag changed since the published event") + event = Path(os.environ["RUNNER_TEMP"]) / "release-metadata" / "event.json" + event.parent.mkdir() + event.write_text(json.dumps({"release": release}, ensure_ascii=False), encoding="utf-8") + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"tag={tag}\nsource_sha={commit}\n") PY - - - name: Set up website tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 - - - name: Verify synchronized release references and release notes - env: - VERSION: ${{ github.event.release.tag_name }} - run: make release-check - - - name: Build release distributions - run: | - uv build --out-dir dist - uvx --from twine twine check dist/* - - - name: Upload release distributions - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + - name: Preserve the Release notes used for validation + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: - name: release-dists - path: dist/* + name: release-metadata + path: ${{ runner.temp }}/release-metadata/event.json if-no-files-found: error + overwrite: true retention-days: 30 + release-build: + needs: resolve-release + uses: ./.github/workflows/upgrade-release.yml + with: + release: true + selection: automatic + source_ref: ${{ needs.resolve-release.outputs.source_sha }} + release_tag: ${{ needs.resolve-release.outputs.tag }} + skip_upgrade_acceptance: ${{ github.event_name == 'workflow_dispatch' && inputs.skip_upgrade_acceptance }} + pypi-publish: runs-on: ubuntu-latest timeout-minutes: 15 @@ -84,23 +112,23 @@ jobs: id-token: write steps: - name: Download release distributions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: release-dists path: dist/ - name: Publish release distributions to PyPI - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + uses: pypa/gh-action-pypi-publish@7245906fb61fa33a13554f9d5fb856fe61758086 # release/v1 release-assets: runs-on: ubuntu-latest timeout-minutes: 15 - needs: release-build + needs: [resolve-release, release-build] permissions: contents: write steps: - name: Download release distributions - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 with: name: release-dists path: dist/ @@ -108,21 +136,21 @@ jobs: - name: Upload GitHub Release assets uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3 with: - tag_name: ${{ github.event.release.tag_name }} + tag_name: ${{ needs.resolve-release.outputs.tag }} files: dist/* fail_on_unmatched_files: true release-verify: - needs: [pypi-publish, release-assets] + needs: [resolve-release, pypi-publish, release-assets] permissions: contents: read uses: ./.github/workflows/release-verify.yml with: - release_tag: ${{ github.event.release.tag_name }} + release_tag: ${{ needs.resolve-release.outputs.tag }} release_package: powercontext deploy-website: - needs: pypi-publish + needs: [resolve-release, pypi-publish] permissions: contents: read pages: write @@ -136,13 +164,21 @@ jobs: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: + ref: ${{ needs.resolve-release.outputs.source_sha }} + fetch-depth: 0 persist-credentials: false - name: Set up the environment uses: ./.github/actions/setup-python-env - name: Set up website tools - uses: jdx/mise-action@c37c93293d6b742fc901e1406b8f764f6fb19dac # v2 + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache website dependencies + uses: ./.github/actions/cache-pnpm + with: + working-directory: website + cache-dependency-glob: website/pnpm-lock.yaml - name: Build website run: make docs-build diff --git a/.github/workflows/seekdb-lock-probe.yml b/.github/workflows/seekdb-lock-probe.yml new file mode 100644 index 0000000000..5172aa0c78 --- /dev/null +++ b/.github/workflows/seekdb-lock-probe.yml @@ -0,0 +1,38 @@ +name: SeekDB migration lock diagnostics + +on: + pull_request: + paths: + - scripts/probe_seekdb_migration_lock.py + - .github/workflows/seekdb-lock-probe.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: seekdb-lock-probe-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + reproduce: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: ./.github/actions/setup-python-env + with: + python-version: '3.11' + - name: Reproduce migration lifecycle with complete native evidence + run: uv run --locked --extra seekdb python scripts/probe_seekdb_migration_lock.py --attempts 12 + - name: Upload native logs, lifecycle and live transaction snapshots + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: seekdb-lock-probe-${{ github.sha }} + path: .powercontext/seekdb-lock-probe + include-hidden-files: true + if-no-files-found: error + retention-days: 7 diff --git a/.github/workflows/skill-guidance.yml b/.github/workflows/skill-guidance.yml new file mode 100644 index 0000000000..aec7b62f7a --- /dev/null +++ b/.github/workflows/skill-guidance.yml @@ -0,0 +1,54 @@ +name: Skill guidance validation + +on: + pull_request: + paths: + - evaluation/skills/skill-up/** + - openapi/powercontext.yaml + - integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/** + - .github/workflows/skill-guidance.yml + push: + branches: [master] + paths: + - evaluation/skills/skill-up/** + - openapi/powercontext.yaml + - integrations/claude-code/plugins/powercontext/skills/powercontext-project-context/** + - .github/workflows/skill-guidance.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + with: + python-version: '3.11' + cache: pip + cache-dependency-path: evaluation/skills/skill-up/requirements-dev.txt + - name: Install validation dependencies + run: python -m pip install -r evaluation/skills/skill-up/requirements-dev.txt + - name: Install checksum-pinned skill-up + shell: bash + run: | + set -euo pipefail + mkdir -p "$RUNNER_TEMP/skill-up" + cd "$RUNNER_TEMP/skill-up" + curl -fsSL https://github.com/alibaba/skill-up/releases/download/v0.12.0/skill-up_0.12.0_linux_amd64.tar.gz -o skill-up.tar.gz + echo 'ec2631e45702d460b69cd700e1847f5562ef2e971f09c23be2efcec4e0f583c6 skill-up.tar.gz' | sha256sum --check - + tar -xzf skill-up.tar.gz + echo "$RUNNER_TEMP/skill-up" >> "$GITHUB_PATH" + - name: Verify pin, fixtures, reports and native schema + working-directory: evaluation/skills/skill-up + run: | + python sync_skill.py --check + python validate_suite.py + python -m unittest discover -s tests -v + skill-up validate evals/eval.yaml + python run.py --dry-run diff --git a/.github/workflows/upgrade-release.yml b/.github/workflows/upgrade-release.yml new file mode 100644 index 0000000000..5373c8a2a7 --- /dev/null +++ b/.github/workflows/upgrade-release.yml @@ -0,0 +1,348 @@ +name: Parallel installed-release upgrade acceptance + +on: + workflow_call: + inputs: + selection: + description: full, automatic, or seekdb-code; manual Code cases do not gate publication + type: string + default: full + release: + description: Enforce release metadata and expose accepted publication artifacts + type: boolean + default: false + source_ref: + description: Pinned release commit; empty uses the calling workflow checkout + type: string + default: '' + release_tag: + description: Published Release tag selected by the release workflow + type: string + default: '' + skip_upgrade_acceptance: + description: Explicit manual publication for new databases without upgrade acceptance + type: boolean + default: false + +permissions: + contents: read + +env: + PC_UPGRADE_SOURCE_SHA: ${{ inputs.source_ref || github.sha }} + +jobs: + prepare: + name: Prepare upgrade artifacts and matrix + runs-on: ubuntu-24.04 + timeout-minutes: 30 + outputs: + matrix: ${{ steps.prepare.outputs.matrix }} + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_ref }} + fetch-depth: 0 + persist-credentials: false + + - name: Require an explicit manual release for the upgrade bypass + env: + SKIP_UPGRADE_ACCEPTANCE: ${{ inputs.skip_upgrade_acceptance }} + RELEASE: ${{ inputs.release }} + run: | + if [ "$SKIP_UPGRADE_ACCEPTANCE" = true ]; then + test "$GITHUB_EVENT_NAME" = workflow_dispatch + test "$RELEASE" = true + fi + + - name: Set up the environment + uses: ./.github/actions/setup-python-env + + - name: Download the frozen Release metadata + if: inputs.release + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: release-metadata + path: ${{ runner.temp }}/release-metadata/ + + - name: Verify package version from Release tag + if: inputs.release + env: + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + python - <<'PY' + import os + import re + import subprocess + + release_tag = os.environ["RELEASE_TAG"] + if release_tag.startswith("powercontext-v"): + release_version = release_tag.removeprefix("powercontext-v") + else: + release_version = release_tag.removeprefix("v") + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:(?:a|b|rc)[0-9]+)?", release_version): + raise SystemExit( + "Release tag must use X.Y.Z with optional aN, bN, or rcN suffix and " + "optional v or powercontext-v prefix (for example, powercontext-v1.1.0)" + ) + + package_version = subprocess.check_output( + ["uvx", "--from", "hatchling", "--with", "hatch-vcs", "hatchling", "version"], + text=True, + ).strip() + if package_version != release_version: + raise SystemExit( + f"Release tag version {release_version!r} does not match VCS version {package_version!r}" + ) + PY + + - name: Set up website tools + if: inputs.release + uses: jdx/mise-action@8418fedb1110381a0d3ba51a7a415d7c1b884550 # v5.2.2 + + - name: Cache website dependencies + if: inputs.release + uses: ./.github/actions/cache-pnpm + with: + working-directory: website + cache-dependency-glob: website/pnpm-lock.yaml + + - name: Verify synchronized release references and release notes + if: inputs.release + env: + VERSION: ${{ inputs.release_tag }} + run: make release-check + + - name: Run upgrade gate regressions + run: uv run pytest -q tests/test_upgrade_release_gate.py tests/test_upgrade_publication.py tests/test_upgrade_schema_equivalence.py + + - name: Verify upgrade policy + run: make upgrade-policy-check + + - name: Build release distributions + run: | + uv build --out-dir dist + uvx --from twine twine check dist/* + + - name: Freeze distributions and resolve the source matrix once + if: ${{ !inputs.skip_upgrade_acceptance }} + id: prepare + env: + RELEASE: ${{ inputs.release }} + SELECTION: ${{ inputs.selection }} + run: | + args=() + if [ "$RELEASE" = true ]; then + args+=(--release --release-event "$RUNNER_TEMP/release-metadata/event.json") + fi + uv run python scripts/parallel_upgrade_release_gate.py prepare \ + --output "$RUNNER_TEMP/upgrade-prepared" --selection "$SELECTION" "${args[@]}" + + - name: Share the exact prepared inputs + if: ${{ !inputs.skip_upgrade_acceptance }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: upgrade-prepared + path: | + ${{ runner.temp }}/upgrade-prepared/inputs/ + ${{ runner.temp }}/upgrade-prepared/plan.json + ${{ runner.temp }}/upgrade-prepared/sdist-build.log + if-no-files-found: error + overwrite: true + retention-days: 30 + + - name: Validate distributions without claiming upgrade acceptance + if: inputs.skip_upgrade_acceptance + env: + PYTHONPATH: scripts + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + uv run python - <<'PY' + import json + import os + import subprocess + from pathlib import Path + from run_upgrade_release_gate import stage_distributions + from upgrade_release_gate import ROOT, load_policy, require, sha256, wheel_version + + policy = load_policy(ROOT) + require(policy["mode"] == "upgrade-required", "The bypass must not declare supported upgrades") + output = Path(os.environ["RUNNER_TEMP"]) / "release-without-upgrades" + output.mkdir() + wheel, files = stage_distributions(Path("dist"), output / "validated-dist", release=True) + require(wheel_version(wheel) == policy["target_version"], "Wheel version differs from release policy") + evidence = { + "release_tag": os.environ["RELEASE_TAG"], + "source_sha": subprocess.check_output(["git", "rev-parse", "HEAD"], text=True).strip(), + "workflow_run_id": os.environ["GITHUB_RUN_ID"], + "actor": os.environ["GITHUB_ACTOR"], + "upgrade_acceptance": "skipped", + "supported_use": "new-databases-only", + "artifacts": {path.name: sha256(path) for path in files}, + } + (output / "publication.json").write_text(json.dumps(evidence, indent=2) + "\n") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary: + summary.write("## Upgrade acceptance skipped\n\n") + summary.write("Published use is limited to new databases. Existing database upgrades are unsupported.\n") + summary.write(f"\nTag: `{evidence['release_tag']}`; source: `{evidence['source_sha']}`.\n") + PY + + - name: Verify a clean installation and new-database Server, MCP and Memory behavior + if: inputs.skip_upgrade_acceptance + run: | + verification="$RUNNER_TEMP/release-fresh-install" + uv venv "$verification" + wheels=("$RUNNER_TEMP"/release-without-upgrades/validated-dist/*.whl) + uv pip install --python "$verification/bin/python" "${wheels[0]}[cli,server]" + uv pip check --python "$verification/bin/python" + version="$("$verification/bin/python" -c 'from importlib.metadata import version; print(version("powercontext"))')" + "$verification/bin/python" scripts/ci_release_smoke.py --version "$version" + + - name: Bind successful fresh-install verification to the uploaded bytes + if: inputs.skip_upgrade_acceptance + env: + PYTHONPATH: scripts + run: | + uv run python - <<'PY' + import json + import os + from pathlib import Path + from upgrade_release_gate import require, sha256 + + output = Path(os.environ["RUNNER_TEMP"]) / "release-without-upgrades" + record = output / "publication.json" + evidence = json.loads(record.read_text()) + actual = {path.name: sha256(path) for path in (output / "validated-dist").iterdir()} + require(actual == evidence["artifacts"], "Distributions changed during fresh-install verification") + evidence["fresh_install_smoke"] = "passed" + record.write_text(json.dumps(evidence, indent=2) + "\n") + PY + + - name: Preserve the explicit upgrade bypass record + if: always() && inputs.skip_upgrade_acceptance + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-without-upgrades + path: | + ${{ runner.temp }}/release-without-upgrades/publication.json + ${{ runner.temp }}/release-without-upgrades/sdist-build.log + if-no-files-found: warn + overwrite: true + retention-days: 30 + + - name: Upload distributions after fresh-install verification only + if: success() && inputs.skip_upgrade_acceptance + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-dists + path: ${{ runner.temp }}/release-without-upgrades/validated-dist/* + if-no-files-found: error + overwrite: true + retention-days: 30 + + cases: + name: Upgrade ${{ matrix.source_version }} / ${{ matrix.backend }} / ${{ matrix.configuration }} + needs: prepare + if: ${{ !inputs.skip_upgrade_acceptance }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.prepare.outputs.matrix) }} + env: + PC_UPGRADE_OCEANBASE_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_test?charset=utf8mb4 + PC_UPGRADE_OCEANBASE_FRESH_URL: mysql+aoceanbase://root%40test:powercontext-e2e@127.0.0.1:2881/pc_upgrade_fresh?charset=utf8mb4 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_ref }} + persist-credentials: false + - name: Set up the environment + uses: ./.github/actions/setup-python-env + - name: Download the shared prepared inputs + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: upgrade-prepared + path: ${{ runner.temp }}/upgrade-prepared/ + - name: Start the isolated upgrade OceanBase environment + if: matrix.backend == 'oceanbase' + uses: ./.github/actions/setup-upgrade-oceanbase + - name: Run this configuration exactly once + env: + CASE_INDEX: ${{ matrix.index }} + run: >- + uv run python scripts/parallel_upgrade_release_gate.py case + --prepared "$RUNNER_TEMP/upgrade-prepared" + --index "$CASE_INDEX" + --output "$RUNNER_TEMP/upgrade-case" + - name: Preserve OceanBase diagnostics and stop the owned container + if: always() && matrix.backend == 'oceanbase' + run: | + mkdir -p "$RUNNER_TEMP/upgrade-case" + docker logs upgrade-oceanbase > "$RUNNER_TEMP/upgrade-case/oceanbase.log" 2>&1 || true + docker inspect --format '{{json .State}}' upgrade-oceanbase \ + > "$RUNNER_TEMP/upgrade-case/oceanbase-state.json" 2>&1 || true + docker rm --force upgrade-oceanbase + - name: Preserve this configuration's evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: upgrade-case-${{ matrix.index }} + path: ${{ runner.temp }}/upgrade-case/ + if-no-files-found: error + overwrite: true + retention-days: 30 + + collect: + name: Verify the selected upgrade matrix + needs: [prepare, cases] + if: always() && needs.prepare.result == 'success' && !inputs.skip_upgrade_acceptance + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.source_ref }} + persist-credentials: false + - name: Set up the environment + uses: ./.github/actions/setup-python-env + - name: Download the shared prepared inputs + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: upgrade-prepared + path: ${{ runner.temp }}/upgrade-prepared/ + - name: Download disjoint case reports + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + pattern: upgrade-case-* + path: ${{ runner.temp }}/upgrade-case-reports/ + - name: Require all selected evidence for the exact prepared distributions + run: >- + uv run python scripts/parallel_upgrade_release_gate.py collect + --prepared "$RUNNER_TEMP/upgrade-prepared" + --reports "$RUNNER_TEMP/upgrade-case-reports" + --output "$RUNNER_TEMP/upgrade-release-evidence" + - name: Require successful provisioning, execution, cleanup and evidence upload + env: + CASE_JOBS_RESULT: ${{ needs.cases.result }} + run: test "$CASE_JOBS_RESULT" = success + - name: Preserve the aggregate verdict + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: upgrade-release-evidence + path: ${{ runner.temp }}/upgrade-release-evidence/ + if-no-files-found: error + overwrite: true + retention-days: 30 + - name: Upload accepted release distributions + if: success() && inputs.release + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: release-dists + path: ${{ runner.temp }}/upgrade-release-evidence/validated-dist/* + if-no-files-found: error + overwrite: true + retention-days: 30 diff --git a/.github/workflows/zcode-acceptance.yml b/.github/workflows/zcode-acceptance.yml new file mode 100644 index 0000000000..fc74e32c7d --- /dev/null +++ b/.github/workflows/zcode-acceptance.yml @@ -0,0 +1,158 @@ +name: ZCode acceptance + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [master] + workflow_dispatch: + schedule: + - cron: '23 3 * * 0' + +permissions: + contents: read + +concurrency: + group: zcode-acceptance-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + # ZCode 3.14.3 contains CLI 0.16.9. Update this pin only with a new acceptance run. + ZCODE_REF: 29628c9acdb81b703bbd4080c207a0e7ce5e276e + ZCODE_CLI_BIN: ${{ github.workspace }}/.powercontext/zcode-ci-source/apps/zcode-cli/packages/cli/dist/zcode.cjs + CI: 'true' + HUSKY: '0' + +jobs: + changes: + uses: ./.github/workflows/ci-changes.yml + + controlled-cli: + needs: changes + if: ${{ !cancelled() && (needs.changes.result != 'success' || fromJSON(needs.changes.outputs.suites || '{}').zcode) }} + runs-on: windows-latest + timeout-minutes: 30 + defaults: + run: + shell: pwsh + steps: + - name: Check out PowerContext + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Check out the supported ZCode host + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + repository: zai-org/ZCode + ref: ${{ env.ZCODE_REF }} + path: .powercontext/zcode-ci-source + persist-credentials: false + + - name: Set up Python + uses: ./.github/actions/setup-python-env + with: + python-version: '3.12' + + - name: Set up Node + uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + # Includes Windows shutdown and file-watcher fixes (nodejs/node#61999, #65118). + node-version: '24.21.0' + + - name: Install the host package manager + run: npm install --global pnpm@10.33.2 + + - name: Validate the ZCode Skill pin and behavior evaluation harness + run: | + uv run --locked --no-sync python -m evaluation.zcode_guidance.pin --check + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv run --locked --no-sync python -m evaluation.zcode_guidance.run --help + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + uv run --locked --no-sync python -m pytest evaluation/zcode_guidance/tests tests/test_zcode_acceptance_protocol.py tests/test_zcode_acceptance_host.py -q + + - name: Record build provenance + run: | + $hostCommit = git -C .powercontext/zcode-ci-source rev-parse HEAD + if ($LASTEXITCODE -ne 0 -or $hostCommit -ne $env:ZCODE_REF) { throw 'Unexpected ZCode commit' } + $reportDir = Join-Path $env:RUNNER_TEMP 'zcode-ci-evidence' + New-Item -ItemType Directory -Path $reportDir | Out-Null + [ordered]@{ + repository_commit = (git rev-parse HEAD) + host_repository = 'zai-org/ZCode' + host_commit = $hostCommit + node_version = (node --version) + pnpm_version = (pnpm --version) + model_mode = 'fixture' + server_mode = 'real-sqlite' + } | ConvertTo-Json | Set-Content -LiteralPath (Join-Path $reportDir 'provenance.json') -Encoding utf8 + + - name: Install frozen host build dependencies + run: >- + pnpm --dir .powercontext/zcode-ci-source + --filter zcode --filter zcode-cli --filter '@zcode/cli...' + install --frozen-lockfile --ignore-scripts + + - name: Build the actual CLI + run: | + pnpm --dir .powercontext/zcode-ci-source exec turbo --skip-infer --cwd apps/zcode-cli run build --filter=@zcode/cli + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + if (-not (Test-Path -LiteralPath $env:ZCODE_CLI_BIN -PathType Leaf)) { throw 'CLI bundle not built' } + node $env:ZCODE_CLI_BIN --version + + - name: Run all plugin Node tests including native host discovery + # Process deadline checks must not compete with other test files for runner CPU and disk. + run: node --test --test-concurrency=1 integrations/zcode/plugins/powercontext/tests/*.test.mjs + + - name: Run two independent controlled CLI acceptances + run: >- + uv run --locked --no-sync pytest tests/e2e/test_zcode_host_acceptance.py + -m 'zcode_host_acceptance and not zcode_live_model' + --zcode-acceptance-output "$env:RUNNER_TEMP/zcode-acceptance" -q + + - name: Install the frozen Pi host and plugin dependencies + run: pnpm --dir integrations/pi/plugins/powercontext install --frozen-lockfile --ignore-scripts + + - name: Verify actual Pi to ZCode Work Continuity + run: >- + uv run --locked --no-sync pytest tests/e2e/test_cross_host_work_continuity.py + -m cross_host_acceptance + --cross-host-acceptance-output "$env:RUNNER_TEMP/cross-host-acceptance" -q + + - name: Retain only summaries and selected evidence + if: always() + run: | + $reportDir = Join-Path $env:RUNNER_TEMP 'zcode-ci-evidence' + foreach ($suite in @('zcode-acceptance', 'cross-host-acceptance')) { + $output = Join-Path $env:RUNNER_TEMP $suite + if (-not (Test-Path -LiteralPath $output)) { continue } + foreach ($run in Get-ChildItem -LiteralPath $output -Directory) { + if ($run.Name -notmatch '^[0-9a-f]{32}$') { continue } + $summary = Join-Path $run.FullName 'summary.json' + if (-not (Test-Path -LiteralPath $summary -PathType Leaf)) { continue } + $target = Join-Path $reportDir $run.Name + New-Item -ItemType Directory -Force -Path $target | Out-Null + Copy-Item -LiteralPath $summary -Destination $target + $evidence = Join-Path $run.FullName 'evidence' + if (Test-Path -LiteralPath $evidence -PathType Container) { + $evidenceTarget = Join-Path $target 'evidence' + New-Item -ItemType Directory -Path $evidenceTarget | Out-Null + Get-ChildItem -LiteralPath $evidence -File -Filter '*.json' | + Copy-Item -Destination $evidenceTarget + } + $result = Get-Content -LiteralPath $summary -Raw | ConvertFrom-Json + "### $suite run $($run.Name): $($result.overall_status)" | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY + foreach ($scenario in $result.scenarios) { + "- $($scenario.id): $($scenario.status)" | Add-Content -LiteralPath $env:GITHUB_STEP_SUMMARY + } + } + } + + - name: Upload controlled acceptance evidence + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: zcode-controlled-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/zcode-ci-evidence/ + if-no-files-found: warn + retention-days: 14 diff --git a/src/powercontext/sources/csv_rows.py b/src/powercontext/sources/csv_rows.py new file mode 100644 index 0000000000..93e5dc37ae --- /dev/null +++ b/src/powercontext/sources/csv_rows.py @@ -0,0 +1,221 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""One durable Source per CSV record, addressed by document and primary key.""" + +from __future__ import annotations + +import csv +import io +import json +from urllib.parse import quote + +from pydantic import BaseModel + +from powercontext.limits import MAX_SOURCE_ID_LENGTH +from powercontext.sources import TEXT_EVIDENCE_PROJECTION_KEY, AdapterSourceDefinition, TextEvidence +from powercontext.sources.models import Source, SourceMaterialization + +CSV_ROW_SOURCE_NAME = "csv-row" +_CSV_IDENTITY_PREFIX = "csv" + + +class CSVDocumentInput(BaseModel): + """Complete offline CSV bytes to split into one Source per record.""" + + document_id: str + primary_key: str + content: bytes + + +class CSVRowInput(BaseModel): + """One validated CSV record in original header order.""" + + document_id: str + primary_key: str + primary_key_value: str + columns: tuple[str, ...] + values: tuple[str, ...] + + +class CSVRowSource(Source): + """Captured record whose identity is its document and primary-key value.""" + + source_id: str + document_id: str + primary_key: str + primary_key_value: str + columns: tuple[str, ...] + values: tuple[str, ...] + + +def parse_csv_document(value: CSVDocumentInput) -> tuple[CSVRowInput, ...]: + """Parse and fully validate a CSV document before returning any record. + + The first record is the header and every later record maps exactly one Source. + All validation happens before the tuple is returned, so a malformed document + never yields a prefix of its rows. + """ + + document_id = _validate_document_id(value.document_id) + primary_key = _validate_primary_key(value.primary_key) + try: + text = value.content.decode("utf-8-sig") + except UnicodeDecodeError as error: + raise ValueError("CSV content must be strict UTF-8") from error + if not text: + raise ValueError("CSV document must contain a header record") + + try: + records = list(csv.reader(io.StringIO(text, newline=""), strict=True)) + except csv.Error as error: + raise ValueError(f"CSV document is malformed: {error}") from error + if not records: + raise ValueError("CSV document must contain a header record") + + header = records[0] + columns = tuple(header) + if not columns: + raise ValueError("CSV header must not be empty") + for column in columns: + if not column.strip(): + raise ValueError("CSV header fields must be non-empty and not whitespace") + if len(set(columns)) != len(columns): + raise ValueError("CSV header fields must be unique") + try: + key_index = columns.index(primary_key) + except ValueError as error: + raise ValueError(f"primary key {primary_key!r} does not match a CSV header field") from error + + rows: list[CSVRowInput] = [] + seen_keys: set[str] = set() + for record in records[1:]: + if not record: + continue + if len(record) != len(columns): + raise ValueError("CSV record width does not match the header") + primary_key_value = record[key_index] + if not primary_key_value.strip(): + raise ValueError("CSV primary-key values must be non-empty and not whitespace") + if primary_key_value in seen_keys: + raise ValueError("CSV primary-key values must be unique in one document") + seen_keys.add(primary_key_value) + rows.append( + CSVRowInput( + document_id=document_id, + primary_key=primary_key, + primary_key_value=primary_key_value, + columns=columns, + values=tuple(record), + ) + ) + return tuple(rows) + + +def csv_row_source_id(document_id: str, primary_key_value: str) -> str: + """Return the stable, delimiter-safe identity of one CSV record.""" + + document_id = _validate_document_id(document_id) + if not isinstance(primary_key_value, str) or not primary_key_value: + raise ValueError("primary-key value must be a non-empty string") + source_id = f"{_CSV_IDENTITY_PREFIX}:{quote(document_id, safe='')}:{quote(primary_key_value, safe='')}" + if len(source_id) > MAX_SOURCE_ID_LENGTH: + raise ValueError("CSV row identity must not exceed the Source identity length limit") + return source_id + + +class CSVRowSourceAdapter: + """Resolve validated rows and read their exact captured fields back.""" + + input_class = CSVRowInput + name = CSV_ROW_SOURCE_NAME + source_class = CSVRowSource + + async def resolve(self, value: CSVRowInput, /) -> CSVRowSource: + return CSVRowSource( + name=csv_row_source_id(value.document_id, value.primary_key_value), + materialization=SourceMaterialization.CAPTURED, + source_id=csv_row_source_id(value.document_id, value.primary_key_value), + document_id=value.document_id, + primary_key=value.primary_key, + primary_key_value=value.primary_key_value, + columns=value.columns, + values=value.values, + ) + + async def read(self, source: CSVRowSource, /) -> CSVRowInput: + return CSVRowInput( + document_id=source.document_id, + primary_key=source.primary_key, + primary_key_value=source.primary_key_value, + columns=source.columns, + values=source.values, + ) + + +class CSVRowTextEvidenceProjection: + """Expose one row as ordered JSON text with its exact record metadata.""" + + name = TEXT_EVIDENCE_PROJECTION_KEY.name + version = TEXT_EVIDENCE_PROJECTION_KEY.version + source_class = CSVRowSource + output_class: type[BaseModel] = TextEvidence + + def project(self, source: CSVRowSource, /) -> TextEvidence: + if len(source.columns) != len(source.values): + raise ValueError("CSV row columns and values must have the same width") + row = dict(zip(source.columns, source.values, strict=True)) + return TextEvidence( + source_type=CSV_ROW_SOURCE_NAME, + source_id=source.name, + content=json.dumps(row, ensure_ascii=False, separators=(",", ":")), + metadata={ + "document_id": source.document_id, + "primary_key": source.primary_key, + "primary_key_value": source.primary_key_value, + "columns": list(source.columns), + "row": row, + }, + ) + + +CSV_ROW_SOURCE_ADAPTER = CSVRowSourceAdapter() +CSV_ROW_SOURCE_DEFINITION = AdapterSourceDefinition( + CSV_ROW_SOURCE_ADAPTER, + projections=(CSVRowTextEvidenceProjection(),), +) + + +def _validate_document_id(document_id: object) -> str: + if not isinstance(document_id, str) or not document_id: + raise ValueError("document_id must be a non-empty string") + if document_id != document_id.strip(): + raise ValueError("document_id must not contain leading or trailing whitespace") + return document_id + + +def _validate_primary_key(primary_key: object) -> str: + if not isinstance(primary_key, str) or not primary_key.strip(): + raise ValueError("primary_key must be a non-empty string") + return primary_key + + +__all__ = [ + "CSV_ROW_SOURCE_DEFINITION", + "CSVDocumentInput", + "CSVRowInput", + "CSVRowSource", + "csv_row_source_id", + "parse_csv_document", +]