release: Kafka-native V2.2.0 and durable consumer SDK

.github/workflows/ci.yml

@@ -96,10 +96,11 @@ jobs:
 
       - name: Build and import standalone SDK wheel
         run: |
-          python scripts/build_qdl_sdk_release.py --output-dir /tmp/qdl-sdk
-          python -m pip install --disable-pip-version-check /tmp/qdl-sdk/qdl_sdk-2.0.3-py3-none-any.whl
+          python scripts/build_qdl_sdk_release.py --output-dir /tmp/qdl-sdk > /tmp/qdl-sdk-build.json
+          WHEEL=$(python -c 'import json; print(json.load(open("/tmp/qdl-sdk-build.json"))["wheel"])')
+          python -m pip install --disable-pip-version-check "/tmp/qdl-sdk/$WHEEL"
           cd "$(mktemp -d)"
-          PYTHONPATH= python -c 'import qdl_sdk; assert qdl_sdk.__version__ == "2.0.3"'
+          PYTHONPATH= python -c 'import json, qdl_sdk; assert qdl_sdk.__version__ == json.load(open("/tmp/qdl-sdk-build.json"))["version"]'
 
   unit-tests:
     runs-on: ubuntu-latest
@@ -145,6 +146,18 @@ jobs:
           docker compose -f docker-compose.yml -f docker-compose.ci.yml
           run --rm test_runner python -m unittest discover -s tests
 
+      - name: Run KN Query and BAR readback against isolated Redis
+        run: |
+          docker run --rm -d --name kn-python-redis --network none \
+            --memory 128m --cpus 0.5 \
+            redis@sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8 \
+            redis-server --save "" --appendonly no
+          trap 'docker stop -t 5 kn-python-redis' EXIT
+          docker run --rm --network container:kn-python-redis \
+            -e QDL_KN_TEST_REDIS=redis://127.0.0.1:6379/0 \
+            "${QDL_CI_IMAGE}" python -m unittest -v \
+            tests.test_kn_query_backend tests.test_kn_bar_readback
+
       - name: Run V2 contract, SDK and shadow-consumer gates
         run: >-
           docker compose -f docker-compose.yml -f docker-compose.ci.yml
@@ -254,3 +267,48 @@ jobs:
         run: >-
           docker compose -f docker-compose.yml -f docker-compose.ci.yml
           down --volumes --remove-orphans
+
+  kn-native-integration:
+    # KN-1 K1.6: native gateway reads against a real, isolated Kafka broker.
+    # The test is `#[ignore]` in the workspace run and fails loudly here if
+    # the broker is missing; it never contacts a shared or production broker.
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+
+      - name: Start isolated Kafka (KRaft, plaintext, no ACLs)
+        run: |
+          docker run -d --name kn-kafka --hostname localhost \
+            -e KAFKA_HEAP_OPTS="-Xmx512m -Xms256m" \
+            apache/kafka@sha256:9516fb7634bad307d17c33b589fde9023003b0cb761374f500002b980a3149b9
+          for attempt in $(seq 1 60); do
+            if docker exec kn-kafka /opt/kafka/bin/kafka-broker-api-versions.sh \
+                --bootstrap-server localhost:9092 >/dev/null 2>&1; then
+              exit 0
+            fi
+            sleep 2
+          done
+          docker logs kn-kafka | tail -50
+          exit 1
+
+      - name: Start isolated Redis (market-cache image; one shared, one dedicated to the memory-full test)
+        run: |
+          for port in 6379 6380; do
+            docker run -d --name kn-redis-$port --network container:kn-kafka \
+              redis@sha256:dfa18828cbc07b3ae6a95ec7343f6c214fdee2d836197b4be8e9904420762cd8 \
+              redis-server --port $port --save "" --appendonly no
+          done
+
+      - name: Committed reads, no-commit reader, projector stage A/B, market cache (Kafka + Redis)
+        run: |
+          docker run --rm --network container:kn-kafka \
+            -e QDL_KN_TEST_KAFKA=localhost:9092 \
+            -e QDL_KN_TEST_REDIS=redis://localhost:6379/0 \
+            -e QDL_KN_TEST_REDIS_EXCLUSIVE=redis://localhost:6380/0 \
+            -v "$PWD:/workspace" -w /workspace rust:1.82-slim \
+            sh -c 'apt-get update -qq && apt-get install -y --no-install-recommends clang cmake libclang-dev libcurl4-openssl-dev libssl-dev libzstd-dev make pkg-config zlib1g-dev >/dev/null && cargo test -p qdl-stream-gateway --test committed_reads --locked -- --ignored && cargo test -p qdl-projector --locked -- --ignored --test-threads=1'
+
+      - name: Remove isolated Kafka and Redis
+        if: always()
+        run: docker rm -f kn-redis-6379 kn-redis-6380 kn-kafka

.github/workflows/release.yml

@@ -25,6 +25,7 @@ jobs:
         run: |
           git merge-base --is-ancestor "$GITHUB_SHA" origin/main
           python3 - <<'PY'
+          import hashlib
           import json
           import os
           from pathlib import Path
@@ -33,11 +34,32 @@ jobs:
           if not re.fullmatch(r'v2\.\d+\.\d+', tag):
               raise SystemExit('Expected a stable v2 semantic-version tag')
           root = Path('upgrade/evidence/releases') / tag
-          if json.loads((root / 'certificate.json').read_text())['status'] != 'PASS':
+          certificate = json.loads((root / 'certificate.json').read_text())
+          if certificate['status'] != 'PASS':
               raise SystemExit('Release certificate is not PASS')
+          if hashlib.sha256((root / 'endpoint-report.json').read_bytes()).hexdigest() != certificate['endpoint_report_sha256']:
+              raise SystemExit('Endpoint report differs from certified evidence')
           if not (root / 'RELEASE_NOTES.md').is_file():
               raise SystemExit('Release notes are missing')
           PY
+      - name: Build and verify certified SDK artifact
+        env:
+          RELEASE_TAG: ${{ github.ref_name }}
+        run: |
+          python3 scripts/build_qdl_sdk_release.py --output-dir "$RUNNER_TEMP/qdl-sdk"
+          python3 - <<'PYCODE'
+          import hashlib
+          import json
+          import os
+          from pathlib import Path
+          certificate = json.loads((Path('upgrade/evidence/releases') / os.environ['RELEASE_TAG'] / 'certificate.json').read_text())
+          root = Path(os.environ['RUNNER_TEMP']) / 'qdl-sdk'
+          wheels = list(root.glob('*.whl'))
+          if len(wheels) != 1:
+              raise SystemExit('Expected exactly one SDK wheel')
+          if hashlib.sha256(wheels[0].read_bytes()).hexdigest() != certificate['sdk']['wheel_sha256']:
+              raise SystemExit('Tagged SDK differs from certified consumer artifact')
+          PYCODE
       - name: Publish immutable tag and public evidence
         env:
           GH_TOKEN: ${{ github.token }}
@@ -47,4 +69,8 @@ jobs:
           gh release create "$RELEASE_TAG" --verify-tag \
             --title "Quant Data Layer $RELEASE_TAG" \
             --notes-file "$evidence/RELEASE_NOTES.md" \
-            "$evidence/certificate.json"
+            "$evidence/certificate.json" \
+            "$evidence/endpoint-report.json" \
+            "$RUNNER_TEMP/qdl-sdk/"*.whl \
+            "$RUNNER_TEMP/qdl-sdk/"*.release.json \
+            "$RUNNER_TEMP/qdl-sdk/"*.cdx.json