feat(scaling): Qdrant and pgvector stores, shared cache, OpenAI-compatible LLM, all off by default (Phase 8)

.env.example

@@ -51,3 +51,24 @@ UPLOADS_DIR=./uploads
 # The native Rust accelerator (rag_accel): preferred (default) uses it and falls back, loudly, to pure Python when it
 # is missing; required refuses to start without it. 'make setup' installs it (wheelhouse/, then cargo).
 # RAG_ACCEL=preferred
+
+# --- Scaling options, all off by default (docs/scaling.md) --------------------------------------------------------
+# Where chunk vectors live: chroma (embedded, default), qdrant or pgvector. Switching needs a re-sync:
+# make sync ADMIN=<admin> FORCE=1. Start a server with: make scale-up SERVICE=qdrant (or pgvector).
+# VECTOR_STORE=chroma
+# QDRANT_URL=http://localhost:6333
+# QDRANT_API_KEY=            # empty = no authentication (loopback only); set one before exposing the port
+# QDRANT_PREFIX=             # prepended to collection names so installs can share a server
+# PGVECTOR_URI=              # default: built from POSTGRES_* for the compose profile (port 5433)
+# PGVECTOR_SCHEMA=public
+
+# The semantic cache: sqlite (default) or postgres, shared by several UI processes. All of them need the same data key.
+# CACHE_BACKEND=sqlite
+# CACHE_DB_URI=postgresql://user:password@host:5432/dbname
+
+# A chat model on vLLM, LM Studio or llama.cpp: LLM_PROVIDER=openai
+# LOCAL_LLM_URL=http://localhost:8000/v1
+# LOCAL_LLM_MODEL=Qwen/Qwen3-8B
+# LOCAL_LLM_API_KEY=
+# LOCAL_LLM_TEMPLATE_KWARGS={"enable_thinking": false}   # empty = send none (a server that rejects unknown fields)
+# VLLM_MODEL=Qwen/Qwen3-8B   # for `make scale-up SERVICE=vllm` (needs an NVIDIA GPU); VLLM_API_KEY, VLLM_PORT

.github/workflows/ci.yml

@@ -15,6 +15,9 @@ jobs:
       # Tests run on throwaway SQLite stores; this server backs the PostgreSQL checkpointer tests.
       # Credentials of a throwaway service container that exists only for this run; real deployments use .env.
       POSTGRES_TEST_URI: postgresql://rag_user:rag_password@localhost:5432/rag_checkpoints
+      # The scaling options (docs/scaling.md) are tested against real servers; a missing one fails the run (REQUIRE_*).
+      QDRANT_TEST_URL: http://localhost:6333
+      PGVECTOR_TEST_URI: postgresql://rag_user:rag_password@localhost:5433/rag_vectors
 
     services:
       postgres:
@@ -31,6 +34,25 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+      qdrant:
+        image: qdrant/qdrant:v1.20.0
+        ports:
+          - 6333:6333
+
+      pgvector:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_DB: rag_vectors
+          POSTGRES_USER: rag_user
+          POSTGRES_PASSWORD: rag_password
+        ports:
+          - 5433:5432
+        options: >-
+          --health-cmd "pg_isready -U rag_user -d rag_vectors"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - name: Check out repository
         uses: actions/checkout@v7
@@ -46,6 +68,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.lock
           pip install maturin
+          pip install "qdrant-client>=1.9"  # optional: the Qdrant adapter is tested here
 
       - name: Lint (ruff, rule set pinned in pyproject.toml)
         run: ruff check .
@@ -75,6 +98,8 @@ jobs:
           # A missing PostgreSQL server or Rust accelerator fails the run instead of skipping its tests.
           REQUIRE_POSTGRES: "1"
           REQUIRE_RUST_ACCEL: "1"
+          REQUIRE_QDRANT: "1"
+          REQUIRE_PGVECTOR: "1"
           # The accelerator is the default here: a missing one stops the process instead of falling back.
           RAG_ACCEL: required
         run: pytest -v --cov=. --cov-report=term-missing:skip-covered
@@ -84,5 +109,7 @@ jobs:
           RAG_OFFLINE: "1"
           REQUIRE_POSTGRES: "1"
           REQUIRE_RUST_ACCEL: "1"
+          REQUIRE_QDRANT: "1"
+          REQUIRE_PGVECTOR: "1"
           RAG_ACCEL: required
         run: pytest -v

AGENTS.md

@@ -84,6 +84,7 @@ make clean                  # Clean caches and temporary test artifacts
 make factory-reset [CONFIRM=yes]
 make up | down | db-password   # PostgreSQL (127.0.0.1, password in .env, created by `make up`); db-password applies it to an old volume
 make lan-tls | lan-tls-down | lan-tls-ca   # optional Caddy HTTPS for the LAN (docs/lan-tls.md)
+make scale-up | scale-down SERVICE=<qdrant|pgvector|vllm>   # optional scaling services (docs/scaling.md); `make up` starts none
 ```
 
 ---
@@ -125,6 +126,7 @@ Graph wiring is in [`rag_lab/pipeline/graph.py`](rag_lab/pipeline/graph.py); sta
 * **Keys & Second Factor**: [`keystore.py`](rag_lab/security/keystore.py) owns where the access and data keys live: a key file holds the raw key, a pointer to an OS-keychain entry (random account per key), or an envelope-wrapped key (interface + stub only). `KEY_STORE=auto|file|keyring`; existing files move only via `make migrate-keys`; a pointer whose entry is gone is an error, never a new key; keychain keys are cached per process. Tests pin `KEY_STORE=file` and an inert keyring backend: nothing may reach the real keychain. TOTP (`access.py`): self-enrolment in two steps, secret sealed with AAD `uid:name`, code spent under the policy lock for a newer time step only, recovery codes stored as keyed tags, enforced for as long as the enrolment exists. Lockouts: an owner's password and second factor are reset by an owner (`set_password`, `disable_totp`), a sole owner's by `recover_account` (CLI only, README table "When someone is locked out"). pyotp 2.10 has no `verify_and_get_timecode`: `_match_totp_step` is ours.
 * **Clearance-Partitioned Sealed Cache**: `SEMANTIC_CACHE` and `SEMANTIC_CACHE_CITATIONS` ([`semantic_cache.py`](rag_lab/retrieval/semantic_cache.py)) provide an LRU in-memory + AES-256-GCM sealed SQLite-persistent cache keyed on `(workspace_id, clearance, index_fingerprint, original_question)` with cosine-similarity semantic retrieval, numeric/negation signature verification, and `reviewed` status tracking so users without `auto_approve` only consume human-reviewed cache entries. Semantic search is [`cache_index.py`](rag_lab/retrieval/cache_index.py): one float32 matrix per (db, workspace, clearance, index version) partition; a lookup searches only the caller's authorized clearances, re-syncs each partition with the table by `(key, rowid, created_at, reviewed)` first, and the winning row is re-read, opened (seal + AAD), re-checked and its cosine recomputed before a hit. The index only proposes candidates; never make it the authority.
 * **Fail-Closed Document Clearance**: Unrecognized explicit `clearance` tags in YAML frontmatter (case-insensitive) or `.clearance` folder files are rejected during ingestion ([`ingest.py`](rag_lab/ingestion/ingest.py)). Both dense ([`retrieval.py`](rag_lab/pipeline/retrieval.py)) and sparse ([`sparse_search.py`](rag_lab/retrieval/sparse_search.py)) retrieval enforce `allowed_clearances` at query time and via fail-closed post-query guards that reject chunks missing a valid clearance attribute.
+* **Scaling options (off by default)**: all vector access goes through [`VectorStore`](rag_lab/storage/vector_store.py) (`VECTOR_STORE=chroma|qdrant|pgvector`); `query()` applies `guard_hits()` after the adapter's server-side filter, so the clearance guard lives in the base class and a new adapter cannot skip it. Similarity must stay cosine on Chroma's `1 - distance` scale (the gates are calibrated on it: tests compare values, not just order). Qdrant/pgvector fail closed (no unfiltered fallback) and publish atomically. `CACHE_BACKEND=postgres` puts the sealed semantic cache in a shared database (L1 off; same data key everywhere). `LLM_PROVIDER=openai` (`LOCAL_LLM_*`) serves vLLM/LM Studio/llama.cpp: thinking off via `chat_template_kwargs`, `<think>` stripped. Tests pin all of it off (conftest, eval redirect) so a `.env` can never make a test reach a real service; adapters are tested in their own namespace (`QDRANT_PREFIX`, `PGVECTOR_SCHEMA`).
 * **Ingest reuse**: a re-sync embeds only chunks whose exact text is new and takes the stored vector of the rest, but only from the same workspace's live collection with the same model fingerprint (collection and manifest) and the same `passage_prefix` recorded in the manifest; `FORCE=1`, an unreadable index or an older manifest embeds everything. Plain-text files (`PLAIN_SUFFIXES`) are decoded in-process; every other format stays in the sandboxed parser child.
 * **Workspaces**: Sanitized slugs (`ws_<workspace_id>`). Each workspace deterministically merges `knowledge_base/<ws>` and `uploads/<ws>` (with `knowledge_base` taking precedence on filename collisions) and performs an atomic Chroma collection swap (`__build` -> live) during `FORCE=1` re-syncs. Deleting a workspace requires admin privileges, escapes SQL `LIKE` wildcards, and prunes its grants from `access_policy.json`.
 * **Human-in-the-Loop & Safety Escalation**: Answers release automatically when the user holds `auto_approve` **and** the generation is grounded, relevant, non-abstaining, and free of blocking degradation (`security_warning`, `dense_retrieval_unavailable`, `sparse_retrieval_unavailable`). Otherwise, execution pauses at `interrupt()` for a reviewer who holds `review` permission, is not the requester (`requester != approver`), and whose clearance dominates (`clearance_covers`) the thread's `principal_clearance`.

CLAUDE.md

@@ -84,6 +84,7 @@ make clean                  # Clean caches and temporary test artifacts
 make factory-reset [CONFIRM=yes]
 make up | down | db-password   # PostgreSQL (127.0.0.1, password in .env, created by `make up`); db-password applies it to an old volume
 make lan-tls | lan-tls-down | lan-tls-ca   # optional Caddy HTTPS for the LAN (docs/lan-tls.md)
+make scale-up | scale-down SERVICE=<qdrant|pgvector|vllm>   # optional scaling services (docs/scaling.md); `make up` starts none
 ```
 
 ---
@@ -125,6 +126,7 @@ Graph wiring is in [`rag_lab/pipeline/graph.py`](rag_lab/pipeline/graph.py); sta
 * **Keys & Second Factor**: [`keystore.py`](rag_lab/security/keystore.py) owns where the access and data keys live: a key file holds the raw key, a pointer to an OS-keychain entry (random account per key), or an envelope-wrapped key (interface + stub only). `KEY_STORE=auto|file|keyring`; existing files move only via `make migrate-keys`; a pointer whose entry is gone is an error, never a new key; keychain keys are cached per process. Tests pin `KEY_STORE=file` and an inert keyring backend: nothing may reach the real keychain. TOTP (`access.py`): self-enrolment in two steps, secret sealed with AAD `uid:name`, code spent under the policy lock for a newer time step only, recovery codes stored as keyed tags, enforced for as long as the enrolment exists. Lockouts: an owner's password and second factor are reset by an owner (`set_password`, `disable_totp`), a sole owner's by `recover_account` (CLI only, README table "When someone is locked out"). pyotp 2.10 has no `verify_and_get_timecode`: `_match_totp_step` is ours.
 * **Clearance-Partitioned Sealed Cache**: `SEMANTIC_CACHE` and `SEMANTIC_CACHE_CITATIONS` ([`semantic_cache.py`](rag_lab/retrieval/semantic_cache.py)) provide an LRU in-memory + AES-256-GCM sealed SQLite-persistent cache keyed on `(workspace_id, clearance, index_fingerprint, original_question)` with cosine-similarity semantic retrieval, numeric/negation signature verification, and `reviewed` status tracking so users without `auto_approve` only consume human-reviewed cache entries. Semantic search is [`cache_index.py`](rag_lab/retrieval/cache_index.py): one float32 matrix per (db, workspace, clearance, index version) partition; a lookup searches only the caller's authorized clearances, re-syncs each partition with the table by `(key, rowid, created_at, reviewed)` first, and the winning row is re-read, opened (seal + AAD), re-checked and its cosine recomputed before a hit. The index only proposes candidates; never make it the authority.
 * **Fail-Closed Document Clearance**: Unrecognized explicit `clearance` tags in YAML frontmatter (case-insensitive) or `.clearance` folder files are rejected during ingestion ([`ingest.py`](rag_lab/ingestion/ingest.py)). Both dense ([`retrieval.py`](rag_lab/pipeline/retrieval.py)) and sparse ([`sparse_search.py`](rag_lab/retrieval/sparse_search.py)) retrieval enforce `allowed_clearances` at query time and via fail-closed post-query guards that reject chunks missing a valid clearance attribute.
+* **Scaling options (off by default)**: all vector access goes through [`VectorStore`](rag_lab/storage/vector_store.py) (`VECTOR_STORE=chroma|qdrant|pgvector`); `query()` applies `guard_hits()` after the adapter's server-side filter, so the clearance guard lives in the base class and a new adapter cannot skip it. Similarity must stay cosine on Chroma's `1 - distance` scale (the gates are calibrated on it: tests compare values, not just order). Qdrant/pgvector fail closed (no unfiltered fallback) and publish atomically. `CACHE_BACKEND=postgres` puts the sealed semantic cache in a shared database (L1 off; same data key everywhere). `LLM_PROVIDER=openai` (`LOCAL_LLM_*`) serves vLLM/LM Studio/llama.cpp: thinking off via `chat_template_kwargs`, `<think>` stripped. Tests pin all of it off (conftest, eval redirect) so a `.env` can never make a test reach a real service; adapters are tested in their own namespace (`QDRANT_PREFIX`, `PGVECTOR_SCHEMA`).
 * **Ingest reuse**: a re-sync embeds only chunks whose exact text is new and takes the stored vector of the rest, but only from the same workspace's live collection with the same model fingerprint (collection and manifest) and the same `passage_prefix` recorded in the manifest; `FORCE=1`, an unreadable index or an older manifest embeds everything. Plain-text files (`PLAIN_SUFFIXES`) are decoded in-process; every other format stays in the sandboxed parser child.
 * **Workspaces**: Sanitized slugs (`ws_<workspace_id>`). Each workspace deterministically merges `knowledge_base/<ws>` and `uploads/<ws>` (with `knowledge_base` taking precedence on filename collisions) and performs an atomic Chroma collection swap (`__build` -> live) during `FORCE=1` re-syncs. Deleting a workspace requires admin privileges, escapes SQL `LIKE` wildcards, and prunes its grants from `access_policy.json`.
 * **Human-in-the-Loop & Safety Escalation**: Answers release automatically when the user holds `auto_approve` **and** the generation is grounded, relevant, non-abstaining, and free of blocking degradation (`security_warning`, `dense_retrieval_unavailable`, `sparse_retrieval_unavailable`). Otherwise, execution pauses at `interrupt()` for a reviewer who holds `review` permission, is not the requester (`requester != approver`), and whose clearance dominates (`clearance_covers`) the thread's `principal_clearance`.

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL := help
-.PHONY: help status setup rust-build models verify-models calibrate-models up down ollama-pull test test-offline lint typecheck coverage workspaces create-workspace delete-workspace sync ingest ui audit-verify audit-tail audit-feedback audit-anchor audit-export retention backup backups backup-verify add-user grant revoke set-clearance set-admin set-role set-default team-mode groups create-group delete-group add-to-group remove-from-group grant-group set-password revoke-sessions delete-user users reset-users factory-reset recover db-password lan-tls lan-tls-down lan-tls-ca migrate-keys totp-enroll totp-recovery totp-disable eval eval-judge eval-llm eval-calibrate graph clean reviews review approve reject
+.PHONY: help status setup rust-build models verify-models calibrate-models up down ollama-pull test test-offline lint typecheck coverage workspaces create-workspace delete-workspace sync ingest ui audit-verify audit-tail audit-feedback audit-anchor audit-export retention backup backups backup-verify add-user grant revoke set-clearance set-admin set-role set-default team-mode groups create-group delete-group add-to-group remove-from-group grant-group set-password revoke-sessions delete-user users reset-users factory-reset recover db-password scale-up scale-down lan-tls lan-tls-down lan-tls-ca migrate-keys totp-enroll totp-recovery totp-disable eval eval-judge eval-llm eval-calibrate graph clean reviews review approve reject
 
 # 0. Command Reference & System Readiness
 help:
@@ -13,6 +13,8 @@ help:
 	@echo "  make rust-build                                Compile the native Rust accelerator (rag_accel) from source"
 	@echo "  make ollama-pull                               Pull the optional Ollama chat model"
 	@echo "  make up / make down                            Start / stop the optional PostgreSQL checkpointer (Docker); 'up' creates .env with a random password"
+	@echo "  make scale-up SERVICE=<qdrant|pgvector|vllm>   Start an optional scaling service (docs/scaling.md); make up never does"
+	@echo "  make scale-down SERVICE=<qdrant|pgvector|vllm> Stop it (its container and data volume are kept)"
 	@echo "  make lan-tls                                   Optional HTTPS for the UI on the LAN (Caddy profile, docs/lan-tls.md)"
 	@echo "  make lan-tls-down                              Stop the LAN TLS proxy"
 	@echo "  make lan-tls-ca                                Export Caddy's local root certificate for client devices"
@@ -129,6 +131,18 @@ up:
 	@$(if $(wildcard venv/bin/python),./venv/bin/python,python3) -m rag_lab.ops.init_env
 	docker compose up -d
 
+# Optional scaling services (compose profiles qdrant, pgvector, vllm). Nothing here is needed on one laptop. The
+# service only starts: turn it on in .env (VECTOR_STORE=qdrant, VECTOR_STORE=pgvector, LLM_PROVIDER=openai).
+scale-up:
+	@case "$(SERVICE)" in qdrant|pgvector|vllm) ;; *) echo "Usage: make scale-up SERVICE=qdrant|pgvector|vllm"; exit 1;; esac
+	@docker info >/dev/null 2>&1 || { echo "Error: Docker daemon is not running."; exit 1; }
+	@$(if $(wildcard venv/bin/python),./venv/bin/python,python3) -m rag_lab.ops.init_env
+	docker compose --profile $(SERVICE) up -d $(SERVICE)
+
+scale-down:
+	@case "$(SERVICE)" in qdrant|pgvector|vllm) ;; *) echo "Usage: make scale-down SERVICE=qdrant|pgvector|vllm"; exit 1;; esac
+	docker compose --profile $(SERVICE) stop $(SERVICE)
+
 # Optional LAN TLS (compose profile "caddy"): Caddy terminates HTTPS with its own local CA and proxies to the
 # Streamlit UI on the host. Never started by `make up`. Set RAG_LAN_HOST and RAG_HTTPS_PORT in .env.
 lan-tls:
@@ -386,13 +400,14 @@ reset-users:
 # 20. Complete zero-trace factory reset (wipes policy, data, audit, checkpoints, keys, vectors, uploads, and caches)
 factory-reset:
 	@if [ "$(CONFIRM)" != "yes" ] && [ "$(YES)" != "1" ] && [ "$(YES)" != "yes" ]; then \
-		printf "WARNING: This permanently deletes all user accounts, audit logs, checkpoints, encryption keys, and indexed vectors.\nType 'yes' to proceed: "; \
+		printf "WARNING: This permanently deletes all user accounts, audit logs, checkpoints, encryption keys, and indexed vectors, including whatever the configured Qdrant, pgvector or shared-cache server holds for this install.\nType 'yes' to proceed: "; \
 		read ans; \
 		if [ "$$ans" != "yes" ]; then \
 			echo "Factory reset cancelled."; \
 			exit 1; \
 		fi; \
 	fi
+	./venv/bin/python -m rag_lab.ops.purge_external || echo "Warning: a configured external store was NOT purged (see above); the documents or cached answers are still on that server."
 	./venv/bin/python -m rag_lab.security.keystore forget || echo "Warning: keychain entries could not be removed; delete the local-rag-lab entries in your keychain app by hand."
 	rm -f access_policy.json access_policy.json.*.bak access_policy.json.lock
 	rm -rf data