diff --git a/Dockerfile b/Dockerfile index 032de1f..f249439 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,11 @@ WORKDIR /app COPY pyproject.toml uv.lock ./ RUN uv sync --frozen --no-dev --no-editable -# Copy server source and config +# Copy server source. config.yaml is intentionally not baked in — mount it at +# runtime (see docker-compose.yaml) if you use it; without one, load_services() +# treats "no config.yaml" as zero configured services, and self-registered (ad-hoc) +# services still work. COPY server/ ./server/ -COPY config.yaml ./ FROM python:3.12-slim AS runtime diff --git a/Makefile b/Makefile index ac4c2fe..49a35bf 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ QDRANT_URL := http://localhost:6333 SEMCODE_URL := http://localhost:8090 +COMPOSE_WITH_CONFIG := -f docker-compose.yaml -f docker-compose.config-yaml.yml .PHONY: qdrant-clean qdrant-dashboard index-code index-history docker-build \ - docker-build-restart docker-build-restart-jina docker-up docker-up-jina docker-logs docker-logs-semcode + docker-build-restart docker-build-restart-jina docker-up docker-up-jina docker-logs docker-logs-semcode \ + docker-build-restart-with-config docker-build-restart-jina-with-config docker-up-with-config docker-up-jina-with-config qdrant-clean: curl -sf -X DELETE $(QDRANT_URL)/collections/code_symbols && \ @@ -37,6 +39,20 @@ docker-up: docker-up-jina: docker compose --profile jina up -d +# "-with-config" variants also mount config.yaml (see docker-compose.config-yaml.yml), +# for curated/static services alongside — or instead of — ad-hoc registration. +docker-build-restart-with-config: + docker compose $(COMPOSE_WITH_CONFIG) down && docker compose $(COMPOSE_WITH_CONFIG) up --build -d + +docker-build-restart-jina-with-config: + docker compose $(COMPOSE_WITH_CONFIG) --profile jina down && docker compose $(COMPOSE_WITH_CONFIG) --profile jina up --build -d + +docker-up-with-config: + docker compose $(COMPOSE_WITH_CONFIG) up -d + +docker-up-jina-with-config: + docker compose $(COMPOSE_WITH_CONFIG) --profile jina up -d + docker-logs: docker compose logs -f diff --git a/README.md b/README.md index 7210f70..4c5d8ee 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,8 @@ uv sync # (a fine-grained PAT with Contents: read on the target repos is sufficient) cp .env.example .env -# Copy services config, then list the repositories you want indexed +# Optional — only if you want curated/static services (see below for the alternative): +# copy the services config, then list the repositories you want indexed cp config.example.yaml config.yaml ``` @@ -93,6 +94,29 @@ The indexer automatically discovers and indexes all files with recognised extens subdirectory within a shared repo, and `exclude` to skip paths you don't want indexed (tests, build artifacts, generated code, etc.). +**Scaling beyond a handful of repos:** `config.yaml` is a curated, static list — great for a small number of services, +but indexing hundreds of repos this way means hundreds of hand-maintained entries. As an alternative (or complement), +`POST /reindex` accepts a repo definition inline and registers it on the fly, no `config.yaml` entry required — see +[`examples/github-actions/reindex-on-merge.yml`](examples/github-actions/reindex-on-merge.yml) for a drop-in workflow +that self-registers a repo and reindexes it on every merge. Details in the [HTTP API](#http-api) section below. If a +name collides between the two, the `config.yaml` entry always wins. + +You don't need a `config.yaml` at all to run this way — a missing file is treated as zero configured services, not an +error. `docker-compose.yaml` reflects this: by default it does **not** mount `config.yaml`, so `make docker-up` / +`make docker-up-jina` work out of the box for ad-hoc-registration-only setups. If you also want curated services, +copy `config.example.yaml` to `config.yaml` (above) and use the `-with-config` targets instead, which layer +`docker-compose.config-yaml.yml` on top to add the mount: `make docker-up-with-config` / `make docker-up-jina-with-config` +(or `docker compose -f docker-compose.yaml -f docker-compose.config-yaml.yml up -d` directly). Don't hand-edit the +volume line in `docker-compose.yaml` itself — bind-mounting a `config.yaml` that doesn't exist on the host silently +creates an empty directory there instead of leaving the path absent, which breaks the server (surfaced as a clear +error if it happens: `CONFIG_PATH (...) is a directory, not a file`). + +**A single `GITHUB_TOKEN` reads every repo you index this way.** For a handful of `config.yaml` entries a +fine-grained PAT scoped to those repos is fine, but for org-wide self-registration — where any repo can onboard +itself just by adding the workflow — a PAT would need its repo access list updated out-of-band every time a new repo +starts using it. A GitHub App installed org-wide (all repos, `Contents: read`) avoids that: new repos are covered +automatically, with no token maintenance per onboarding. + ## Running There are two ways to run, depending on whether you want embeddings to come from a local @@ -114,6 +138,9 @@ make docker-up # or: docker-compose up ``` +Using `config.yaml` for curated services? Use the `-with-config` variant of whichever target above applies +(`make docker-up-with-config` / `make docker-up-jina-with-config`) — see the [Setup](#setup) section. + > ⚠ The default `EMBEDDINGS_PROVIDER` is `jina`. If you start without `--profile jina` but leave > the provider on the default, semcode will boot (Jina is `required: false` in compose) but the > first embedding call will fail with a connection error — there's no auto-fallback. @@ -124,7 +151,7 @@ Services started with health checks and persistent volumes: |---------------------------|---------|------------------------------|----------------------------------|------------------------| | **Qdrant** | always | `6333` (HTTP), `6334` (gRPC) | `qdrant_data` | Vector DB | | **Jina Embeddings** (TEI) | `jina` | `8087` | `embeddings_cache` | Embedding model server | -| **semcode MCP** | always | `8090` | mounts `./config.yaml` read-only | MCP + HTTP server | +| **semcode MCP** | always | `8090` | mounts `./config.yaml` read-only with `-with-config` | MCP + HTTP server | The MCP server starts with empty collections — trigger an initial index by calling the `reindex` MCP tool or `POST /reindex` (see below). @@ -280,13 +307,23 @@ from CI/CD or external schedulers: | Endpoint | Body | Description | |-------------------------|--------------------------------------------|----------------------------------------------| -| `POST /reindex` | `{"service": ""?, "force": ?}` | Reindex one or all services — returns NDJSON | +| `POST /reindex` | `{"service": ""?, "force": ?, "github_repo": ""?, "github_ref": ""?, "root": ""?, "exclude": [, ...]?}` | Reindex one or all services — returns NDJSON | | `POST /reindex-history` | `{"service": ""?, "force": ?}` | Index git commit history — returns NDJSON | All bodies are optional — omit `service` to act on all services, omit `force` for incremental indexing. Both endpoints stream **newline-delimited JSON** (one frame per line) so you can consume progress in real time from CI/CD pipelines or any other client. +**Registering a repo without `config.yaml`:** if `POST /reindex`'s body includes `github_repo`, the +`service` name is registered with that repo definition (persisted, so it survives restarts and behaves +like a `config.yaml` service from then on) before indexing runs — `service` is required in this case. +`github_ref` defaults to `main`; `root`/`exclude` mirror the same fields in `config.yaml`. A `service` +name already defined in `config.yaml` always wins over one registered this way. There's no +authentication on this endpoint — same as the rest of `/reindex` — so put it behind your own network +boundary before exposing it. See +[`examples/github-actions/reindex-on-merge.yml`](examples/github-actions/reindex-on-merge.yml) for a +ready-to-use workflow. + Frame shapes: ```jsonc diff --git a/docker-compose.config-yaml.yml b/docker-compose.config-yaml.yml new file mode 100644 index 0000000..460680a --- /dev/null +++ b/docker-compose.config-yaml.yml @@ -0,0 +1,8 @@ +# Optional overlay: adds the config.yaml mount for curated/static services. +# Requires config.yaml to exist (cp config.example.yaml config.yaml first). +# Use via the `-with-config` Makefile targets, or directly: +# docker compose -f docker-compose.yaml -f docker-compose.config-yaml.yml up -d +services: + semcode: + volumes: + - ./config.yaml:/app/config.yaml:ro diff --git a/docker-compose.yaml b/docker-compose.yaml index 681da2b..d682b97 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -50,8 +50,13 @@ services: jina-embeddings: condition: service_healthy required: false + # No config.yaml mount here on purpose — this base file works standalone for + # ad-hoc-registration-only setups (see examples/github-actions). If you also want + # curated services from config.yaml, layer docker-compose.config-yaml.yml on top + # (see the `-with-config` Makefile targets) instead of editing this file — bind-mounting + # a config.yaml that doesn't exist on the host silently creates an empty directory + # there, which breaks the server at startup. volumes: - - ./config.yaml:/app/config.yaml:ro - fastembed_cache:/fastembed_cache volumes: diff --git a/docs/README.md b/docs/README.md index fa4db99..2942b58 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,13 +16,16 @@ semcode is an MCP server that provides **hybrid semantic search over code** from | [dense-vectors.md](dense-vectors.md) | Dense embedding providers (Jina, Voyage, OpenAI, Ollama), the embedding text strategy, and provider selection | | [sparse-vectors.md](sparse-vectors.md) | BM25 sparse embeddings, the code identifier tokenizer, and the sparse vector format | | [retrieval-rrf.md](retrieval-rrf.md) | Hybrid search architecture, RRF fusion, name lookup, and the four MCP tool entry points | -| [configuration.md](configuration.md) | All environment variables, `config.yaml` structure, and startup validation | +| [configuration.md](configuration.md) | All environment variables, `config.yaml` structure, dynamic service registration, and startup validation | --- ## Quick Start -1. **Configure services** — copy `config.example.yaml` to `config.yaml` and add your GitHub repositories. See [configuration.md](configuration.md) for all fields. +1. **Configure services** — either copy `config.example.yaml` to `config.yaml` and add your GitHub repositories, or + skip `config.yaml` entirely and register repos on the fly via `POST /reindex` (see the + [GitHub Actions example](../examples/github-actions/reindex-on-merge.yml)) — the two can also be combined. See + [configuration.md](configuration.md) for all fields and how the two interact. 2. **Set environment variables** — copy `.env.example` to `.env` and set at minimum `GITHUB_TOKEN`. The default embedding provider (`jina`) requires a locally running TEI container; for a hosted alternative, set `EMBEDDINGS_PROVIDER=voyage` and `VOYAGE_API_KEY=...`. 3. **Start Qdrant and the server** — `make docker-up-jina` (local Jina) or `make docker-up-voyage` (Voyage API), then connect your MCP client to `http://localhost:8090`. diff --git a/docs/configuration.md b/docs/configuration.md index b4942da..88fdc36 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,17 +1,18 @@ # Configuration -This document covers every configuration knob in semcode: environment variables read from `.env`, the `config.yaml` service definitions, and the startup validation that fires when the embedding provider and Qdrant collection dimensions conflict. +This document covers every configuration knob in semcode: environment variables read from `.env`, the `config.yaml` service definitions, the dynamic service registry (the alternative to `config.yaml`), and the startup validation that fires when the embedding provider and Qdrant collection dimensions conflict. --- ## Overview -semcode is configured through two files: +semcode is configured through: - **`.env`** — environment variables for infrastructure settings (embedding provider, Qdrant URL, GitHub token, server port). Loaded by `pydantic-settings` at startup. -- **`config.yaml`** — service definitions: which GitHub repositories to index, under what names, and with what filters. Loaded on demand by `settings.load_services()`. +- **`config.yaml`** *(optional)* — a static, curated list of service definitions: which GitHub repositories to index, under what names, and with what filters. Loaded on demand by `settings.load_services()`. +- **The dynamic service registry** *(optional, alternative or complement to `config.yaml`)* — services registered on the fly via `POST /reindex`'s `github_repo` field, persisted in a Qdrant collection instead of a file. See [Dynamic Service Registration](#dynamic-service-registration) below. -A `config.example.yaml` is provided in the repository root as a starting point. +`config.yaml` is entirely optional — a missing file is treated as zero configured services, not an error. A `config.example.yaml` is provided in the repository root as a starting point if you do want it. --- @@ -97,8 +98,8 @@ Used when `EMBEDDINGS_PROVIDER=ollama`. Requires a running [Ollama](https://olla | Variable | Default | Description | |----------|---------|-------------| -| `GITHUB_TOKEN` | `""` | GitHub personal access token. Required for all indexing operations. Without it, GitHub API calls return 403. | -| `CONFIG_PATH` | `./config.yaml` | Path to the services config file. Relative to the working directory at server start. | +| `GITHUB_TOKEN` | `""` | GitHub personal access token (or GitHub App installation token). Required for all indexing operations. Without it, GitHub API calls return 403. A single token is used for every service, whether from `config.yaml` or the dynamic registry — see the note in [Dynamic Service Registration](#dynamic-service-registration). | +| `CONFIG_PATH` | `./config.yaml` | Path to the optional services config file. Relative to the working directory at server start. A missing file means zero `config.yaml`-defined services, not an error. | | `GIT_HISTORY_MAX_COMMITS` | `500` | Maximum number of commits fetched per service for git history indexing. | | `EMBEDDING_MAX_CHARS` | provider-aware — see below | Max characters of a symbol's dense-embedding text (preamble + signature + docstring + source). Oversized symbols are truncated (with a logged `WARNING`). Set this explicitly to override the derived default for any provider. | @@ -144,6 +145,64 @@ services: --- +## Dynamic Service Registration + +`config.yaml` doesn't scale to hundreds of repos — every addition means hand-editing one file. As an alternative, +`POST /reindex` accepts the same fields inline in the request body, registering the service on the fly instead of +requiring a `config.yaml` entry: + +```jsonc +POST /reindex +{ + "service": "catalog-service", // required when github_repo is present + "github_repo": "my-org/my-repo", + "github_ref": "main", // optional, defaults to "main" + "root": "services/catalog", // optional + "exclude": ["**/test/**"] // optional +} +``` + +This is the mechanism behind the [GitHub Actions example](../examples/github-actions/reindex-on-merge.yml) — a repo +adds that workflow to its own CI, and every merge to `main`/`master` both registers it and triggers indexing, with +no central file to edit. + +**Where it's stored**: registrations are persisted in a dedicated Qdrant collection (`service_registry`) via +`ServiceRegistry` (`server/store/service_registry.py`), not a file — this is what makes them survive server +restarts and, unlike `config.yaml`, safe to write to from an unattended, unauthenticated HTTP request without +needing a writable file mount. + +**Resolution**: `load_effective_services()` merges `config.yaml` services with everything in the registry every +time services are resolved (on every `index_service`/`index_all` call, and in `get_code_context`) — there's no +in-memory cache, same as `config.yaml`. **`config.yaml` always wins on a name collision** — if a `POST /reindex` +tries to register a name that's already defined in `config.yaml`, the registry entry is stored but ignored when +resolving what to index. This stops a stray or malicious request from silently repointing a curated service to a +different repo. + +**Visibility**: registered services appear in `list_indexed_services` (once they have indexed symbols) and in +`index_stats`, under "Registered dynamically via API" — listed separately from `config.yaml`'s "From config.yaml". + +**Not covered**: `POST /reindex-history` does not accept these inline fields — it only registers through +`/reindex`. Once a service exists in the registry, `/reindex-history` picks it up automatically (same +`load_effective_services()` call), but nothing registers a service for you if you only ever call the history +endpoint. + +**No deregistration**: there's currently no way to remove a registered service short of deleting its point +directly from the `service_registry` Qdrant collection. It will never be auto-pruned — `prune_orphaned_services` +treats every merged service (config.yaml + registry) as known-good. + +**No new access control**: `POST /reindex` has no authentication, same as before this feature existed — providing +`github_repo` doesn't gate behind anything new. This means an unauthenticated caller can make the server index and +permanently register any repo `GITHUB_TOKEN` can read, not just repos already known to it. Put the HTTP API behind +your own network boundary or reverse-proxy auth if that's a concern. + +**`GITHUB_TOKEN` at scale**: the same single token is used for every repo, `config.yaml` or registry. A +fine-grained PAT scoped to an explicit repo list works for a handful of curated services, but for org-wide +self-registration — where any repo can onboard itself just by adding the workflow — a PAT needs its scope updated +out-of-band every time a new repo starts using it, or that repo's first indexing run fails with a 403. A GitHub +App installed org-wide (`Contents: read`, all repositories) avoids that upkeep. + +--- + ## Startup Validation `QdrantStore.ensure_collection()` runs in the server's **lifespan context** (`server/main.py:39`) — at boot, before any requests are served. If the Qdrant collection already exists, its vector dimension is compared against the configured provider's `dimensions` value. A mismatch raises: @@ -168,11 +227,13 @@ This error aborts server startup — the server will not accept connections unti ## Observations -**`load_services()` reads from disk on every call** — there is no in-memory cache for `config.yaml`. Adding, removing, or renaming services takes effect on the next index run without restarting. The downside is a file I/O operation on every indexing request. +**`load_services()` reads from disk on every call** — there is no in-memory cache for `config.yaml`. Adding, removing, or renaming services takes effect on the next index run without restarting. The downside is a file I/O operation on every indexing request. A missing file returns `[]` (see below); an empty file (`yaml.safe_load` returning `None`) is also treated as zero services. **API keys are not validated at startup** — unlike dimension validation (which crashes startup), `JINA_API_KEY`, `VOYAGE_API_KEY`, and `OPENAI_API_KEY` are checked only in the provider constructor, which is deferred to first use. A missing key causes a `RuntimeError` on the first embedding request, not at boot. A server configured with a valid Qdrant collection but a missing API key will start successfully and fail only when indexing is first attempted. -**`CONFIG_PATH` is cwd-relative** — the default `./config.yaml` is resolved relative to the working directory at server start, not relative to the binary or the project root. If the server is started from a different directory, the config file will not be found. +**`CONFIG_PATH` is cwd-relative** — the default `./config.yaml` is resolved relative to the working directory at server start, not relative to the binary or the project root. If the server is started from a different directory, the config file will not be found — same effect as not having one: zero `config.yaml`-defined services. + +**Docker bind-mount footgun**: if `CONFIG_PATH` resolves to a directory instead of a file, `load_services()` raises a clear `RuntimeError` rather than crashing with a cryptic `IsADirectoryError`. This specifically guards against a Docker bind mount pointing at a host `config.yaml` that doesn't exist — Docker silently creates an empty directory there instead of leaving the path absent. `docker-compose.yaml` avoids this by not mounting `config.yaml` at all by default; use the `-with-config` Makefile targets (which layer `docker-compose.config-yaml.yml` on top) if you want it mounted, rather than hand-editing the volume line. **`GITHUB_TOKEN` defaults to empty string** — a missing token doesn't prevent server startup; it causes a 403 from the GitHub API on the first indexing request. diff --git a/docs/ingestion.md b/docs/ingestion.md index 50411e2..45b1894 100644 --- a/docs/ingestion.md +++ b/docs/ingestion.md @@ -14,6 +14,10 @@ Ingestion is managed by `IndexPipeline` (`server/indexer/pipeline.py`). For each 4. Removes index entries for files that have been deleted from the repository The pipeline is triggered via the `/reindex` HTTP endpoint (streaming NDJSON progress) or the `index_all` MCP admin tool. +"Configured service" means resolved from `load_effective_services()` — the union of `config.yaml` and the dynamic +service registry (see [configuration.md](configuration.md#dynamic-service-registration)). `POST /reindex`'s +`github_repo` field registers a service in the registry inline, in the same request that triggers indexing for it — +no separate registration step. --- diff --git a/docs/retrieval-rrf.md b/docs/retrieval-rrf.md index 84c3d9b..ef0a9d2 100644 --- a/docs/retrieval-rrf.md +++ b/docs/retrieval-rrf.md @@ -144,7 +144,8 @@ get_code_context(file_path: str, symbol_name: str | None) -> str Returns full source code for a file or a specific symbol. Unlike the other tools, it **fetches live from GitHub** rather than returning Qdrant payload content: 1. Calls `store.get_file_info(file_path)` to resolve the service name -2. Looks up the service's `github_repo` and `github_ref` from `config.yaml` +2. Looks up the service's `github_repo` and `github_ref` via `load_effective_services()` — `config.yaml` or the + dynamic service registry (see [configuration.md](configuration.md#dynamic-service-registration)) 3. Fetches the raw file content from GitHub (path-based, not blob SHA) 4. If `symbol_name` is given: calls `find_by_name(exact=True)` to get stored line numbers, then slices the file; falls back to a text search if the symbol isn't in the index diff --git a/examples/github-actions/reindex-on-merge.yml b/examples/github-actions/reindex-on-merge.yml new file mode 100644 index 0000000..a53af01 --- /dev/null +++ b/examples/github-actions/reindex-on-merge.yml @@ -0,0 +1,52 @@ +# Example: keep this repo indexed in semcode automatically, with no config.yaml entry. +# +# Copy this file into the repo you want semcode to index, as +# .github/workflows/reindex-on-merge.yml. It self-registers the repo with a running +# semcode server on every push to main/master and triggers indexing in the same call — +# an alternative to listing the repo as a `services` entry in semcode's config.yaml. +# Use whichever fits: config.yaml for a small, curated set of repos you manage centrally; +# this workflow for scaling to many repos without a central file to edit. +# +# Setup: in the repo (or org) running this workflow, set the "SEMCODE_URL" Actions +# variable to the base URL of your running semcode server, e.g. https://semcode.example.com. +# Optionally set "SEMCODE_SERVICE" to override the auto-derived service name, and +# "SEMCODE_ROOT" to scope indexing to a subdirectory (useful for monorepos). +# +# Note: semcode's /reindex endpoint is unauthenticated by default — put it behind your +# own network boundary or reverse-proxy auth before exposing it to CI. +name: Trigger semcode reindex + +on: + push: + branches: [ main, master ] + +jobs: + reindex: + name: Reindex this repo in semcode + runs-on: ubuntu-latest + steps: + - name: POST /reindex and watch for errors + env: + SEMCODE_URL: ${{ vars.SEMCODE_URL }} + SEMCODE_SERVICE: ${{ vars.SEMCODE_SERVICE }} + SEMCODE_ROOT: ${{ vars.SEMCODE_ROOT }} + GITHUB_REPO: ${{ github.repository }} + GITHUB_REF_NAME: ${{ github.ref_name }} + run: | + # Default service name: full repo name with "/" replaced by "-", so it stays + # unique across orgs without any setup. + service="${SEMCODE_SERVICE:-${GITHUB_REPO//\//-}}" + + body=$(jq -n \ + --arg service "$service" \ + --arg repo "$GITHUB_REPO" \ + --arg ref "$GITHUB_REF_NAME" \ + --arg root "$SEMCODE_ROOT" \ + '{service: $service, github_repo: $repo, github_ref: $ref} + + (if $root != "" then {root: $root} else {} end)') + + curl -sf -N -X POST "$SEMCODE_URL/reindex" \ + -H "Content-Type: application/json" \ + -d "$body" | tee /dev/stderr | while IFS= read -r line; do + echo "$line" | grep -q '"type":\s*"error"' && { echo "semcode reported an error: $line" >&2; exit 1; } + done diff --git a/server/config.py b/server/config.py index 16966d7..c4b0d2b 100644 --- a/server/config.py +++ b/server/config.py @@ -114,10 +114,20 @@ def _apply_default_embedding_max_chars(self) -> Settings: github_token: str = Field(default="", alias="GITHUB_TOKEN") def load_services(self) -> list[ServiceConfig]: - with open(self.config_path) as f: - data = yaml.safe_load(f) + try: + with open(self.config_path) as f: + data = yaml.safe_load(f) + except FileNotFoundError: + return [] + except IsADirectoryError as exc: + raise RuntimeError( + f"CONFIG_PATH ({self.config_path!r}) is a directory, not a file. This " + "usually means a Docker bind mount pointed at a config.yaml that doesn't " + "exist on the host, which makes Docker create an empty directory there " + "instead. Either create that file, or remove the mount." + ) from exc services = [] - for svc in data.get("services", []): + for svc in (data or {}).get("services", []): services.append( ServiceConfig( name=svc["name"], diff --git a/server/indexer/git_history.py b/server/indexer/git_history.py index fd3fcfb..ca0a0b5 100644 --- a/server/indexer/git_history.py +++ b/server/indexer/git_history.py @@ -17,8 +17,9 @@ list_commits, ) from server.indexer.pipeline import ProgressEvent -from server.state import get_reindex_lock +from server.state import get_reindex_lock, get_service_registry from server.store.commit_store import CommitStore +from server.store.service_registry import ServiceRegistry, load_effective_services logger = logging.getLogger(__name__) @@ -66,9 +67,12 @@ def _commit_to_payload(commit: GitHubCommit, service_name: str) -> dict[str, Any class GitHistoryPipeline: - def __init__(self, store: CommitStore) -> None: + def __init__( + self, store: CommitStore, registry: ServiceRegistry | None = None + ) -> None: self._store = store self._embedder: EmbeddingProvider = get_embedding_provider() + self._registry = registry or get_service_registry() async def index_service( self, @@ -77,7 +81,7 @@ async def index_service( progress_callback: Callable[[ProgressEvent], Awaitable[None]] | None = None, ) -> dict[str, int]: await self._store.ensure_collection() - services = settings.load_services() + services = await load_effective_services(self._registry) svc = next((s for s in services if s.name == service_name), None) if svc is None: return {"error": 1, "new": 0, "skipped": 0} @@ -205,7 +209,7 @@ async def index_all( force: bool = False, progress_callback: Callable[[ProgressEvent], Awaitable[None]] | None = None, ) -> dict[str, Any]: - services = settings.load_services() + services = await load_effective_services(self._registry) await self._store.ensure_collection() await prune_orphaned_services( self._store, {s.name for s in services}, label="git commits" diff --git a/server/indexer/pipeline.py b/server/indexer/pipeline.py index a25700f..7670458 100644 --- a/server/indexer/pipeline.py +++ b/server/indexer/pipeline.py @@ -18,8 +18,9 @@ from server.indexer.github_source import fetch_blob_content, list_github_files from server.parser.base import CodeSymbol, ParseError from server.parser.registry import parse_file -from server.state import get_reindex_lock +from server.state import get_reindex_lock, get_service_registry from server.store.qdrant import QdrantStore +from server.store.service_registry import ServiceRegistry, load_effective_services logger = logging.getLogger(__name__) @@ -155,10 +156,13 @@ def _symbol_to_payload( class IndexPipeline: - def __init__(self, store: QdrantStore) -> None: + def __init__( + self, store: QdrantStore, registry: ServiceRegistry | None = None + ) -> None: self._store = store self._embedder: EmbeddingProvider = get_embedding_provider() self._sparse_embedder: BM25SparseProvider = get_sparse_embedding_provider() + self._registry = registry or get_service_registry() async def index_service( self, @@ -167,7 +171,7 @@ async def index_service( progress_callback: Callable[[ProgressEvent], Awaitable[None]] | None = None, ) -> dict[str, int]: await self._store.ensure_collection() - services = settings.load_services() + services = await load_effective_services(self._registry) svc = next((s for s in services if s.name == service_name), None) if svc is None: return {"error": 1, "files": 0, "chunks": 0} @@ -303,7 +307,7 @@ async def index_all( force: bool = False, progress_callback: Callable[[ProgressEvent], Awaitable[None]] | None = None, ) -> dict[str, Any]: - services = settings.load_services() + services = await load_effective_services(self._registry) await self._store.ensure_collection() await prune_orphaned_services( self._store, {s.name for s in services}, label="code symbols" diff --git a/server/main.py b/server/main.py index 696fcbe..1c73807 100644 --- a/server/main.py +++ b/server/main.py @@ -13,13 +13,16 @@ from server.embeddings.bm25 import BM25SparseProvider, close_sparse_embedding_provider from server.state import ( get_commit_store, + get_service_registry, get_store, set_commit_store, + set_service_registry, set_sparse_provider, set_store, ) from server.store.commit_store import CommitStore from server.store.qdrant import QdrantStore +from server.store.service_registry import ServiceRegistry logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -46,6 +49,8 @@ async def lifespan(_: FastMCP) -> AsyncIterator[None]: sparse_provider = BM25SparseProvider() set_sparse_provider(sparse_provider) + set_service_registry(ServiceRegistry()) + logger.info( "Qdrant collections ready. Use `reindex` / `index_history` MCP tools to index services." ) @@ -53,6 +58,7 @@ async def lifespan(_: FastMCP) -> AsyncIterator[None]: try: await get_store().close() await get_commit_store().close() + await get_service_registry().close() except RuntimeError: pass await close_embedding_provider() diff --git a/server/routes/reindex.py b/server/routes/reindex.py index dfc2101..6366371 100644 --- a/server/routes/reindex.py +++ b/server/routes/reindex.py @@ -7,11 +7,12 @@ from mcp.server.fastmcp import FastMCP from starlette.requests import Request -from starlette.responses import StreamingResponse +from starlette.responses import JSONResponse, StreamingResponse +from server.config import ServiceConfig from server.indexer.git_history import GitHistoryPipeline from server.indexer.pipeline import IndexPipeline, ProgressEvent -from server.state import get_commit_store, get_store +from server.state import get_commit_store, get_service_registry, get_store logger = logging.getLogger(__name__) @@ -28,8 +29,16 @@ async def reindex(request: Request) -> StreamingResponse: {"type": "done", "result": {"files": int, "chunks": int, "skipped": int}} Body (optional JSON): - service: str — service name; omit to reindex all - force: bool — re-embed unchanged files (default false) + service: str — service name; omit to reindex all + force: bool — re-embed unchanged files (default false) + github_repo: str — "owner/repo"; if present, registers `service` as this repo + (no config.yaml entry needed) before indexing it. Requires `service`. + github_ref: str — branch, tag, or commit SHA (default "main"); only used with github_repo + root: str — optional subdirectory to scope indexing to; only used with github_repo + exclude: list[str] — optional glob patterns to skip; only used with github_repo + + A service name already defined in config.yaml always wins over a same-named + registration made this way. """ body: dict = {} if request.headers.get("content-type", "").startswith("application/json"): @@ -39,6 +48,30 @@ async def reindex(request: Request) -> StreamingResponse: service: str | None = body.get("service") force: bool = bool(body.get("force", False)) + github_repo: str | None = body.get("github_repo") + + if github_repo: + if not service: + return JSONResponse( + {"error": "'service' is required when 'github_repo' is provided"}, + status_code=400, + ) + github_ref = body.get("github_ref", "main") + await get_service_registry().upsert( + ServiceConfig( + name=service, + github_repo=github_repo, + github_ref=github_ref, + root=body.get("root"), + exclude=body.get("exclude", []), + ) + ) + logger.info( + "Registered ad-hoc service %s -> %s@%s", + service, + github_repo, + github_ref, + ) pipeline = IndexPipeline(get_store()) logger.info( diff --git a/server/state.py b/server/state.py index 2f95af4..f7335cc 100644 --- a/server/state.py +++ b/server/state.py @@ -6,10 +6,12 @@ from server.embeddings.bm25 import BM25SparseProvider from server.store.commit_store import CommitStore from server.store.qdrant import QdrantStore +from server.store.service_registry import ServiceRegistry _store: QdrantStore | None = None _commit_store: CommitStore | None = None _sparse_provider: BM25SparseProvider | None = None +_service_registry: ServiceRegistry | None = None _reindex_locks: defaultdict[str, asyncio.Lock] = defaultdict(asyncio.Lock) @@ -46,6 +48,17 @@ def set_sparse_provider(provider: BM25SparseProvider) -> None: _sparse_provider = provider +def get_service_registry() -> ServiceRegistry: + if _service_registry is None: + raise RuntimeError("Service registry not initialized") + return _service_registry + + +def set_service_registry(registry: ServiceRegistry) -> None: + global _service_registry + _service_registry = registry + + def get_reindex_lock(key: str) -> asyncio.Lock: """Returns the lock guarding reindexing for `key` (e.g. "code:my-service"), so concurrent reindex requests for the same service serialize instead of diff --git a/server/store/service_registry.py b/server/store/service_registry.py new file mode 100644 index 0000000..f92b997 --- /dev/null +++ b/server/store/service_registry.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import uuid +from datetime import UTC, datetime + +from qdrant_client import AsyncQdrantClient +from qdrant_client.models import Distance, PointStruct, VectorParams + +from server.config import ServiceConfig, settings + +_COLLECTION = "service_registry" + + +def _service_point_id(name: str) -> str: + return str(uuid.uuid5(uuid.NAMESPACE_URL, f"service_registry:{name}")) + + +def _to_service_config(payload: dict) -> ServiceConfig: + return ServiceConfig( + name=payload["name"], + github_repo=payload["github_repo"], + github_ref=payload.get("github_ref", "main"), + root=payload.get("root"), + exclude=payload.get("exclude") or [], + ) + + +class ServiceRegistry: + """Persists ad-hoc service definitions registered inline through `POST /reindex` + (`github_repo` in the body), so a repo can be indexed without a `config.yaml` entry. + + Registered services behave like `config.yaml` ones: they're picked up by + `load_effective_services`, so they show up in `list_indexed_services`/`index_stats` + and survive `index_all`'s orphan cleanup. + """ + + def __init__(self) -> None: + self._client = AsyncQdrantClient(url=settings.qdrant_url) + + async def ensure_collection(self) -> None: + if not await self._client.collection_exists(_COLLECTION): + await self._client.create_collection( + collection_name=_COLLECTION, + vectors_config=VectorParams(size=1, distance=Distance.COSINE), + ) + + async def upsert(self, svc: ServiceConfig) -> None: + await self.ensure_collection() + await self._client.upsert( + collection_name=_COLLECTION, + points=[ + PointStruct( + id=_service_point_id(svc.name), + vector=[0.0], + payload={ + "name": svc.name, + "github_repo": svc.github_repo, + "github_ref": svc.github_ref, + "root": svc.root, + "exclude": svc.exclude, + "registered_at": datetime.now(UTC).isoformat(), + }, + ) + ], + ) + + async def list_all(self) -> list[ServiceConfig]: + if not await self._client.collection_exists(_COLLECTION): + return [] + + services: list[ServiceConfig] = [] + offset = None + while True: + results, offset = await self._client.scroll( + collection_name=_COLLECTION, + limit=1000, + offset=offset, + with_payload=True, + with_vectors=False, + ) + services.extend(_to_service_config(point.payload) for point in results) + if offset is None: + break + return services + + async def close(self) -> None: + await self._client.close() + + +async def load_effective_services(registry: ServiceRegistry) -> list[ServiceConfig]: + """Merge config.yaml services with dynamically-registered ones. + + config.yaml wins on name collisions — it's the curated/authoritative source, so an + inline registration can never silently repoint an existing configured service. + """ + config_services = settings.load_services() + registry_services = await registry.list_all() + + by_name = {s.name: s for s in registry_services} + by_name.update({s.name: s for s in config_services}) + return list(by_name.values()) diff --git a/server/tools/search.py b/server/tools/search.py index abeb954..46a389a 100644 --- a/server/tools/search.py +++ b/server/tools/search.py @@ -8,7 +8,8 @@ from server.config import settings from server.embeddings import get_embedding_provider from server.indexer.github_source import fetch_file_content -from server.state import get_sparse_provider, get_store +from server.state import get_service_registry, get_sparse_provider, get_store +from server.store.service_registry import load_effective_services logger = logging.getLogger(__name__) @@ -194,10 +195,13 @@ async def get_code_context( return f"File not found in index: `{file_path}`" service_name = file_info["service"] - services = settings.load_services() + services = await load_effective_services(get_service_registry()) svc = next((s for s in services if s.name == service_name), None) if not svc: - return f"Service `{service_name}` is no longer in config.yaml." + return ( + f"Service `{service_name}` is no longer in config.yaml or the " + "dynamic service registry." + ) # Strip the "{service_name}/" prefix, then restore the root prefix so the # path matches the actual location in the GitHub repo tree. diff --git a/server/tools/stats.py b/server/tools/stats.py index 3cd9e87..b65928e 100644 --- a/server/tools/stats.py +++ b/server/tools/stats.py @@ -5,7 +5,7 @@ from mcp.server.fastmcp import FastMCP from server.config import settings -from server.state import get_store +from server.state import get_service_registry, get_store logger = logging.getLogger(__name__) @@ -44,6 +44,7 @@ async def index_stats() -> str: return f"Could not reach Qdrant: {exc}" configured = settings.load_services() + registered = await get_service_registry().list_all() lines = [ "## Code Search Index Stats\n", @@ -52,11 +53,16 @@ async def index_stats() -> str: f"**Vector dimensions**: {info['vector_size']}", f"**Status**: {info['status']}", "", - f"**Configured services** ({len(configured)}):", + f"**From config.yaml** ({len(configured)}):", ] for svc in configured: lines.append(f"- `{svc.name}` — `{svc.github_repo}@{svc.github_ref}`") + lines.append("") + lines.append(f"**Registered dynamically via API** ({len(registered)}):") + for svc in registered: + lines.append(f"- `{svc.name}` — `{svc.github_repo}@{svc.github_ref}`") + lines.append("") lines.append(f"**Embeddings provider**: {settings.embeddings_provider}") provider_endpoint = { diff --git a/tests/test_config.py b/tests/test_config.py index daba632..81a7455 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,7 @@ from __future__ import annotations +import pytest + from server.config import Settings @@ -24,3 +26,36 @@ def test_embedding_max_chars_explicit_override_wins() -> None: _env_file=None, EMBEDDINGS_PROVIDER="voyage", EMBEDDING_MAX_CHARS=12345 ) assert settings.embedding_max_chars == 12345 + + +def test_load_services_returns_empty_when_config_file_missing() -> None: + settings = Settings(_env_file=None, CONFIG_PATH="/nonexistent/config.yaml") + assert settings.load_services() == [] + + +def test_load_services_returns_empty_for_empty_file(tmp_path) -> None: + config_path = tmp_path / "config.yaml" + config_path.write_text("") + settings = Settings(_env_file=None, CONFIG_PATH=str(config_path)) + assert settings.load_services() == [] + + +def test_load_services_parses_services_from_file(tmp_path) -> None: + config_path = tmp_path / "config.yaml" + config_path.write_text("services:\n - name: svc\n github_repo: org/svc\n") + settings = Settings(_env_file=None, CONFIG_PATH=str(config_path)) + services = settings.load_services() + assert len(services) == 1 + assert services[0].name == "svc" + assert services[0].github_repo == "org/svc" + + +def test_load_services_raises_clear_error_when_config_path_is_a_directory( + tmp_path, +) -> None: + config_path = tmp_path / "config.yaml" + config_path.mkdir() + settings = Settings(_env_file=None, CONFIG_PATH=str(config_path)) + + with pytest.raises(RuntimeError, match="is a directory, not a file"): + settings.load_services() diff --git a/tests/test_git_history.py b/tests/test_git_history.py index 6a64835..5aa36a7 100644 --- a/tests/test_git_history.py +++ b/tests/test_git_history.py @@ -32,12 +32,18 @@ def _file(filename: str = "src/Foo.java", patch: str | None = None) -> CommitFil ) +def _empty_registry() -> AsyncMock: + registry = AsyncMock() + registry.list_all = AsyncMock(return_value=[]) + return registry + + async def test_index_all_prunes_orphaned_services_before_indexing() -> None: store = AsyncMock() store.ensure_collection = AsyncMock() store.get_indexed_services = AsyncMock(return_value=["kept", "renamed-away"]) store.delete_by_service = AsyncMock() - pipeline = GitHistoryPipeline(store) + pipeline = GitHistoryPipeline(store, registry=_empty_registry()) pipeline.index_service = AsyncMock(return_value={"commits": 0}) with patch.object( @@ -58,7 +64,7 @@ async def test_index_all_skips_prune_when_no_services_configured() -> None: store.ensure_collection = AsyncMock() store.get_indexed_services = AsyncMock(return_value=["kept"]) store.delete_by_service = AsyncMock() - pipeline = GitHistoryPipeline(store) + pipeline = GitHistoryPipeline(store, registry=_empty_registry()) with patch.object( type(git_history_module.settings), "load_services", return_value=[] @@ -68,6 +74,57 @@ async def test_index_all_skips_prune_when_no_services_configured() -> None: store.delete_by_service.assert_not_awaited() +async def test_index_all_includes_registry_only_service() -> None: + store = AsyncMock() + store.ensure_collection = AsyncMock() + store.get_indexed_services = AsyncMock(return_value=["adhoc"]) + store.delete_by_service = AsyncMock() + registry = AsyncMock() + registry.list_all = AsyncMock( + return_value=[ServiceConfig(name="adhoc", github_repo="org/adhoc", exclude=[])] + ) + pipeline = GitHistoryPipeline(store, registry=registry) + pipeline.index_service = AsyncMock(return_value={"commits": 0}) + + with patch.object( + type(git_history_module.settings), "load_services", return_value=[] + ): + await pipeline.index_all() + + store.delete_by_service.assert_not_awaited() + pipeline.index_service.assert_awaited_once_with( + "adhoc", force=False, progress_callback=None + ) + + +async def test_config_yaml_wins_over_registry_on_name_collision() -> None: + store = AsyncMock() + store.ensure_collection = AsyncMock() + store.get_indexed_services = AsyncMock(return_value=["shared"]) + store.delete_by_service = AsyncMock() + registry = AsyncMock() + registry.list_all = AsyncMock( + return_value=[ + ServiceConfig(name="shared", github_repo="attacker/repo", exclude=[]) + ] + ) + pipeline = GitHistoryPipeline(store, registry=registry) + pipeline.index_service = AsyncMock(return_value={"commits": 0}) + + with patch.object( + type(git_history_module.settings), + "load_services", + return_value=[ + ServiceConfig(name="shared", github_repo="org/legit", exclude=[]) + ], + ): + await pipeline.index_all() + + pipeline.index_service.assert_awaited_once_with( + "shared", force=False, progress_callback=None + ) + + def test_embedding_text_contains_service_and_author() -> None: text = _build_embedding_text(_commit(), "auth-server") assert "auth-server" in text diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 6e0c5c3..49d2721 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -23,11 +23,15 @@ async def embed_batch(self, texts: list[str]) -> list[list[float]]: return [[0.0]] * len(texts) -def _make_pipeline(store) -> IndexPipeline: +def _make_pipeline(store, registry=None) -> IndexPipeline: pipeline = IndexPipeline.__new__(IndexPipeline) pipeline._store = store pipeline._embedder = _StubEmbedder() pipeline._sparse_embedder = _StubEmbedder() + if registry is None: + registry = AsyncMock() + registry.list_all = AsyncMock(return_value=[]) + pipeline._registry = registry return pipeline @@ -93,6 +97,48 @@ async def test_index_all_skips_prune_when_no_services_configured() -> None: store.delete_by_service.assert_not_awaited() +async def test_index_all_includes_registry_only_service() -> None: + store = AsyncMock() + store.ensure_collection = AsyncMock() + store.get_indexed_services = AsyncMock(return_value=["adhoc"]) + store.delete_by_service = AsyncMock() + registry = AsyncMock() + registry.list_all = AsyncMock( + return_value=[ServiceConfig(name="adhoc", github_repo="org/adhoc", exclude=[])] + ) + pipeline = _make_pipeline(store, registry=registry) + pipeline.index_service = AsyncMock( + return_value={"files": 1, "chunks": 1, "skipped": 0} + ) + + with patch.object(type(pipeline_module.settings), "load_services", return_value=[]): + await pipeline.index_all() + + store.delete_by_service.assert_not_awaited() + pipeline.index_service.assert_awaited_once_with( + "adhoc", force=False, progress_callback=None + ) + + +async def test_index_service_resolves_registry_only_service() -> None: + store = AsyncMock() + store.ensure_collection = AsyncMock() + store.get_indexed_file_hashes = AsyncMock(return_value={}) + registry = AsyncMock() + registry.list_all = AsyncMock( + return_value=[ServiceConfig(name="adhoc", github_repo="org/adhoc", exclude=[])] + ) + pipeline = _make_pipeline(store, registry=registry) + + with ( + patch.object(type(pipeline_module.settings), "load_services", return_value=[]), + patch.object(pipeline_module, "list_github_files", AsyncMock(return_value=[])), + ): + result = await pipeline.index_service("adhoc") + + assert result == {"files": 0, "chunks": 0, "skipped": 0} + + def test_python_triple_double_quote_stripped() -> None: text = _build_embedding_text(_sym('"""Hello world"""'), "svc") assert "Hello world" in text diff --git a/tests/test_reindex_route.py b/tests/test_reindex_route.py index 043cd6c..08a04c3 100644 --- a/tests/test_reindex_route.py +++ b/tests/test_reindex_route.py @@ -172,6 +172,101 @@ async def test_reindex_unknown_service_returns_pipeline_result( assert _done_result(response.text) == {"error": 1} +async def test_reindex_with_github_repo_registers_then_indexes( + client, mock_pipeline +) -> None: + mock_registry = AsyncMock() + store_patch = patch("server.routes.reindex.get_store", return_value=MagicMock()) + registry_patch = patch( + "server.routes.reindex.get_service_registry", return_value=mock_registry + ) + pipeline_patch = patch( + "server.routes.reindex.IndexPipeline", return_value=mock_pipeline + ) + with store_patch, registry_patch, pipeline_patch: + response = await client.post( + "/reindex", + json={ + "service": "adhoc", + "github_repo": "org/adhoc", + "github_ref": "develop", + "root": "src", + "exclude": ["**/vendor/**"], + }, + ) + + assert response.status_code == 200 + mock_registry.upsert.assert_awaited_once() + (svc,), _ = mock_registry.upsert.call_args + assert svc.name == "adhoc" + assert svc.github_repo == "org/adhoc" + assert svc.github_ref == "develop" + assert svc.root == "src" + assert svc.exclude == ["**/vendor/**"] + mock_pipeline.index_service.assert_called_once_with( + "adhoc", force=False, progress_callback=ANY + ) + + +async def test_reindex_with_github_repo_defaults_ref_to_main( + client, mock_pipeline +) -> None: + mock_registry = AsyncMock() + store_patch = patch("server.routes.reindex.get_store", return_value=MagicMock()) + registry_patch = patch( + "server.routes.reindex.get_service_registry", return_value=mock_registry + ) + pipeline_patch = patch( + "server.routes.reindex.IndexPipeline", return_value=mock_pipeline + ) + with store_patch, registry_patch, pipeline_patch: + response = await client.post( + "/reindex", json={"service": "adhoc", "github_repo": "org/adhoc"} + ) + + assert response.status_code == 200 + (svc,), _ = mock_registry.upsert.call_args + assert svc.github_ref == "main" + + +async def test_reindex_with_github_repo_without_service_returns_400( + client, mock_pipeline +) -> None: + mock_registry = AsyncMock() + store_patch = patch("server.routes.reindex.get_store", return_value=MagicMock()) + registry_patch = patch( + "server.routes.reindex.get_service_registry", return_value=mock_registry + ) + pipeline_patch = patch( + "server.routes.reindex.IndexPipeline", return_value=mock_pipeline + ) + with store_patch, registry_patch, pipeline_patch: + response = await client.post("/reindex", json={"github_repo": "org/adhoc"}) + + assert response.status_code == 400 + mock_registry.upsert.assert_not_awaited() + mock_pipeline.index_service.assert_not_called() + mock_pipeline.index_all.assert_not_called() + + +async def test_reindex_without_github_repo_never_touches_registry( + client, mock_pipeline +) -> None: + mock_registry = AsyncMock() + store_patch = patch("server.routes.reindex.get_store", return_value=MagicMock()) + registry_patch = patch( + "server.routes.reindex.get_service_registry", return_value=mock_registry + ) + pipeline_patch = patch( + "server.routes.reindex.IndexPipeline", return_value=mock_pipeline + ) + with store_patch, registry_patch, pipeline_patch: + response = await client.post("/reindex", json={"service": "svc-a"}) + + assert response.status_code == 200 + mock_registry.upsert.assert_not_awaited() + + HISTORY_SERVICE_RESULT = {"new": 25, "skipped": 0} HISTORY_ALL_RESULT = { "svc-a": HISTORY_SERVICE_RESULT, diff --git a/tests/test_service_registry.py b/tests/test_service_registry.py new file mode 100644 index 0000000..74ed8d6 --- /dev/null +++ b/tests/test_service_registry.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +from server.config import ServiceConfig, settings +from server.store.service_registry import ServiceRegistry, load_effective_services + + +def _make_registry() -> ServiceRegistry: + registry = ServiceRegistry.__new__(ServiceRegistry) + registry._client = MagicMock() + return registry + + +def _point(payload: dict) -> SimpleNamespace: + return SimpleNamespace(payload=payload) + + +async def test_list_all_returns_empty_when_collection_missing() -> None: + registry = _make_registry() + registry._client.collection_exists = AsyncMock(return_value=False) + + services = await registry.list_all() + + assert services == [] + + +async def test_upsert_creates_collection_then_writes_point() -> None: + registry = _make_registry() + registry._client.collection_exists = AsyncMock(return_value=False) + registry._client.create_collection = AsyncMock() + registry._client.upsert = AsyncMock() + + svc = ServiceConfig( + name="adhoc", github_repo="org/adhoc", github_ref="main", root=None, exclude=[] + ) + await registry.upsert(svc) + + registry._client.create_collection.assert_awaited_once() + registry._client.upsert.assert_awaited_once() + _, kwargs = registry._client.upsert.call_args + point = kwargs["points"][0] + assert point.payload["name"] == "adhoc" + assert point.payload["github_repo"] == "org/adhoc" + assert point.payload["github_ref"] == "main" + + +async def test_list_all_paginates_and_converts_payloads() -> None: + registry = _make_registry() + registry._client.collection_exists = AsyncMock(return_value=True) + + page1 = [_point({"name": "a", "github_repo": "org/a", "github_ref": "main"})] + page2 = [ + _point( + { + "name": "b", + "github_repo": "org/b", + "github_ref": "dev", + "root": "src", + "exclude": ["**/vendor/**"], + } + ) + ] + registry._client.scroll = AsyncMock(side_effect=[(page1, "cursor"), (page2, None)]) + + services = await registry.list_all() + + assert [s.name for s in services] == ["a", "b"] + assert services[1].root == "src" + assert services[1].exclude == ["**/vendor/**"] + + +async def test_load_effective_services_merges_config_and_registry() -> None: + registry = AsyncMock() + registry.list_all = AsyncMock( + return_value=[ServiceConfig(name="adhoc", github_repo="org/adhoc", exclude=[])] + ) + + with patch.object( + type(settings), + "load_services", + return_value=[ + ServiceConfig(name="curated", github_repo="org/curated", exclude=[]) + ], + ): + services = await load_effective_services(registry) + + names = {s.name for s in services} + assert names == {"adhoc", "curated"} + + +async def test_load_effective_services_config_yaml_wins_on_name_collision() -> None: + registry = AsyncMock() + registry.list_all = AsyncMock( + return_value=[ + ServiceConfig(name="shared", github_repo="attacker/repo", exclude=[]) + ] + ) + + with patch.object( + type(settings), + "load_services", + return_value=[ + ServiceConfig(name="shared", github_repo="org/legit", exclude=[]) + ], + ): + services = await load_effective_services(registry) + + assert len(services) == 1 + assert services[0].github_repo == "org/legit" diff --git a/tests/tools/test_search.py b/tests/tools/test_search.py index e43fae0..2d7af60 100644 --- a/tests/tools/test_search.py +++ b/tests/tools/test_search.py @@ -254,12 +254,19 @@ async def test_get_code_context_service_removed_from_config() -> None: with ( patch("server.tools.search.get_store", return_value=store), - patch("server.tools.search.settings") as mock_settings, + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[], + ), ): - mock_settings.load_services.return_value = [] result = await get_code_context("orders/Order.java") - assert result == "Service `orders` is no longer in config.yaml." + assert ( + result == "Service `orders` is no longer in config.yaml or the " + "dynamic service registry." + ) async def test_get_code_context_reconstructs_path_with_root_prefix() -> None: @@ -276,12 +283,17 @@ async def test_get_code_context_reconstructs_path_with_root_prefix() -> None: with ( patch("server.tools.search.get_store", return_value=store), + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[svc], + ), patch("server.tools.search.settings") as mock_settings, patch( "server.tools.search.fetch_file_content", new_callable=AsyncMock ) as mock_fetch, ): - mock_settings.load_services.return_value = [svc] mock_settings.github_token = "token" mock_fetch.return_value = b"class Order {}" @@ -300,12 +312,17 @@ async def test_get_code_context_returns_full_file_without_symbol_name() -> None: with ( patch("server.tools.search.get_store", return_value=store), + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[svc], + ), patch("server.tools.search.settings") as mock_settings, patch( "server.tools.search.fetch_file_content", new_callable=AsyncMock ) as mock_fetch, ): - mock_settings.load_services.return_value = [svc] mock_settings.github_token = "token" mock_fetch.return_value = b"class Order {}" @@ -322,12 +339,17 @@ async def test_get_code_context_github_fetch_error_is_reported() -> None: with ( patch("server.tools.search.get_store", return_value=store), + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[svc], + ), patch("server.tools.search.settings") as mock_settings, patch( "server.tools.search.fetch_file_content", new_callable=AsyncMock ) as mock_fetch, ): - mock_settings.load_services.return_value = [svc] mock_settings.github_token = "token" mock_fetch.side_effect = httpx.HTTPError("boom") @@ -356,12 +378,17 @@ async def test_get_code_context_uses_qdrant_line_numbers_for_symbol() -> None: with ( patch("server.tools.search.get_store", return_value=store), + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[svc], + ), patch("server.tools.search.settings") as mock_settings, patch( "server.tools.search.fetch_file_content", new_callable=AsyncMock ) as mock_fetch, ): - mock_settings.load_services.return_value = [svc] mock_settings.github_token = "token" mock_fetch.return_value = content @@ -383,12 +410,17 @@ async def test_get_code_context_falls_back_to_text_search_when_symbol_not_in_ind with ( patch("server.tools.search.get_store", return_value=store), + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[svc], + ), patch("server.tools.search.settings") as mock_settings, patch( "server.tools.search.fetch_file_content", new_callable=AsyncMock ) as mock_fetch, ): - mock_settings.load_services.return_value = [svc] mock_settings.github_token = "token" mock_fetch.return_value = content @@ -406,12 +438,17 @@ async def test_get_code_context_symbol_not_found_anywhere() -> None: with ( patch("server.tools.search.get_store", return_value=store), + patch("server.tools.search.get_service_registry", return_value=AsyncMock()), + patch( + "server.tools.search.load_effective_services", + new_callable=AsyncMock, + return_value=[svc], + ), patch("server.tools.search.settings") as mock_settings, patch( "server.tools.search.fetch_file_content", new_callable=AsyncMock ) as mock_fetch, ): - mock_settings.load_services.return_value = [svc] mock_settings.github_token = "token" mock_fetch.return_value = b"nothing relevant here" diff --git a/tests/tools/test_stats.py b/tests/tools/test_stats.py index 1dc4223..5d77eca 100644 --- a/tests/tools/test_stats.py +++ b/tests/tools/test_stats.py @@ -75,9 +75,12 @@ async def test_index_stats_includes_provider_endpoint() -> None: "status": "green", } svc = ServiceConfig(name="orders", github_repo="org/orders", exclude=[]) + registry = AsyncMock() + registry.list_all.return_value = [] with ( patch("server.tools.stats.get_store", return_value=store), + patch("server.tools.stats.get_service_registry", return_value=registry), patch("server.tools.stats.settings") as mock_settings, ): mock_settings.load_services.return_value = [svc] @@ -87,3 +90,30 @@ async def test_index_stats_includes_provider_endpoint() -> None: assert "**Embeddings provider**: voyage" in result assert "https://api.voyageai.com/v1/embeddings" in result assert "`orders` — `org/orders@main`" in result + + +async def test_index_stats_lists_dynamically_registered_services() -> None: + index_stats = _tool("index_stats") + store = AsyncMock() + store.collection_info.return_value = { + "collection": "code_symbols", + "total_vectors": 100, + "vector_size": 768, + "status": "green", + } + registry = AsyncMock() + registry.list_all.return_value = [ + ServiceConfig(name="adhoc", github_repo="org/adhoc", exclude=[]) + ] + + with ( + patch("server.tools.stats.get_store", return_value=store), + patch("server.tools.stats.get_service_registry", return_value=registry), + patch("server.tools.stats.settings") as mock_settings, + ): + mock_settings.load_services.return_value = [] + mock_settings.embeddings_provider = "voyage" + result = await index_stats() + + assert "**Registered dynamically via API** (1)" in result + assert "`adhoc` — `org/adhoc@main`" in result