Add dynamic service registration + GitHub Actions reindex example

Dockerfile

@@ -8,9 +8,11 @@ WORKDIR /app
 COPY pyproject.toml uv.lock ./
 RUN uv sync --frozen --no-dev --no-editable
 
-# Copy server source and config
+# Copy server source. config.yaml is intentionally not baked in — mount it at
+# runtime (see docker-compose.yaml) if you use it; without one, load_services()
+# treats "no config.yaml" as zero configured services, and self-registered (ad-hoc)
+# services still work.
 COPY server/ ./server/
-COPY config.yaml ./
 
 FROM python:3.12-slim AS runtime
 

Makefile

@@ -1,8 +1,10 @@
 QDRANT_URL      := http://localhost:6333
 SEMCODE_URL     := http://localhost:8090
+COMPOSE_WITH_CONFIG := -f docker-compose.yaml -f docker-compose.config-yaml.yml
 
 .PHONY: qdrant-clean qdrant-dashboard index-code index-history docker-build \
-        docker-build-restart docker-build-restart-jina docker-up docker-up-jina docker-logs docker-logs-semcode
+        docker-build-restart docker-build-restart-jina docker-up docker-up-jina docker-logs docker-logs-semcode \
+        docker-build-restart-with-config docker-build-restart-jina-with-config docker-up-with-config docker-up-jina-with-config
 
 qdrant-clean:
 	curl -sf -X DELETE $(QDRANT_URL)/collections/code_symbols && \
@@ -37,6 +39,20 @@ docker-up:
 docker-up-jina:
 	docker compose --profile jina up -d
 
+# "-with-config" variants also mount config.yaml (see docker-compose.config-yaml.yml),
+# for curated/static services alongside — or instead of — ad-hoc registration.
+docker-build-restart-with-config:
+	docker compose $(COMPOSE_WITH_CONFIG) down && docker compose $(COMPOSE_WITH_CONFIG) up --build -d
+
+docker-build-restart-jina-with-config:
+	docker compose $(COMPOSE_WITH_CONFIG) --profile jina down && docker compose $(COMPOSE_WITH_CONFIG) --profile jina up --build -d
+
+docker-up-with-config:
+	docker compose $(COMPOSE_WITH_CONFIG) up -d
+
+docker-up-jina-with-config:
+	docker compose $(COMPOSE_WITH_CONFIG) --profile jina up -d
+
 docker-logs:
 	docker compose logs -f
 

README.md

@@ -72,7 +72,8 @@ uv sync
 # (a fine-grained PAT with Contents: read on the target repos is sufficient)
 cp .env.example .env
 
-# Copy services config, then list the repositories you want indexed
+# Optional — only if you want curated/static services (see below for the alternative):
+# copy the services config, then list the repositories you want indexed
 cp config.example.yaml config.yaml
 ```
 
@@ -93,6 +94,29 @@ The indexer automatically discovers and indexes all files with recognised extens
 subdirectory within a shared repo, and `exclude` to skip paths you don't want indexed (tests, build artifacts, generated
 code, etc.).
 
+**Scaling beyond a handful of repos:** `config.yaml` is a curated, static list — great for a small number of services,
+but indexing hundreds of repos this way means hundreds of hand-maintained entries. As an alternative (or complement),
+`POST /reindex` accepts a repo definition inline and registers it on the fly, no `config.yaml` entry required — see
+[`examples/github-actions/reindex-on-merge.yml`](examples/github-actions/reindex-on-merge.yml) for a drop-in workflow
+that self-registers a repo and reindexes it on every merge. Details in the [HTTP API](#http-api) section below. If a
+name collides between the two, the `config.yaml` entry always wins.
+
+You don't need a `config.yaml` at all to run this way — a missing file is treated as zero configured services, not an
+error. `docker-compose.yaml` reflects this: by default it does **not** mount `config.yaml`, so `make docker-up` /
+`make docker-up-jina` work out of the box for ad-hoc-registration-only setups. If you also want curated services,
+copy `config.example.yaml` to `config.yaml` (above) and use the `-with-config` targets instead, which layer
+`docker-compose.config-yaml.yml` on top to add the mount: `make docker-up-with-config` / `make docker-up-jina-with-config`
+(or `docker compose -f docker-compose.yaml -f docker-compose.config-yaml.yml up -d` directly). Don't hand-edit the
+volume line in `docker-compose.yaml` itself — bind-mounting a `config.yaml` that doesn't exist on the host silently
+creates an empty directory there instead of leaving the path absent, which breaks the server (surfaced as a clear
+error if it happens: `CONFIG_PATH (...) is a directory, not a file`).
+
+**A single `GITHUB_TOKEN` reads every repo you index this way.** For a handful of `config.yaml` entries a
+fine-grained PAT scoped to those repos is fine, but for org-wide self-registration — where any repo can onboard
+itself just by adding the workflow — a PAT would need its repo access list updated out-of-band every time a new repo
+starts using it. A GitHub App installed org-wide (all repos, `Contents: read`) avoids that: new repos are covered
+automatically, with no token maintenance per onboarding.
+
 ## Running
 
 There are two ways to run, depending on whether you want embeddings to come from a local
@@ -114,6 +138,9 @@ make docker-up
 # or: docker-compose up
 ```
 
+Using `config.yaml` for curated services? Use the `-with-config` variant of whichever target above applies
+(`make docker-up-with-config` / `make docker-up-jina-with-config`) — see the [Setup](#setup) section.
+
 > ⚠ The default `EMBEDDINGS_PROVIDER` is `jina`. If you start without `--profile jina` but leave
 > the provider on the default, semcode will boot (Jina is `required: false` in compose) but the
 > first embedding call will fail with a connection error — there's no auto-fallback.
@@ -124,7 +151,7 @@ Services started with health checks and persistent volumes:
 |---------------------------|---------|------------------------------|----------------------------------|------------------------|
 | **Qdrant**                | always  | `6333` (HTTP), `6334` (gRPC) | `qdrant_data`                    | Vector DB              |
 | **Jina Embeddings** (TEI) | `jina`  | `8087`                       | `embeddings_cache`               | Embedding model server |
-| **semcode MCP**           | always  | `8090`                       | mounts `./config.yaml` read-only | MCP + HTTP server      |
+| **semcode MCP**           | always  | `8090`                       | mounts `./config.yaml` read-only with `-with-config` | MCP + HTTP server |
 
 The MCP server starts with empty collections — trigger an initial index by calling the `reindex` MCP tool
 or `POST /reindex` (see below).
@@ -280,13 +307,23 @@ from CI/CD or external schedulers:
 
 | Endpoint                | Body                                       | Description                                  |
 |-------------------------|--------------------------------------------|----------------------------------------------|
-| `POST /reindex`         | `{"service": "<name>"?, "force": <bool>?}` | Reindex one or all services — returns NDJSON |
+| `POST /reindex`         | `{"service": "<name>"?, "force": <bool>?, "github_repo": "<owner/repo>"?, "github_ref": "<ref>"?, "root": "<path>"?, "exclude": [<glob>, ...]?}` | Reindex one or all services — returns NDJSON |
 | `POST /reindex-history` | `{"service": "<name>"?, "force": <bool>?}` | Index git commit history — returns NDJSON    |
 
 All bodies are optional — omit `service` to act on all services, omit `force` for incremental indexing.
 Both endpoints stream **newline-delimited JSON** (one frame per line) so you can consume progress
 in real time from CI/CD pipelines or any other client.
 
+**Registering a repo without `config.yaml`:** if `POST /reindex`'s body includes `github_repo`, the
+`service` name is registered with that repo definition (persisted, so it survives restarts and behaves
+like a `config.yaml` service from then on) before indexing runs — `service` is required in this case.
+`github_ref` defaults to `main`; `root`/`exclude` mirror the same fields in `config.yaml`. A `service`
+name already defined in `config.yaml` always wins over one registered this way. There's no
+authentication on this endpoint — same as the rest of `/reindex` — so put it behind your own network
+boundary before exposing it. See
+[`examples/github-actions/reindex-on-merge.yml`](examples/github-actions/reindex-on-merge.yml) for a
+ready-to-use workflow.
+
 Frame shapes:
 
 ```jsonc

docker-compose.config-yaml.yml

@@ -0,0 +1,8 @@
+# Optional overlay: adds the config.yaml mount for curated/static services.
+# Requires config.yaml to exist (cp config.example.yaml config.yaml first).
+# Use via the `-with-config` Makefile targets, or directly:
+#   docker compose -f docker-compose.yaml -f docker-compose.config-yaml.yml up -d
+services:
+  semcode:
+    volumes:
+      - ./config.yaml:/app/config.yaml:ro

docker-compose.yaml

@@ -50,8 +50,13 @@ services:
       jina-embeddings:
         condition: service_healthy
         required: false
+    # No config.yaml mount here on purpose — this base file works standalone for
+    # ad-hoc-registration-only setups (see examples/github-actions). If you also want
+    # curated services from config.yaml, layer docker-compose.config-yaml.yml on top
+    # (see the `-with-config` Makefile targets) instead of editing this file — bind-mounting
+    # a config.yaml that doesn't exist on the host silently creates an empty directory
+    # there, which breaks the server at startup.
     volumes:
-      - ./config.yaml:/app/config.yaml:ro
       - fastembed_cache:/fastembed_cache
 
 volumes:

docs/README.md

@@ -16,13 +16,16 @@ semcode is an MCP server that provides **hybrid semantic search over code** from
 | [dense-vectors.md](dense-vectors.md) | Dense embedding providers (Jina, Voyage, OpenAI, Ollama), the embedding text strategy, and provider selection |
 | [sparse-vectors.md](sparse-vectors.md) | BM25 sparse embeddings, the code identifier tokenizer, and the sparse vector format |
 | [retrieval-rrf.md](retrieval-rrf.md) | Hybrid search architecture, RRF fusion, name lookup, and the four MCP tool entry points |
-| [configuration.md](configuration.md) | All environment variables, `config.yaml` structure, and startup validation |
+| [configuration.md](configuration.md) | All environment variables, `config.yaml` structure, dynamic service registration, and startup validation |
 
 ---
 
 ## Quick Start
 
-1. **Configure services** — copy `config.example.yaml` to `config.yaml` and add your GitHub repositories. See [configuration.md](configuration.md) for all fields.
+1. **Configure services** — either copy `config.example.yaml` to `config.yaml` and add your GitHub repositories, or
+   skip `config.yaml` entirely and register repos on the fly via `POST /reindex` (see the
+   [GitHub Actions example](../examples/github-actions/reindex-on-merge.yml)) — the two can also be combined. See
+   [configuration.md](configuration.md) for all fields and how the two interact.
 2. **Set environment variables** — copy `.env.example` to `.env` and set at minimum `GITHUB_TOKEN`. The default embedding provider (`jina`) requires a locally running TEI container; for a hosted alternative, set `EMBEDDINGS_PROVIDER=voyage` and `VOYAGE_API_KEY=...`.
 3. **Start Qdrant and the server** — `make docker-up-jina` (local Jina) or `make docker-up-voyage` (Voyage API), then connect your MCP client to `http://localhost:8090`.