diff --git a/README.md b/README.md index 9d8293a9..627a5e87 100644 --- a/README.md +++ b/README.md @@ -1289,9 +1289,15 @@ The crawler reads source files such as: - Grafana unified-alerting provisioning YAML and Loki rule files - Grafana dashboard JSON - Alertmanager config -- OTel Collector config +- OTel Collector config, its prometheus receivers' scrape jobs included (with + or without a `service:` block) - Helm and Kubernetes manifests - Docker Compose files +- A prebuilt exporter's metrics: its `metrics-manifest.json`, a saved + `/metrics` (`.prom`, `.metrics`), or its config read against an exporter + catalogue (`--exporter-catalogue ` adds or replaces entries). + `packc emit --exporter ` writes the manifest from a live endpoint + ([`docs/DOWNSTREAM.md`](docs/DOWNSTREAM.md) §18) It emits a canonical v1.4 pack plus crawler annotations describing what was scanned and what was inferred. A fresh crawl validates (exit 0) and needs no diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 8e34a15c..fe5c4a01 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "6705a45caf5253caf795b0c23b5d1ea8cdbe75d8f91b4d748fec20911265c16f", - "bytes": 110301, + "sha256": "d0387b62b132f273fc622aaa7ca0411c3e90dc907db4bd07e4a98c89f552455e", + "bytes": 133296, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", @@ -395,6 +395,7 @@ "imports": [ "tools/lib/alert-routes.mjs", "tools/lib/backend-products.mjs", + "tools/lib/exporter-metrics.mjs", "tools/lib/l2x.mjs", "tools/lib/mini-yaml.mjs", "tools/lib/promql.mjs", @@ -510,6 +511,42 @@ "npm": [], "changedSinceRelease": false }, + "tools/lib/exporter-catalogue.mjs": { + "sha256": "594c0bb91e68a1a87a598ff57532d388e40f17a0133de6eec64db8e06c82cb9a", + "bytes": 5963, + "exports": [ + "EXPORTER_CATALOGUE" + ], + "imports": [], + "npm": [], + "changedSinceRelease": true, + "releasedExports": null + }, + "tools/lib/exporter-metrics.mjs": { + "sha256": "57be2804bd81a9c9c4666839540d0c22d00486f292a85d54b852226099a88471", + "bytes": 20774, + "exports": [ + "EXPOSITION_TYPES", + "asResolvedCatalogue", + "inferExporterFamilies", + "isExpositionSnapshotPath", + "isMetricsManifestPath", + "isTestOrFixturePath", + "manifestFromExposition", + "matchExporterConfig", + "mayBeExporterConfig", + "parseExposition", + "readMetricsManifest", + "resolveExporterCatalogue", + "safeSourceUrl" + ], + "imports": [ + "tools/lib/exporter-catalogue.mjs" + ], + "npm": [], + "changedSinceRelease": true, + "releasedExports": null + }, "tools/lib/good-when.mjs": { "sha256": "f89b81ef3c6afc30a60453deee4cc659503ec16f720351fcb61991766337e474", "bytes": 2557, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d9c32663..96c133c6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,18 @@ ## Unreleased +### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs +- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `asResolvedCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `e2e/`, `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, `testing/`, `test-data/`, `__snapshots__/`, …), a camelCase test source set (`componentTest/`, `smokeTest/`, `testUtils/`) or Maven's `src/it/`, nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). +- **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. +- **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart, unless the subchart is disabled: its `enabled` toggles merged across the in-scope values files, an overlay winning as with `helm -f`, so an overlay can disable or re-enable it), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess, settled after every file is walked. A scrape job leaves by the exporters of its own file's metrics pipelines that list its receiver, else by those of the declared metrics pipelines in any other file that do (a config split across `--config` files), else by the guess. A collector job that cannot be read is warned about and skipped; its siblings and the rest of the config are read. The guess sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold only when nothing else set it: no declared pipeline and no metrics backend image (those two fill the metrics exporter from whichever file the walk reads first, as before; a guess comes after both). The config's scrape receivers become `pipelines.receivers`, once. A first-of-several guess that stands is warned about. +- **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. +- **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template that carries either, a values file that carries a collector config) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. +- **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. +- No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). +- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value with a `?` or `#` before its last `@` is recorded as nothing: the `@` may be in a query (`?user=ops@corp&api_key=…`) or a `?` in a password, and text cannot tell which. An endpoint or source that leaves nothing past its scheme is left out. A value whose path holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument, a `--from`, `--out` or `--generated-at` value that looks like a URL (`://`, `@`, `?`, or `host:port`). `--from` takes a file (a file whose name holds `@` is read); a URL-like value that is no file is refused. An `--out` it cannot write is refused, not left as a stack trace. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as the keys of its `key=value` parts only (`?token=…`; a bare `?` is named as "a query part"), and a `Location` that is not a URL is named as such. Neither one is followed. + ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. - **`server/test-store.mjs` Concurrency: a parent waits out the lock-holding child for as long as the child takes to commit** (a test fix; no runtime, `tools/lib`, golden or output change — the store's `BUSY_TIMEOUT_MS`, 5000 ms, is unchanged and still pinned on `openStore`). `Concurrency: a migration waits out a child holding BEGIN IMMEDIATE, then applies` failed the `windows (node 22)` job of run 38737678276 with `database is locked` (9.9 s; the same run's write sibling waited 2355 ms of a 700 ms hold). Each of the three Concurrency tests races two clocks: the parent's busy_timeout, 5000 ms from the child's "locked", against the child's hold of 700 ms plus its COMMIT plus however long a loaded runner keeps it off the CPU; the migration's child pays the most, since `closeStore` removed the `-wal` and `openRaw` leaves `synchronous` at FULL, so its commit syncs a new WAL header and then the commit inside the window. When the child's window outlasts the parent's budget, SQLite's busy handler gives up and `BEGIN IMMEDIATE` fails. The lock holder now sets `synchronous=OFF` (its durability is not under test, its lock is: no fsync between "locked" and its exit), and every parent that waits one out — the write, the migration and the repository write — waits up to `LOCK_WAIT_LIVENESS_MS`, 30 s, a liveness bound nothing asserts (the tests assert only that the wait was at least 300 ms, and what was written); a successful wait still ends at the child's COMMIT, about 700 ms. Reproduced on Linux (node 22.22.2) by delaying every `fsync` 1.8 s (`strace -f -e inject=fsync:delay_exit=1800000`) over the three tests, ten runs each: before, the write and the repository write waited 4.2–4.3 s and passed and the migration failed with `database is locked` in every run, as on CI; after, all three passed in every run (the write and the repository write waiting 0.6–0.7 s; the migration's 4.3 s is its own commit's syncs, outside the lock wait). And with the child's hold raised to 6 s, a window past the 5000 ms budget as a stalled child's would be, all three failed with `database is locked` before and pass after. No test added or removed: the suite total stays 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 553c36bb..6d5b333a 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1828,3 +1828,144 @@ no `refusedEnvNames` and binds every name the journey declares, as before. - *`journey-records-principal`* — journey records readable only by the member whose run wrote them (today every member of the org reads them, as dashboards). + +## 18. Metrics from prebuilt exporters (follow-up exporter-metrics) + +A service whose metrics come from a prebuilt exporter — IBM's +mq-metric-samples, a node_exporter, a vendor sidecar — has no client-library +call or PromQL that names them: the exporter's repository is a Dockerfile and +a config file, and the collector that scrapes it lives in another chart. The +crawler (`tools/lib/crawler.mjs`) reads the exporter's metrics from three +places, strongest first, through `tools/lib/exporter-metrics.mjs` (listed, +pure): + +| Source | Files | `origin_kind` | Confidence | +|---|---|---|---| +| a metrics manifest | `metrics-manifest.json`, `*.metrics-manifest.json` | `exporter-manifest` | `declared` | +| a saved exposition | `*.prom`, `*.metrics` (never under a test, fixture or mock folder; an OpenMetrics counter or info named as stored, `_total` / `_info`) | `exporter-snapshot` | `declared` | +| the exporter's config, read against the catalogue | any YAML whose top-level keys match a catalogue entry | `exporter-config` | `inferred` (a candidate) | + +A metric named in several files reads as the strongest: manifest, snapshot, +source code, source fragment, recording-rule output, PromQL reference, and +an exporter config last. Every exporter the crawl learned from is listed in +`crawler.discovered.exporters` (`{ : { sources: [{ origin_kind, +file, metrics, … }] } }`) and counted in `crawler.discovered.exporters_count`. +A manifest and a snapshot are read in every environment; an exporter config +is scoped like any other config, and a crawl whose environment scope leaves +out a file with scrape jobs or exporter evidence says which in a warning. + +### 18.1 The manifest + +```json +{ + "exporter": "ibm-mq", + "version": "5.6.0", + "source": "http://mq-exporter:9157/metrics", + "generatedAt": "2026-10-01T00:00:00.000Z", + "metrics": [ + { "name": "ibmmq_queue_depth", "type": "gauge", "help": "Current depth of the queue", "labels": ["qmgr", "queue"] } + ] +} +``` + +`exporter` and `metrics` are required; each entry needs a metric name and an +exposition type (`counter`, `gauge`, `histogram`, `summary`, `untyped`, +`unknown`, `gaugehistogram`, `info`, `stateset`). An entry that has neither, +or repeats a name, is not read and the crawl says how many; a file that is not +a manifest is named in a warning. `source` is recorded without userinfo, +query or fragment: tabs and line breaks, everything before the last `@`, and +everything from the first `?` or `#` are dropped (`safeSourceUrl`, textual — +a password may hold `/` or `@`). A value with a `?` or `#` before its last +`@` is not recorded at all. +Label values are never part of a manifest. + +`packc emit --exporter ` (`tools/emit-manifest.mjs`) writes one from a +live endpoint — run it in the exporter's build pipeline against the image it +just built: + +```bash +packc emit --exporter http://mq-exporter:9157/metrics --name ibm-mq --version 5.6.0 --out mq-exporter/metrics-manifest.json +packc emit --from saved.prom --name ibm-mq --out - # a saved exposition, to stdout +``` + +It scrapes once and records each family's name, type, help and label names. +A redirect is refused, not followed: the refusal names its target with the +query as the keys of its `key=value` parts only (`?token=…`) — pass that URL +with the values the exporter expects. A URL with credentials is refused (save the exposition with your own +client and pass it with `--from`). A status other than 200 or a response +with no metric family exits 1. + +### 18.2 The catalogue + +An exporter config names no metric; the catalogue says which families a +config switches on. The built-in catalogue (`tools/lib/exporter-catalogue.mjs`, +data only, `EXPORTER_CATALOGUE`) carries a small IBM MQ entry. A downstream +carries a larger one as a JSON file of the same shape, never as an engine +diff: + +```json +{ + "exporters": [ + { + "id": "ibm-mq", + "match": { "keys": ["global", "connection", "objects", "prometheus"] }, + "namespace": { "path": "prometheus.namespace", "default": "ibmmq" }, + "gates": { + "objectStatus": { "path": "global.useObjectStatus", "default": true }, + "publications": { "path": "global.usePublications", "default": true } + }, + "scopes": { "queues": { "path": "objects.queues" } }, + "families": [ + { "name": "queue_depth", "type": "gauge", "help": "Messages on the queue", "labels": ["qmgr", "queue"], "gate": "publications", "scope": "queues" } + ] + } + ] +} +``` + +- `match.keys` — the top-level keys a config must all carry. The first entry + that matches reads the file. +- `namespace` — the metric-name prefix: the value at `path` in the config, + else `default`; a family's name is `_`. +- `gates` — boolean switches (`true`/`false`, `yes`/`no`, `on`/`off`), the + `default` when the config is silent; a family with `gate` is emitted only + when it is on. +- `scopes` — object selections; a family with `scope` is emitted only when + the value at `path` selects something (a non-empty list, string or + mapping). + +Pass it with `packc x-ray --exporter-catalogue ` or +`crawlFiles(files, { exporterCatalogue })`. Its entries replace a built-in +entry with the same `id` and add the rest. An entry that is not well-formed +is dropped with a crawl warning naming what it lacks. A config-inferred +family stays a candidate (`confidence: inferred`): the config says the +exporter is switched on, not that the name is exact. Publish the real list +with `packc emit` and the manifest outranks it. + +### 18.3 The collector's scrape jobs + +An OTel collector's prometheus receivers (`prometheus`, +`prometheus/`) are read for their `config.scrape_configs`, with or +without a `service:` block (a Helm chart's `files/` copy has none), in a +config file, a Helm ConfigMap, or a chart's `values*.yaml` (at its top level, +under `config:`, or under a subchart's `.config:` unless the subchart +is disabled — its `enabled` toggles merged across the in-scope values files, +an overlay winning). + +- Each job is in `crawler.discovered.scrape_jobs`. Its entry in + `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the + `receiver`, the `target_count` of its static targets, and the `exporters` + its samples leave by: the exporters of the metrics pipelines that list the + receiver, in the same file or, when none there does, another (a config + split across `--config` files), or, when none does, the metrics exporter a + pipeline-less config names. A job that cannot be read is warned about and + skipped; the rest of the config is read. + Each exporter is given as `{ name, kind, topic | endpoint }`: a kafka + exporter's metrics topic, any other's endpoint reduced by `safeSourceUrl` + (no userinfo, query or fragment). +- That exporter's kind is `spec.pipelines.exporters.metrics`, in place of the + `prometheusremotewrite` scaffold. A config with no pipelines only proposes + one: it applies after every file is walked, and only when nothing else set + the metrics exporter — no pipeline in any file declares one, and no metrics + backend image implies one. When it names several, the first is taken and + the crawl warns. diff --git a/package.json b/package.json index 200e7db0..2f0bb0cf 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "lint:recorder": "node --check tools/record-mcp-fixtures.mjs", "test:fetch": "node tools/test-fetch-live.mjs", "lint:adapter": "node --check tools/adapt-spec-pack.mjs && node --check tools/lib/adapter.mjs && node --check tools/lib/mini-yaml.mjs && node --check tools/lib/promql.mjs && node --check tools/lib/promql-lezer.mjs && node --check tools/lib/traceability.mjs && node --check tools/lib/traceability-graph.mjs && node --check tools/lib/promql-canon.mjs", - "lint:crawler": "node --check tools/crawl-repo.mjs && node --check tools/lib/crawler.mjs", + "lint:crawler": "node --check tools/crawl-repo.mjs && node --check tools/lib/crawler.mjs && node --check tools/lib/exporter-metrics.mjs && node --check tools/emit-manifest.mjs", "lint:server": "node --check server/index.mjs", "lint:studio": "node --check studio/app.mjs", "lint:cli": "node --check tools/cli.mjs", @@ -118,7 +118,7 @@ "test:store:prestore:strict": "node tools/test-store-prestore-live.mjs --strict", "test:auth": "node server/test-auth-local.mjs && node server/test-auth-oidc.mjs && node --test server/test-authz.mjs && node --test server/test-identity-api.mjs", "users": "node tools/user-admin.mjs", - "test": "node --test server/test-auth-local.mjs server/test-auth-oidc.mjs server/test-audit-api.mjs server/test-audit-report-api.mjs server/test-auth-proxy.mjs server/test-authz.mjs server/test-brand-shell.mjs server/test-deploy-helpers.mjs server/test-glossary-shell.mjs server/test-hermetic-suites.mjs server/test-identity-api.mjs server/test-live-studio.mjs server/test-malformed-json.mjs server/test-mcp-jobs.mjs server/test-mcp-ping.mjs server/test-mcp-settings.mjs server/test-mcp-settings-studio.mjs server/test-mcp-target-policy.mjs server/test-notify-target-policy.mjs server/test-mcp-url.mjs server/test-service-admin.mjs server/test-services-api.mjs server/test-services-studio.mjs server/test-settings-studio.mjs server/test-smoke.mjs server/test-store-boot.mjs server/test-store-guards.mjs server/test-store-import.mjs server/test-store-ops.mjs server/test-store-paths.mjs server/test-store.mjs server/test-taxonomy.mjs server/test-tenancy.mjs server/test-transport-hook.mjs server/test-verdict-admin.mjs server/test-verdicts-api.mjs server/test-waivers-api.mjs server/test-workspace.mjs tools/test-adapt.mjs tools/test-artefact-classify.mjs tools/test-audit-report.mjs tools/test-backend-validate.mjs tools/test-blast-radius.mjs tools/test-brand.mjs tools/test-build-editor.mjs tools/test-build-info.mjs tools/test-build-model.mjs tools/test-bundle-compare.mjs tools/test-bundle-compare-pairs.mjs tools/test-chain-history.mjs tools/test-compile.mjs tools/test-contract-guard.mjs tools/test-contract-shapes.mjs tools/test-crawl.mjs tools/test-crawl-alerting-rules.mjs tools/test-crawl-canonical.mjs tools/test-declared-type.mjs tools/test-deploy-manifests.mjs tools/test-design-system.mjs tools/test-diagnostic-grade.mjs tools/test-diff.mjs tools/test-doc-test-totals.mjs tools/test-discover-rows.mjs tools/test-fetch-live.mjs tools/test-gen-pack.mjs tools/test-gen-site.mjs tools/test-glossary.mjs tools/test-golden-board.mjs tools/test-golden-compile.mjs tools/test-golden-crawl.mjs tools/test-golden-diff.mjs tools/test-identity-modes.mjs tools/test-inventory-coverage.mjs tools/test-journey-notify.mjs tools/test-journey-scope.mjs tools/test-journey.mjs tools/test-legacy-pack.mjs tools/test-library.mjs tools/test-live-model.mjs tools/test-mcp-settings-model.mjs tools/test-live-snapshot.mjs tools/test-mcp-ping.mjs tools/test-mcp-server-settings.mjs tools/test-mcp-transport.mjs tools/test-metric-readers.mjs tools/test-mini-yaml.mjs tools/test-neuron-model.mjs tools/test-pack-conformance.mjs tools/test-packs.mjs tools/test-platform.mjs tools/test-profiles.mjs tools/test-promql-canon.mjs tools/test-promql.mjs tools/test-record-fixtures.mjs tools/test-remediation-flow.mjs tools/test-remediation-flow-view.mjs tools/test-retrofeed.mjs tools/test-scan-live.mjs tools/test-schedule-snippets.mjs tools/test-schedule.mjs tools/test-service-keys.mjs tools/test-services-model.mjs tools/test-settings-model.mjs tools/test-stack-evidence.mjs tools/test-stack-self-metrics.mjs tools/test-static-remedies.mjs tools/test-studio-bundle.mjs tools/test-studio-graph.mjs tools/test-studio-layout.mjs tools/test-svg-charts.mjs tools/test-tap-summary.mjs tools/test-traceability-graph.mjs tools/test-upconvert-merge.mjs tools/test-ux-kit.mjs tools/test-validator.mjs tools/test-vendor-manifest.mjs tools/test-verdicts-panel.mjs tools/test-verify-deploy.mjs tools/test-waivers.mjs tools/test-zip.mjs", + "test": "node --test server/test-auth-local.mjs server/test-auth-oidc.mjs server/test-audit-api.mjs server/test-audit-report-api.mjs server/test-auth-proxy.mjs server/test-authz.mjs server/test-brand-shell.mjs server/test-deploy-helpers.mjs server/test-glossary-shell.mjs server/test-hermetic-suites.mjs server/test-identity-api.mjs server/test-live-studio.mjs server/test-malformed-json.mjs server/test-mcp-jobs.mjs server/test-mcp-ping.mjs server/test-mcp-settings.mjs server/test-mcp-settings-studio.mjs server/test-mcp-target-policy.mjs server/test-notify-target-policy.mjs server/test-mcp-url.mjs server/test-service-admin.mjs server/test-services-api.mjs server/test-services-studio.mjs server/test-settings-studio.mjs server/test-smoke.mjs server/test-store-boot.mjs server/test-store-guards.mjs server/test-store-import.mjs server/test-store-ops.mjs server/test-store-paths.mjs server/test-store.mjs server/test-taxonomy.mjs server/test-tenancy.mjs server/test-transport-hook.mjs server/test-verdict-admin.mjs server/test-verdicts-api.mjs server/test-waivers-api.mjs server/test-workspace.mjs tools/test-adapt.mjs tools/test-artefact-classify.mjs tools/test-audit-report.mjs tools/test-backend-validate.mjs tools/test-blast-radius.mjs tools/test-brand.mjs tools/test-build-editor.mjs tools/test-build-info.mjs tools/test-build-model.mjs tools/test-bundle-compare.mjs tools/test-bundle-compare-pairs.mjs tools/test-chain-history.mjs tools/test-compile.mjs tools/test-contract-guard.mjs tools/test-contract-shapes.mjs tools/test-crawl.mjs tools/test-crawl-alerting-rules.mjs tools/test-crawl-canonical.mjs tools/test-crawl-exporters.mjs tools/test-declared-type.mjs tools/test-deploy-manifests.mjs tools/test-design-system.mjs tools/test-diagnostic-grade.mjs tools/test-diff.mjs tools/test-doc-test-totals.mjs tools/test-discover-rows.mjs tools/test-fetch-live.mjs tools/test-gen-pack.mjs tools/test-gen-site.mjs tools/test-glossary.mjs tools/test-golden-board.mjs tools/test-golden-compile.mjs tools/test-golden-crawl.mjs tools/test-golden-diff.mjs tools/test-identity-modes.mjs tools/test-inventory-coverage.mjs tools/test-journey-notify.mjs tools/test-journey-scope.mjs tools/test-journey.mjs tools/test-legacy-pack.mjs tools/test-library.mjs tools/test-live-model.mjs tools/test-mcp-settings-model.mjs tools/test-live-snapshot.mjs tools/test-mcp-ping.mjs tools/test-mcp-server-settings.mjs tools/test-mcp-transport.mjs tools/test-metric-readers.mjs tools/test-mini-yaml.mjs tools/test-neuron-model.mjs tools/test-pack-conformance.mjs tools/test-packs.mjs tools/test-platform.mjs tools/test-profiles.mjs tools/test-promql-canon.mjs tools/test-promql.mjs tools/test-record-fixtures.mjs tools/test-remediation-flow.mjs tools/test-remediation-flow-view.mjs tools/test-retrofeed.mjs tools/test-scan-live.mjs tools/test-schedule-snippets.mjs tools/test-schedule.mjs tools/test-service-keys.mjs tools/test-services-model.mjs tools/test-settings-model.mjs tools/test-stack-evidence.mjs tools/test-stack-self-metrics.mjs tools/test-static-remedies.mjs tools/test-studio-bundle.mjs tools/test-studio-graph.mjs tools/test-studio-layout.mjs tools/test-svg-charts.mjs tools/test-tap-summary.mjs tools/test-traceability-graph.mjs tools/test-upconvert-merge.mjs tools/test-ux-kit.mjs tools/test-validator.mjs tools/test-vendor-manifest.mjs tools/test-verdicts-panel.mjs tools/test-verify-deploy.mjs tools/test-waivers.mjs tools/test-zip.mjs", "lint": "eslint .", "coverage": "c8 npm test", "sync-spec": "node tools/sync-spec.mjs", diff --git a/server/github-crawl.mjs b/server/github-crawl.mjs index 479f36a0..b8f2935c 100644 --- a/server/github-crawl.mjs +++ b/server/github-crawl.mjs @@ -1,10 +1,13 @@ // server/github-crawl.mjs — helpers for the public-GitHub crawl route. // // Server-only (ghFetch reads GITHUB_TOKEN from the environment, which the -// browser-safe tools/lib layer is not allowed to do). parseGithubUrl and -// isCrawlerFile are pure; ghFetch is the single authenticated wrapper +// browser-safe tools/lib layer is not allowed to do). parseGithubUrl, +// isCrawlerFile, isExporterCandidateFile and selectCrawlerFiles are pure; +// ghFetch is the single authenticated wrapper // around the GitHub REST API. +import { isTestOrFixturePath } from '../tools/lib/exporter-metrics.mjs'; + export function parseGithubUrl(input) { if (typeof input !== 'string' || !input.trim()) return null; const cleaned = input.trim().replace(/\.git$/, '').replace(/\/$/, ''); @@ -30,6 +33,9 @@ export function isCrawlerFile(path) { return ( sourceMetricCandidate || /(^|\/)(application|bootstrap)[\w.-]*\.ya?ml$/.test(p) || + // A prebuilt exporter's metrics manifest (named precisely; the broad + // exporter patterns are isExporterCandidateFile's, fetched last). + /(^|[/.])metrics-manifest\.json$/.test(p) || /(^|\/)docker[-_]compose[a-z0-9._-]*\.(ya?ml)$/.test(p) || /(^|\/)compose[a-z0-9._-]*\.(ya?ml)$/.test(p) || /\.rules\.(ya?ml)$/.test(p) || @@ -49,6 +55,37 @@ export function isCrawlerFile(path) { ); } +// Files that MAY say what a prebuilt exporter exposes or how a collector +// scrapes it, by names too common to fetch first: a saved /metrics (`.prom`, +// `.metrics`, never from a test or fixture folder), any `config.yaml` (an +// mq-metric-samples config), a Helm chart's `files/*.yaml` (a collector +// config). Never one isCrawlerFile already takes. +export function isExporterCandidateFile(path) { + if (typeof path !== 'string' || isCrawlerFile(path)) return false; + const p = path.toLowerCase(); + if (p.includes('node_modules/') || p.startsWith('.git/') || p.includes('/.git/')) return false; + // The test-folder rule reads the path as written (camelCase source sets). + return (/\.(prom|metrics)$/.test(p) && !isTestOrFixturePath(path)) + || /(^|\/)config\.ya?ml$/.test(p) + || /(^|\/)files\/.*\.ya?ml$/.test(p); +} + +/** + * The blobs of a GitHub tree a crawl downloads, at most `maxFiles` of at most + * `fileCapBytes` each: every isCrawlerFile blob first, in tree order, then + * the exporter candidates — so a monorepo's many `config.yaml` files never + * push out a rule file, an Alertmanager config or a collector config. + */ +export function selectCrawlerFiles(tree, { maxFiles = 200, fileCapBytes = Infinity } = {}) { + const blobs = (Array.isArray(tree) ? tree : []).filter(node => node?.type === 'blob' && typeof node.path === 'string'); + return [ + ...blobs.filter(node => isCrawlerFile(node.path)), + ...blobs.filter(node => isExporterCandidateFile(node.path)), + ] + .filter(node => !node.size || node.size <= fileCapBytes) + .slice(0, maxFiles); +} + export async function ghFetch(path, init = {}) { const headers = { 'Accept': 'application/vnd.github+json', diff --git a/server/index.mjs b/server/index.mjs index 0bfd22ca..37bf7eee 100644 --- a/server/index.mjs +++ b/server/index.mjs @@ -68,7 +68,7 @@ import { describeProxyAuth } from './auth-proxy.mjs'; import { stripMcpUrl, mcpUrlOrigin, droppedNote } from './mcp-url.mjs'; import { mcpCallerOf, mcpRefusalBody, redactTarget } from './mcp-target-policy.mjs'; import { captureNotifyRefusal, hooksAllTaken, notifyBootReport, notifyHookOf, notifyHooksOf, notifyNamesRefusal, notifyPostureRefusal, resolveJourneyNotify } from './notify-target-policy.mjs'; -import { parseGithubUrl, isCrawlerFile, ghFetch } from './github-crawl.mjs'; +import { parseGithubUrl, selectCrawlerFiles, ghFetch } from './github-crawl.mjs'; import { deployRoutes } from './routes/deploy.mjs'; import { auditAfter, actorForRecord, bounded, finite } from './audit-after.mjs'; import { identityRoutes } from './routes/identity.mjs'; @@ -1880,7 +1880,7 @@ app.post('/api/crawl', authorize('POST /api/crawl'), (req, res) => { // // Bandwidth guards: max 200 files, max 16 MB total, max 1 MB per file. // ---------------------------------------------------------------- -// parseGithubUrl / isCrawlerFile / ghFetch live in server/github-crawl.mjs. +// parseGithubUrl / selectCrawlerFiles / ghFetch live in server/github-crawl.mjs. app.post('/api/crawl-github', authorize('POST /api/crawl-github'), async (req, res) => { const body = req.body || {}; @@ -1914,10 +1914,8 @@ app.post('/api/crawl-github', authorize('POST /api/crawl-github'), async (req, r const FILE_CAP_BYTES = 1 * 1024 * 1024; // 1 MB / file const TOTAL_CAP_BYTES = 16 * 1024 * 1024; // 16 MB total const MAX_FILES = 200; - const candidates = (treeResp.tree || []) - .filter(node => node.type === 'blob' && isCrawlerFile(node.path)) - .filter(node => !node.size || node.size <= FILE_CAP_BYTES) - .slice(0, MAX_FILES); + // isCrawlerFile's blobs first, the exporter candidates after them. + const candidates = selectCrawlerFiles(treeResp.tree, { maxFiles: MAX_FILES, fileCapBytes: FILE_CAP_BYTES }); if (candidates.length === 0) { return res.json({ diff --git a/tools/cli.mjs b/tools/cli.mjs index 8dba0914..2c6e6e9d 100644 --- a/tools/cli.mjs +++ b/tools/cli.mjs @@ -10,6 +10,7 @@ // packc validate → tools/validate-pack.mjs // packc adapt [env] → tools/adapt-spec-pack.mjs // packc x-ray → tools/crawl-repo.mjs +// packc emit --exporter → tools/emit-manifest.mjs (a prebuilt exporter's metrics-manifest.json) // packc compile [target] → tools/lib/compile.mjs (programmatic) // packc init … → tools/pack-init.mjs (build a pack from the library) // packc audit-report … → tools/audit-report.mjs (a pack's service audit report, JSON or HTML) @@ -114,6 +115,7 @@ Usage: packc audit-report [--env ] [--format json|html|both] [--out ] [--brand ] [--taxonomy ] [--top ] [--generated-at | --no-timestamp] [--verdicts ] [--waivers ] [--schema ] A pack's service audit report: conformance, placeholders, verdicts, waivers, coverage, goes-blind risks, response path packc adapt [env] Adapt a pack into the layered projection packc x-ray Crawl a repo into a draft pack + packc emit --exporter [--name ] [--version ] [--out |-] Scrape an exporter's /metrics once and write its metrics-manifest.json (--from reads a saved one) packc compile [target] Compile a pack into a backend artefact packc init --list List the library entries (products and archetypes) a pack can be built from packc init --show An entry's params, SLIs per tier, per-tier objectives and evidence @@ -282,6 +284,9 @@ switch (command) { case 'crawl': delegate('tools/crawl-repo.mjs', rest); break; + case 'emit': + delegate('tools/emit-manifest.mjs', rest); + break; case 'compile': await runCompile(rest); break; diff --git a/tools/crawl-repo.mjs b/tools/crawl-repo.mjs index a39df1c4..7f71eb54 100644 --- a/tools/crawl-repo.mjs +++ b/tools/crawl-repo.mjs @@ -78,6 +78,10 @@ manifest by introspecting common observability artefacts. (the original is kept in crawler.nameNormalizedFrom). --diff-scope live-drift comparison lens: service | family | all (default: service) + --exporter-catalogue + a JSON exporter family catalogue whose entries + replace or extend the built-in ones + (tools/lib/exporter-catalogue.mjs; docs/DOWNSTREAM.md §18) -h, --help print this message Output: @@ -96,6 +100,12 @@ manifest by introspecting common observability artefacts. - OTel Collector configs → spec.pipelines - Grafana dashboard JSONs → spec.dashboards[] - Helm values / K8s workloads→ spec.telemetry.backends[] + - metrics-manifest.json, saved + /metrics (.prom, .metrics), + a prebuilt exporter's config → the exporter's metrics (crawler.discovered.exporters) + - an OTel Collector's prometheus + receiver scrape jobs → crawler.discovered.scrape_jobs, and its + metrics exporter → spec.pipelines.exporters.metrics Example: npm run crawl -- ./payments-svc --name payments --criticality tier-2 > payments.pack.yaml @@ -121,6 +131,7 @@ function parseArgs(argv) { else if (a === '--binding') out.binding = argv[++i]; else if (a === '--owners') out.owners = (argv[++i] ?? '').split(','); else if (a === '--diff-scope' || a === '--live-scope') out.diffScopeMode = argv[++i]; + else if (a === '--exporter-catalogue') out.exporterCatalogueFile = argv[++i] ?? ''; } return out; } @@ -165,6 +176,14 @@ async function main() { process.exit(2); } if (!opts.repoName) opts.repoName = basename(opts.repoPath.replace(/[\\/]$/, '')); + if (opts.exporterCatalogueFile !== undefined) { + try { + opts.exporterCatalogue = JSON.parse(readFileSync(opts.exporterCatalogueFile, 'utf8')); + } catch (e) { + process.stderr.write(`crawl-repo: --exporter-catalogue ${JSON.stringify(opts.exporterCatalogueFile)}: ${e instanceof SyntaxError ? `not JSON (${e.message})` : (e.code || e.message)}\n`); + process.exit(2); + } + } const files = await walk(opts.repoPath); const { yaml, canonical, summary, evidence } = crawlToYaml(files, opts); @@ -195,6 +214,7 @@ async function main() { `# alert rules : ${summary.discovered.alertRules} (operational, alerting.rules)`, `# metric definitions: ${summary.discovered.metricDefinitions}`, `# scrape jobs : ${summary.discovered.scrapeJobs}`, + ...(summary.discovered.exporters ? [`# exporters : ${summary.discovered.exporters}`] : []), `# dashboards : ${summary.discovered.dashboards}`, `# alerting routes : ${summary.discovered.alertingRoutes}`, `# pipelines : ${summary.discovered.pipelines}`, diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs new file mode 100644 index 00000000..2049996c --- /dev/null +++ b/tools/emit-manifest.mjs @@ -0,0 +1,158 @@ +#!/usr/bin/env node +/** + * tools/emit-manifest.mjs — `packc emit --exporter `: a prebuilt + * exporter's metrics manifest, read off its live /metrics endpoint. + * + * A repository that ships a prebuilt exporter (a Dockerfile and a config + * file) names none of the exporter's metrics; the crawler reads them from a + * `metrics-manifest.json` beside the config (tools/lib/exporter-metrics.mjs). + * This writes one: it scrapes the endpoint once and records each metric + * family's name, type, help and label names — never a label value, never a + * sample. Run it in the exporter's build pipeline against the image it just + * built and commit (or publish) the result. + * + * node tools/emit-manifest.mjs --exporter [--name ] [--version ] [--out |-] + * node tools/emit-manifest.mjs --from [--name ] [--version ] [--out |-] + * --exporter the exporter's metrics URL (http or https), e.g. http://mq-exporter:9157/metrics + * --from read a saved exposition instead (one the scrape needs credentials for: save it + * with your own client, then pass it here) + * --name the exporter's name in the manifest (default: the URL's host, or the file's name) + * --version the exporter's version, recorded as given + * --out where to write (default: metrics-manifest.json); `-` writes to stdout + * --generated-at the stamp (default: the clock); --no-timestamp leaves it out (reproducible bytes) + * --timeout the scrape timeout in seconds (default 15) + * + * exit 0 the manifest was written + * exit 1 the scrape failed: unreachable, a redirect, a status other than 200, no metric families + * exit 2 usage: no source, both sources, an unknown flag, a URL that is not http(s) or carries credentials + * + * The recorded `source` is the URL without userinfo, query or fragment + * (tools/lib/exporter-metrics.mjs safeSourceUrl). A redirect is refused, not + * followed: the refusal names its target, the query as `key=value` keys only — pass that + * URL with the values the exporter expects. + */ + +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { resolve, basename } from 'node:path'; +import { manifestFromExposition, safeSourceUrl } from './lib/exporter-metrics.mjs'; + +// A URL as a message may name it: safeSourceUrl's form, or a stand-in when +// that leaves nothing (the URL is then not shown at all). +const shown = (url) => safeSourceUrl(url) || 'the exporter URL (not shown)'; + +// A value that may be a URL with its credentials or token: never echoed. +const urlLike = (v) => /:\/\/|@|\?|^[^/\\\s]+:\d+(?:[/?#]|$)/.test(String(v ?? '')); +// A path argument as a message may name it. +const showArg = (v) => (urlLike(v) ? '(a URL-like value, not shown)' : v); + +const USAGE = 'usage: packc emit --exporter | --from [--name ] [--version ] [--out |-] [--generated-at | --no-timestamp] [--timeout ]'; + +function fail(code, message) { + process.stderr.write(`packc emit: ${message}\n`); + process.exit(code); +} + +function parseArgs(argv) { + const out = { out: 'metrics-manifest.json', timeout: 15, timestamp: true }; + const takes = new Set(['--exporter', '--from', '--name', '--version', '--out', '--generated-at', '--timeout']); + for (let i = 0; i < argv.length; i++) { + let a = argv[i]; + if (a === '-h' || a === '--help') { out.help = true; continue; } + if (a === '--no-timestamp') { out.timestamp = false; continue; } + // `--flag=value` as well as `--flag value`. + const eq = a.startsWith('--') ? a.indexOf('=') : -1; + let inline; + if (eq > 0) { inline = a.slice(eq + 1); a = a.slice(0, eq); } + // An unknown argument may be a URL with its credentials: never echoed. + if (!takes.has(a)) fail(2, `unknown argument ${urlLike(a) ? '(a URL-like value, not shown)' : `'${a}'`}\n${USAGE}`); + if (inline !== undefined) { out[a.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = inline; continue; } + const v = argv[++i]; + if (v === undefined || (v.startsWith('--') && v !== '-')) fail(2, `${a} needs a value\n${USAGE}`); + out[a.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = v; + } + return out; +} + +// A redirect is named, never followed: its target without userinfo, and its +// query as the keys of `key=value` parts only (`?format=…`; a bare `?` is +// a value) — a value may be a token — so the URL to +// pass is the one shown with the values the exporter expects. +function redirectRefusal(url, status, location) { + let target = null; + try { target = location ? new URL(location, url) : null; } catch { /* not a URL: named as such */ } + // Keys of `key=value` parts only: a bare `?` is a value, not a key. + const parts = target?.search ? target.search.slice(1).split('&').filter(Boolean) : []; + const keys = [...new Set(parts.filter(p => p.includes('=')).map(p => p.slice(0, p.indexOf('='))))]; + const bare = parts.some(p => !p.includes('=')); + const query = !parts.length ? '' + : `${keys.length ? `?${keys.map(k => `${k}=…`).join('&')}` : ''}${bare ? `${keys.length ? ' and' : ' with'} a query part` : ''} (query values not shown)`; + const to = !location ? '' + : !target ? ' to a location that is not a URL' + : ` to ${shown(target.href)}${query}`; + return `${shown(url)} answered ${status}, a redirect${to}; it is not followed — pass the URL the exporter serves its metrics at${parts.length ? ', with the query values it expects' : ''}`; +} + +async function scrape(url, timeoutSeconds) { + let res; + try { + res = await fetch(url, { + redirect: 'manual', + headers: { accept: 'text/plain;version=0.0.4;q=1, application/openmetrics-text;q=0.5, */*;q=0.1' }, + signal: AbortSignal.timeout(timeoutSeconds * 1000), + }); + } catch (e) { + const why = e?.name === 'TimeoutError' ? `no answer within ${timeoutSeconds}s (raise --timeout)` : (e?.cause?.message || e?.message || String(e)); + fail(1, `could not scrape ${shown(url)}: ${why}`); + } + if (res.status >= 300 && res.status < 400) fail(1, redirectRefusal(url, res.status, res.headers.get('location'))); + if (res.status !== 200) fail(1, `${shown(url)} answered ${res.status}; a metrics endpoint answers 200`); + try { + return await res.text(); + } catch (e) { + fail(1, `could not read the response of ${shown(url)}: ${e?.name === 'TimeoutError' ? `not complete within ${timeoutSeconds}s (raise --timeout)` : 'the connection ended before the body did'}`); + } +} + +const args = parseArgs(process.argv.slice(2)); +if (args.help) { process.stdout.write(`${USAGE}\n`); process.exit(0); } +if (!args.exporter && !args.from) fail(2, `name the exporter's metrics URL (--exporter) or a saved exposition (--from)\n${USAGE}`); +if (args.exporter && args.from) fail(2, 'pass --exporter or --from, not both'); +const timeout = Number(args.timeout); +if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 600) fail(2, '--timeout is a number of seconds, 1 to 600'); +let generatedAt; +if (args.timestamp) { + const at = args.generatedAt === undefined ? new Date() : new Date(args.generatedAt); + if (Number.isNaN(at.getTime())) fail(2, `--generated-at ${urlLike(args.generatedAt) ? '(a URL-like value, not shown)' : `'${args.generatedAt}'`} is not a date`); + generatedAt = at.toISOString(); +} else if (args.generatedAt !== undefined) fail(2, 'pass --generated-at or --no-timestamp, not both'); + +let text, source, defaultName; +if (args.exporter) { + let url; + try { url = new URL(args.exporter); } catch { fail(2, `--exporter '${shown(args.exporter)}' is not a URL (e.g. http://mq-exporter:9157/metrics); shown without anything before an @ or after a ? or #`); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') fail(2, `--exporter must be an http or https URL, not ${url.protocol}`); + if (url.username || url.password) fail(2, 'the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from'); + text = await scrape(url.href, timeout); + source = url.href; + defaultName = url.hostname; +} else { + // A file whose name holds `@` or `?` is read; a URL-like value that is not + // a file is refused unshown. + if (urlLike(args.from) && !existsSync(resolve(args.from))) fail(2, '--from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter'); + try { text = readFileSync(resolve(args.from), 'utf8'); } catch (e) { fail(2, `cannot read --from ${showArg(args.from)}: ${e.code || 'read failed'}`); } + defaultName = basename(args.from).replace(/\.[^.]+$/, ''); +} + +const name = String(args.name ?? defaultName ?? '').trim(); +if (!name) fail(2, 'the exporter has no name; pass --name'); +const manifest = manifestFromExposition(text, { exporter: name, version: args.version, source, generatedAt }); +if (!manifest.metrics.length) fail(1, `${source ? shown(source) : showArg(args.from)} holds no metric family (no # TYPE, # HELP or sample line)`); + +const json = `${JSON.stringify(manifest, null, 2)}\n`; +if (args.out === '-') { + process.stdout.write(json, () => process.exit(0)); +} else { + try { writeFileSync(resolve(args.out), json); } + catch (e) { fail(2, `cannot write --out ${showArg(args.out)}: ${e.code || 'write failed'}`); } + process.stdout.write(`wrote ${showArg(args.out)}: ${manifest.metrics.length} metric famil${manifest.metrics.length === 1 ? 'y' : 'ies'} of ${name}\n`, () => process.exit(0)); +} diff --git a/tools/fixtures/crawl/exporters/README.md b/tools/fixtures/crawl/exporters/README.md new file mode 100644 index 00000000..234c7884 --- /dev/null +++ b/tools/fixtures/crawl/exporters/README.md @@ -0,0 +1,15 @@ +# Exporter fixture + +A service whose metrics come from a prebuilt exporter (IBM's +mq-metric-samples) that an OTel collector scrapes and ships to Kafka. +Nothing here registers or queries a metric: the crawler learns the names +from the exporter's `metrics-manifest.json`, the families its `config.yaml` +switches on, and the collector's `qmgrs` scrape job. Read by +`tools/test-crawl-exporters.mjs`. + +- `mq-exporter/` — the exporter's repository: a Dockerfile (not scanned), + its config and the manifest `packc emit --exporter` writes. +- `charts/otel-collector/files/collector.yaml` — the collector config a + Helm chart keeps under `files/`, with no `service:` block. +- `preprod/otel-collector.yaml` — the preprod collector; a `prod` crawl + leaves it out and says so. diff --git a/tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml b/tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml new file mode 100644 index 00000000..626eb7f4 --- /dev/null +++ b/tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: otel-collector +version: 0.1.0 diff --git a/tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml b/tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml new file mode 100644 index 00000000..e93189b9 --- /dev/null +++ b/tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml @@ -0,0 +1,21 @@ +receivers: + prometheus/mq: + config: + global: + scrape_interval: 30s + scrape_configs: + - job_name: qmgrs + metrics_path: /metrics + static_configs: + - targets: ["mq-exp-qm1:9157", "mq-exp-qm2:9157", "mq-exp-qm3:9157"] + labels: + env: prod +processors: + batch: {} +exporters: + debug: {} + kafka: + brokers: ["kafka-0.kafka:9092", "kafka-1.kafka:9092"] + metrics: + topic: mq-metrics + encoding: otlp_proto diff --git a/tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml b/tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml new file mode 100644 index 00000000..10a5d50d --- /dev/null +++ b/tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "otel-collector.fullname" . }} +data: + relay: | +{{ .Files.Get "files/collector.yaml" | indent 4 }} diff --git a/tools/fixtures/crawl/exporters/mq-exporter/Dockerfile b/tools/fixtures/crawl/exporters/mq-exporter/Dockerfile new file mode 100644 index 00000000..f21c8c4a --- /dev/null +++ b/tools/fixtures/crawl/exporters/mq-exporter/Dockerfile @@ -0,0 +1,2 @@ +FROM ibmcom/mq-metric-samples:5.6.0 +COPY config.yaml /opt/config/mq_prometheus.yaml diff --git a/tools/fixtures/crawl/exporters/mq-exporter/config.yaml b/tools/fixtures/crawl/exporters/mq-exporter/config.yaml new file mode 100644 index 00000000..1887cb80 --- /dev/null +++ b/tools/fixtures/crawl/exporters/mq-exporter/config.yaml @@ -0,0 +1,25 @@ +global: + useObjectStatus: true + useResetQStats: false + usePublications: true + logLevel: INFO + pollInterval: 30s + rediscoverInterval: 1h +connection: + queueManager: QM1 + clientConnection: true + user: "" + replyQueue: SYSTEM.DEFAULT.MODEL.QUEUE +objects: + queues: + - "APP.*" + - "!SYSTEM.*" + channels: + - "TO.*" + topics: + subscriptions: + showInactiveChannels: false +prometheus: + port: 9157 + metricsPath: /metrics + namespace: ibmmq diff --git a/tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json b/tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json new file mode 100644 index 00000000..99e709f3 --- /dev/null +++ b/tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json @@ -0,0 +1,14 @@ +{ + "exporter": "ibm-mq", + "version": "5.6.0", + "source": "http://mq-exporter:9157/metrics", + "generatedAt": "2026-10-01T00:00:00.000Z", + "metrics": [ + { "name": "ibmmq_qmgr_status", "type": "gauge", "help": "Queue manager status (1 = running)", "labels": ["qmgr", "platform"] }, + { "name": "ibmmq_qmgr_uptime", "type": "gauge", "help": "Queue manager uptime in seconds", "labels": ["qmgr", "platform"] }, + { "name": "ibmmq_queue_depth", "type": "gauge", "help": "Current depth of the queue", "labels": ["qmgr", "queue", "usage", "description"] }, + { "name": "ibmmq_queue_mqget_count", "type": "counter", "help": "MQGET calls", "labels": ["qmgr", "queue"] }, + { "name": "ibmmq_channel_status", "type": "gauge", "help": "Channel status", "labels": ["qmgr", "channel", "type", "connname"] }, + { "name": "ibmmq_qmgr_log_write_size_bytes", "type": "gauge", "help": "Log write size", "labels": ["qmgr", "platform"] } + ] +} diff --git a/tools/fixtures/crawl/exporters/preprod/otel-collector.yaml b/tools/fixtures/crawl/exporters/preprod/otel-collector.yaml new file mode 100644 index 00000000..171eae09 --- /dev/null +++ b/tools/fixtures/crawl/exporters/preprod/otel-collector.yaml @@ -0,0 +1,16 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: qmgrs-preprod + scrape_interval: 60s + static_configs: + - targets: ["mq-exp-pp1:9157"] +exporters: + kafka: + topic: mq-metrics-preprod +service: + pipelines: + metrics: + receivers: [prometheus] + exporters: [kafka] diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 55797ca1..a54e3c35 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -33,6 +33,10 @@ import { routesFromAlertmanagerConfig } from './alert-routes.mjs'; import { BACKEND_PATTERNS } from './backend-products.mjs'; import { PROMQL_KEYWORDS, extractPromqlMetricNames } from './promql.mjs'; import { symbolSlug as slug, packSlug } from './slug.mjs'; +import { + isMetricsManifestPath, isExpositionSnapshotPath, isTestOrFixturePath, parseExposition, readMetricsManifest, safeSourceUrl, + resolveExporterCatalogue, asResolvedCatalogue, mayBeExporterConfig, matchExporterConfig, inferExporterFamilies, +} from './exporter-metrics.mjs'; // The spec's closed vocabularies, mirrored by hand (a browser-safe module // cannot read the schema file; tools/test-crawl-canonical.mjs pins the @@ -58,7 +62,8 @@ function suffixedId(base, suffix) { // before it used this rule it staged 75,000 files (430 MB) of a repository // whose scannable sources are 600. Where the browser hands over the folder // itself, walkScanFolder (below) does not list them in the first place. -export const SCAN_EXT = /\.(ya?ml|json|cjs|mjs|js|jsx|ts|tsx|py|go|java|kt|rs|cs)$/i; +// `.prom` / `.metrics`: a saved /metrics response (an exposition snapshot). +export const SCAN_EXT = /\.(ya?ml|json|cjs|mjs|js|jsx|ts|tsx|py|go|java|kt|rs|cs|prom|metrics)$/i; export const SCAN_MAX_FILE_BYTES = 5 * 1024 * 1024; export const SCAN_IGNORE_DIRS = new Set([ 'node_modules', 'vendor', 'venv', @@ -140,6 +145,12 @@ const K8S_WORKLOAD_KINDS = new Set([ const METRIC_SOURCE_EXT_RE = /\.(?:cjs|mjs|js|jsx|ts|tsx|py|go|java|kt|rs|cs)$/i; +// The built-in exporter catalogue (tools/lib/exporter-catalogue.mjs), the one +// detectArtefactKind reads when its caller names none. +const BUILTIN_EXPORTER_CATALOGUE = resolveExporterCatalogue(); + +const isMapping = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); + // Match an image reference (`repository[:tag]`) against the known backend // catalog and register it on the backends bucket. Shared by the Helm values, // Helm template, and plain-K8s workload walkers. When `dedupeByProduct` is set @@ -191,12 +202,28 @@ function registerMetricsExporterFromBackend(pattern, pipelines, relPath, evidenc * Detect what kind of observability artefact a file is. * @param {string} relPath - repo-relative path * @param {string} content - file content + * @param {object} [opts] + * @param {object|Array} [opts.exporterCatalogue] - a downstream's exporter + * catalogue ({ exporters: [...] } or the array), merged with the built-in + * entries; the built-in one when omitted * @returns {string} kind: 'prometheus-rules' | 'alertmanager' | 'grafana-dashboard' | * 'otel-collector' | 'docker-compose' | 'helm-chart' | - * 'helm-template' | 'unknown' + * 'helm-template' | 'metrics-manifest' | 'exposition-snapshot' | + * 'exporter-config' | 'unknown' */ -export function detectArtefactKind(relPath, content) { +export function detectArtefactKind(relPath, content, opts) { + const { exporterCatalogue = BUILTIN_EXPORTER_CATALOGUE } = opts ?? {}; const lower = relPath.toLowerCase(); + const catalogue = asResolvedCatalogue(exporterCatalogue); + // A prebuilt exporter's metric list, as `packc emit --exporter` writes it. + // Named by its file, so a malformed one is walked and warned about, never + // silently skipped. + if (isMetricsManifestPath(lower)) return 'metrics-manifest'; + // A saved /metrics response — not from a test or fixture path, where an + // exposition is sample input, not what a deployed exporter serves. + if (isExpositionSnapshotPath(lower)) { + return !isTestOrFixturePath(relPath) && parseExposition(content).families.length ? 'exposition-snapshot' : 'unknown'; + } // JSON first — Grafana dashboards are the only json we care about. if (lower.endsWith('.json')) { try { @@ -232,6 +259,15 @@ export function detectArtefactKind(relPath, content) { // config second. if (looksLikeHelm(content)) return 'helm-template'; + // A prebuilt exporter's own config (mq-metric-samples' config.yaml), by + // the top-level keys a catalogue entry names — before the filename + // heuristics, which would read `mq-collector.yaml` as an OTel collector. + if (mayBeExporterConfig(content, catalogue)) { + let cfg = null; + try { cfg = parseYaml(content); } catch { /* not YAML: fall through */ } + if (matchExporterConfig(cfg, catalogue)) return 'exporter-config'; + } + // Filename heuristics first — fastest. if (/(^|\/)(docker-)?compose\.ya?ml$/.test(lower)) return 'docker-compose'; if (/(^|\/)chart\.ya?ml$/.test(lower)) return 'helm-chart'; @@ -255,7 +291,7 @@ export function detectArtefactKind(relPath, content) { && obj.scrape_configs.some(s => s?.job_name)) return 'prometheus-scrape-config'; if (looksLikeActuatorMetricsConfig(lower, content, obj)) return 'actuator-metrics-config'; if (obj.route && obj.receivers && Array.isArray(obj.receivers)) return 'alertmanager'; - if (obj.receivers && obj.exporters && obj.service?.pipelines) return 'otel-collector'; + if (looksLikeOtelCollector(obj)) return 'otel-collector'; if (obj.services && typeof obj.services === 'object' && Object.values(obj.services).some(s => s?.image)) return 'docker-compose'; // Plain (non-templated) Kubernetes workload manifest carrying concrete @@ -369,6 +405,9 @@ function isSourceCodePath(relPath) { function shouldIncludeForEnvironment(relPath, profile, surfaces) { const p = normalizeRepoPath(relPath); if (isSourceCodePath(p)) return true; + // What an exporter exposes is the exporter's, not a deployment surface's: + // a metrics manifest and a saved exposition are read in every environment. + if (isMetricsManifestPath(p) || isExpositionSnapshotPath(p)) return true; if (profile === 'local-docker') { if (!surfaces.docker) return true; @@ -437,6 +476,9 @@ function scopeFilesForEnvironment(files, environment) { * @param {string} [opts.criticality] - 'tier-1'|'tier-2'|'tier-3'. Inferred if omitted. * @param {string} [opts.binding='otel-elastic-prometheus-grafana'] * @param {Array} [opts.owners=['team-platform']] + * @param {object|Array} [opts.exporterCatalogue] - exporter family catalogue + * entries ({ exporters: [...] } or the array) that replace or extend the + * built-in ones (tools/lib/exporter-catalogue.mjs) * @returns {{canonical: object, summary: object, evidence: Record}} */ export function crawlFiles(filesInput, opts = {}) { @@ -464,7 +506,8 @@ export function crawlFiles(filesInput, opts = {}) { // Profile detection reads the environment as given (it lowercases itself). const envScope = scopeFilesForEnvironment(rawFiles, envRaw); const files = envScope.files; - const inputWarnings = []; + const catalogue = resolveExporterCatalogue(opts.exporterCatalogue); + const inputWarnings = [...catalogue.warnings]; const normalized = { name: null, environment: null, owners: [] }; if (repoName !== String(nameRaw)) { normalized.name = { from: String(nameRaw), to: repoName }; @@ -522,6 +565,15 @@ export function crawlFiles(filesInput, opts = {}) { summary.warnings.push(...inputWarnings); if (envScope.applied) { summary.warnings.push(`Environment scope "${envScope.profile}" excluded ${envScope.excluded.length} file(s) from other deployment surfaces.`); + // A file the scope leaves out may be the only one that shows how the + // metrics are collected: say which, never drop them silently. + const collection = envScope.excluded + .map(relPath => collectionEvidenceOf(relPath, rawFiles.get(relPath), catalogue)) + .filter(Boolean); + if (collection.length) { + const shown = collection.slice(0, 8).join('; ') + (collection.length > 8 ? `; … (${collection.length - 8} more)` : ''); + summary.warnings.push(`Environment scope "${envScope.profile}" excluded ${collection.length} file(s) that carry scrape jobs or exporter evidence: ${shown}. An environment that names no deployment profile (e.g. \`all\`) reads every file.`); + } } const evidence = {}; // -> @@ -531,6 +583,12 @@ export function crawlFiles(filesInput, opts = {}) { const alertRules = []; // every alert rule found, raw, with its file and group (classified below) const metricDefinitions = []; const scrapeJobs = []; + const exporters = new Map(); // exporter name → the files that say what it exposes + // A collector config with no pipelines names its metrics exporter only by + // guess; the guesses are settled after every file is walked against the + // metrics pipelines the walk found (a config split across --config files + // declares them in another file) — settleCollectorGuesses. + const collectorState = newCollectorState(); const dashboards = []; const alertingRoutes = []; const pipelines = { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }; @@ -555,7 +613,7 @@ export function crawlFiles(filesInput, opts = {}) { // ----- pass 1: classify ----- const classified = []; for (const [relPath, content] of files) { - const kind = detectArtefactKind(relPath, content); + const kind = detectArtefactKind(relPath, content, { exporterCatalogue: catalogue }); if (kind === 'unknown') continue; classified.push({ relPath, content, kind }); summary.files.classified++; @@ -583,10 +641,13 @@ export function crawlFiles(filesInput, opts = {}) { case 'actuator-metrics-config': walkActuatorMetricsConfig(f, scrapeJobs, evidence, summary, repoName); break; case 'metric-source-code': walkMetricSourceCode(f, metricDefinitions, evidence, summary, repoName); break; case 'alertmanager': walkAlertmanager(f, alertingRoutes, evidence, summary); break; - case 'otel-collector': walkOtelCollector(f, pipelines, evidence, summary); break; + case 'otel-collector': walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, collectorState); break; + case 'metrics-manifest': walkMetricsManifest(f, metricDefinitions, exporters, summary, evidence); break; + case 'exposition-snapshot': walkExpositionSnapshot(f, metricDefinitions, exporters, summary, evidence); break; + case 'exporter-config': walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence); break; case 'grafana-dashboard':walkGrafanaDashboard(f, dashboards, metricDefinitions, evidence, summary); break; - case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, repoName }, evidence, summary); break; - case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents); break; + case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorState, repoName }, evidence, summary); break; + case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents, { scrapeJobs, collectorState }); break; case 'k8s-workload': walkK8sWorkload(f, backends, pipelines, evidence, summary); break; // Chart.yaml itself carries no observability contracts — the payloads // live in the templated manifests (handled as 'helm-template'). We @@ -598,6 +659,8 @@ export function crawlFiles(filesInput, opts = {}) { } } + settleCollectorGuesses(collectorState, pipelines, evidence, summary); + // ----- infer SLIs/SLOs (decompiler ⇄ compiler symmetry) ----- // PRIMARY source: recording rules. compile.mjs materialises every SLI in // spec.slis as `::` recording rules, so reading those names @@ -896,6 +959,7 @@ export function crawlFiles(filesInput, opts = {}) { curateMetricDefinitions(metricDefinitions, summary); const metricAnnotations = buildMetricDefinitionAnnotations(metricDefinitions); const scrapeAnnotations = buildScrapeJobAnnotations(scrapeJobs); + const exporterAnnotations = buildExporterAnnotations(exporters); // ----- canonical assembly ----- const canonical = { @@ -947,6 +1011,7 @@ export function crawlFiles(filesInput, opts = {}) { ...(sdkLanguages.length ? { 'crawler.discovered.sdk_languages': annotationJson(sdkLanguages) } : {}), ...metricAnnotations, ...scrapeAnnotations, + ...exporterAnnotations, ...scaffoldAnnotations, }, }, @@ -1117,7 +1182,7 @@ function sniffObservabilityKind(content) { if (Array.isArray(obj.scrape_configs) && obj.scrape_configs.some(s => s?.job_name)) return 'prometheus-scrape-config'; if (looksLikeActuatorMetricsConfig('', content, obj)) return 'actuator-metrics-config'; if (obj.route && Array.isArray(obj.receivers)) return 'alertmanager'; - if (obj.receivers && obj.exporters && obj.service?.pipelines) return 'otel-collector'; + if (looksLikeOtelCollector(obj)) return 'otel-collector'; if (obj.services && typeof obj.services === 'object' && Object.values(obj.services).some(s => s?.image)) return 'docker-compose'; if (K8S_WORKLOAD_KINDS.has(obj.kind) @@ -1410,7 +1475,8 @@ function addMetricDefinition(metricDefinitions, evidence, summary, item) { sourceName: item.sourceName && item.sourceName !== name ? item.sourceName : item.sourceName || null, originKind: item.originKind || 'source-code', candidateOnly: Boolean(item.candidateOnly), - confidence: item.candidateOnly ? 'candidate' : 'declared', + // A caller may state it ('inferred': an exporter config's family). + confidence: item.confidence || (item.candidateOnly ? 'candidate' : 'declared'), references: Array.isArray(item.references) ? item.references : [], usedBy: Array.isArray(item.usedBy) ? item.usedBy.filter(Boolean) : [], }; @@ -1435,7 +1501,9 @@ function mergeMetricEvidence(target, source) { target.usedBy = dedupe([...(target.usedBy || []), ...(source.usedBy || [])]).slice(0, 24); target.references = [...(target.references || []), ...(source.references || [])].slice(0, 12); target.candidateOnly = Boolean(target.candidateOnly && source.candidateOnly); - target.confidence = target.candidateOnly ? 'candidate' : 'declared'; + target.confidence = target.candidateOnly + ? (target.confidence === 'inferred' ? 'inferred' : 'candidate') + : 'declared'; } function dedupe(values) { @@ -1634,8 +1702,11 @@ function curateMetricDefinitions(metricDefinitions, summary) { let unreferenced = 0; for (const metric of metricDefinitions) { if (!metric.candidateOnly) continue; - metric.confidence = referenced.has(metric.name) ? 'referenced-candidate' : 'candidate'; if (!referenced.has(metric.name)) unreferenced++; + // An exporter config's family stays 'inferred': the config says the + // exporter is switched on, not that the name is right. + if (metric.confidence === 'inferred') continue; + metric.confidence = referenced.has(metric.name) ? 'referenced-candidate' : 'candidate'; } summary.discovered.metricEvidence = metricDefinitions.length; summary.discovered.metricDefinitions = new Set(metricDefinitions.map(m => m.name)).size; @@ -1693,12 +1764,19 @@ function annotationJson(value) { : compact; } +// Which file speaks for a metric named in several: what the exporter says it +// exposes first (its manifest, then a saved exposition), then the code that +// registers it, then what reads it, and last a name only inferred from an +// exporter's config. function metricOriginRank(metric) { const kind = metric?.originKind || 'source-code'; - if (kind === 'source-code') return 0; - if (kind === 'source-code-fragment') return 1; - if (kind === 'recording-rule-output') return 2; - if (kind === 'promql-reference') return 3; + if (kind === 'exporter-manifest') return 0; + if (kind === 'exporter-snapshot') return 1; + if (kind === 'source-code') return 2; + if (kind === 'source-code-fragment') return 3; + if (kind === 'recording-rule-output') return 4; + if (kind === 'promql-reference') return 5; + if (kind === 'exporter-config') return 6; return 9; } @@ -1717,6 +1795,14 @@ function buildScrapeJobAnnotations(scrapeJobs) { metrics_path: item.metrics_path || '', interval: item.interval || '', targets: item.targets || [], + // A job an OTel collector's prometheus receiver scrapes: the receiver, + // its static target count, and where the collector ships the samples. + ...(item.via ? { + via: item.via, + receiver: item.receiver, + target_count: item.target_count, + exporters: item.exporters, + } : {}), }; } return { @@ -1726,6 +1812,26 @@ function buildScrapeJobAnnotations(scrapeJobs) { }; } +// crawler.discovered.exporters: each prebuilt exporter the crawl learned +// metrics from, and the files that said what it exposes (a manifest, a saved +// exposition, its own config). +function buildExporterAnnotations(exporters) { + if (!exporters.size) return {}; + const names = [...exporters.keys()].sort(); + return { + 'crawler.discovered.exporters': annotationJson(Object.fromEntries(names.map(n => [n, exporters.get(n)]))), + 'crawler.discovered.exporters_count': String(names.length), + }; +} + +function recordExporter(exporters, name, source, summary) { + if (!exporters.has(name)) { + exporters.set(name, { sources: [] }); + summary.discovered.exporters = (summary.discovered.exporters || 0) + 1; + } + exporters.get(name).sources.push(source); +} + // Components disabled via Helm `enabled: false`, merged across the in-scope // values files. Base `values.yaml` is applied first; environment overlays // (`values-.yaml`) are applied after and win — mirroring `helm -f` order. @@ -1758,7 +1864,7 @@ function collectDisabledComponents(valuesFiles) { return disabled; } -function walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents = new Set()) { +function walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents = new Set(), { scrapeJobs = [], collectorState = newCollectorState() } = {}) { const visit = (node, path = []) => { if (!node) return; // Prune the subtree of any top-level component the environment disables @@ -1789,7 +1895,35 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo } }; - for (const obj of parseYamlDocs(f.content)) visit(obj); + for (const obj of parseYamlDocs(f.content)) { + // A collector chart's values carry the collector config: at the top + // level, under `config:` (the opentelemetry-collector chart's idiom), or + // under a subchart's key in an umbrella chart. Read first, so a pipeline + // it declares beats an image's inference; a config this walk cannot read + // (the chart's `pipelines.metrics: null` drops a pipeline) is warned + // about and never costs the values file its images. + for (const collector of valuesCollectorConfigs(obj, { disabled: disabledComponents })) { + try { walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorState); } + catch (e) { summary.warnings.push(`The collector config in ${f.relPath} could not be read (${e.message}); its images are read as before.`); } + } + visit(obj); + } +} + +// The collector configs a Helm values document carries: the document +// itself, its `config:`, or a subchart's `.config:` (an umbrella +// chart) — never a subchart that is disabled. A crawl knows that from +// `disabled`, the `enabled` toggles merged across the in-scope values files +// (an overlay wins, as with `helm -f`); a file read alone (the scope warning) +// from its own `enabled: false`. +function valuesCollectorConfigs(obj, { disabled = null } = {}) { + if (!isMapping(obj)) return []; + if (looksLikeOtelCollector(obj)) return [obj]; + if (isMapping(obj.config) && looksLikeOtelCollector(obj.config)) return [obj.config]; + const off = (key, v) => (disabled ? disabled.has(key) : v?.enabled === false); + return Object.entries(obj) + .filter(([key, v]) => !off(key, v) && isMapping(v?.config) && looksLikeOtelCollector(v.config)) + .map(([, v]) => v.config); } function workloadPodSpec(obj) { @@ -1820,14 +1954,14 @@ function walkK8sWorkload(f, backends, pipelines, evidence, summary) { // like a rule shipped as a standalone file. If the template embeds nothing, we // strip the scaffolding and treat the document itself as a single manifest. function walkHelmTemplate(f, buckets, evidence, summary) { - const { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, repoName } = buckets; + const { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorState, repoName } = buckets; const route = (kind, sub) => { switch (kind) { case 'prometheus-rules': walkPrometheusRules(sub, recordingRules, burnRateAlerts, alertRules, metricDefinitions, evidence, summary); return true; case 'prometheus-scrape-config': walkPrometheusScrapeConfig(sub, scrapeJobs, evidence, summary); return true; case 'actuator-metrics-config': walkActuatorMetricsConfig(sub, scrapeJobs, evidence, summary, repoName); return true; case 'alertmanager': walkAlertmanager(sub, alertingRoutes, evidence, summary); return true; - case 'otel-collector': walkOtelCollector(sub, pipelines, evidence, summary); return true; + case 'otel-collector': walkOtelCollector(sub, pipelines, scrapeJobs, evidence, summary, collectorState); return true; case 'grafana-dashboard':walkGrafanaDashboard(sub, dashboards, metricDefinitions, evidence, summary); return true; case 'docker-compose': walkDockerCompose(sub, backends, pipelines, evidence, summary); return true; case 'k8s-workload': walkK8sWorkload(sub, backends, pipelines, evidence, summary); return true; @@ -1849,6 +1983,9 @@ function walkHelmTemplate(f, buckets, evidence, summary) { else if (lk.endsWith('.json') || /"schemaVersion"\s*:/.test(clean)) kind = 'grafana-dashboard'; else if (/^route:/m.test(clean) && /^receivers:/m.test(clean)) kind = 'alertmanager'; else if (/^receivers:/m.test(clean) && /^exporters:/m.test(clean) && /pipelines:/.test(clean)) kind = 'otel-collector'; + // A collector config with no service block: a prometheus receiver's + // scrape jobs, or its exporters alone. + else if (/^receivers:/m.test(clean) && (/^exporters:/m.test(clean) || /^\s+scrape_configs:/m.test(clean))) kind = 'otel-collector'; // A ConfigMap whose data key looks like an observability artefact but // can't be parsed is a genuine signal worth surfacing; ordinary @@ -2066,28 +2203,49 @@ function walkPrometheusRule(f, group, rule, recordingRules, burnRateAlerts, aler } function walkPrometheusScrapeConfig(f, scrapeJobs, evidence, summary) { - for (const obj of parseYamlDocs(f.content)) { - if (!Array.isArray(obj?.scrape_configs)) continue; - for (const cfg of obj.scrape_configs) { - const job = String(cfg?.job_name || '').trim(); - if (!job) continue; + for (const obj of parseYamlDocs(f.content)) walkScrapeConfigDoc(obj, f.relPath, scrapeJobs, evidence, summary); +} + +// One Prometheus config document's scrape jobs — a standalone prometheus.yml +// or the `config:` of an OTel collector's prometheus receiver (`collector` +// then names the receiver and the exporters its samples leave by). +// Answers how many jobs it added. In a collector config a job this walk +// cannot read is warned about and skipped (its siblings and the rest of the +// config are still read); a standalone prometheus.yml reads as before. +function walkScrapeConfigDoc(obj, origin, scrapeJobs, evidence, summary, collector = null) { + if (!Array.isArray(obj?.scrape_configs)) return 0; + let added = 0; + for (const cfg of obj.scrape_configs) { + const job = String(cfg?.job_name || '').trim(); + if (!job) continue; + const read = () => { // Promtail and Alloy use the same `scrape_configs` / `job_name` keys // to say which LOG FILES to tail. Such an entry is not a metrics // scrape job and never appears among a metrics store's targets; // declaring it made it read "not live" on every platform. if (isLogScrapeConfig(cfg, obj)) { summary.discovered.logScrapeConfigs = (summary.discovered.logScrapeConfigs || 0) + 1; - continue; + return; } - addScrapeJob(scrapeJobs, evidence, summary, { + if (addScrapeJob(scrapeJobs, evidence, summary, { job, metrics_path: cfg.metrics_path || '/metrics', interval: cfg.scrape_interval || obj.global?.scrape_interval || null, targets: scrapeTargets(cfg), - origin: f.relPath, - }); - } + origin, + ...(collector ? { + via: 'otel-collector', + receiver: collector.receiver, + target_count: (Array.isArray(cfg.static_configs) ? cfg.static_configs : []).reduce((n, sc) => n + (Array.isArray(sc?.targets) ? sc.targets.filter(Boolean).length : 0), 0), + exporters: collector.exporters, + } : {}), + })) added++; + }; + if (!collector) { read(); continue; } + try { read(); } + catch (e) { summary.warnings.push(`Scrape job '${job}' of receiver ${collector.receiver} in ${origin} could not be read (${e.message}); the rest of the config is read.`); } } + return added; } // A log-collection scrape config (Promtail, Alloy's loki.source): it has @@ -2117,11 +2275,12 @@ function walkActuatorMetricsConfig(f, scrapeJobs, evidence, summary, repoName = } function addScrapeJob(scrapeJobs, evidence, summary, item) { - if (!item?.job) return; - if (scrapeJobs.some(j => j.job === item.job && j.origin === item.origin)) return; + if (!item?.job) return false; + if (scrapeJobs.some(j => j.job === item.job && j.origin === item.origin)) return false; scrapeJobs.push(item); if (!evidence[`scrape.${item.job}`]) evidence[`scrape.${item.job}`] = item.origin; summary.discovered.scrapeJobs++; + return true; } function scrapeTargets(cfg) { @@ -2169,11 +2328,31 @@ function walkAlertmanager(f, routes, evidence, summary) { } } -function walkOtelCollector(f, pipelines, evidence, summary) { - for (const obj of parseYamlDocs(f.content)) { - if (!obj?.service?.pipelines) continue; +// An OTel collector config: its pipelines (receivers, processors, the +// exporter per signal) and the scrape jobs of its prometheus receivers. A +// config with no `service:` block (a Helm chart's files/ copy, assembled +// with the pipelines elsewhere) still names its receivers and exporters: +// the prometheus receivers' jobs are read, and the metrics exporter it +// configures is a guess, settled after the walk (settleCollectorGuesses). +function walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, state = newCollectorState()) { + for (const obj of parseYamlDocs(f.content)) walkOtelCollectorDoc(obj, f.relPath, pipelines, scrapeJobs, evidence, summary, state); +} + +// What the walk learns about collector configs across files: each metrics +// pipeline declared (its receivers and exporters, with the exporters' +// configs), and each pipeline-less config's guess. +function newCollectorState() { + return { declared: [], guesses: [] }; +} + +function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, summary, state) { + const scrapeReceivers = collectorScrapeReceivers(obj); + const exporterCfgs = isMapping(obj?.exporters) ? obj.exporters : {}; + if (obj?.service?.pipelines) { const seen = new Set(); for (const pipeline of Object.values(obj.service.pipelines)) { + // `metrics: null` removes a pipeline (the Helm chart's idiom). + if (!isMapping(pipeline)) continue; for (const recv of pipeline.receivers || []) { if (!seen.has(`r:${recv}`)) { seen.add(`r:${recv}`); @@ -2197,11 +2376,269 @@ function walkOtelCollector(f, pipelines, evidence, summary) { } } } - evidence[`PIP-${summary.discovered.pipelines + 1}`] = f.relPath; + evidence[`PIP-${summary.discovered.pipelines + 1}`] = relPath; + summary.discovered.pipelines++; + for (const [sig, p] of Object.entries(obj.service.pipelines)) { + if (/^metrics/i.test(sig) && isMapping(p) && Array.isArray(p.receivers)) { + state.declared.push({ receivers: p.receivers, exporters: Array.isArray(p.exporters) ? p.exporters : [], cfgs: exporterCfgs }); + } + } + // Each scrape receiver's samples leave by the exporters of the metrics + // pipelines that list it. + for (const { key, config } of scrapeReceivers) { + const names = []; + for (const [sig, p] of Object.entries(obj.service.pipelines)) { + if (!/^metrics/i.test(sig) || !Array.isArray(p?.receivers) || !p.receivers.includes(key)) continue; + for (const exp of p.exporters || []) if (!names.includes(exp)) names.push(exp); + } + const exporters = names.map(n => collectorExporterDetail(n, exporterCfgs[n])); + const jobs = walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + // No metrics pipeline here lists the receiver: another file's may (a + // config split across --config files), settled after the walk. + if (!names.length) state.guesses.push({ relPath, candidates: [], cfgs: exporterCfgs, groups: [{ receiver: key, exporters, jobs }], receivers: [] }); + } + return; + } + + // No pipelines: the metrics exporter is the metrics-capable exporter the + // config names (the first when it names several) — a guess. Its scrape + // jobs carry it for now; settleCollectorGuesses replaces it with a + // declared pipeline's exporters when one lists the receiver, and applies + // it to spec.pipelines only when nothing else set the metrics exporter. + const candidates = Object.keys(exporterCfgs).filter(n => METRICS_EXPORTER_TYPES.has(n.split('/')[0].toLowerCase())); + const chosen = candidates[0] || null; + if (!scrapeReceivers.length && !chosen) return; + const groups = []; + for (const { key, config } of scrapeReceivers) { + // One array per receiver, shared with its jobs' entries (patched in place). + const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; + const jobs = walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + groups.push({ receiver: key, exporters, jobs }); + } + state.guesses.push({ relPath, candidates, cfgs: exporterCfgs, groups, receivers: scrapeReceivers.map(({ key }) => key.split('/')[0]) }); +} + +// After every file is walked: a pipeline-less config's scrape jobs leave by +// the exporters of the declared metrics pipelines that list their receiver +// (a config split across --config files), else by its guess; its receivers +// join spec.pipelines; the first guess sets spec.pipelines.exporters.metrics +// only when nothing else set it (no declared pipeline, no metrics backend +// image). A first-of-several guess left standing is warned about. +function settleCollectorGuesses(state, pipelines, evidence, summary) { + const warned = new Set(); + const warnFirstOfSeveral = (g) => { + if (g.candidates.length < 2 || warned.has(g.relPath)) return; + warned.add(g.relPath); + summary.warnings.push(`${g.relPath} has no service pipelines and ${g.candidates.length} metrics-capable exporters (${g.candidates.join(', ')}); the metrics exporter is taken to be the first, '${g.candidates[0]}'.`); + }; + for (const g of state.guesses) { + for (const group of g.groups) { + const names = []; + const cfgOf = {}; + for (const d of state.declared) { + if (!d.receivers.includes(group.receiver)) continue; + for (const n of d.exporters) { + if (typeof n !== 'string') continue; + if (!names.includes(n)) names.push(n); + if (isMapping(d.cfgs?.[n]) && Object.keys(d.cfgs[n]).length && !cfgOf[n]) cfgOf[n] = d.cfgs[n]; + } + } + if (names.length) { + group.exporters.splice(0, group.exporters.length, + ...names.map(n => collectorExporterDetail(n, isMapping(g.cfgs[n]) && Object.keys(g.cfgs[n]).length ? g.cfgs[n] : cfgOf[n]))); + } else if (group.exporters.length && group.jobs) { + warnFirstOfSeveral(g); + } + } + for (const name of g.receivers) { + if (!pipelines.receivers.some(r => r.name === name)) pipelines.receivers.push({ name }); + } + } + const first = state.guesses.find(g => g.candidates.length); + if (!pipelines.exporters.metrics && first) { + pipelines.exporters.metrics = { kind: mapExporterKind(first.candidates[0].split('/')[0]) }; + evidence['pipelines.exporters.metrics'] = first.relPath; + warnFirstOfSeveral(first); + evidence[`PIP-${summary.discovered.pipelines + 1}`] = first.relPath; summary.discovered.pipelines++; } } +// Collector exporters that carry metrics, by component type. +const METRICS_EXPORTER_TYPES = new Set([ + 'kafka', 'prometheusremotewrite', 'prometheus', 'otlp', 'otlphttp', + 'googlemanagedprometheus', 'awsemf', 'datadog', 'influxdb', 'clickhouse', + 'signalfx', 'carbon', 'azuremonitor', 'splunk_hec', 'sumologic', +]); + +// A collector's prometheus receivers (`prometheus`, `prometheus/`) +// that state scrape jobs: [{ key, config }]. +function collectorScrapeReceivers(obj) { + if (!isMapping(obj?.receivers)) return []; + return Object.entries(obj.receivers) + .filter(([key, r]) => /^prometheus(\/|$)/.test(key) && isMapping(r?.config) && Array.isArray(r.config.scrape_configs)) + .map(([key, r]) => ({ key, config: r.config })); +} + +// An OTel collector config by shape: receivers, exporters and pipelines (a +// full config), or a receivers mapping with a prometheus receiver's scrape +// jobs or with exporters (one with no service block). Alertmanager's +// `receivers` is a list, never a mapping. +function looksLikeOtelCollector(obj) { + if (obj.receivers && obj.exporters && obj.service?.pipelines) return true; + if (!isMapping(obj.receivers)) return false; + return collectorScrapeReceivers(obj).length > 0 || isMapping(obj.exporters); +} + +// What a collector exporter is and where it sends: a kafka exporter's +// metrics topic, any other's endpoint — reduced by safeSourceUrl, so no +// userinfo, query or fragment is recorded. +function collectorExporterDetail(name, cfg) { + const type = String(name).split('/')[0]; + const out = { name: String(name), kind: mapExporterKind(type) }; + if (/^kafka$/i.test(type)) { + const topic = cfg?.metrics?.topic ?? cfg?.topic; + if (typeof topic === 'string' && topic.trim()) out.topic = topic.trim(); + } else { + const endpoint = cfg?.endpoint ?? cfg?.metrics_endpoint; + const safe = typeof endpoint === 'string' ? safeSourceUrl(endpoint) : ''; + if (safe) out.endpoint = safe; + } + return out; +} + +// The scrape jobs or exporter evidence a file carries, as one line for the +// environment-scope warning (` (scrape jobs: a, b)`), or null. +function collectionEvidenceOf(relPath, content, catalogue) { + if (typeof content !== 'string') return null; + let kind; + try { kind = detectArtefactKind(relPath, content, { exporterCatalogue: catalogue }); } catch { return null; } + const jobsOf = (docs) => docs.flatMap(d => (Array.isArray(d?.scrape_configs) ? d.scrape_configs : [])) + .map(c => String(c?.job_name || '').trim()).filter(Boolean); + try { + if (kind === 'prometheus-scrape-config') { + const jobs = jobsOf(parseYamlDocs(content)); + return jobs.length ? `${relPath} (scrape jobs: ${jobs.join(', ')})` : null; + } + if (kind === 'otel-collector') { + const jobs = jobsOf(parseYamlDocs(content).flatMap(d => collectorScrapeReceivers(d).map(r => r.config))); + return jobs.length ? `${relPath} (collector scrape jobs: ${jobs.join(', ')})` : null; + } + if (kind === 'exporter-config') { + const entry = matchExporterConfig(parseYaml(content), catalogue); + return `${relPath} (exporter config: ${entry?.id ?? 'unknown'})`; + } + if (kind === 'helm-values') { + const jobs = jobsOf(parseYamlDocs(content).flatMap(d => valuesCollectorConfigs(d)) + .flatMap(c => collectorScrapeReceivers(c).map(r => r.config))); + return jobs.length ? `${relPath} (collector scrape jobs: ${jobs.join(', ')})` : null; + } + if (kind === 'helm-template') { + // The jobs the template's walker would read (its ConfigMap values or + // the whole template), walked into throwaway buckets. + const buckets = { + backends: [], recordingRules: [], burnRateAlerts: [], alertRules: [], metricDefinitions: [], scrapeJobs: [], + alertingRoutes: [], dashboards: [], collectorState: newCollectorState(), repoName: null, + pipelines: { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }, + }; + const scratch = { + discovered: {}, warnings: [], invented: { endpoints: [], channels: [] }, + omitted: { syntheticRecordingRules: [], unresolvedChannels: [], recordingRules: [], ruleIntervals: [] }, + }; + walkHelmTemplate({ relPath, content }, buckets, {}, scratch); + const jobs = [...new Set(buckets.scrapeJobs.map(j => j.job))]; + return jobs.length ? `${relPath} (template scrape jobs: ${jobs.join(', ')})` : null; + } + } catch { /* unreadable: nothing to report */ } + return null; +} + +// ---------- prebuilt exporters ---------- + +// A metrics manifest: each entry is a declared metric of the exporter. +function walkMetricsManifest(f, metricDefinitions, exporters, summary, evidence) { + const read = readMetricsManifest(f.content); + if (!read.ok) { + summary.warnings.push(`${f.relPath} is not a metrics manifest (${read.error}); it is not read.`); + return; + } + const m = read.manifest; + for (const metric of m.metrics) { + addMetricDefinition(metricDefinitions, evidence, summary, { + name: metric.name, + type: metric.type, + help: metric.help, + labels: metric.labels, + service: m.exporter, + origin: f.relPath, + originKind: 'exporter-manifest', + confidence: 'declared', + }); + } + if (read.skipped) { + summary.warnings.push(`${f.relPath}: ${read.skipped} metric entr${read.skipped === 1 ? 'y is' : 'ies are'} not read (each needs a metric name not listed before and an exposition type: ${['counter', 'gauge', 'histogram', 'summary', 'untyped'].join(', ')}, …).`); + } + recordExporter(exporters, m.exporter, { + origin_kind: 'exporter-manifest', + file: f.relPath, + metrics: m.metrics.length, + ...(m.version ? { version: m.version } : {}), + ...(m.source ? { source: m.source } : {}), + ...(m.generatedAt ? { generated_at: m.generatedAt } : {}), + }, summary); +} + +// A saved exposition: the exporter is named by the file (`ibm-mq.prom` → +// ibm-mq). +function walkExpositionSnapshot(f, metricDefinitions, exporters, summary, evidence) { + const { families } = parseExposition(f.content); + const exporter = basenameOf(f.relPath).replace(/\.(prom|metrics)$/i, '') || 'exporter'; + for (const fam of families) { + addMetricDefinition(metricDefinitions, evidence, summary, { + name: fam.name, + type: fam.type, + help: fam.help, + labels: fam.labels, + service: exporter, + origin: f.relPath, + originKind: 'exporter-snapshot', + confidence: 'declared', + }); + } + recordExporter(exporters, exporter, { origin_kind: 'exporter-snapshot', file: f.relPath, metrics: families.length }, summary); +} + +// An exporter's own config, read against the catalogue: the families it +// switches on, as candidates. +function walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence) { + for (const obj of parseYamlDocs(f.content)) { + const entry = matchExporterConfig(obj, catalogue); + if (!entry) continue; + const inferred = inferExporterFamilies(obj, entry); + for (const fam of inferred.families) { + addMetricDefinition(metricDefinitions, evidence, summary, { + name: fam.name, + type: fam.type, + help: fam.help, + labels: fam.labels, + service: entry.id, + origin: f.relPath, + originKind: 'exporter-config', + confidence: 'inferred', + candidateOnly: true, + }); + } + recordExporter(exporters, entry.id, { + origin_kind: 'exporter-config', + file: f.relPath, + metrics: inferred.families.length, + namespace: inferred.namespace, + gates: inferred.gates, + scopes: inferred.scopes, + }, summary); + } +} + function mapExporterKind(name) { // The spec's exporter kind taxonomy is small; map common collector // exporters to it. diff --git a/tools/lib/exporter-catalogue.mjs b/tools/lib/exporter-catalogue.mjs new file mode 100644 index 00000000..d9776b59 --- /dev/null +++ b/tools/lib/exporter-catalogue.mjs @@ -0,0 +1,74 @@ +// ============================================================ +// exporter-catalogue.mjs — the built-in exporter family catalogue. +// +// DATA, not engine code: the crawler (tools/lib/crawler.mjs, through +// tools/lib/exporter-metrics.mjs) reads a prebuilt exporter's config file +// against these entries and emits the metric families the config switches +// on, as candidates (`originKind: 'exporter-config'`, `confidence: +// 'inferred'`). A downstream carries a larger catalogue as a JSON file of +// the same shape (`packc x-ray --exporter-catalogue `, or +// `crawlFiles(files, { exporterCatalogue })`): its entries replace a +// built-in entry with the same `id` and add the rest, so the engine is +// never diffed. docs/DOWNSTREAM.md §18 has the format. +// +// An entry: +// id the exporter's name in crawler.discovered.exporters +// match.keys top-level mapping keys a config file must all carry +// namespace { path, default } — the metric name prefix the config sets +// gates { : { path, default } } — boolean switches; a family +// with `gate` is emitted only when that switch is on +// scopes { : { path } } — object selections; a family with +// `scope` is emitted only when that selection is non-empty +// families [{ name, type, help, labels, gate?, scope? }] — `name` is +// joined to the namespace with `_` +// +// The IBM entry is a small subset of what mq-metric-samples' Prometheus +// collector exposes, enough to name the queue manager, queue and channel +// families a config enables. The names are inferred, not observed: publish +// the exporter's real list with `packc emit --exporter ` and the +// manifest outranks every name here. +// ============================================================ + +export const EXPORTER_CATALOGUE = { + version: 1, + exporters: [ + { + id: 'ibm-mq', + title: 'IBM MQ — mq-metric-samples (Prometheus collector)', + source: 'https://github.com/ibm-messaging/mq-metric-samples', + match: { keys: ['global', 'connection', 'objects', 'prometheus'] }, + namespace: { path: 'prometheus.namespace', default: 'ibmmq' }, + gates: { + objectStatus: { path: 'global.useObjectStatus', default: true }, + publications: { path: 'global.usePublications', default: true }, + }, + scopes: { + queues: { path: 'objects.queues' }, + channels: { path: 'objects.channels' }, + }, + families: [ + { name: 'qmgr_status', type: 'gauge', help: 'Queue manager status', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_uptime', type: 'gauge', help: 'Queue manager uptime in seconds', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_connection_count', type: 'gauge', help: 'Connections to the queue manager', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_channel_initiator_status', type: 'gauge', help: 'Channel initiator status', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_command_server_status', type: 'gauge', help: 'Command server status', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_user_cpu_time_percentage', type: 'gauge', help: 'User CPU time of the queue manager', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_system_cpu_time_percentage', type: 'gauge', help: 'System CPU time of the queue manager', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_log_in_use_bytes', type: 'gauge', help: 'Recovery log in use', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_log_write_latency_seconds', type: 'gauge', help: 'Recovery log write latency', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_mqconn_mqconnx_count', type: 'counter', help: 'MQCONN and MQCONNX calls', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'queue_depth', type: 'gauge', help: 'Messages on the queue', labels: ['qmgr', 'queue'], gate: 'publications', scope: 'queues' }, + { name: 'queue_mqput_mqput1_count', type: 'counter', help: 'MQPUT and MQPUT1 calls on the queue', labels: ['qmgr', 'queue'], gate: 'publications', scope: 'queues' }, + { name: 'queue_mqget_count', type: 'counter', help: 'MQGET calls on the queue', labels: ['qmgr', 'queue'], gate: 'publications', scope: 'queues' }, + { name: 'queue_oldest_message_age', type: 'gauge', help: 'Age of the oldest message on the queue in seconds', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'queue_uncommitted_messages', type: 'gauge', help: 'Uncommitted messages on the queue', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'queue_time_since_put', type: 'gauge', help: 'Seconds since the last put', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'queue_time_since_get', type: 'gauge', help: 'Seconds since the last get', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'channel_status', type: 'gauge', help: 'Channel status', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + { name: 'channel_messages', type: 'counter', help: 'Messages sent over the channel', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + { name: 'channel_bytes_sent', type: 'counter', help: 'Bytes sent over the channel', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + { name: 'channel_bytes_rcvd', type: 'counter', help: 'Bytes received over the channel', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + ], + }, + ], +}; diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs new file mode 100644 index 00000000..194adaac --- /dev/null +++ b/tools/lib/exporter-metrics.mjs @@ -0,0 +1,436 @@ +// ============================================================ +// exporter-metrics.mjs — what a prebuilt exporter exposes, read three ways. +// +// A service that runs a prebuilt exporter (IBM's mq-metric-samples, a +// node_exporter, a vendor's sidecar) has no client-library call or PromQL +// that names the exporter's metrics: its repository is a Dockerfile and a +// config file. The crawler (tools/lib/crawler.mjs) reads the metric names +// from, strongest first: +// +// 1. a metrics manifest — `metrics-manifest.json` / `*.metrics-manifest.json`, +// { exporter, version?, source?, generatedAt?, metrics: [{ name, type, +// help?, labels? }] }; `packc emit --exporter ` writes one from a +// live /metrics endpoint (originKind 'exporter-manifest', declared); +// 2. an exposition snapshot — a saved /metrics response (`.prom`, +// `.metrics`), its # HELP / # TYPE lines and the label names of its +// samples (originKind 'exporter-snapshot', declared); +// 3. the exporter's own config file, matched against a family catalogue +// (tools/lib/exporter-catalogue.mjs, or a downstream's JSON file of the +// same shape): the families the config switches on, as candidates +// (originKind 'exporter-config', confidence 'inferred'). +// +// Pure: no fs, no network — the crawler, the CLI (tools/emit-manifest.mjs) +// and the browser share it. Label VALUES are never kept (a channel's +// connname is an address); only label names, types and help text. +// ============================================================ + +import { EXPORTER_CATALOGUE as BUILTIN_CATALOGUE } from './exporter-catalogue.mjs'; + +/** The exposition types a manifest entry or a # TYPE line may state (text format 0.0.4 and OpenMetrics). */ +export const EXPOSITION_TYPES = Object.freeze(['counter', 'gauge', 'histogram', 'summary', 'untyped', 'unknown', 'gaugehistogram', 'info', 'stateset']); +const TYPE_SET = new Set(EXPOSITION_TYPES); +const NAME_RE = /^[A-Za-z_:][A-Za-z0-9_:]*$/; +const LABEL_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; +const LABEL_CAP = 32; + +// A sample of a family is named the family plus one of these suffixes. +const FAMILY_SUFFIXES = { + counter: ['_total', '_created'], + histogram: ['_bucket', '_sum', '_count', '_created'], + summary: ['_sum', '_count', '_created'], + gaugehistogram: ['_bucket', '_gsum', '_gcount'], + info: ['_info'], +}; +// Labels the exposition adds to a family's samples, not the family's own. +const SAMPLE_ONLY_LABELS = { histogram: ['le'], gaugehistogram: ['le'], summary: ['quantile'] }; + +/** `metrics-manifest.json` or `.metrics-manifest.json`, any folder. */ +export function isMetricsManifestPath(relPath) { + return /(^|[/\\.])metrics-manifest\.json$/i.test(String(relPath ?? '')); +} + +/** + * A path a saved exposition is never read from: a folder whose name is a + * test, spec, fixture, mock or test-data word, alone or joined to others by + * `-` / `_` and wrapped in `__` or not (`test/`, `__tests__/`, `e2e/`, + * `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, + * `testing/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `testdata/`, + * `test-data/`, `__snapshots__/`, …), a camelCase test source set + * (`componentTest/`, `smokeTest/`, `testUtils/`), Maven's `src/it/`, or a + * `*.test.*` / `*.spec.*` / `*_test.*` / `*_spec.*` file — there an + * exposition is sample input, not what a deployed exporter serves. + */ +export function isTestOrFixturePath(relPath) { + const segs = String(relPath ?? '').replace(/\\/g, '/').split('/'); + const word = '(?:tests?|specs?|fixtures?|mocks?|testing|e2e|test[-_]?(?:data|fixtures?|resources|utils?|support|helpers?)|(?:integration|unit|e2e|functional|acceptance)[-_]?tests?)'; + const folder = new RegExp(`^(?:__)?(?:[a-z0-9]+[-_])*${word}(?:[-_][a-z0-9]+)*(?:__)?$|^__snapshots__$`); + // A camelCase test source set (Gradle: componentTest, smokeTest, intTest, + // testFixtures, testUtils) and Maven's src/it. + const camel = /^[a-z][a-zA-Z0-9]*Tests?$|^tests?[A-Z][a-zA-Z0-9]*$/; + return segs.slice(0, -1).some((seg, i) => folder.test(seg.toLowerCase()) || camel.test(seg) + || (/^(?:it|int)$/i.test(seg) && segs[i - 1]?.toLowerCase() === 'src')) + || /[._](?:test|spec)\.[a-z]+$/i.test(segs[segs.length - 1] || ''); +} + +/** A saved exposition: `.prom` or `.metrics`. */ +export function isExpositionSnapshotPath(relPath) { + return /\.(prom|metrics)$/i.test(String(relPath ?? '')); +} + +/** + * A URL as it may be recorded: never the userinfo, query or fragment, where a + * credential rides. Textual, never a URL parse: a password may hold `/`, `?` + * or `@`, which end a parsed authority early and leave the rest of the secret + * in the "host" or "path". So everything before the LAST `@` goes (a value + * whose path holds an `@` keeps only what follows it, which loses the host + * rather than risk a credential), then everything from the first `?` or + * `#`. A value with a `?` or `#` before its last `@` (the `@` in a query, or + * a `?` in a password: text cannot tell which) is recorded as nothing (''); + * so is one with nothing left past its scheme. A leading `scheme://` or `//` is kept; a value that is not a + * URL at all (`prometheus:9090`, `${env:ENDPOINT}`) passes through the same + * rule. + */ +export function safeSourceUrl(raw) { + // Tab and line breaks go first, as a URL parser drops them (the value + // would still send its query). + const s = String(raw ?? '').replace(/[\t\n\r]/g, '').trim(); + if (!s) return ''; + const prefix = /^(?:[A-Za-z][A-Za-z0-9+.-]*:)?\/\//.exec(s)?.[0] ?? ''; + let rest = s.slice(prefix.length); + const at = rest.lastIndexOf('@'); + const query = rest.search(/[?#]/); + // A `?` or `#` before the last `@`: the `@` is in a query or fragment, or + // a password holds the `?` — text cannot tell which, so nothing after the + // scheme is kept. + if (at >= 0 && query >= 0 && query < at) return ''; + if (at >= 0) rest = rest.slice(at + 1); + rest = rest.replace(/[?#][\s\S]*$/, ''); + // Nothing past the scheme: nothing to record. + return rest ? prefix + rest : ''; +} + +function parseSampleLine(line) { + const m = /^[A-Za-z_:][A-Za-z0-9_:]*/.exec(line); + if (!m) return null; + const name = m[0]; + const labels = []; + let i = name.length; + // A sample ends in its value: a number, NaN or ±Inf (then an optional + // timestamp). A line that does not is not a sample. + const valued = (at) => (/^[ \t]+[-+]?(?:\d|\.\d|Inf|NaN)/i.test(line.slice(at)) ? { name, labels } : null); + while (line[i] === ' ' || line[i] === '\t') i++; + if (line[i] !== '{') return valued(name.length); + i++; + for (;;) { + while (line[i] === ' ' || line[i] === '\t' || line[i] === ',') i++; + if (i >= line.length) return null; + if (line[i] === '}') return valued(i + 1); + const lm = /^[A-Za-z_][A-Za-z0-9_]*/.exec(line.slice(i)); + if (!lm) return null; + labels.push(lm[0]); + i += lm[0].length; + while (line[i] === ' ' || line[i] === '\t') i++; + if (line[i] !== '=') return null; + i++; + while (line[i] === ' ' || line[i] === '\t') i++; + if (line[i] !== '"') return null; + i++; + while (i < line.length && line[i] !== '"') i += line[i] === '\\' ? 2 : 1; + if (i >= line.length) return null; + i++; + } +} + +function unescapeHelp(s) { + return s.replace(/\\(\\|n)/g, (_, c) => (c === 'n' ? '\n' : '\\')); +} + +/** + * Parse a Prometheus / OpenMetrics text exposition into its metric families: + * `{ families: [{ name, type, help, labels }], samples }`, families sorted by + * name, labels in first-seen order. A family's name is the one Prometheus + * stores: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is + * `build_info`. A sample with no # TYPE for it (or for + * the family its suffix names) is its own `untyped` family. + */ +export function parseExposition(text) { + const families = new Map(); + const family = (name) => { + if (!families.has(name)) families.set(name, { name, type: 'untyped', help: '', labels: [], typed: false, stored: '' }); + return families.get(name); + }; + let samples = 0; + for (const raw of String(text ?? '').replace(/\r\n?/g, '\n').split('\n')) { + const line = raw.trim(); + if (!line) continue; + if (line.startsWith('#')) { + const meta = /^#\s*(HELP|TYPE)\s+([A-Za-z_:][A-Za-z0-9_:]*)(?:\s+(.*))?$/.exec(line); + if (!meta) continue; + const f = family(meta[2]); + if (meta[1] === 'HELP') f.help = unescapeHelp(meta[3] ?? ''); + else { + const t = String(meta[3] ?? '').trim().toLowerCase(); + f.type = TYPE_SET.has(t) ? t : 'untyped'; + f.typed = true; + } + continue; + } + const sample = parseSampleLine(line); + if (!sample) continue; + samples++; + let f = families.get(sample.name); + if (!f || (!f.typed && !f.help)) { + for (const [type, suffixes] of Object.entries(FAMILY_SUFFIXES)) { + const suffix = suffixes.find(s => sample.name.endsWith(s) && sample.name.length > s.length); + const base = suffix ? families.get(sample.name.slice(0, -suffix.length)) : null; + if (base && base.type === type) { + f = base; + // An OpenMetrics counter or info family is stored (and queried) + // under its sample's name: `jobs` → jobs_total, `build` → build_info. + if ((suffix === '_total' || suffix === '_info') && !f.stored) f.stored = sample.name; + break; + } + } + } + if (!f) f = family(sample.name); + const skip = SAMPLE_ONLY_LABELS[f.type] || []; + for (const l of sample.labels) { + if (!skip.includes(l) && !f.labels.includes(l) && f.labels.length < LABEL_CAP) f.labels.push(l); + } + } + // OpenMetrics (`# EOF`) names a counter family without `_total` and an + // info family without `_info`; with no sample to say so, the stored name + // is the family's plus that suffix. + const openMetrics = /^#\s*EOF\s*$/m.test(String(text ?? '')); + const storedName = (f) => f.stored + || (openMetrics && f.type === 'counter' && !f.name.endsWith('_total') ? `${f.name}_total` + : openMetrics && f.type === 'info' && !f.name.endsWith('_info') ? `${f.name}_info` : f.name); + const out = [...families.values()] + .map((f) => ({ name: storedName(f), type: f.type, help: f.help, labels: f.labels })) + .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + return { families: out, samples }; +} + +/** + * A metrics manifest built from an exposition (what `packc emit --exporter` + * writes): `{ exporter, version?, source?, generatedAt?, metrics }`, the + * metrics sorted by name, `source` reduced by safeSourceUrl. + */ +export function manifestFromExposition(text, { exporter, version, source, generatedAt } = {}) { + const { families } = parseExposition(text); + const manifest = { exporter: String(exporter ?? '').trim() }; + if (version) manifest.version = String(version); + const src = safeSourceUrl(source); + if (src) manifest.source = src; + if (generatedAt) manifest.generatedAt = String(generatedAt); + manifest.metrics = families.map(f => ({ + name: f.name, + type: f.type, + ...(f.help ? { help: f.help } : {}), + ...(f.labels.length ? { labels: f.labels } : {}), + })); + return manifest; +} + +/** + * Read a metrics manifest's text. `{ ok: true, manifest, skipped }` with the + * entries that are not `{ name: , type: }` + * counted in `skipped` (a repeated name keeps its first entry), or + * `{ ok: false, error }` when the text is not a manifest at all. + */ +export function readMetricsManifest(text) { + let obj; + // Never the parser's message: it quotes the start of the text, which in a + // file that is not a manifest may be a key. + try { obj = JSON.parse(stripBom(text)); } catch { return { ok: false, error: 'not JSON' }; } + if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return { ok: false, error: 'not a JSON object' }; + const exporter = typeof obj.exporter === 'string' ? obj.exporter.trim() : ''; + if (!exporter) return { ok: false, error: 'no `exporter` name' }; + if (!Array.isArray(obj.metrics)) return { ok: false, error: 'no `metrics` array' }; + const metrics = []; + const seen = new Set(); + let skipped = 0; + for (const m of obj.metrics) { + const name = typeof m?.name === 'string' ? m.name.trim() : ''; + const type = typeof m?.type === 'string' ? m.type.trim().toLowerCase() : ''; + if (!NAME_RE.test(name) || !TYPE_SET.has(type) || seen.has(name)) { skipped++; continue; } + seen.add(name); + metrics.push({ + name, + type, + help: typeof m.help === 'string' ? m.help : '', + labels: Array.isArray(m.labels) ? [...new Set(m.labels.filter(l => typeof l === 'string' && LABEL_RE.test(l)))].slice(0, LABEL_CAP) : [], + }); + } + const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : ''); + return { + ok: true, + skipped, + manifest: { + exporter, + version: str(obj.version) || (typeof obj.version === 'number' ? String(obj.version) : ''), + source: safeSourceUrl(str(obj.source)), + generatedAt: str(obj.generatedAt), + metrics, + }, + }; +} + +// ---------- the exporter config catalogue ---------- + +const isMapping = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); +// One leading U+FEFF, as mini-yaml strips it (Windows editors write one). +const stripBom = (text) => String(text ?? '').replace(/^\uFEFF/, ''); + +function pathValue(obj, path) { + let cur = obj; + for (const key of String(path ?? '').split('.').filter(Boolean)) { + if (!isMapping(cur) || !Object.hasOwn(cur, key)) return undefined; + cur = cur[key]; + } + return cur; +} + +function normalizeEntry(e, i) { + const where = `exporter catalogue entry ${i + 1}${typeof e?.id === 'string' && e.id ? ` ('${e.id}')` : ''}`; + if (!isMapping(e)) return { error: `${where} is not an object` }; + if (typeof e.id !== 'string' || !/^[a-z][a-z0-9_.-]{0,62}$/.test(e.id)) return { error: `${where}: \`id\` must be a lowercase name (letters, digits, \`_\`, \`.\`, \`-\`)` }; + const keys = Array.isArray(e.match?.keys) ? e.match.keys.filter(k => typeof k === 'string' && /^[A-Za-z_][A-Za-z0-9_-]*$/.test(k)) : []; + if (!keys.length || keys.length !== e.match.keys.length) return { error: `${where}: \`match.keys\` must list the top-level keys a config carries` }; + const switches = (o, kind) => { + if (o === undefined) return {}; + if (!isMapping(o) || Object.values(o).some(v => !isMapping(v) || typeof v.path !== 'string')) throw new Error(`${where}: every ${kind} needs a \`path\``); + return Object.fromEntries(Object.entries(o).map(([k, v]) => [k, { path: v.path, default: v.default !== false }])); + }; + let gates, scopes; + try { gates = switches(e.gates, 'gate'); scopes = switches(e.scopes, 'scope'); } + catch (err) { return { error: err.message }; } + const ns = isMapping(e.namespace) ? e.namespace : {}; + if (ns.default !== undefined && ns.default !== '' && !(typeof ns.default === 'string' && NAME_RE.test(ns.default))) { + return { error: `${where}: \`namespace.default\` must be a metric name prefix` }; + } + if (!Array.isArray(e.families) || !e.families.length) return { error: `${where}: \`families\` must list at least one family` }; + const families = []; + for (const f of e.families) { + const bad = !isMapping(f) || typeof f.name !== 'string' || !/^[A-Za-z0-9_:]+$/.test(f.name) + || !TYPE_SET.has(String(f.type ?? '').toLowerCase()) + || (f.gate !== undefined && !Object.hasOwn(gates, f.gate)) || (f.scope !== undefined && !Object.hasOwn(scopes, f.scope)); + if (bad) return { error: `${where}: family ${JSON.stringify(f?.name ?? f)} needs a name, an exposition type, and only the gates and scopes the entry defines` }; + families.push({ + name: f.name, + type: String(f.type).toLowerCase(), + help: typeof f.help === 'string' ? f.help : '', + labels: Array.isArray(f.labels) ? f.labels.filter(l => typeof l === 'string' && LABEL_RE.test(l)).slice(0, LABEL_CAP) : [], + ...(f.gate !== undefined ? { gate: f.gate } : {}), + ...(f.scope !== undefined ? { scope: f.scope } : {}), + }); + } + return { + entry: { + id: e.id, + title: typeof e.title === 'string' ? e.title : e.id, + match: { keys }, + namespace: { path: typeof ns.path === 'string' ? ns.path : '', default: typeof ns.default === 'string' ? ns.default : '' }, + gates, + scopes, + families, + }, + }; +} + +// The catalogues resolveExporterCatalogue made, so a resolved one is never +// resolved again (asResolvedCatalogue). +const RESOLVED = new WeakSet(); + +/** + * The catalogue a crawl reads: the built-in entries, then a caller's (an + * object `{ exporters: [...] }` or the array itself) — an entry replaces the + * built-in one with the same id, the rest are added. An entry that is not + * well-formed is dropped with a warning naming what it lacks; a catalogue + * that is not an object or array is ignored with a warning. + * → `{ exporters, warnings }`. + */ +export function resolveExporterCatalogue(custom) { + const warnings = []; + const byId = new Map(); + for (const [i, e] of BUILTIN_CATALOGUE.exporters.entries()) { + const { entry } = normalizeEntry(e, i); + if (entry) byId.set(entry.id, entry); + } + if (custom !== undefined && custom !== null) { + const list = Array.isArray(custom) ? custom : (isMapping(custom) && Array.isArray(custom.exporters) ? custom.exporters : null); + if (!list) warnings.push('the exporter catalogue is neither `{ exporters: [...] }` nor an array; the built-in catalogue is used.'); + else { + list.forEach((e, i) => { + const { entry, error } = normalizeEntry(e, i); + if (error) warnings.push(`${error}; dropped.`); + else byId.set(entry.id, entry); + }); + } + } + const resolved = { exporters: [...byId.values()], warnings }; + RESOLVED.add(resolved); + return resolved; +} + +/** + * A catalogue as a caller hands it over — resolveExporterCatalogue's result, + * or a downstream's `{ exporters: [...] }` / array (resolved here: merged + * with the built-in entries, a malformed entry dropped) — ready to read. + */ +export function asResolvedCatalogue(catalogue) { + return catalogue && RESOLVED.has(catalogue) ? catalogue : resolveExporterCatalogue(catalogue); +} + +/** + * Whether a YAML text can be an exporter config of the catalogue — every + * key of some entry's `match.keys` starts a line — before the text is + * parsed. Cheap; `matchExporterConfig` decides. + */ +export function mayBeExporterConfig(content, catalogue) { + const text = stripBom(content); + return catalogue.exporters.some(e => e.match.keys.every(k => new RegExp(`^${k}\\s*:`, 'm').test(text))); +} + +/** The first catalogue entry whose `match.keys` the parsed config all carries, or null. */ +export function matchExporterConfig(obj, catalogue) { + if (!isMapping(obj)) return null; + return catalogue.exporters.find(e => e.match.keys.every(k => Object.prototype.hasOwnProperty.call(obj, k))) || null; +} + +function switchOn(value, dflt) { + if (typeof value === 'boolean') return value; + if (typeof value === 'number') return value !== 0; + if (typeof value === 'string') { + const v = value.trim().toLowerCase(); + if (v === 'true' || v === 'yes' || v === 'on') return true; + if (v === 'false' || v === 'no' || v === 'off') return false; + } + return dflt; +} + +function selectionNonEmpty(value) { + if (Array.isArray(value)) return value.some(v => String(v ?? '').trim() !== ''); + if (typeof value === 'string') return value.trim() !== ''; + if (isMapping(value)) return Object.keys(value).length > 0; + return false; +} + +/** + * The families an exporter config switches on: `{ namespace, gates, scopes, + * families: [{ name, type, help, labels }] }` — each family's name joined to + * the namespace the config sets (else the entry's default), a family kept + * only when its gate is on and its scope selects something. + */ +export function inferExporterFamilies(obj, entry) { + const nsRaw = entry.namespace.path ? pathValue(obj, entry.namespace.path) : undefined; + const namespace = typeof nsRaw === 'string' && NAME_RE.test(nsRaw.trim()) ? nsRaw.trim() : entry.namespace.default; + const gates = Object.fromEntries(Object.entries(entry.gates).map(([k, g]) => [k, switchOn(pathValue(obj, g.path), g.default)])); + const scopes = Object.fromEntries(Object.entries(entry.scopes).map(([k, s]) => [k, selectionNonEmpty(pathValue(obj, s.path))])); + const families = entry.families + .filter(f => (f.gate === undefined || (Object.hasOwn(gates, f.gate) && gates[f.gate])) + && (f.scope === undefined || (Object.hasOwn(scopes, f.scope) && scopes[f.scope]))) + .map(f => ({ name: namespace ? `${namespace}_${f.name}` : f.name, type: f.type, help: f.help, labels: f.labels })) + .filter(f => NAME_RE.test(f.name)); + return { namespace, gates, scopes, families }; +} diff --git a/tools/lib/journey.mjs b/tools/lib/journey.mjs index 35f44a91..b2841a43 100644 --- a/tools/lib/journey.mjs +++ b/tools/lib/journey.mjs @@ -517,7 +517,7 @@ function loadPackFile(path, baseDir, scope = null) { // Minimal repo walk for the crawl source — mirrors tools/crawl-repo.mjs's // filters (that script runs main() on import, so it can't be imported). const SKIP_DIRS = new Set(['.git', 'node_modules', 'dist', 'build', 'out', '.observogram', '.tomograph', 'coverage', 'vendor']); -const SCAN_EXT = /\.(ya?ml|json|cs|go|java|py|ts|tsx|js|mjs|rs|kt)$/i; +const SCAN_EXT = /\.(ya?ml|json|cs|go|java|py|ts|tsx|js|mjs|rs|kt|prom|metrics)$/i; const MAX_FILE_BYTES = 1024 * 1024; const MAX_TOTAL_BYTES = 16 * 1024 * 1024; diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs new file mode 100644 index 00000000..cb02ee65 --- /dev/null +++ b/tools/test-crawl-exporters.mjs @@ -0,0 +1,669 @@ +#!/usr/bin/env node +/** + * tools/test-crawl-exporters.mjs + * + * The crawler learns metrics from prebuilt exporters and from an OTel + * collector's scrape jobs (tools/lib/exporter-metrics.mjs, the exporter + * kinds and the collector walker in tools/lib/crawler.mjs): + * A. the fixture repository (tools/fixtures/crawl/exporters/) — an + * mq-metric-samples config, its metrics manifest and a collector config + * under a chart's files/ — crawls into a valid pack holding the + * manifest's metrics, the collector's `qmgrs` job and the kafka + * exporter, and says which files the prod scope left out; + * B. a manifest's metric outranks the same name inferred from the config; + * C. the config's gates, scopes and namespace decide the families; + * D. a saved exposition (.prom / .metrics) is read, never from a test path; + * E. a downstream catalogue replaces and extends the built-in one; + * F. a collector with pipelines: the scrape job's exporters are the + * metrics pipelines' that list the receiver, endpoints without secrets; + * G. a malformed manifest is warned about, not dropped silently; + * H. a crawl with no exporter evidence carries none of the new keys; + * I. `packc emit --exporter` writes a manifest the crawler reads, and + * refuses what it cannot honestly record. + * Exit 0 = pass. + */ + +import { readFileSync, readdirSync, statSync, mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { spawnSync, spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import { crawlFiles, detectArtefactKind, scanReadsPath } from './lib/crawler.mjs'; +import { + parseExposition, manifestFromExposition, readMetricsManifest, safeSourceUrl, resolveExporterCatalogue, +} from './lib/exporter-metrics.mjs'; +import { EXPORTER_CATALOGUE as BUILTIN } from './lib/exporter-catalogue.mjs'; +import { validateCanonical, SPEC_SCHEMA_PATH } from './lib/validator.mjs'; +import { isCrawlerFile, isExporterCandidateFile, selectCrawlerFiles } from '../server/github-crawl.mjs'; +import { createHarness } from './lib/harness.mjs'; + +const { assert, report } = createHarness({ truncate: 600 }); +const SCHEMA = JSON.parse(readFileSync(new URL(`../${SPEC_SCHEMA_PATH}`, import.meta.url), 'utf8')); +const NOW = '2026-10-15T00:00:00.000Z'; +const EMIT = fileURLToPath(new URL('./emit-manifest.mjs', import.meta.url)); +const CLI = fileURLToPath(new URL('./cli.mjs', import.meta.url)); + +const ROOT = fileURLToPath(new URL('./fixtures/crawl/exporters/', import.meta.url)); +const FILES = {}; +(function walk(dir) { + for (const name of readdirSync(dir)) { + const p = join(dir, name); + if (statSync(p).isDirectory()) walk(p); + else if (scanReadsPath(name)) FILES[relative(ROOT, p).replace(/\\/g, '/')] = readFileSync(p, 'utf8'); + } +})(ROOT); + +const ann = (r) => r.canonical.metadata.annotations; +const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); + +// ---------- A. the acceptance fixture, the default (prod) scope ---------- +{ + assert(!('mq-exporter/Dockerfile' in FILES) && 'mq-exporter/config.yaml' in FILES && 'mq-exporter/metrics-manifest.json' in FILES + && 'charts/otel-collector/files/collector.yaml' in FILES, + 'the fixture: the exporter config and manifest, the chart\'s collector config; the Dockerfile is not scanned', Object.keys(FILES)); + assert(detectArtefactKind('mq-exporter/config.yaml', FILES['mq-exporter/config.yaml']) === 'exporter-config', 'an mq-metric-samples config is an exporter config'); + assert(detectArtefactKind('mq-exporter/metrics-manifest.json', FILES['mq-exporter/metrics-manifest.json']) === 'metrics-manifest', 'metrics-manifest.json is a metrics manifest'); + assert(detectArtefactKind('x/ibm-mq.metrics-manifest.json', '{}') === 'metrics-manifest', '*.metrics-manifest.json is a metrics manifest'); + assert(detectArtefactKind('charts/otel-collector/files/collector.yaml', FILES['charts/otel-collector/files/collector.yaml']) === 'otel-collector', + 'a collector config with no service block is an OTel collector'); + + const r = crawlFiles(FILES, { repoName: 'mq-platform', now: NOW }); + assert(validateCanonical(r.canonical, SCHEMA).length === 0, 'the crawl validates against the vendored schema', validateCanonical(r.canonical, SCHEMA).slice(0, 3)); + const names = json(r, 'crawler.discovered.metric_names'); + const manifest = JSON.parse(FILES['mq-exporter/metrics-manifest.json']); + assert(manifest.metrics.every(m => names.includes(m.name)), 'every manifest metric is in crawler.discovered.metric_names', names); + const origins = json(r, 'crawler.discovered.metric_origins'); + assert(manifest.metrics.every(m => origins[m.name].origin_kind === 'exporter-manifest' && origins[m.name].confidence === 'declared' + && origins[m.name].candidate === false && origins[m.name].service === 'ibm-mq' && origins[m.name].file === 'mq-exporter/metrics-manifest.json'), + 'each is declared by the manifest, service ibm-mq', origins.ibmmq_queue_depth); + assert(JSON.stringify(origins.ibmmq_queue_depth.labels) === '["qmgr","queue","usage","description"]' && origins.ibmmq_queue_depth.type === 'gauge', + 'with the manifest\'s type and labels', origins.ibmmq_queue_depth); + + const jobs = json(r, 'crawler.discovered.scrape_job_origins'); + assert(JSON.stringify(json(r, 'crawler.discovered.scrape_jobs')) === '["qmgrs"]', 'the collector\'s qmgrs job is a discovered scrape job', json(r, 'crawler.discovered.scrape_jobs')); + assert(JSON.stringify(jobs.qmgrs) === JSON.stringify({ + file: 'charts/otel-collector/files/collector.yaml', metrics_path: '/metrics', interval: '30s', + targets: ['mq-exp-qm1:9157', 'mq-exp-qm2:9157', 'mq-exp-qm3:9157'], + via: 'otel-collector', receiver: 'prometheus/mq', target_count: 3, + exporters: [{ name: 'kafka', kind: 'kafka', topic: 'mq-metrics' }], + }), 'its interval, path, target count, receiver and the kafka topic it leaves by', jobs.qmgrs); + assert(r.canonical.spec.pipelines.exporters.metrics.kind === 'kafka' && JSON.stringify(r.canonical.spec.pipelines.exporters.metrics) === '{"kind":"kafka"}', + 'pipelines.exporters.metrics is the kafka exporter, in the shape every exporter has', r.canonical.spec.pipelines.exporters.metrics); + assert(!('crawler.scaffold.pipelines.exporters.metrics' in ann(r)), 'and is no longer the prometheusremotewrite scaffold'); + assert(r.evidence['pipelines.exporters.metrics'] === 'charts/otel-collector/files/collector.yaml', 'its evidence is the collector config'); + assert(JSON.stringify(r.canonical.spec.pipelines.receivers) === '[{"name":"prometheus"}]' && !('crawler.scaffold.pipelines.receivers[0]' in ann(r)), + 'the prometheus receiver is the pipeline\'s receiver', r.canonical.spec.pipelines.receivers); + assert(!r.summary.warnings.some(w => /metrics-capable/.test(w)), 'debug is not a metrics exporter: one candidate, no ambiguity warning'); + + const exporters = json(r, 'crawler.discovered.exporters'); + assert(JSON.stringify(exporters) === JSON.stringify({ 'ibm-mq': { sources: [{ + origin_kind: 'exporter-manifest', file: 'mq-exporter/metrics-manifest.json', metrics: 6, version: '5.6.0', + source: 'http://mq-exporter:9157/metrics', generated_at: '2026-10-01T00:00:00.000Z' }] } }) + && ann(r)['crawler.discovered.exporters_count'] === '1', + 'crawler.discovered.exporters names ibm-mq and its manifest', exporters); + + const scoped = r.summary.warnings.find(w => /carry scrape jobs or exporter evidence/.test(w)); + assert(scoped === 'Environment scope "prod" excluded 2 file(s) that carry scrape jobs or exporter evidence: mq-exporter/config.yaml (exporter config: ibm-mq); preprod/otel-collector.yaml (collector scrape jobs: qmgrs-preprod). An environment that names no deployment profile (e.g. `all`) reads every file.', + 'the prod scope says which collection evidence it left out, and how to read it', scoped); + assert(r.summary.environment.excluded.includes('mq-exporter/config.yaml') && !r.summary.environment.excluded.includes('mq-exporter/metrics-manifest.json'), + 'the manifest is read in every environment; the exporter config is scoped like any config', r.summary.environment.excluded); + + // The way out, followed literally. + const all = crawlFiles(FILES, { repoName: 'mq-platform', environment: 'all', now: NOW }); + assert(!all.summary.warnings.some(w => /Environment scope/.test(w)) && all.summary.files.excludedByEnvironment === 0, + '`all` reads every file: no scope, no warning'); + assert(JSON.stringify(json(all, 'crawler.discovered.scrape_jobs')) === '["qmgrs","qmgrs-preprod"]', 'and the preprod job is read', json(all, 'crawler.discovered.scrape_jobs')); + assert(validateCanonical(all.canonical, SCHEMA).length === 0, 'that crawl validates too'); + + // ---------- B. the manifest outranks the config ---------- + const o = json(all, 'crawler.discovered.metric_origins'); + assert(o.ibmmq_queue_depth.origin_kind === 'exporter-manifest' && o.ibmmq_queue_depth.confidence === 'declared' && o.ibmmq_queue_depth.candidate === false + && o.ibmmq_queue_depth.help === 'Current depth of the queue', + 'a name both the manifest and the config give reads as the manifest\'s', o.ibmmq_queue_depth); + assert(o.ibmmq_channel_bytes_sent.origin_kind === 'exporter-config' && o.ibmmq_channel_bytes_sent.confidence === 'inferred' + && o.ibmmq_channel_bytes_sent.candidate === true && o.ibmmq_channel_bytes_sent.file === 'mq-exporter/config.yaml', + 'a name only the config gives is an inferred candidate', o.ibmmq_channel_bytes_sent); + assert(all.summary.discovered.metricCandidatesUnreferenced > 0, 'curation counts the inferred names among the unreferenced candidates', all.summary.discovered.metricCandidatesUnreferenced); + const sources = json(all, 'crawler.discovered.exporters')['ibm-mq'].sources; + assert(sources.length === 2 && sources[0].origin_kind === 'exporter-config' && sources[0].metrics === BUILTIN.exporters[0].families.length + && JSON.stringify(sources[0].gates) === '{"objectStatus":true,"publications":true}' && sources[0].namespace === 'ibmmq', + 'ibm-mq lists its config (every built-in family on) beside its manifest', sources); +} + +// A config-inferred name a dashboard reads stays an inferred candidate (the +// config says the exporter runs, not that the name is right). +{ + const dash = JSON.stringify({ uid: 'mq', title: 'MQ', schemaVersion: 39, panels: [{ type: 'timeseries', title: 'depth', targets: [{ expr: 'max(ibmmq_queue_time_since_get)' }] }] }); + const r = crawlFiles({ 'config.yaml': FILES['mq-exporter/config.yaml'], 'grafana/mq.json': dash }, { repoName: 'mq', now: NOW }); + const o = json(r, 'crawler.discovered.metric_origins').ibmmq_queue_time_since_get; + assert(o.origin_kind === 'promql-reference' && o.used_by.includes('dashboard:mq/depth'), 'a name the dashboard reads is primarily the reference', o); +} + +// ---------- C. gates, scopes, namespace ---------- +{ + const cfg = (g, objects, ns) => `global:\n${g}\nconnection:\n queueManager: QM1\nobjects:\n${objects}\nprometheus:\n port: 9157\n${ns}`; + const fams = (text) => Object.keys(json(crawlFiles({ 'config.yaml': text }, { repoName: 'mq', now: NOW }), 'crawler.discovered.metric_origins') || {}).sort(); + const noStatus = fams(cfg(' useObjectStatus: false', ' queues: ["APP.*"]\n channels: ["TO.*"]', '')); + assert(noStatus.length && noStatus.every(n => !/status|uptime|connection_count|oldest|uncommitted|time_since|channel_/.test(n)), + 'useObjectStatus: false drops the status families', noStatus); + const noPubs = fams(cfg(' usePublications: "false"', ' queues: ["APP.*"]\n channels: ["TO.*"]', '')); + assert(noPubs.length && !noPubs.some(n => /cpu|log_|mqconn|queue_depth|mqput|mqget/.test(n)), 'usePublications: "false" drops the published families', noPubs); + const noObjects = fams(cfg(' logLevel: INFO', ' queues:\n channels: []', '')); + assert(noObjects.length && noObjects.every(n => /^ibmmq_qmgr_/.test(n)), 'with no queues or channels selected only the queue manager families remain', noObjects); + const ns = fams(cfg(' logLevel: INFO', ' queues: APP.*', ' namespace: mqprod')); + assert(ns.includes('mqprod_queue_depth') && ns.every(n => n.startsWith('mqprod_')) && !ns.some(n => /^mqprod_channel_/.test(n)), + 'prometheus.namespace prefixes every family; a string selection counts', ns); + const bothOff = crawlFiles({ 'config.yaml': cfg(' useObjectStatus: off\n usePublications: no', ' queues: ["A"]', '') }, { repoName: 'mq', now: NOW }); + assert(!('crawler.discovered.metric_origins' in ann(bothOff)) && json(bothOff, 'crawler.discovered.exporters')['ibm-mq'].sources[0].metrics === 0, + 'both switches off: no family, the exporter still recorded with 0', ann(bothOff)['crawler.discovered.exporters']); + assert(detectArtefactKind('mq-collector.yaml', FILES['mq-exporter/config.yaml']) === 'exporter-config', + 'a config named like a collector is read by its shape first'); +} + +// ---------- D. exposition snapshots ---------- +{ + const PROM = [ + '# HELP ibmmq_queue_depth Current depth', + '# TYPE ibmmq_queue_depth gauge', + 'ibmmq_queue_depth{qmgr="QM1",queue="APP.IN",description="a, \\"quoted\\" } value"} 4', + 'ibmmq_queue_depth{qmgr="QM1",queue="APP.OUT"} 0', + '# HELP http_request_duration_seconds Latency\\nsecond line', + '# TYPE http_request_duration_seconds histogram', + 'http_request_duration_seconds_bucket{le="0.1",route="/a"} 1', + 'http_request_duration_seconds_sum{route="/a"} 0.2', + 'http_request_duration_seconds_count{route="/a"} 2', + '# TYPE rpc_latency summary', + 'rpc_latency{quantile="0.5",method="get"} 1', + 'rpc_latency_count{method="get"} 3', + '# TYPE jobs counter', + 'jobs_total{kind="x"} 7', + 'process_up 1', + ].join('\n'); + const { families, samples } = parseExposition(PROM); + const by = Object.fromEntries(families.map(f => [f.name, f])); + assert(samples === 9 && families.map(f => f.name).join() === 'http_request_duration_seconds,ibmmq_queue_depth,jobs_total,process_up,rpc_latency', + 'families by name; suffixed samples fold into their family; an undeclared sample is its own', families.map(f => f.name)); + assert(JSON.stringify(by.ibmmq_queue_depth.labels) === '["qmgr","queue","description"]', 'label names only, a quoted value with `,` `}` and `\\"` parsed past', by.ibmmq_queue_depth); + assert(JSON.stringify(by.http_request_duration_seconds.labels) === '["route"]' && by.http_request_duration_seconds.help === 'Latency\nsecond line', + 'a histogram family drops `le`; HELP unescaped', by.http_request_duration_seconds); + assert(JSON.stringify(by.rpc_latency.labels) === '["method"]' && by.jobs_total.type === 'counter' && JSON.stringify(by.jobs_total.labels) === '["kind"]' + && by.process_up.type === 'untyped', 'a summary drops `quantile`; `_total` folds into a counter, named as stored; no TYPE is untyped', [by.rpc_latency, by.jobs_total, by.process_up]); + // OpenMetrics: a counter or info family is named without its suffix and + // stored with it; the declared names are the ones rules and dashboards read. + const om = parseExposition('# TYPE jobs counter\njobs_total{k="x"} 7\njobs_created{k="x"} 1\n# TYPE build info\nbuild_info{version="1"} 1\n# TYPE idle counter\n# TYPE g gauge\ng 1\n# EOF\n'); + assert(om.families.map(f => f.name).join() === 'build_info,g,idle_total,jobs_total', + 'OpenMetrics counters and infos are named as Prometheus stores them, with or without a sample', om.families.map(f => f.name)); + assert(parseExposition('# TYPE jobs_total counter\njobs_total 1\n').families[0].name === 'jobs_total', 'a 0.0.4 counter keeps its name'); + assert(!JSON.stringify(families).includes('APP.IN'), 'no label value is kept'); + + assert(scanReadsPath('exporter/ibm-mq.prom') && scanReadsPath('snapshots/node.metrics'), 'a scan reads .prom and .metrics files'); + assert(detectArtefactKind('snapshots/ibm-mq.prom', PROM) === 'exposition-snapshot', 'a .prom file with families is an exposition snapshot'); + assert(detectArtefactKind('test/fixtures/ibm-mq.prom', PROM) === 'unknown' && detectArtefactKind('src/x.test.metrics', PROM) === 'unknown' + && detectArtefactKind('testdata/a.prom', PROM) === 'unknown' + && ['src/__fixtures__/node.prom', 'test-data/node.prom', 'test_fixtures/a.prom', 'x/__mocks__/a.metrics', 'mocks/a.prom', 'spec/a.prom', + 'e2e/mq.prom', 'integration-tests/mq.prom', 'src/testFixtures/resources/node.prom', 'testfixtures/a.prom', 'src/integrationTest/resources/mq.prom', 'unittests/a.prom', 'e2etests/a.prom', + 'src/componentTest/resources/a.prom', 'src/smokeTest/a.prom', 'src/intTest/a.prom', 'src/systemTest/a.prom', 'src/testUtils/a.prom', 'src/it/resources/a.prom', 'module/testResources/a.prom', 'e2e-tests/mq.prom', 'unit_tests/mq.prom', 'testing/mq.prom', 'src/__snapshots__/mq.prom', 'exp/mq_spec.prom', 'exp/mq.spec.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), + 'never from a test, fixture or mock path, in any of the usual spellings'); + assert(['snapshots/ibm-mq.prom', 'exporters/latest/mq.prom', 'contest/mq.prom', 'attestation/mq.prom', 'i18n/it/mq.prom', 'prod/mq.prom', 'Latest/mq.prom'].every(p => detectArtefactKind(p, PROM) === 'exposition-snapshot'), + 'a folder that only contains such a word in a longer one is not a test folder'); + assert(detectArtefactKind('notes.metrics', 'nothing to see here\n') === 'unknown', 'a .metrics file with no family is not a snapshot'); + + const r = crawlFiles({ 'snapshots/ibm-mq.prom': PROM, 'test/ibm-mq.prom': '# TYPE test_only gauge\ntest_only 1\n' }, { repoName: 'snap', now: NOW }); + const o = json(r, 'crawler.discovered.metric_origins'); + assert(o.ibmmq_queue_depth.origin_kind === 'exporter-snapshot' && o.ibmmq_queue_depth.confidence === 'declared' && o.ibmmq_queue_depth.service === 'ibm-mq' + && !('test_only' in o), 'a crawl declares the snapshot\'s families, named by the file; the test path is not read', o.ibmmq_queue_depth); + assert(o.jobs_total.origin_kind === 'exporter-snapshot', 'the counter is declared under its stored name', Object.keys(o)); + assert(JSON.stringify(json(r, 'crawler.discovered.exporters')) === '{"ibm-mq":{"sources":[{"origin_kind":"exporter-snapshot","file":"snapshots/ibm-mq.prom","metrics":5}]}}', + 'the snapshot is a source of the exporter', ann(r)['crawler.discovered.exporters']); + // Rank: manifest > snapshot > source code. + const ranked = crawlFiles({ + 'snapshots/ibm-mq.prom': PROM, + 'metrics-manifest.json': JSON.stringify({ exporter: 'ibm-mq', metrics: [{ name: 'ibmmq_queue_depth', type: 'gauge', help: 'from the manifest' }] }), + 'src/metrics.ts': "import { Counter } from 'prom-client';\nexport const j = new Counter({ name: 'jobs_total', help: 'from code' });\n", + }, { repoName: 'snap', now: NOW }); + const ro = json(ranked, 'crawler.discovered.metric_origins'); + assert(ro.ibmmq_queue_depth.help === 'from the manifest' && ro.ibmmq_queue_depth.origin_kind === 'exporter-manifest' + && ro.jobs_total.origin_kind === 'exporter-snapshot', 'the manifest outranks the snapshot, the snapshot outranks source code', [ro.ibmmq_queue_depth, ro.jobs_total]); +} + +// ---------- E. a downstream catalogue ---------- +{ + const custom = { + exporters: [ + { id: 'ibm-mq', match: { keys: ['global', 'connection', 'objects', 'prometheus'] }, namespace: { path: 'prometheus.namespace', default: 'ibmmq' }, + families: [{ name: 'qmgr_custom_only', type: 'gauge', labels: ['qmgr'] }] }, + { id: 'acme-gw', match: { keys: ['acme', 'listen'] }, namespace: { default: 'acme' }, + gates: { tls: { path: 'acme.tls', default: false } }, + families: [{ name: 'requests_total', type: 'counter' }, { name: 'tls_handshakes_total', type: 'counter', gate: 'tls' }] }, + { id: 'Bad Id', match: { keys: ['x'] }, families: [{ name: 'a', type: 'gauge' }] }, + { id: 'nofam', match: { keys: ['x'] }, families: [] }, + { id: 'badgate', match: { keys: ['x'] }, families: [{ name: 'a', type: 'gauge', gate: 'nope' }] }, + ], + }; + const read = resolveExporterCatalogue(custom); + assert(read.exporters.map(e => e.id).join() === 'ibm-mq,acme-gw' && read.warnings.length === 3 + && read.warnings.every(w => /; dropped\.$/.test(w)), 'an entry replaces the built-in one with its id, a new one is added, malformed ones dropped with a warning', read.warnings); + assert(BUILTIN.exporters[0].families.length > 1 && resolveExporterCatalogue().exporters[0].families.length === BUILTIN.exporters[0].families.length, + 'the built-in catalogue is untouched by a resolution'); + const r = crawlFiles({ + 'mq/config.yaml': FILES['mq-exporter/config.yaml'], + 'gw/acme.yaml': 'acme:\n tls: true\nlisten: ":8080"\n', + }, { repoName: 'cat', now: NOW, exporterCatalogue: custom }); + const o = json(r, 'crawler.discovered.metric_origins'); + assert(Object.keys(o).sort().join() === 'acme_requests_total,acme_tls_handshakes_total,ibmmq_qmgr_custom_only', + 'the crawl reads both configs with the downstream catalogue', Object.keys(o)); + assert(r.summary.warnings.filter(w => /exporter catalogue entry/.test(w)).length === 3, 'the dropped entries are crawl warnings'); + assert(detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n') === 'unknown', 'without the catalogue the gateway config is nothing'); + // detectArtefactKind takes the same downstream catalogue: merged with the + // built-in entries, in either form, a malformed entry dropped, never thrown. + const acmeOnly = [custom.exporters[1]]; + assert(detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n', { exporterCatalogue: { exporters: acmeOnly } }) === 'exporter-config' + && detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n', { exporterCatalogue: acmeOnly }) === 'exporter-config' + && detectArtefactKind('mq/config.yaml', FILES['mq-exporter/config.yaml'], { exporterCatalogue: acmeOnly }) === 'exporter-config' + && detectArtefactKind('a.yaml', 'acme: 1\n', { exporterCatalogue: { exporters: [{ id: 'nokeys' }] } }) === 'unknown', + 'detectArtefactKind merges a downstream catalogue (object or array) with the built-in one and never throws on a malformed entry'); + // A gate or scope the entry never defines is refused, whatever its name + // (toString and constructor are on every object's prototype). + const proto = resolveExporterCatalogue({ exporters: [ + { id: 'p1', match: { keys: ['p'] }, families: [{ name: 'a', type: 'gauge', gate: 'toString' }] }, + { id: 'p2', match: { keys: ['p'] }, families: [{ name: 'b', type: 'gauge', scope: 'constructor' }] }, + ] }); + assert(proto.warnings.length === 2 && !proto.exporters.some(e => /^p\d$/.test(e.id)), 'a family gated or scoped on a prototype name is dropped', proto.warnings); + const junk = crawlFiles({ 'a.yaml': 'a: 1\n' }, { repoName: 'cat', now: NOW, exporterCatalogue: 'not a catalogue' }); + assert(junk.summary.warnings.some(w => /neither `\{ exporters: \[\.\.\.\] \}` nor an array/.test(w)), 'a catalogue of the wrong shape is warned about'); +} + +// ---------- F. a collector with pipelines ---------- +{ + const COL = `receivers: + otlp: + protocols: { grpc: {} } + prometheus: + config: + scrape_configs: + - job_name: node + kubernetes_sd_configs: [{ role: pod }] + - job_name: qmgrs + scrape_interval: 15s + static_configs: + - targets: ["a:9157", "b:9157"] + - targets: ["c:9157"] +exporters: + prometheusremotewrite: + endpoint: https://writer:s3cret@mimir.example:443/api/v1/push?token=abc#frag + otlphttp/tempo: + endpoint: http://tempo:4318 + kafka/metrics: + topic: legacy-topic +service: + pipelines: + metrics: + receivers: [otlp, prometheus] + exporters: [prometheusremotewrite] + metrics/archive: + receivers: [prometheus] + exporters: [kafka/metrics] + traces: + receivers: [otlp] + exporters: [otlphttp/tempo] +`; + const r = crawlFiles({ 'deploy/otel-collector.yaml': COL }, { repoName: 'col', now: NOW }); + const jobs = json(r, 'crawler.discovered.scrape_job_origins'); + const leave = [{ name: 'prometheusremotewrite', kind: 'prometheusremotewrite', endpoint: 'https://mimir.example:443/api/v1/push' }, { name: 'kafka/metrics', kind: 'kafka', topic: 'legacy-topic' }]; + assert(JSON.stringify(jobs.qmgrs.exporters) === JSON.stringify(leave) && jobs.qmgrs.target_count === 3 && jobs.qmgrs.interval === '15s', + 'the job leaves by every metrics pipeline that lists its receiver; the endpoint loses userinfo, query and fragment', jobs.qmgrs); + assert(jobs.node.target_count === 0 && JSON.stringify(jobs.node.targets) === '["kubernetes:pod"]', 'a discovered-target job counts no static target', jobs.node); + assert(!ann(r)['crawler.discovered.scrape_job_origins'].includes('s3cret') && !ann(r)['crawler.discovered.scrape_job_origins'].includes('token'), + 'no credential reaches an annotation'); + const pe = r.canonical.spec.pipelines.exporters; + assert(JSON.stringify([pe.metrics, pe.traces, pe.logs]) === '[{"kind":"prometheusremotewrite"},{"kind":"otlp"},{"kind":"elasticsearch"}]', + 'the pipelines read as before', r.canonical.spec.pipelines.exporters); + // Every spelling of a credential: a password holding `@`, `/` or `?`, no + // scheme, a scheme-relative URL, one a URL parser reads as host:port/path. + const SECRETS = [ + ['user:p@ss@otel:4317', 'otel:4317'], + ['user:pa/ss@otel:4317', 'otel:4317'], + ['https://user:pa/ss@otel:4317/v1', 'https://otel:4317/v1'], + ['https://user:12/ss@otel/v1', 'https://otel/v1'], + // A `?` or `#` before the last `@`: a query holding an `@`, or a password + // holding a `?` — nothing is kept. + ['https://u:pa?ss@h/x', ''], + ['https://mimir/api/v1/push?user=ops@corp.example&api_key=SECRET', ''], + ['https://h/push#frag@SECRET', ''], + ['https://', ''], + ['//u:p@h/metrics', '//h/metrics'], + ['HTTPS://U:P@H:1/m?t=1#f', 'HTTPS://H:1/m'], + ['svc:Zx9/Qk+SECRET==@otel-gw:4317', 'otel-gw:4317'], + // A line break or tab, which a URL parser drops (the value still sends its query). + ['http://mq:9157/metrics?token=SECRET1\nx', 'http://mq:9157/metrics'], + ['https://i/v1?k=SECRET\u2028x', 'https://i/v1'], + ['https://u:p\t@h/v1#frag\r\nx', 'https://h/v1'], + ]; + assert(SECRETS.every(([raw, want]) => safeSourceUrl(raw) === want), 'safeSourceUrl drops everything before the last @ and from the first ? or #', + SECRETS.map(([raw]) => safeSourceUrl(raw)), SECRETS.map(([, want]) => want)); + const queryAt = crawlFiles({ + 'otel/collector.yaml': 'receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlphttp:\n endpoint: "https://mimir/push?user=ops@corp.example&api_key=SECRETQ"\n', + 'metrics-manifest.json': JSON.stringify({ exporter: 'x', source: 'http://mq/metrics?u=a@b&token=SECRETR', metrics: [{ name: 'm', type: 'gauge' }] }), + }, { repoName: 'col', now: NOW }); + assert(!JSON.stringify(queryAt).includes('SECRET') && !('endpoint' in json(queryAt, 'crawler.discovered.scrape_job_origins').mq.exporters[0]) + && !('source' in json(queryAt, 'crawler.discovered.exporters').x.sources[0]), + 'an endpoint or source whose query holds an @ is not recorded at all', [json(queryAt, 'crawler.discovered.scrape_job_origins').mq, json(queryAt, 'crawler.discovered.exporters')]); + const leak = crawlFiles({ 'otel/collector.yaml': `receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp:\n endpoint: "svc:Zx9/Qk+SECRETPW@otel-gw:4317"\n` }, { repoName: 'col', now: NOW }); + assert(!JSON.stringify(leak.canonical).includes('SECRET') && json(leak, 'crawler.discovered.scrape_job_origins').mq.exporters[0].endpoint === 'otel-gw:4317', + 'a collector endpoint whose password holds `/` reaches the pack without it', json(leak, 'crawler.discovered.scrape_job_origins').mq); + assert(safeSourceUrl('prometheus:9090') === 'prometheus:9090' && safeSourceUrl('${env:ENDPOINT}') === '${env:ENDPOINT}' + && safeSourceUrl('u:p@host:4317?x=1') === 'host:4317' && safeSourceUrl('http://h:9157') === 'http://h:9157' && safeSourceUrl('http://h:9157/') === 'http://h:9157/', + 'safeSourceUrl: a non-URL keeps its spelling less userinfo and query'); + + // Two metrics-capable exporters and no pipelines: the first, with a warning. + const two = crawlFiles({ 'files/collector.yaml': 'receivers:\n otlp: {}\nexporters:\n otlp:\n endpoint: http://gw:4317\n kafka:\n topic: t\n' }, { repoName: 'col', now: NOW }); + assert(two.canonical.spec.pipelines.exporters.metrics.kind === 'otlp' + && two.summary.warnings.includes("files/collector.yaml has no service pipelines and 2 metrics-capable exporters (otlp, kafka); the metrics exporter is taken to be the first, 'otlp'."), + 'two candidates: the first is taken and the warning says so', two.summary.warnings); + // A config split across --config files: the components in one file, the + // service in another. The declared metrics pipeline wins over the guess, + // whichever file is walked first, and the guess leaves no trace. + for (const [a, b] of [['otel/a-components.yaml', 'otel/b-service.yaml'], ['otel/b-components.yaml', 'otel/a-service.yaml']]) { + const split = crawlFiles({ + [a]: 'receivers:\n otlp: {}\nexporters:\n otlp/tempo:\n endpoint: http://tempo:4317\n prometheusremotewrite:\n endpoint: http://mimir:9009/api/v1/push\n', + [b]: 'receivers:\n otlp: {}\nexporters:\n otlp/tempo: {}\n prometheusremotewrite: {}\nservice:\n pipelines:\n traces: { receivers: [otlp], exporters: [otlp/tempo] }\n metrics: { receivers: [otlp], exporters: [prometheusremotewrite] }\n', + }, { repoName: 'col', now: NOW }); + assert(split.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && split.summary.discovered.pipelines === 1 + && !split.summary.warnings.some(w => /metrics-capable/.test(w)) && split.evidence['pipelines.exporters.metrics'] === undefined, + `a split config (${a} first): the declared metrics exporter, one pipeline, no guess warning`, [split.canonical.spec.pipelines.exporters.metrics, split.summary.discovered.pipelines]); + } + // The same split with a scrape receiver: the job leaves by the declared + // pipeline that lists its receiver (endpoint from the file that configures + // the exporter), not by the guess; one receiver, one pipeline, no warning. + const A = 'receivers:\n prometheus/mq:\n config:\n scrape_configs:\n - job_name: qmgrs\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp:\n endpoint: http://gw:4317\n prometheusremotewrite:\n endpoint: https://w:SECRET@mimir:9009/api/v1/push\n'; + const B = 'receivers:\n prometheus/mq: {}\nexporters:\n otlp: {}\n prometheusremotewrite: {}\nservice:\n pipelines:\n metrics: { receivers: [prometheus/mq], exporters: [prometheusremotewrite] }\n'; + for (const [a, b] of [['otel/a.yaml', 'otel/b.yaml'], ['otel/b.yaml', 'otel/a.yaml']]) { + const split = crawlFiles({ [a]: A, [b]: B }, { repoName: 'col', now: NOW }); + const job = json(split, 'crawler.discovered.scrape_job_origins').qmgrs; + assert(JSON.stringify(job.exporters) === '[{"name":"prometheusremotewrite","kind":"prometheusremotewrite","endpoint":"https://mimir:9009/api/v1/push"}]' + && split.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && split.summary.discovered.pipelines === 1 + && JSON.stringify(split.canonical.spec.pipelines.receivers) === '[{"name":"prometheus"}]' && !split.summary.warnings.some(w => /metrics-capable/.test(w)), + `a split config with a scrape receiver (${a} first): the job leaves by the declared pipeline`, [job, split.canonical.spec.pipelines, split.summary.discovered.pipelines]); + } + // A config WITH pipelines whose receiver no metrics pipeline lists (only + // traces here, or `metrics: null`): another file's metrics pipeline that + // lists it says where its jobs go. + for (const pipes of [' traces: { receivers: [otlp], exporters: [otlp] }\n', ' metrics: null\n traces: { receivers: [otlp], exporters: [otlp] }\n']) { + const tracesOnly = 'receivers:\n otlp: {}\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp: {}\nservice:\n pipelines:\n' + pipes; + const metricsFile = 'receivers:\n prometheus: {}\nexporters:\n kafka:\n topic: mq-metrics\nservice:\n pipelines:\n metrics: { receivers: [prometheus], exporters: [kafka] }\n'; + const r = crawlFiles({ 'otel/a.yaml': tracesOnly, 'otel/b.yaml': metricsFile }, { repoName: 'col', now: NOW }); + assert(JSON.stringify(json(r, 'crawler.discovered.scrape_job_origins').mq.exporters) === '[{"name":"kafka","kind":"kafka","topic":"mq-metrics"}]', + 'a receiver its own file\'s pipelines do not list for metrics leaves by the other file\'s metrics pipeline', json(r, 'crawler.discovered.scrape_job_origins').mq); + } + // A guess no job carries (its receiver's jobs are settled elsewhere, or it + // has none) is not warned about. + const spur = crawlFiles({ + 'otel/a.yaml': A.replace('exporters:\n', 'receivers2: {}\nexporters:\n').replace('receivers:\n prometheus/mq:', 'receivers:\n prometheus/empty:\n config:\n scrape_configs: []\n prometheus/mq:'), + 'otel/b.yaml': B, + }, { repoName: 'col', now: NOW }); + assert(!spur.summary.warnings.some(w => /metrics-capable/.test(w)) && spur.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite', + 'no first-of-several warning for a guess no job and no spec field carries', spur.summary.warnings); + // A malformed job in a collector config is warned about and skipped; its + // sibling and the rest of the file (a second document) are read. + const mal = crawlFiles({ 'otel/otel-collector.yaml': 'receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: bad\n static_configs: { targets: [x] }\n - job_name: good\n static_configs: [{ targets: ["g:1"] }]\nexporters:\n kafka: { topic: t }\nservice:\n pipelines:\n metrics: { receivers: [prometheus], exporters: [kafka] }\n---\nreceivers:\n otlp: {}\nexporters:\n otlp: {}\nservice:\n pipelines:\n traces: { receivers: [otlp], exporters: [otlp] }\n' }, { repoName: 'col', now: NOW }); + assert(JSON.stringify(json(mal, 'crawler.discovered.scrape_jobs')) === '["good"]' && mal.canonical.spec.pipelines.exporters.traces.kind === 'otlp' && mal.summary.discovered.pipelines === 2 + && mal.summary.warnings.some(w => /^Scrape job 'bad' of receiver prometheus in otel\/otel-collector\.yaml could not be read \(.*\); the rest of the config is read\.$/.test(w)) + && !mal.summary.warnings.some(w => /Failed to parse/.test(w)), 'a malformed collector job costs only itself', mal.summary.warnings); + // Alone, the same config's job carries the guess, and the warning says so. + const alone = crawlFiles({ 'otel/a.yaml': A }, { repoName: 'col', now: NOW }); + assert(json(alone, 'crawler.discovered.scrape_job_origins').qmgrs.exporters[0].name === 'otlp' && alone.summary.warnings.some(w => /taken to be the first, 'otlp'/.test(w)), + 'alone, a first-of-several guess stands on the job and is warned about'); + // A metrics backend image already set the metrics exporter: the guess does + // not override it (the walk's precedence, as for a declared pipeline); the + // job still names the exporter its collector configures. + const img = crawlFiles({ + 'charts/vm/values.yaml': 'vm:\n image:\n repository: victoriametrics/victoria-metrics\n tag: v1.99.0\n', + 'charts/c/files/collector.yaml': FILES['charts/otel-collector/files/collector.yaml'], + }, { repoName: 'col', now: NOW }); + assert(img.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && json(img, 'crawler.discovered.scrape_job_origins').qmgrs.exporters[0].kind === 'kafka', + 'a metrics backend image keeps the metrics exporter it implies; the job names the kafka exporter', img.canonical.spec.pipelines.exporters.metrics); + // An Alertmanager config (receivers is a list) is still Alertmanager. + assert(detectArtefactKind('am.yaml', 'route:\n receiver: x\nreceivers:\n - name: x\n') === 'alertmanager', 'Alertmanager receivers are not a collector\'s'); + + // A collector embedded in a Helm ConfigMap with no service block. + const TPL = `apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "c.fullname" . }} +data: + relay.yaml: | + receivers: + prometheus: + config: + scrape_configs: + - job_name: qmgrs + static_configs: + - targets: ["q:9157"] + exporters: + kafka: + topic: mq +`; + const h = crawlFiles({ 'charts/c/templates/cm.yaml': TPL, 'charts/c/Chart.yaml': 'apiVersion: v2\nname: c\nversion: 0.1.0\n' }, { repoName: 'col', now: NOW }); + assert(JSON.stringify(json(h, 'crawler.discovered.scrape_jobs')) === '["qmgrs"]' && h.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', + 'a Helm ConfigMap\'s collector with no service block: its job and its kafka exporter', ann(h)['crawler.discovered.scrape_jobs']); +} + +// A collector config in a chart's values: at the top level, or under +// `config:` (the opentelemetry-collector chart's idiom). +{ + const COL = 'receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: qmgrs\n static_configs:\n - targets: ["a:1", "b:1"]\nexporters:\n kafka:\n topic: mq\n'; + const nested = 'mode: deployment\nimage:\n repository: otel/opentelemetry-collector-contrib\nconfig:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); + for (const [label, text] of [['top level', COL], ['under config:', nested]]) { + const r = crawlFiles({ 'charts/otel-collector/values.yaml': text }, { repoName: 'vals', now: NOW }); + const job = json(r, 'crawler.discovered.scrape_job_origins')?.qmgrs; + assert(job?.target_count === 2 && job.exporters[0].topic === 'mq' && r.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', + `a collector in a chart's values.yaml (${label}): its job and its kafka exporter`, job); + } + // A disabled subchart's collector config is not read: in the same file, or + // by the environment's overlay. + const sub = 'opentelemetry-collector:\n enabled: ENABLED\n config:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); + const off = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'false') }, { repoName: 'vals', now: NOW }); + const overlay = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'true'), 'charts/u/values-prod.yaml': 'opentelemetry-collector:\n enabled: false\n' }, { repoName: 'vals', environment: 'prod', now: NOW }); + // The overlay wins both ways: re-enabled by values-prod.yaml, it is read. + const reenabled = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'false'), 'charts/u/values-prod.yaml': 'opentelemetry-collector:\n enabled: true\n' }, { repoName: 'vals', environment: 'prod', now: NOW }); + assert(JSON.stringify(json(reenabled, 'crawler.discovered.scrape_jobs')) === '["qmgrs"]', 'a subchart an overlay re-enables has its collector read', ann(reenabled)['crawler.discovered.scrape_jobs']); + assert(!('crawler.discovered.scrape_jobs' in ann(off)) && !('crawler.discovered.scrape_jobs' in ann(overlay)) + && off.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && overlay.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite', + 'a disabled umbrella subchart\'s collector is not read (same file, prod overlay)', [ann(off)['crawler.discovered.scrape_jobs'], ann(overlay)['crawler.discovered.scrape_jobs']]); + // An umbrella chart: the collector under its subchart's key. + const umbrella = crawlFiles({ 'charts/u/values.yaml': 'opentelemetry-collector:\n config:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n') }, { repoName: 'vals', now: NOW }); + assert(json(umbrella, 'crawler.discovered.scrape_jobs')?.includes('qmgrs') && umbrella.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', + 'an umbrella chart\'s subchart collector config is read', ann(umbrella)['crawler.discovered.scrape_jobs']); + // A collector config this walk cannot read never costs the values file + // its images; `pipelines.metrics: null` (the chart's way to drop a + // pipeline) is skipped, in values and in a plain collector file alike. + const LOKI = 'loki:\n image:\n repository: grafana/loki\n tag: 3.0.0\n'; + const nullPipe = crawlFiles({ 'charts/c/values.yaml': `${LOKI}config:\n receivers:\n otlp: {}\n exporters:\n kafka: {topic: t}\n service:\n pipelines:\n metrics: null\n traces: { receivers: [otlp], exporters: [kafka] }\n` }, { repoName: 'vals', now: NOW }); + assert(nullPipe.canonical.spec.telemetry.backends.some(b => b.product === 'loki') && nullPipe.canonical.spec.pipelines.exporters.traces.kind === 'kafka' + && !nullPipe.summary.warnings.some(w => /could not be read|Failed to parse/.test(w)), 'a null pipeline is skipped; the images are read', nullPipe.summary.warnings); + const broken = crawlFiles({ 'charts/c/values.yaml': `${LOKI}config:\n receivers:\n otlp: {}\n exporters:\n kafka: {topic: t}\n service:\n pipelines:\n metrics: { receivers: [1], exporters: [kafka] }\n` }, { repoName: 'vals', now: NOW }); + assert(broken.canonical.spec.telemetry.backends.some(b => b.product === 'loki') + && broken.summary.warnings.some(w => /^The collector config in charts\/c\/values\.yaml could not be read \(.*\); its images are read as before\.$/.test(w)), + 'a collector config the walk cannot read is warned about and the values file keeps its images', broken.summary.warnings); + assert(detectArtefactKind('a.yaml', 'a: 1\n', null) === 'unknown', 'detectArtefactKind takes null options, as before'); + // The environment scope names an excluded values file or Helm template + // that carries scrape jobs, as it names a plain collector config. + const TPL = 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: {{ include "c.fullname" . }}\ndata:\n relay.yaml: |\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); + const scoped = crawlFiles({ + 'charts/c/templates/cm.yaml': TPL, 'charts/c/Chart.yaml': 'apiVersion: v2\nname: c\nversion: 0.1.0\n', + 'charts/v/values.yaml': nested, 'docker-compose.yml': 'services:\n p:\n image: prom/prometheus:v2.51.0\n', + }, { repoName: 'vals', environment: 'local-docker', now: NOW }); + const w = scoped.summary.warnings.find(x => /carry scrape jobs/.test(x)) || ''; + assert(w.includes('charts/c/templates/cm.yaml (template scrape jobs: qmgrs)') && w.includes('charts/v/values.yaml (collector scrape jobs: qmgrs)'), + 'an excluded Helm template or values file with scrape jobs is named in the scope warning', w); +} + +// ---------- G. a malformed manifest ---------- +{ + const r = crawlFiles({ + 'a/metrics-manifest.json': '{ "metrics": [] }', + 'b/metrics-manifest.json': JSON.stringify({ exporter: 'x', metrics: [{ name: 'ok_metric', type: 'Gauge' }, { name: '9bad', type: 'gauge' }, { name: 'ok_metric', type: 'counter' }, { name: 'no_type' }] }), + 'c/metrics-manifest.json': 'ghp_SECRETxyz: not json', + }, { repoName: 'bad', now: NOW }); + const w = r.summary.warnings; + assert(w.includes('a/metrics-manifest.json is not a metrics manifest (no `exporter` name); it is not read.') + && w.includes('c/metrics-manifest.json is not a metrics manifest (not JSON); it is not read.') && !JSON.stringify(r.summary).includes('ghp_'), + 'a manifest that is not one is named in a warning, none of its text quoted', w); + const bom = crawlFiles({ + 'metrics-manifest.json': `\uFEFF${JSON.stringify({ exporter: 'x', metrics: [{ name: 'bom_metric', type: 'gauge' }] })}`, + 'mq/config.yaml': `\uFEFF${FILES['mq-exporter/config.yaml']}`, + }, { repoName: 'bom', now: NOW }); + assert(json(bom, 'crawler.discovered.metric_names').includes('bom_metric') && bom.summary.files.byKind['exporter-config'] === 1, + 'a manifest and an exporter config saved with a BOM are read', bom.summary.files.byKind); + assert(w.some(x => x.startsWith('b/metrics-manifest.json: 3 metric entries are not read')), 'skipped entries are counted', w); + assert(JSON.stringify(json(r, 'crawler.discovered.metric_names')) === '["ok_metric"]', 'the valid entry is read (the type case-insensitive)'); + const m = readMetricsManifest(JSON.stringify({ exporter: 'x', source: 'https://u:p@h/metrics?t=1', metrics: [] })); + assert(m.ok && m.manifest.source === 'https://h/metrics', 'a manifest\'s source is recorded without userinfo or query', m.manifest); +} + +// ---------- H. no exporter evidence: none of the new keys ---------- +{ + const r = crawlFiles({ + 'otel-collector.yaml': 'receivers:\n otlp: {}\nexporters:\n otlp:\n endpoint: http://x:4317\nservice:\n pipelines:\n metrics: { receivers: [otlp], exporters: [otlp] }\n', + 'src/metrics.ts': "import { Counter } from 'prom-client';\nexport const c = new Counter({ name: 'reqs_total', help: 'r' });\n", + }, { repoName: 'plain', now: NOW }); + const keys = Object.keys(ann(r)); + assert(!keys.some(k => /^crawler\.discovered\.(exporters|scrape_job)/.test(k)) && !('exporters' in r.summary.discovered) + && JSON.stringify(r.canonical.spec.pipelines.exporters.metrics) === '{"kind":"otlp"}', + 'a crawl with no exporter or collector scrape evidence carries no new key', keys.filter(k => /discovered/.test(k))); +} + +// ---------- the GitHub crawl's file budget ---------- +// The exporter patterns match names too common to fetch first: a monorepo's +// 210 config.yaml files never push out its rules, Alertmanager or collector. +{ + const tree = [ + ...Array.from({ length: 210 }, (_, i) => ({ type: 'blob', path: `apps/svc-${String(i).padStart(3, '0')}/config.yaml`, size: 100 })), + { type: 'blob', path: 'monitoring/alertmanager.yml' }, { type: 'blob', path: 'monitoring/slo.rules.yaml' }, + { type: 'blob', path: 'otel/otel-collector-config.yaml' }, { type: 'blob', path: 'mq/metrics-manifest.json' }, + { type: 'blob', path: 'snap/ibm-mq.prom' }, { type: 'blob', path: 'test/fixtures/x.prom' }, { type: 'blob', path: 'charts/c/files/collector.yaml' }, + { type: 'tree', path: 'apps' }, { type: 'blob', path: 'big/config.yaml', size: 2e6 }, + ]; + const picked = selectCrawlerFiles(tree, { maxFiles: 200, fileCapBytes: 1e6 }).map(n => n.path); + assert(picked.length === 200 && picked.slice(0, 4).join() === 'monitoring/alertmanager.yml,monitoring/slo.rules.yaml,otel/otel-collector-config.yaml,mq/metrics-manifest.json', + 'the established files and the manifest come first; the exporter candidates fill what is left', picked.slice(0, 5)); + const all = selectCrawlerFiles(tree, { maxFiles: 1000, fileCapBytes: 1e6 }).map(n => n.path); + assert(all.includes('snap/ibm-mq.prom') && all.includes('charts/c/files/collector.yaml') && !all.includes('test/fixtures/x.prom') && !all.includes('big/config.yaml') && !all.includes('apps'), + 'with room, a snapshot and a chart\'s files/ collector are fetched; a fixture snapshot, an oversized file and a tree never'); + assert(!isExporterCandidateFile('src/componentTest/resources/node.prom') && !isExporterCandidateFile('src/smokeTest/a.prom') && isExporterCandidateFile('snap/a.prom'), + 'the GitHub crawl skips a camelCase test source set\'s snapshot, as the crawler does'); + assert(isCrawlerFile('mq/metrics-manifest.json') && !isCrawlerFile('apps/x/config.yaml') && isExporterCandidateFile('apps/x/config.yaml') + && !isExporterCandidateFile('monitoring/prometheus.yml'), 'isCrawlerFile is develop\'s rule plus the manifest; the broad names are candidates only'); +} + +// ---------- I. packc emit --exporter ---------- +const EXPOSITION = '# HELP ibmmq_qmgr_status Status\n# TYPE ibmmq_qmgr_status gauge\nibmmq_qmgr_status{qmgr="QM1",platform="UNIX"} 1\n' + + '# TYPE ibmmq_queue_depth gauge\nibmmq_queue_depth{qmgr="QM1",queue="APP.IN"} 3\n'; +const server = createServer((req, res) => { + const path = new URL(req.url, 'http://x').pathname; + if (path === '/metrics') { res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' }); res.end(EXPOSITION); return; } + if (path === '/moved') { res.writeHead(302, { location: '/metrics?token=abc' }); res.end(); return; } + if (path === '/empty') { res.writeHead(200); res.end('\n'); return; } + if (path === '/cut') { res.writeHead(200, { 'content-length': '1000' }); res.write('# TYPE x gauge\n'); setTimeout(() => res.destroy(), 50); return; } + if (path === '/bare') { res.writeHead(302, { location: '/m?SECRETBARE&format=prom' }); res.end(); return; } + if (path === '/badloc') { res.writeHead(302, { location: 'http://[oops/metrics' }); res.end(); return; } + res.writeHead(500); res.end('boom'); +}); +await new Promise(r => server.listen(0, '127.0.0.1', r)); +const base = `http://127.0.0.1:${server.address().port}`; +// The child must not block the event loop the server answers on. +const run = (args, script = EMIT) => new Promise((resolveRun) => { + const child = spawn(process.execPath, [script, ...args], { cwd: tmp, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = '', stderr = ''; + child.stdout.on('data', d => { stdout += d; }); + child.stderr.on('data', d => { stderr += d; }); + child.on('close', status => resolveRun({ status, stdout, stderr })); +}); +const tmp = mkdtempSync(join(tmpdir(), 'emit-manifest-')); +try { + const ok = await run(['--exporter', `${base}/metrics?token=abc`, '--name', 'ibm-mq', '--version', '5.6.0', '--generated-at', NOW]); + assert(ok.status === 0 && ok.stdout === 'wrote metrics-manifest.json: 2 metric families of ibm-mq\n', 'emit writes metrics-manifest.json', ok); + const written = JSON.parse(readFileSync(join(tmp, 'metrics-manifest.json'), 'utf8')); + assert(JSON.stringify(written) === JSON.stringify({ + exporter: 'ibm-mq', version: '5.6.0', source: `${base}/metrics`, generatedAt: NOW, + metrics: [ + { name: 'ibmmq_qmgr_status', type: 'gauge', help: 'Status', labels: ['qmgr', 'platform'] }, + { name: 'ibmmq_queue_depth', type: 'gauge', labels: ['qmgr', 'queue'] }, + ], + }), 'the manifest: names, types, help, label names; the source without its query', written); + assert(!readFileSync(join(tmp, 'metrics-manifest.json'), 'utf8').includes('APP.IN'), 'no label value is written'); + const crawled = crawlFiles({ 'metrics-manifest.json': readFileSync(join(tmp, 'metrics-manifest.json'), 'utf8') }, { repoName: 'emitted', now: NOW }); + assert(JSON.stringify(json(crawled, 'crawler.discovered.metric_names')) === '["ibmmq_qmgr_status","ibmmq_queue_depth"]', 'the crawler reads what emit writes'); + assert(JSON.stringify(manifestFromExposition(EXPOSITION, { exporter: 'ibm-mq', version: '5.6.0', source: `${base}/metrics`, generatedAt: NOW })) === JSON.stringify(written), + 'the CLI writes exactly manifestFromExposition\'s manifest'); + + const viaPackc = await run(['emit', `--exporter=${base}/metrics`, '--out=-', '--no-timestamp'], CLI); + const piped = JSON.parse(viaPackc.stdout); + assert(viaPackc.status === 0 && piped.exporter === '127.0.0.1' && !('generatedAt' in piped) && piped.metrics.length === 2, + '`packc emit` dispatches; `--flag=value` is read; --out - writes to stdout; the host names the exporter; --no-timestamp leaves the stamp out', viaPackc); + + writeFileSync(join(tmp, 'qm1.prom'), EXPOSITION); + writeFileSync(join(tmp, 'node@1.0.prom'), EXPOSITION); + writeFileSync(join(tmp, 'svcSECRETF@h.prom'), '\n'); + mkdirSync(join(tmp, 'svcSECRETD@h')); + const named = await run(['--from', 'qm1.prom', '--out', 'outSECRETO@h.json', '--no-timestamp']); + assert(named.status === 0 && named.stdout === 'wrote (a URL-like value, not shown): 2 metric families of qm1\n', 'the success line names no URL-like --out', named); + const atName = await run(['--from', 'node@1.0.prom', '--out', '-', '--no-timestamp']); + assert(atName.status === 0 && JSON.parse(atName.stdout).exporter === 'node@1.0', 'a --from file whose name holds an @ is read', atName); + const from = await run(['--from', 'qm1.prom', '--out', '-', '--no-timestamp']); + assert(from.status === 0 && JSON.parse(from.stdout).exporter === 'qm1' && !('source' in JSON.parse(from.stdout)), '--from reads a saved exposition, named by the file', from); + + const refusals = [ + [['--exporter', `${base}/moved`], 1, `packc emit: ${base}/moved answered 302, a redirect to ${base}/metrics?token=… (query values not shown); it is not followed — pass the URL the exporter serves its metrics at, with the query values it expects\n`], + [['--exporter', `${base}/badloc?token=SECRET3`], 1, `packc emit: ${base}/badloc answered 302, a redirect to a location that is not a URL; it is not followed — pass the URL the exporter serves its metrics at\n`], + [['--exporter', '//admin:SECRET1@127.0.0.1:1/metrics'], 2, "packc emit: --exporter '//127.0.0.1:1/metrics' is not a URL (e.g. http://mq-exporter:9157/metrics); shown without anything before an @ or after a ? or #\n"], + [['--exporter', `${base}/broken`], 1, `packc emit: ${base}/broken answered 500; a metrics endpoint answers 200\n`], + [['--exporter', `${base}/empty`], 1, `packc emit: ${base}/empty holds no metric family (no # TYPE, # HELP or sample line)\n`], + [['--exporter', 'http://u:p@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], + [['--exporter', 'file:///etc/passwd'], 2, 'packc emit: --exporter must be an http or https URL, not file:\n'], + [['--exporter', `${base}/metrics`, '--from', 'qm1.prom'], 2, 'packc emit: pass --exporter or --from, not both\n'], + [['--exporter', `${base}/metrics`, '--bogus'], 2, null], + [['http://user:SECRET4@127.0.0.1:1/metrics?token=SECRET5'], 2, null], + [['mq:9157/metrics?token=SECRET8'], 2, null], + [['--from', 'http://svc:SECRET9@mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], + [['--from', 'x.prom', '--generated-at', 'http://u:SECRETG@h'], 2, 'packc emit: --generated-at (a URL-like value, not shown) is not a date\n'], + // The error code is the platform's (a ':' in a Windows path is invalid): only the exit and the unshown value are pinned. + [['--from', 'qm1.prom', '--out', 'http://u:SECRETO@h/x/y'], 2, null], + [['--exporter', `${base}/bare`], 1, `packc emit: ${base}/bare answered 302, a redirect to ${base}/m?format=… and a query part (query values not shown); it is not followed — pass the URL the exporter serves its metrics at, with the query values it expects\n`], + [['--from', 'svcSECRETF@h.prom', '--out', '-'], 1, 'packc emit: (a URL-like value, not shown) holds no metric family (no # TYPE, # HELP or sample line)\n'], + [['--from', 'svcSECRETD@h'], 2, null], // a directory: the code is the platform's + [['--from', 'qm1.prom', '--out', 'no/such/dir/m.json'], 2, 'packc emit: cannot write --out no/such/dir/m.json: ENOENT\n'], + [['--from=mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], + [['--exporter=http://user:SECRET6@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], + [['--exporter', `${base}/cut?token=SECRET7`], 1, `packc emit: could not read the response of ${base}/cut: the connection ended before the body did\n`], + [[], 2, null], + ]; + for (const [args, code, message] of refusals) { + const out = await run(args); + assert(out.status === code && out.stdout === '' && (message === null || out.stderr === message) && !/SECRET/.test(out.stderr), + `emit ${args.join(' ') || '(no args)'} → exit ${code}${message ? ', the reason named' : ''}`, out, { status: code, message }); + } + // The refusals' ways out work: save it yourself and pass --from (above); + // the redirect's target with the query values it expects (the first run). + assert(from.status === 0 && ok.status === 0, 'the credentials refusal\'s --from and the redirect refusal\'s URL with its query values are the runs that worked above'); +} finally { + server.close(); + rmSync(tmp, { recursive: true, force: true }); +} + +// The crawl CLI reads a downstream catalogue file; an unreadable one is a usage error. +{ + const dir = mkdtempSync(join(tmpdir(), 'crawl-cat-')); + try { + writeFileSync(join(dir, 'acme.yaml'), 'acme:\n tls: true\nlisten: ":8080"\n'); + writeFileSync(join(dir, 'cat.json'), JSON.stringify({ exporters: [{ id: 'acme-gw', match: { keys: ['acme', 'listen'] }, namespace: { default: 'acme' }, families: [{ name: 'up', type: 'gauge' }] }] })); + const crawl = fileURLToPath(new URL('./crawl-repo.mjs', import.meta.url)); + const okRun = spawnSync(process.execPath, [crawl, dir, '--name', 'gw', '--exporter-catalogue', join(dir, 'cat.json')], { encoding: 'utf8' }); + assert(okRun.status === 0 && okRun.stdout.includes('acme_up') && okRun.stderr.includes('# exporters : 1'), 'x-ray --exporter-catalogue reads the file', okRun.stderr.slice(-400)); + const bad = spawnSync(process.execPath, [crawl, dir, '--name', 'gw', '--exporter-catalogue', join(dir, 'missing.json')], { encoding: 'utf8' }); + assert(bad.status === 2 && bad.stdout === '' && /--exporter-catalogue .*missing\.json.*ENOENT/.test(bad.stderr), 'a missing catalogue file is a usage error', bad.stderr); + } finally { rmSync(dir, { recursive: true, force: true }); } +} + +report('crawl-exporters'); diff --git a/tools/test-platform.mjs b/tools/test-platform.mjs index 22cd0089..cf6576b3 100644 --- a/tools/test-platform.mjs +++ b/tools/test-platform.mjs @@ -67,6 +67,7 @@ const PATHNAME_ALLOWED = Object.freeze({ 'tools/build-studio-bundle.mjs': 1, // a --pack-url (http) path, for an id 'tools/lib/journey.mjs': 1, // a notify hook URL's path and query, redacted from a delivery error 'tools/record-mcp-fixtures.mjs': 1, // an MCP URL, printed without its query + 'tools/test-crawl-exporters.mjs': 1, // the fake exporter's HTTP request path }); // P6: the fixture's two call forms and what a well-formed argument is. const SKIP_CALL_RE = /\b(win32Skip|skipOnWin32)\(([^)]*)\)/g; diff --git a/tools/test-vendor-manifest.mjs b/tools/test-vendor-manifest.mjs index 18024511..846c186d 100644 --- a/tools/test-vendor-manifest.mjs +++ b/tools/test-vendor-manifest.mjs @@ -36,7 +36,8 @@ const EXPECTED_MODULES = [ 'tools/lib/backend-products.mjs', 'tools/lib/blast-radius.mjs', 'tools/lib/brand.mjs', 'tools/lib/burn-rules.mjs', 'tools/lib/chain-history.mjs', 'tools/lib/compile.mjs', 'tools/lib/conformance.mjs', 'tools/lib/contracts/mcp-capabilities.mjs', 'tools/lib/contracts/response-shapes.mjs', 'tools/lib/contracts/stack-self-metrics.mjs', 'tools/lib/crawler.mjs', - 'tools/lib/dashboards/generic.mjs', 'tools/lib/dashboards/lib.mjs', 'tools/lib/diff.mjs', 'tools/lib/good-when.mjs', 'tools/lib/identity-modes.mjs', + 'tools/lib/dashboards/generic.mjs', 'tools/lib/dashboards/lib.mjs', 'tools/lib/diff.mjs', 'tools/lib/exporter-catalogue.mjs', 'tools/lib/exporter-metrics.mjs', + 'tools/lib/good-when.mjs', 'tools/lib/identity-modes.mjs', 'tools/lib/inventory-coverage.mjs', 'tools/lib/journey-notify.mjs', 'tools/lib/l2x.mjs', 'tools/lib/legacy.mjs', 'tools/lib/library.mjs', 'tools/lib/live-fetch.mjs', 'tools/lib/mcp-client.mjs', 'tools/lib/mcp-server-settings.mjs', 'tools/lib/mcp-url-safety.mjs', 'tools/lib/mini-yaml.mjs', 'tools/lib/neuron-model.mjs', 'tools/lib/pack-conformance.mjs', 'tools/lib/profiles.mjs', 'tools/lib/promql-canon.mjs', 'tools/lib/promql-lezer.mjs', 'tools/lib/promql.mjs',