From f487910bc01c4dddbe86082434d58d75bca2d3b5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:12:40 +0000 Subject: [PATCH 01/17] crawler: learn metrics from prebuilt exporters and OTel collector scrape jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - metrics-manifest.json / *.metrics-manifest.json: a new artefact kind, entries declared (origin_kind exporter-manifest), ranked first - .prom / .metrics exposition snapshots: # HELP / # TYPE and label names, never label values, never from test or fixture paths - an exporter's config read against a pluggable family catalogue (tools/lib/exporter-catalogue.mjs, built-in IBM mq-metric-samples entry; --exporter-catalogue / opts.exporterCatalogue): inferred candidates - an OTel collector's prometheus receiver scrape jobs, with or without service.pipelines; the job records receiver, target count and the exporters it leaves by (kafka topic / endpoint without secrets); the collector's metrics exporter replaces the prometheusremotewrite scaffold - crawler.discovered.exporters; an environment scope warns about the scrape-job and exporter-config files it leaves out - packc emit --exporter (tools/emit-manifest.mjs) writes the manifest from a live /metrics endpoint; --from reads a saved one Goldens byte-identical. New suite tools/test-crawl-exporters.mjs; vendor manifest regenerated; CHANGELOG, DOWNSTREAM §18, README. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- README.md | 8 +- VENDOR-MANIFEST.json | 39 +- docs/CHANGELOG.md | 11 + docs/DOWNSTREAM.md | 126 +++++ package.json | 4 +- server/github-crawl.mjs | 6 + tools/cli.mjs | 5 + tools/crawl-repo.mjs | 20 + tools/emit-manifest.mjs | 117 +++++ tools/fixtures/crawl/exporters/README.md | 15 + .../charts/otel-collector/Chart.yaml | 3 + .../otel-collector/files/collector.yaml | 21 + .../otel-collector/templates/configmap.yaml | 7 + .../crawl/exporters/mq-exporter/Dockerfile | 2 + .../crawl/exporters/mq-exporter/config.yaml | 25 + .../mq-exporter/metrics-manifest.json | 14 + .../exporters/preprod/otel-collector.yaml | 16 + tools/lib/crawler.mjs | 424 ++++++++++++++--- tools/lib/exporter-catalogue.mjs | 74 +++ tools/lib/exporter-metrics.mjs | 366 +++++++++++++++ tools/lib/journey.mjs | 2 +- tools/test-crawl-exporters.mjs | 441 ++++++++++++++++++ tools/test-vendor-manifest.mjs | 3 +- 23 files changed, 1685 insertions(+), 64 deletions(-) create mode 100644 tools/emit-manifest.mjs create mode 100644 tools/fixtures/crawl/exporters/README.md create mode 100644 tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml create mode 100644 tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml create mode 100644 tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml create mode 100644 tools/fixtures/crawl/exporters/mq-exporter/Dockerfile create mode 100644 tools/fixtures/crawl/exporters/mq-exporter/config.yaml create mode 100644 tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json create mode 100644 tools/fixtures/crawl/exporters/preprod/otel-collector.yaml create mode 100644 tools/lib/exporter-catalogue.mjs create mode 100644 tools/lib/exporter-metrics.mjs create mode 100644 tools/test-crawl-exporters.mjs diff --git a/README.md b/README.md index 9d8293a9..627a5e87 100644 --- a/README.md +++ b/README.md @@ -1289,9 +1289,15 @@ The crawler reads source files such as: - Grafana unified-alerting provisioning YAML and Loki rule files - Grafana dashboard JSON - Alertmanager config -- OTel Collector config +- OTel Collector config, its prometheus receivers' scrape jobs included (with + or without a `service:` block) - Helm and Kubernetes manifests - Docker Compose files +- A prebuilt exporter's metrics: its `metrics-manifest.json`, a saved + `/metrics` (`.prom`, `.metrics`), or its config read against an exporter + catalogue (`--exporter-catalogue ` adds or replaces entries). + `packc emit --exporter ` writes the manifest from a live endpoint + ([`docs/DOWNSTREAM.md`](docs/DOWNSTREAM.md) §18) It emits a canonical v1.4 pack plus crawler annotations describing what was scanned and what was inferred. A fresh crawl validates (exit 0) and needs no diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 8e34a15c..c46e113e 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "6705a45caf5253caf795b0c23b5d1ea8cdbe75d8f91b4d748fec20911265c16f", - "bytes": 110301, + "sha256": "c4f564f2c0968120500defba9f61f0776e34342b2536011883a12553a7724e1f", + "bytes": 125803, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", @@ -395,6 +395,7 @@ "imports": [ "tools/lib/alert-routes.mjs", "tools/lib/backend-products.mjs", + "tools/lib/exporter-metrics.mjs", "tools/lib/l2x.mjs", "tools/lib/mini-yaml.mjs", "tools/lib/promql.mjs", @@ -510,6 +511,40 @@ "npm": [], "changedSinceRelease": false }, + "tools/lib/exporter-catalogue.mjs": { + "sha256": "594c0bb91e68a1a87a598ff57532d388e40f17a0133de6eec64db8e06c82cb9a", + "bytes": 5963, + "exports": [ + "EXPORTER_CATALOGUE" + ], + "imports": [], + "npm": [], + "changedSinceRelease": true, + "releasedExports": null + }, + "tools/lib/exporter-metrics.mjs": { + "sha256": "6a9cff6f68206ae8a43e0f6b1066cc3f595a0df128e4bf8d2bfdea6dd6e49ba5", + "bytes": 16534, + "exports": [ + "EXPOSITION_TYPES", + "inferExporterFamilies", + "isExpositionSnapshotPath", + "isMetricsManifestPath", + "manifestFromExposition", + "matchExporterConfig", + "mayBeExporterConfig", + "parseExposition", + "readMetricsManifest", + "resolveExporterCatalogue", + "safeSourceUrl" + ], + "imports": [ + "tools/lib/exporter-catalogue.mjs" + ], + "npm": [], + "changedSinceRelease": true, + "releasedExports": null + }, "tools/lib/good-when.mjs": { "sha256": "f89b81ef3c6afc30a60453deee4cc659503ec16f720351fcb61991766337e474", "bytes": 2557, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d9c32663..56338cd7 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -2,6 +2,17 @@ ## Unreleased +### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs +- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest, or entries without a name and an exposition type, are named in a warning. +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches them, manifests, `config.yaml` and a chart's `files/*.yaml`). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. It is never read from a test or fixture path. +- **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest. A malformed entry is dropped with a warning. `docs/DOWNSTREAM.md` §18 has the format. +- **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the one metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold, and its scrape receivers become `pipelines.receivers`. +- **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. +- **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. +- **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. +- No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. + ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. - **`server/test-store.mjs` Concurrency: a parent waits out the lock-holding child for as long as the child takes to commit** (a test fix; no runtime, `tools/lib`, golden or output change — the store's `BUSY_TIMEOUT_MS`, 5000 ms, is unchanged and still pinned on `openStore`). `Concurrency: a migration waits out a child holding BEGIN IMMEDIATE, then applies` failed the `windows (node 22)` job of run 38737678276 with `database is locked` (9.9 s; the same run's write sibling waited 2355 ms of a 700 ms hold). Each of the three Concurrency tests races two clocks: the parent's busy_timeout, 5000 ms from the child's "locked", against the child's hold of 700 ms plus its COMMIT plus however long a loaded runner keeps it off the CPU; the migration's child pays the most, since `closeStore` removed the `-wal` and `openRaw` leaves `synchronous` at FULL, so its commit syncs a new WAL header and then the commit inside the window. When the child's window outlasts the parent's budget, SQLite's busy handler gives up and `BEGIN IMMEDIATE` fails. The lock holder now sets `synchronous=OFF` (its durability is not under test, its lock is: no fsync between "locked" and its exit), and every parent that waits one out — the write, the migration and the repository write — waits up to `LOCK_WAIT_LIVENESS_MS`, 30 s, a liveness bound nothing asserts (the tests assert only that the wait was at least 300 ms, and what was written); a successful wait still ends at the child's COMMIT, about 700 ms. Reproduced on Linux (node 22.22.2) by delaying every `fsync` 1.8 s (`strace -f -e inject=fsync:delay_exit=1800000`) over the three tests, ten runs each: before, the write and the repository write waited 4.2–4.3 s and passed and the migration failed with `database is locked` in every run, as on CI; after, all three passed in every run (the write and the repository write waiting 0.6–0.7 s; the migration's 4.3 s is its own commit's syncs, outside the lock wait). And with the child's hold raised to 6 s, a window past the 5000 ms budget as a stalled child's would be, all three failed with `database is locked` before and pass after. No test added or removed: the suite total stays 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 553c36bb..7eed3b95 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1828,3 +1828,129 @@ no `refusedEnvNames` and binds every name the journey declares, as before. - *`journey-records-principal`* — journey records readable only by the member whose run wrote them (today every member of the org reads them, as dashboards). + +## 18. Metrics from prebuilt exporters (follow-up exporter-metrics) + +A service whose metrics come from a prebuilt exporter — IBM's +mq-metric-samples, a node_exporter, a vendor sidecar — has no client-library +call or PromQL that names them: the exporter's repository is a Dockerfile and +a config file, and the collector that scrapes it lives in another chart. The +crawler (`tools/lib/crawler.mjs`) reads the exporter's metrics from three +places, strongest first, through `tools/lib/exporter-metrics.mjs` (listed, +pure): + +| Source | Files | `origin_kind` | Confidence | +|---|---|---|---| +| a metrics manifest | `metrics-manifest.json`, `*.metrics-manifest.json` | `exporter-manifest` | `declared` | +| a saved exposition | `*.prom`, `*.metrics` (never under a test or fixture path) | `exporter-snapshot` | `declared` | +| the exporter's config, read against the catalogue | any YAML whose top-level keys match a catalogue entry | `exporter-config` | `inferred` (a candidate) | + +A metric named in several files reads as the strongest: manifest, snapshot, +source code, source fragment, recording-rule output, PromQL reference, and +an exporter config last. Every exporter the crawl learned from is listed in +`crawler.discovered.exporters` (`{ : { sources: [{ origin_kind, +file, metrics, … }] } }`) and counted in `crawler.discovered.exporters_count`. +A manifest and a snapshot are read in every environment; an exporter config +is scoped like any other config, and a crawl whose environment scope leaves +out a file with scrape jobs or exporter evidence says which in a warning. + +### 18.1 The manifest + +```json +{ + "exporter": "ibm-mq", + "version": "5.6.0", + "source": "http://mq-exporter:9157/metrics", + "generatedAt": "2026-10-01T00:00:00.000Z", + "metrics": [ + { "name": "ibmmq_queue_depth", "type": "gauge", "help": "Current depth of the queue", "labels": ["qmgr", "queue"] } + ] +} +``` + +`exporter` and `metrics` are required; each entry needs a metric name and an +exposition type (`counter`, `gauge`, `histogram`, `summary`, `untyped`, +`unknown`, `gaugehistogram`, `info`, `stateset`). An entry that has neither, +or repeats a name, is not read and the crawl says how many; a file that is not +a manifest is named in a warning. `source` is recorded without userinfo, +query or fragment. Label values are never part of a manifest. + +`packc emit --exporter ` (`tools/emit-manifest.mjs`) writes one from a +live endpoint — run it in the exporter's build pipeline against the image it +just built: + +```bash +packc emit --exporter http://mq-exporter:9157/metrics --name ibm-mq --version 5.6.0 --out mq-exporter/metrics-manifest.json +packc emit --from saved.prom --name ibm-mq --out - # a saved exposition, to stdout +``` + +It scrapes once and records each family's name, type, help and label names. +A redirect is refused, not followed (pass the URL it names); a URL with +credentials is refused (save the exposition with your own client and pass it +with `--from`); a status other than 200 or a response with no metric family +exits 1. + +### 18.2 The catalogue + +An exporter config names no metric; the catalogue says which families a +config switches on. The built-in catalogue (`tools/lib/exporter-catalogue.mjs`, +data only, `EXPORTER_CATALOGUE`) carries a small IBM MQ entry. A downstream +carries a larger one as a JSON file of the same shape, never as an engine +diff: + +```json +{ + "exporters": [ + { + "id": "ibm-mq", + "match": { "keys": ["global", "connection", "objects", "prometheus"] }, + "namespace": { "path": "prometheus.namespace", "default": "ibmmq" }, + "gates": { + "objectStatus": { "path": "global.useObjectStatus", "default": true }, + "publications": { "path": "global.usePublications", "default": true } + }, + "scopes": { "queues": { "path": "objects.queues" } }, + "families": [ + { "name": "queue_depth", "type": "gauge", "help": "Messages on the queue", "labels": ["qmgr", "queue"], "gate": "publications", "scope": "queues" } + ] + } + ] +} +``` + +- `match.keys` — the top-level keys a config must all carry. The first entry + that matches reads the file. +- `namespace` — the metric-name prefix: the value at `path` in the config, + else `default`; a family's name is `_`. +- `gates` — boolean switches (`true`/`false`, `yes`/`no`, `on`/`off`), the + `default` when the config is silent; a family with `gate` is emitted only + when it is on. +- `scopes` — object selections; a family with `scope` is emitted only when + the value at `path` selects something (a non-empty list, string or + mapping). + +Pass it with `packc x-ray --exporter-catalogue ` or +`crawlFiles(files, { exporterCatalogue })`. Its entries replace a built-in +entry with the same `id` and add the rest. An entry that is not well-formed +is dropped with a crawl warning naming what it lacks. A config-inferred +family stays a candidate (`confidence: inferred`): the config says the +exporter is switched on, not that the name is exact. Publish the real list +with `packc emit` and the manifest outranks it. + +### 18.3 The collector's scrape jobs + +An OTel collector's prometheus receivers (`prometheus`, +`prometheus/`) are read for their `config.scrape_configs`, with or +without a `service:` block (a Helm chart's `files/` copy has none). + +- Each job is in `crawler.discovered.scrape_jobs`. Its entry in + `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the + `receiver`, the `target_count` of its static targets, and the `exporters` + its samples leave by: the exporters of the metrics pipelines that list the + receiver, or, with no pipelines, the one metrics exporter the config names. + Each exporter is given as `{ name, kind, topic | endpoint }`: a kafka + exporter's metrics topic, any other's endpoint without userinfo, query or + fragment. +- That exporter's kind is `spec.pipelines.exporters.metrics`, in place of the + `prometheusremotewrite` scaffold. When a config with no pipelines names + several metrics exporters, the first is taken and the crawl warns. diff --git a/package.json b/package.json index 200e7db0..2f0bb0cf 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "lint:recorder": "node --check tools/record-mcp-fixtures.mjs", "test:fetch": "node tools/test-fetch-live.mjs", "lint:adapter": "node --check tools/adapt-spec-pack.mjs && node --check tools/lib/adapter.mjs && node --check tools/lib/mini-yaml.mjs && node --check tools/lib/promql.mjs && node --check tools/lib/promql-lezer.mjs && node --check tools/lib/traceability.mjs && node --check tools/lib/traceability-graph.mjs && node --check tools/lib/promql-canon.mjs", - "lint:crawler": "node --check tools/crawl-repo.mjs && node --check tools/lib/crawler.mjs", + "lint:crawler": "node --check tools/crawl-repo.mjs && node --check tools/lib/crawler.mjs && node --check tools/lib/exporter-metrics.mjs && node --check tools/emit-manifest.mjs", "lint:server": "node --check server/index.mjs", "lint:studio": "node --check studio/app.mjs", "lint:cli": "node --check tools/cli.mjs", @@ -118,7 +118,7 @@ "test:store:prestore:strict": "node tools/test-store-prestore-live.mjs --strict", "test:auth": "node server/test-auth-local.mjs && node server/test-auth-oidc.mjs && node --test server/test-authz.mjs && node --test server/test-identity-api.mjs", "users": "node tools/user-admin.mjs", - "test": "node --test server/test-auth-local.mjs server/test-auth-oidc.mjs server/test-audit-api.mjs server/test-audit-report-api.mjs server/test-auth-proxy.mjs server/test-authz.mjs server/test-brand-shell.mjs server/test-deploy-helpers.mjs server/test-glossary-shell.mjs server/test-hermetic-suites.mjs server/test-identity-api.mjs server/test-live-studio.mjs server/test-malformed-json.mjs server/test-mcp-jobs.mjs server/test-mcp-ping.mjs server/test-mcp-settings.mjs server/test-mcp-settings-studio.mjs server/test-mcp-target-policy.mjs server/test-notify-target-policy.mjs server/test-mcp-url.mjs server/test-service-admin.mjs server/test-services-api.mjs server/test-services-studio.mjs server/test-settings-studio.mjs server/test-smoke.mjs server/test-store-boot.mjs server/test-store-guards.mjs server/test-store-import.mjs server/test-store-ops.mjs server/test-store-paths.mjs server/test-store.mjs server/test-taxonomy.mjs server/test-tenancy.mjs server/test-transport-hook.mjs server/test-verdict-admin.mjs server/test-verdicts-api.mjs server/test-waivers-api.mjs server/test-workspace.mjs tools/test-adapt.mjs tools/test-artefact-classify.mjs tools/test-audit-report.mjs tools/test-backend-validate.mjs tools/test-blast-radius.mjs tools/test-brand.mjs tools/test-build-editor.mjs tools/test-build-info.mjs tools/test-build-model.mjs tools/test-bundle-compare.mjs tools/test-bundle-compare-pairs.mjs tools/test-chain-history.mjs tools/test-compile.mjs tools/test-contract-guard.mjs tools/test-contract-shapes.mjs tools/test-crawl.mjs tools/test-crawl-alerting-rules.mjs tools/test-crawl-canonical.mjs tools/test-declared-type.mjs tools/test-deploy-manifests.mjs tools/test-design-system.mjs tools/test-diagnostic-grade.mjs tools/test-diff.mjs tools/test-doc-test-totals.mjs tools/test-discover-rows.mjs tools/test-fetch-live.mjs tools/test-gen-pack.mjs tools/test-gen-site.mjs tools/test-glossary.mjs tools/test-golden-board.mjs tools/test-golden-compile.mjs tools/test-golden-crawl.mjs tools/test-golden-diff.mjs tools/test-identity-modes.mjs tools/test-inventory-coverage.mjs tools/test-journey-notify.mjs tools/test-journey-scope.mjs tools/test-journey.mjs tools/test-legacy-pack.mjs tools/test-library.mjs tools/test-live-model.mjs tools/test-mcp-settings-model.mjs tools/test-live-snapshot.mjs tools/test-mcp-ping.mjs tools/test-mcp-server-settings.mjs tools/test-mcp-transport.mjs tools/test-metric-readers.mjs tools/test-mini-yaml.mjs tools/test-neuron-model.mjs tools/test-pack-conformance.mjs tools/test-packs.mjs tools/test-platform.mjs tools/test-profiles.mjs tools/test-promql-canon.mjs tools/test-promql.mjs tools/test-record-fixtures.mjs tools/test-remediation-flow.mjs tools/test-remediation-flow-view.mjs tools/test-retrofeed.mjs tools/test-scan-live.mjs tools/test-schedule-snippets.mjs tools/test-schedule.mjs tools/test-service-keys.mjs tools/test-services-model.mjs tools/test-settings-model.mjs tools/test-stack-evidence.mjs tools/test-stack-self-metrics.mjs tools/test-static-remedies.mjs tools/test-studio-bundle.mjs tools/test-studio-graph.mjs tools/test-studio-layout.mjs tools/test-svg-charts.mjs tools/test-tap-summary.mjs tools/test-traceability-graph.mjs tools/test-upconvert-merge.mjs tools/test-ux-kit.mjs tools/test-validator.mjs tools/test-vendor-manifest.mjs tools/test-verdicts-panel.mjs tools/test-verify-deploy.mjs tools/test-waivers.mjs tools/test-zip.mjs", + "test": "node --test server/test-auth-local.mjs server/test-auth-oidc.mjs server/test-audit-api.mjs server/test-audit-report-api.mjs server/test-auth-proxy.mjs server/test-authz.mjs server/test-brand-shell.mjs server/test-deploy-helpers.mjs server/test-glossary-shell.mjs server/test-hermetic-suites.mjs server/test-identity-api.mjs server/test-live-studio.mjs server/test-malformed-json.mjs server/test-mcp-jobs.mjs server/test-mcp-ping.mjs server/test-mcp-settings.mjs server/test-mcp-settings-studio.mjs server/test-mcp-target-policy.mjs server/test-notify-target-policy.mjs server/test-mcp-url.mjs server/test-service-admin.mjs server/test-services-api.mjs server/test-services-studio.mjs server/test-settings-studio.mjs server/test-smoke.mjs server/test-store-boot.mjs server/test-store-guards.mjs server/test-store-import.mjs server/test-store-ops.mjs server/test-store-paths.mjs server/test-store.mjs server/test-taxonomy.mjs server/test-tenancy.mjs server/test-transport-hook.mjs server/test-verdict-admin.mjs server/test-verdicts-api.mjs server/test-waivers-api.mjs server/test-workspace.mjs tools/test-adapt.mjs tools/test-artefact-classify.mjs tools/test-audit-report.mjs tools/test-backend-validate.mjs tools/test-blast-radius.mjs tools/test-brand.mjs tools/test-build-editor.mjs tools/test-build-info.mjs tools/test-build-model.mjs tools/test-bundle-compare.mjs tools/test-bundle-compare-pairs.mjs tools/test-chain-history.mjs tools/test-compile.mjs tools/test-contract-guard.mjs tools/test-contract-shapes.mjs tools/test-crawl.mjs tools/test-crawl-alerting-rules.mjs tools/test-crawl-canonical.mjs tools/test-crawl-exporters.mjs tools/test-declared-type.mjs tools/test-deploy-manifests.mjs tools/test-design-system.mjs tools/test-diagnostic-grade.mjs tools/test-diff.mjs tools/test-doc-test-totals.mjs tools/test-discover-rows.mjs tools/test-fetch-live.mjs tools/test-gen-pack.mjs tools/test-gen-site.mjs tools/test-glossary.mjs tools/test-golden-board.mjs tools/test-golden-compile.mjs tools/test-golden-crawl.mjs tools/test-golden-diff.mjs tools/test-identity-modes.mjs tools/test-inventory-coverage.mjs tools/test-journey-notify.mjs tools/test-journey-scope.mjs tools/test-journey.mjs tools/test-legacy-pack.mjs tools/test-library.mjs tools/test-live-model.mjs tools/test-mcp-settings-model.mjs tools/test-live-snapshot.mjs tools/test-mcp-ping.mjs tools/test-mcp-server-settings.mjs tools/test-mcp-transport.mjs tools/test-metric-readers.mjs tools/test-mini-yaml.mjs tools/test-neuron-model.mjs tools/test-pack-conformance.mjs tools/test-packs.mjs tools/test-platform.mjs tools/test-profiles.mjs tools/test-promql-canon.mjs tools/test-promql.mjs tools/test-record-fixtures.mjs tools/test-remediation-flow.mjs tools/test-remediation-flow-view.mjs tools/test-retrofeed.mjs tools/test-scan-live.mjs tools/test-schedule-snippets.mjs tools/test-schedule.mjs tools/test-service-keys.mjs tools/test-services-model.mjs tools/test-settings-model.mjs tools/test-stack-evidence.mjs tools/test-stack-self-metrics.mjs tools/test-static-remedies.mjs tools/test-studio-bundle.mjs tools/test-studio-graph.mjs tools/test-studio-layout.mjs tools/test-svg-charts.mjs tools/test-tap-summary.mjs tools/test-traceability-graph.mjs tools/test-upconvert-merge.mjs tools/test-ux-kit.mjs tools/test-validator.mjs tools/test-vendor-manifest.mjs tools/test-verdicts-panel.mjs tools/test-verify-deploy.mjs tools/test-waivers.mjs tools/test-zip.mjs", "lint": "eslint .", "coverage": "c8 npm test", "sync-spec": "node tools/sync-spec.mjs", diff --git a/server/github-crawl.mjs b/server/github-crawl.mjs index 479f36a0..d30f3a51 100644 --- a/server/github-crawl.mjs +++ b/server/github-crawl.mjs @@ -30,6 +30,12 @@ export function isCrawlerFile(path) { return ( sourceMetricCandidate || /(^|\/)(application|bootstrap)[\w.-]*\.ya?ml$/.test(p) || + // A prebuilt exporter: its manifest, a saved /metrics, its config; a + // collector config a Helm chart keeps under files/. + /(^|[/.])metrics-manifest\.json$/.test(p) || + (/\.(prom|metrics)$/.test(p) && !/(^|\/)(tests?|__tests__|specs?|fixtures?|testdata)\//.test(p)) || + /(^|\/)config\.ya?ml$/.test(p) || + /(^|\/)files\/.*\.ya?ml$/.test(p) || /(^|\/)docker[-_]compose[a-z0-9._-]*\.(ya?ml)$/.test(p) || /(^|\/)compose[a-z0-9._-]*\.(ya?ml)$/.test(p) || /\.rules\.(ya?ml)$/.test(p) || diff --git a/tools/cli.mjs b/tools/cli.mjs index 8dba0914..2c6e6e9d 100644 --- a/tools/cli.mjs +++ b/tools/cli.mjs @@ -10,6 +10,7 @@ // packc validate → tools/validate-pack.mjs // packc adapt [env] → tools/adapt-spec-pack.mjs // packc x-ray → tools/crawl-repo.mjs +// packc emit --exporter → tools/emit-manifest.mjs (a prebuilt exporter's metrics-manifest.json) // packc compile [target] → tools/lib/compile.mjs (programmatic) // packc init … → tools/pack-init.mjs (build a pack from the library) // packc audit-report … → tools/audit-report.mjs (a pack's service audit report, JSON or HTML) @@ -114,6 +115,7 @@ Usage: packc audit-report [--env ] [--format json|html|both] [--out ] [--brand ] [--taxonomy ] [--top ] [--generated-at | --no-timestamp] [--verdicts ] [--waivers ] [--schema ] A pack's service audit report: conformance, placeholders, verdicts, waivers, coverage, goes-blind risks, response path packc adapt [env] Adapt a pack into the layered projection packc x-ray Crawl a repo into a draft pack + packc emit --exporter [--name ] [--version ] [--out |-] Scrape an exporter's /metrics once and write its metrics-manifest.json (--from reads a saved one) packc compile [target] Compile a pack into a backend artefact packc init --list List the library entries (products and archetypes) a pack can be built from packc init --show An entry's params, SLIs per tier, per-tier objectives and evidence @@ -282,6 +284,9 @@ switch (command) { case 'crawl': delegate('tools/crawl-repo.mjs', rest); break; + case 'emit': + delegate('tools/emit-manifest.mjs', rest); + break; case 'compile': await runCompile(rest); break; diff --git a/tools/crawl-repo.mjs b/tools/crawl-repo.mjs index a39df1c4..7f71eb54 100644 --- a/tools/crawl-repo.mjs +++ b/tools/crawl-repo.mjs @@ -78,6 +78,10 @@ manifest by introspecting common observability artefacts. (the original is kept in crawler.nameNormalizedFrom). --diff-scope live-drift comparison lens: service | family | all (default: service) + --exporter-catalogue + a JSON exporter family catalogue whose entries + replace or extend the built-in ones + (tools/lib/exporter-catalogue.mjs; docs/DOWNSTREAM.md §18) -h, --help print this message Output: @@ -96,6 +100,12 @@ manifest by introspecting common observability artefacts. - OTel Collector configs → spec.pipelines - Grafana dashboard JSONs → spec.dashboards[] - Helm values / K8s workloads→ spec.telemetry.backends[] + - metrics-manifest.json, saved + /metrics (.prom, .metrics), + a prebuilt exporter's config → the exporter's metrics (crawler.discovered.exporters) + - an OTel Collector's prometheus + receiver scrape jobs → crawler.discovered.scrape_jobs, and its + metrics exporter → spec.pipelines.exporters.metrics Example: npm run crawl -- ./payments-svc --name payments --criticality tier-2 > payments.pack.yaml @@ -121,6 +131,7 @@ function parseArgs(argv) { else if (a === '--binding') out.binding = argv[++i]; else if (a === '--owners') out.owners = (argv[++i] ?? '').split(','); else if (a === '--diff-scope' || a === '--live-scope') out.diffScopeMode = argv[++i]; + else if (a === '--exporter-catalogue') out.exporterCatalogueFile = argv[++i] ?? ''; } return out; } @@ -165,6 +176,14 @@ async function main() { process.exit(2); } if (!opts.repoName) opts.repoName = basename(opts.repoPath.replace(/[\\/]$/, '')); + if (opts.exporterCatalogueFile !== undefined) { + try { + opts.exporterCatalogue = JSON.parse(readFileSync(opts.exporterCatalogueFile, 'utf8')); + } catch (e) { + process.stderr.write(`crawl-repo: --exporter-catalogue ${JSON.stringify(opts.exporterCatalogueFile)}: ${e instanceof SyntaxError ? `not JSON (${e.message})` : (e.code || e.message)}\n`); + process.exit(2); + } + } const files = await walk(opts.repoPath); const { yaml, canonical, summary, evidence } = crawlToYaml(files, opts); @@ -195,6 +214,7 @@ async function main() { `# alert rules : ${summary.discovered.alertRules} (operational, alerting.rules)`, `# metric definitions: ${summary.discovered.metricDefinitions}`, `# scrape jobs : ${summary.discovered.scrapeJobs}`, + ...(summary.discovered.exporters ? [`# exporters : ${summary.discovered.exporters}`] : []), `# dashboards : ${summary.discovered.dashboards}`, `# alerting routes : ${summary.discovered.alertingRoutes}`, `# pipelines : ${summary.discovered.pipelines}`, diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs new file mode 100644 index 00000000..21e8448b --- /dev/null +++ b/tools/emit-manifest.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node +/** + * tools/emit-manifest.mjs — `packc emit --exporter `: a prebuilt + * exporter's metrics manifest, read off its live /metrics endpoint. + * + * A repository that ships a prebuilt exporter (a Dockerfile and a config + * file) names none of the exporter's metrics; the crawler reads them from a + * `metrics-manifest.json` beside the config (tools/lib/exporter-metrics.mjs). + * This writes one: it scrapes the endpoint once and records each metric + * family's name, type, help and label names — never a label value, never a + * sample. Run it in the exporter's build pipeline against the image it just + * built and commit (or publish) the result. + * + * node tools/emit-manifest.mjs --exporter [--name ] [--version ] [--out |-] + * node tools/emit-manifest.mjs --from [--name ] [--version ] [--out |-] + * --exporter the exporter's metrics URL (http or https), e.g. http://mq-exporter:9157/metrics + * --from read a saved exposition instead (one the scrape needs credentials for: save it + * with your own client, then pass it here) + * --name the exporter's name in the manifest (default: the URL's host, or the file's name) + * --version the exporter's version, recorded as given + * --out where to write (default: metrics-manifest.json); `-` writes to stdout + * --generated-at the stamp (default: the clock); --no-timestamp leaves it out (reproducible bytes) + * --timeout the scrape timeout in seconds (default 15) + * + * exit 0 the manifest was written + * exit 1 the scrape failed: unreachable, a redirect, a status other than 200, no metric families + * exit 2 usage: no source, both sources, an unknown flag, a URL that is not http(s) or carries credentials + * + * The recorded `source` is the URL without userinfo, query or fragment. A + * redirect is refused, not followed: pass the URL it names. + */ + +import { readFileSync, writeFileSync } from 'node:fs'; +import { resolve, basename } from 'node:path'; +import { manifestFromExposition, safeSourceUrl } from './lib/exporter-metrics.mjs'; + +const USAGE = 'usage: packc emit --exporter | --from [--name ] [--version ] [--out |-] [--generated-at | --no-timestamp] [--timeout ]'; + +function fail(code, message) { + process.stderr.write(`packc emit: ${message}\n`); + process.exit(code); +} + +function parseArgs(argv) { + const out = { out: 'metrics-manifest.json', timeout: 15, timestamp: true }; + const takes = new Set(['--exporter', '--from', '--name', '--version', '--out', '--generated-at', '--timeout']); + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '-h' || a === '--help') { out.help = true; continue; } + if (a === '--no-timestamp') { out.timestamp = false; continue; } + if (!takes.has(a)) fail(2, `unknown argument '${a}'\n${USAGE}`); + const v = argv[++i]; + if (v === undefined || (v.startsWith('--') && v !== '-')) fail(2, `${a} needs a value\n${USAGE}`); + out[a.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = v; + } + return out; +} + +async function scrape(url, timeoutSeconds) { + let res; + try { + res = await fetch(url, { + redirect: 'manual', + headers: { accept: 'text/plain;version=0.0.4;q=1, application/openmetrics-text;q=0.5, */*;q=0.1' }, + signal: AbortSignal.timeout(timeoutSeconds * 1000), + }); + } catch (e) { + const why = e?.name === 'TimeoutError' ? `no answer within ${timeoutSeconds}s (raise --timeout)` : (e?.cause?.message || e?.message || String(e)); + fail(1, `could not scrape ${safeSourceUrl(url)}: ${why}`); + } + if (res.status >= 300 && res.status < 400) { + const to = res.headers.get('location'); + fail(1, `${safeSourceUrl(url)} answered ${res.status}, a redirect${to ? ` to ${safeSourceUrl(new URL(to, url).href)}` : ''}; it is not followed — pass the exporter's own metrics URL`); + } + if (res.status !== 200) fail(1, `${safeSourceUrl(url)} answered ${res.status}; a metrics endpoint answers 200`); + return res.text(); +} + +const args = parseArgs(process.argv.slice(2)); +if (args.help) { process.stdout.write(`${USAGE}\n`); process.exit(0); } +if (!args.exporter && !args.from) fail(2, `name the exporter's metrics URL (--exporter) or a saved exposition (--from)\n${USAGE}`); +if (args.exporter && args.from) fail(2, 'pass --exporter or --from, not both'); +const timeout = Number(args.timeout); +if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 600) fail(2, '--timeout is a number of seconds, 1 to 600'); +let generatedAt; +if (args.timestamp) { + const at = args.generatedAt === undefined ? new Date() : new Date(args.generatedAt); + if (Number.isNaN(at.getTime())) fail(2, `--generated-at '${args.generatedAt}' is not a date`); + generatedAt = at.toISOString(); +} else if (args.generatedAt !== undefined) fail(2, 'pass --generated-at or --no-timestamp, not both'); + +let text, source, defaultName; +if (args.exporter) { + let url; + try { url = new URL(args.exporter); } catch { fail(2, `--exporter '${args.exporter}' is not a URL (e.g. http://mq-exporter:9157/metrics)`); } + if (url.protocol !== 'http:' && url.protocol !== 'https:') fail(2, `--exporter must be an http or https URL, not ${url.protocol}`); + if (url.username || url.password) fail(2, 'the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from'); + text = await scrape(url.href, timeout); + source = url.href; + defaultName = url.hostname; +} else { + try { text = readFileSync(resolve(args.from), 'utf8'); } catch (e) { fail(2, `cannot read --from ${args.from}: ${e.code || e.message}`); } + defaultName = basename(args.from).replace(/\.[^.]+$/, ''); +} + +const name = String(args.name ?? defaultName ?? '').trim(); +if (!name) fail(2, 'the exporter has no name; pass --name'); +const manifest = manifestFromExposition(text, { exporter: name, version: args.version, source, generatedAt }); +if (!manifest.metrics.length) fail(1, `${source ? safeSourceUrl(source) : args.from} holds no metric family (no # TYPE, # HELP or sample line)`); + +const json = `${JSON.stringify(manifest, null, 2)}\n`; +if (args.out === '-') { + process.stdout.write(json, () => process.exit(0)); +} else { + writeFileSync(resolve(args.out), json); + process.stdout.write(`wrote ${args.out}: ${manifest.metrics.length} metric famil${manifest.metrics.length === 1 ? 'y' : 'ies'} of ${name}\n`, () => process.exit(0)); +} diff --git a/tools/fixtures/crawl/exporters/README.md b/tools/fixtures/crawl/exporters/README.md new file mode 100644 index 00000000..234c7884 --- /dev/null +++ b/tools/fixtures/crawl/exporters/README.md @@ -0,0 +1,15 @@ +# Exporter fixture + +A service whose metrics come from a prebuilt exporter (IBM's +mq-metric-samples) that an OTel collector scrapes and ships to Kafka. +Nothing here registers or queries a metric: the crawler learns the names +from the exporter's `metrics-manifest.json`, the families its `config.yaml` +switches on, and the collector's `qmgrs` scrape job. Read by +`tools/test-crawl-exporters.mjs`. + +- `mq-exporter/` — the exporter's repository: a Dockerfile (not scanned), + its config and the manifest `packc emit --exporter` writes. +- `charts/otel-collector/files/collector.yaml` — the collector config a + Helm chart keeps under `files/`, with no `service:` block. +- `preprod/otel-collector.yaml` — the preprod collector; a `prod` crawl + leaves it out and says so. diff --git a/tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml b/tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml new file mode 100644 index 00000000..626eb7f4 --- /dev/null +++ b/tools/fixtures/crawl/exporters/charts/otel-collector/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: otel-collector +version: 0.1.0 diff --git a/tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml b/tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml new file mode 100644 index 00000000..e93189b9 --- /dev/null +++ b/tools/fixtures/crawl/exporters/charts/otel-collector/files/collector.yaml @@ -0,0 +1,21 @@ +receivers: + prometheus/mq: + config: + global: + scrape_interval: 30s + scrape_configs: + - job_name: qmgrs + metrics_path: /metrics + static_configs: + - targets: ["mq-exp-qm1:9157", "mq-exp-qm2:9157", "mq-exp-qm3:9157"] + labels: + env: prod +processors: + batch: {} +exporters: + debug: {} + kafka: + brokers: ["kafka-0.kafka:9092", "kafka-1.kafka:9092"] + metrics: + topic: mq-metrics + encoding: otlp_proto diff --git a/tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml b/tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml new file mode 100644 index 00000000..10a5d50d --- /dev/null +++ b/tools/fixtures/crawl/exporters/charts/otel-collector/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "otel-collector.fullname" . }} +data: + relay: | +{{ .Files.Get "files/collector.yaml" | indent 4 }} diff --git a/tools/fixtures/crawl/exporters/mq-exporter/Dockerfile b/tools/fixtures/crawl/exporters/mq-exporter/Dockerfile new file mode 100644 index 00000000..f21c8c4a --- /dev/null +++ b/tools/fixtures/crawl/exporters/mq-exporter/Dockerfile @@ -0,0 +1,2 @@ +FROM ibmcom/mq-metric-samples:5.6.0 +COPY config.yaml /opt/config/mq_prometheus.yaml diff --git a/tools/fixtures/crawl/exporters/mq-exporter/config.yaml b/tools/fixtures/crawl/exporters/mq-exporter/config.yaml new file mode 100644 index 00000000..1887cb80 --- /dev/null +++ b/tools/fixtures/crawl/exporters/mq-exporter/config.yaml @@ -0,0 +1,25 @@ +global: + useObjectStatus: true + useResetQStats: false + usePublications: true + logLevel: INFO + pollInterval: 30s + rediscoverInterval: 1h +connection: + queueManager: QM1 + clientConnection: true + user: "" + replyQueue: SYSTEM.DEFAULT.MODEL.QUEUE +objects: + queues: + - "APP.*" + - "!SYSTEM.*" + channels: + - "TO.*" + topics: + subscriptions: + showInactiveChannels: false +prometheus: + port: 9157 + metricsPath: /metrics + namespace: ibmmq diff --git a/tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json b/tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json new file mode 100644 index 00000000..99e709f3 --- /dev/null +++ b/tools/fixtures/crawl/exporters/mq-exporter/metrics-manifest.json @@ -0,0 +1,14 @@ +{ + "exporter": "ibm-mq", + "version": "5.6.0", + "source": "http://mq-exporter:9157/metrics", + "generatedAt": "2026-10-01T00:00:00.000Z", + "metrics": [ + { "name": "ibmmq_qmgr_status", "type": "gauge", "help": "Queue manager status (1 = running)", "labels": ["qmgr", "platform"] }, + { "name": "ibmmq_qmgr_uptime", "type": "gauge", "help": "Queue manager uptime in seconds", "labels": ["qmgr", "platform"] }, + { "name": "ibmmq_queue_depth", "type": "gauge", "help": "Current depth of the queue", "labels": ["qmgr", "queue", "usage", "description"] }, + { "name": "ibmmq_queue_mqget_count", "type": "counter", "help": "MQGET calls", "labels": ["qmgr", "queue"] }, + { "name": "ibmmq_channel_status", "type": "gauge", "help": "Channel status", "labels": ["qmgr", "channel", "type", "connname"] }, + { "name": "ibmmq_qmgr_log_write_size_bytes", "type": "gauge", "help": "Log write size", "labels": ["qmgr", "platform"] } + ] +} diff --git a/tools/fixtures/crawl/exporters/preprod/otel-collector.yaml b/tools/fixtures/crawl/exporters/preprod/otel-collector.yaml new file mode 100644 index 00000000..171eae09 --- /dev/null +++ b/tools/fixtures/crawl/exporters/preprod/otel-collector.yaml @@ -0,0 +1,16 @@ +receivers: + prometheus: + config: + scrape_configs: + - job_name: qmgrs-preprod + scrape_interval: 60s + static_configs: + - targets: ["mq-exp-pp1:9157"] +exporters: + kafka: + topic: mq-metrics-preprod +service: + pipelines: + metrics: + receivers: [prometheus] + exporters: [kafka] diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 55797ca1..8e9dee42 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -33,6 +33,10 @@ import { routesFromAlertmanagerConfig } from './alert-routes.mjs'; import { BACKEND_PATTERNS } from './backend-products.mjs'; import { PROMQL_KEYWORDS, extractPromqlMetricNames } from './promql.mjs'; import { symbolSlug as slug, packSlug } from './slug.mjs'; +import { + isMetricsManifestPath, isExpositionSnapshotPath, parseExposition, readMetricsManifest, safeSourceUrl, + resolveExporterCatalogue, mayBeExporterConfig, matchExporterConfig, inferExporterFamilies, +} from './exporter-metrics.mjs'; // The spec's closed vocabularies, mirrored by hand (a browser-safe module // cannot read the schema file; tools/test-crawl-canonical.mjs pins the @@ -58,7 +62,8 @@ function suffixedId(base, suffix) { // before it used this rule it staged 75,000 files (430 MB) of a repository // whose scannable sources are 600. Where the browser hands over the folder // itself, walkScanFolder (below) does not list them in the first place. -export const SCAN_EXT = /\.(ya?ml|json|cjs|mjs|js|jsx|ts|tsx|py|go|java|kt|rs|cs)$/i; +// `.prom` / `.metrics`: a saved /metrics response (an exposition snapshot). +export const SCAN_EXT = /\.(ya?ml|json|cjs|mjs|js|jsx|ts|tsx|py|go|java|kt|rs|cs|prom|metrics)$/i; export const SCAN_MAX_FILE_BYTES = 5 * 1024 * 1024; export const SCAN_IGNORE_DIRS = new Set([ 'node_modules', 'vendor', 'venv', @@ -140,6 +145,12 @@ const K8S_WORKLOAD_KINDS = new Set([ const METRIC_SOURCE_EXT_RE = /\.(?:cjs|mjs|js|jsx|ts|tsx|py|go|java|kt|rs|cs)$/i; +// The built-in exporter catalogue (tools/lib/exporter-catalogue.mjs), the one +// detectArtefactKind reads when its caller names none. +const BUILTIN_EXPORTER_CATALOGUE = resolveExporterCatalogue(); + +const isMapping = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); + // Match an image reference (`repository[:tag]`) against the known backend // catalog and register it on the backends bucket. Shared by the Helm values, // Helm template, and plain-K8s workload walkers. When `dedupeByProduct` is set @@ -191,12 +202,25 @@ function registerMetricsExporterFromBackend(pattern, pipelines, relPath, evidenc * Detect what kind of observability artefact a file is. * @param {string} relPath - repo-relative path * @param {string} content - file content + * @param {object} [opts] + * @param {object} [opts.exporterCatalogue] - a resolved exporter catalogue + * (resolveExporterCatalogue); the built-in one when omitted * @returns {string} kind: 'prometheus-rules' | 'alertmanager' | 'grafana-dashboard' | * 'otel-collector' | 'docker-compose' | 'helm-chart' | - * 'helm-template' | 'unknown' + * 'helm-template' | 'metrics-manifest' | 'exposition-snapshot' | + * 'exporter-config' | 'unknown' */ -export function detectArtefactKind(relPath, content) { +export function detectArtefactKind(relPath, content, { exporterCatalogue = BUILTIN_EXPORTER_CATALOGUE } = {}) { const lower = relPath.toLowerCase(); + // A prebuilt exporter's metric list, as `packc emit --exporter` writes it. + // Named by its file, so a malformed one is walked and warned about, never + // silently skipped. + if (isMetricsManifestPath(lower)) return 'metrics-manifest'; + // A saved /metrics response — not from a test or fixture path, where an + // exposition is sample input, not what a deployed exporter serves. + if (isExpositionSnapshotPath(lower)) { + return !isTestPath(lower) && parseExposition(content).families.length ? 'exposition-snapshot' : 'unknown'; + } // JSON first — Grafana dashboards are the only json we care about. if (lower.endsWith('.json')) { try { @@ -232,6 +256,15 @@ export function detectArtefactKind(relPath, content) { // config second. if (looksLikeHelm(content)) return 'helm-template'; + // A prebuilt exporter's own config (mq-metric-samples' config.yaml), by + // the top-level keys a catalogue entry names — before the filename + // heuristics, which would read `mq-collector.yaml` as an OTel collector. + if (mayBeExporterConfig(content, exporterCatalogue)) { + let cfg = null; + try { cfg = parseYaml(content); } catch { /* not YAML: fall through */ } + if (matchExporterConfig(cfg, exporterCatalogue)) return 'exporter-config'; + } + // Filename heuristics first — fastest. if (/(^|\/)(docker-)?compose\.ya?ml$/.test(lower)) return 'docker-compose'; if (/(^|\/)chart\.ya?ml$/.test(lower)) return 'helm-chart'; @@ -255,7 +288,7 @@ export function detectArtefactKind(relPath, content) { && obj.scrape_configs.some(s => s?.job_name)) return 'prometheus-scrape-config'; if (looksLikeActuatorMetricsConfig(lower, content, obj)) return 'actuator-metrics-config'; if (obj.route && obj.receivers && Array.isArray(obj.receivers)) return 'alertmanager'; - if (obj.receivers && obj.exporters && obj.service?.pipelines) return 'otel-collector'; + if (looksLikeOtelCollector(obj)) return 'otel-collector'; if (obj.services && typeof obj.services === 'object' && Object.values(obj.services).some(s => s?.image)) return 'docker-compose'; // Plain (non-templated) Kubernetes workload manifest carrying concrete @@ -369,6 +402,9 @@ function isSourceCodePath(relPath) { function shouldIncludeForEnvironment(relPath, profile, surfaces) { const p = normalizeRepoPath(relPath); if (isSourceCodePath(p)) return true; + // What an exporter exposes is the exporter's, not a deployment surface's: + // a metrics manifest and a saved exposition are read in every environment. + if (isMetricsManifestPath(p) || isExpositionSnapshotPath(p)) return true; if (profile === 'local-docker') { if (!surfaces.docker) return true; @@ -437,6 +473,9 @@ function scopeFilesForEnvironment(files, environment) { * @param {string} [opts.criticality] - 'tier-1'|'tier-2'|'tier-3'. Inferred if omitted. * @param {string} [opts.binding='otel-elastic-prometheus-grafana'] * @param {Array} [opts.owners=['team-platform']] + * @param {object|Array} [opts.exporterCatalogue] - exporter family catalogue + * entries ({ exporters: [...] } or the array) that replace or extend the + * built-in ones (tools/lib/exporter-catalogue.mjs) * @returns {{canonical: object, summary: object, evidence: Record}} */ export function crawlFiles(filesInput, opts = {}) { @@ -464,7 +503,9 @@ export function crawlFiles(filesInput, opts = {}) { // Profile detection reads the environment as given (it lowercases itself). const envScope = scopeFilesForEnvironment(rawFiles, envRaw); const files = envScope.files; - const inputWarnings = []; + const catalogueRead = resolveExporterCatalogue(opts.exporterCatalogue); + const catalogue = { exporters: catalogueRead.exporters }; + const inputWarnings = [...catalogueRead.warnings]; const normalized = { name: null, environment: null, owners: [] }; if (repoName !== String(nameRaw)) { normalized.name = { from: String(nameRaw), to: repoName }; @@ -522,6 +563,15 @@ export function crawlFiles(filesInput, opts = {}) { summary.warnings.push(...inputWarnings); if (envScope.applied) { summary.warnings.push(`Environment scope "${envScope.profile}" excluded ${envScope.excluded.length} file(s) from other deployment surfaces.`); + // A file the scope leaves out may be the only one that shows how the + // metrics are collected: say which, never drop them silently. + const collection = envScope.excluded + .map(relPath => collectionEvidenceOf(relPath, rawFiles.get(relPath), catalogue)) + .filter(Boolean); + if (collection.length) { + const shown = collection.slice(0, 8).join('; ') + (collection.length > 8 ? `; … (${collection.length - 8} more)` : ''); + summary.warnings.push(`Environment scope "${envScope.profile}" excluded ${collection.length} file(s) that carry scrape jobs or exporter evidence: ${shown}. An environment that names no deployment profile (e.g. \`all\`) reads every file.`); + } } const evidence = {}; // -> @@ -531,6 +581,7 @@ export function crawlFiles(filesInput, opts = {}) { const alertRules = []; // every alert rule found, raw, with its file and group (classified below) const metricDefinitions = []; const scrapeJobs = []; + const exporters = new Map(); // exporter name → the files that say what it exposes const dashboards = []; const alertingRoutes = []; const pipelines = { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }; @@ -555,7 +606,7 @@ export function crawlFiles(filesInput, opts = {}) { // ----- pass 1: classify ----- const classified = []; for (const [relPath, content] of files) { - const kind = detectArtefactKind(relPath, content); + const kind = detectArtefactKind(relPath, content, { exporterCatalogue: catalogue }); if (kind === 'unknown') continue; classified.push({ relPath, content, kind }); summary.files.classified++; @@ -583,7 +634,10 @@ export function crawlFiles(filesInput, opts = {}) { case 'actuator-metrics-config': walkActuatorMetricsConfig(f, scrapeJobs, evidence, summary, repoName); break; case 'metric-source-code': walkMetricSourceCode(f, metricDefinitions, evidence, summary, repoName); break; case 'alertmanager': walkAlertmanager(f, alertingRoutes, evidence, summary); break; - case 'otel-collector': walkOtelCollector(f, pipelines, evidence, summary); break; + case 'otel-collector': walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary); break; + case 'metrics-manifest': walkMetricsManifest(f, metricDefinitions, exporters, summary, evidence); break; + case 'exposition-snapshot': walkExpositionSnapshot(f, metricDefinitions, exporters, summary, evidence); break; + case 'exporter-config': walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence); break; case 'grafana-dashboard':walkGrafanaDashboard(f, dashboards, metricDefinitions, evidence, summary); break; case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, repoName }, evidence, summary); break; case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents); break; @@ -896,6 +950,7 @@ export function crawlFiles(filesInput, opts = {}) { curateMetricDefinitions(metricDefinitions, summary); const metricAnnotations = buildMetricDefinitionAnnotations(metricDefinitions); const scrapeAnnotations = buildScrapeJobAnnotations(scrapeJobs); + const exporterAnnotations = buildExporterAnnotations(exporters); // ----- canonical assembly ----- const canonical = { @@ -947,6 +1002,7 @@ export function crawlFiles(filesInput, opts = {}) { ...(sdkLanguages.length ? { 'crawler.discovered.sdk_languages': annotationJson(sdkLanguages) } : {}), ...metricAnnotations, ...scrapeAnnotations, + ...exporterAnnotations, ...scaffoldAnnotations, }, }, @@ -1117,7 +1173,7 @@ function sniffObservabilityKind(content) { if (Array.isArray(obj.scrape_configs) && obj.scrape_configs.some(s => s?.job_name)) return 'prometheus-scrape-config'; if (looksLikeActuatorMetricsConfig('', content, obj)) return 'actuator-metrics-config'; if (obj.route && Array.isArray(obj.receivers)) return 'alertmanager'; - if (obj.receivers && obj.exporters && obj.service?.pipelines) return 'otel-collector'; + if (looksLikeOtelCollector(obj)) return 'otel-collector'; if (obj.services && typeof obj.services === 'object' && Object.values(obj.services).some(s => s?.image)) return 'docker-compose'; if (K8S_WORKLOAD_KINDS.has(obj.kind) @@ -1410,7 +1466,8 @@ function addMetricDefinition(metricDefinitions, evidence, summary, item) { sourceName: item.sourceName && item.sourceName !== name ? item.sourceName : item.sourceName || null, originKind: item.originKind || 'source-code', candidateOnly: Boolean(item.candidateOnly), - confidence: item.candidateOnly ? 'candidate' : 'declared', + // A caller may state it ('inferred': an exporter config's family). + confidence: item.confidence || (item.candidateOnly ? 'candidate' : 'declared'), references: Array.isArray(item.references) ? item.references : [], usedBy: Array.isArray(item.usedBy) ? item.usedBy.filter(Boolean) : [], }; @@ -1435,7 +1492,9 @@ function mergeMetricEvidence(target, source) { target.usedBy = dedupe([...(target.usedBy || []), ...(source.usedBy || [])]).slice(0, 24); target.references = [...(target.references || []), ...(source.references || [])].slice(0, 12); target.candidateOnly = Boolean(target.candidateOnly && source.candidateOnly); - target.confidence = target.candidateOnly ? 'candidate' : 'declared'; + target.confidence = target.candidateOnly + ? (target.confidence === 'inferred' ? 'inferred' : 'candidate') + : 'declared'; } function dedupe(values) { @@ -1634,8 +1693,11 @@ function curateMetricDefinitions(metricDefinitions, summary) { let unreferenced = 0; for (const metric of metricDefinitions) { if (!metric.candidateOnly) continue; - metric.confidence = referenced.has(metric.name) ? 'referenced-candidate' : 'candidate'; if (!referenced.has(metric.name)) unreferenced++; + // An exporter config's family stays 'inferred': the config says the + // exporter is switched on, not that the name is right. + if (metric.confidence === 'inferred') continue; + metric.confidence = referenced.has(metric.name) ? 'referenced-candidate' : 'candidate'; } summary.discovered.metricEvidence = metricDefinitions.length; summary.discovered.metricDefinitions = new Set(metricDefinitions.map(m => m.name)).size; @@ -1693,12 +1755,19 @@ function annotationJson(value) { : compact; } +// Which file speaks for a metric named in several: what the exporter says it +// exposes first (its manifest, then a saved exposition), then the code that +// registers it, then what reads it, and last a name only inferred from an +// exporter's config. function metricOriginRank(metric) { const kind = metric?.originKind || 'source-code'; - if (kind === 'source-code') return 0; - if (kind === 'source-code-fragment') return 1; - if (kind === 'recording-rule-output') return 2; - if (kind === 'promql-reference') return 3; + if (kind === 'exporter-manifest') return 0; + if (kind === 'exporter-snapshot') return 1; + if (kind === 'source-code') return 2; + if (kind === 'source-code-fragment') return 3; + if (kind === 'recording-rule-output') return 4; + if (kind === 'promql-reference') return 5; + if (kind === 'exporter-config') return 6; return 9; } @@ -1717,6 +1786,14 @@ function buildScrapeJobAnnotations(scrapeJobs) { metrics_path: item.metrics_path || '', interval: item.interval || '', targets: item.targets || [], + // A job an OTel collector's prometheus receiver scrapes: the receiver, + // its static target count, and where the collector ships the samples. + ...(item.via ? { + via: item.via, + receiver: item.receiver, + target_count: item.target_count, + exporters: item.exporters, + } : {}), }; } return { @@ -1726,6 +1803,26 @@ function buildScrapeJobAnnotations(scrapeJobs) { }; } +// crawler.discovered.exporters: each prebuilt exporter the crawl learned +// metrics from, and the files that said what it exposes (a manifest, a saved +// exposition, its own config). +function buildExporterAnnotations(exporters) { + if (!exporters.size) return {}; + const names = [...exporters.keys()].sort(); + return { + 'crawler.discovered.exporters': annotationJson(Object.fromEntries(names.map(n => [n, exporters.get(n)]))), + 'crawler.discovered.exporters_count': String(names.length), + }; +} + +function recordExporter(exporters, name, source, summary) { + if (!exporters.has(name)) { + exporters.set(name, { sources: [] }); + summary.discovered.exporters = (summary.discovered.exporters || 0) + 1; + } + exporters.get(name).sources.push(source); +} + // Components disabled via Helm `enabled: false`, merged across the in-scope // values files. Base `values.yaml` is applied first; environment overlays // (`values-.yaml`) are applied after and win — mirroring `helm -f` order. @@ -1827,7 +1924,7 @@ function walkHelmTemplate(f, buckets, evidence, summary) { case 'prometheus-scrape-config': walkPrometheusScrapeConfig(sub, scrapeJobs, evidence, summary); return true; case 'actuator-metrics-config': walkActuatorMetricsConfig(sub, scrapeJobs, evidence, summary, repoName); return true; case 'alertmanager': walkAlertmanager(sub, alertingRoutes, evidence, summary); return true; - case 'otel-collector': walkOtelCollector(sub, pipelines, evidence, summary); return true; + case 'otel-collector': walkOtelCollector(sub, pipelines, scrapeJobs, evidence, summary); return true; case 'grafana-dashboard':walkGrafanaDashboard(sub, dashboards, metricDefinitions, evidence, summary); return true; case 'docker-compose': walkDockerCompose(sub, backends, pipelines, evidence, summary); return true; case 'k8s-workload': walkK8sWorkload(sub, backends, pipelines, evidence, summary); return true; @@ -1849,6 +1946,9 @@ function walkHelmTemplate(f, buckets, evidence, summary) { else if (lk.endsWith('.json') || /"schemaVersion"\s*:/.test(clean)) kind = 'grafana-dashboard'; else if (/^route:/m.test(clean) && /^receivers:/m.test(clean)) kind = 'alertmanager'; else if (/^receivers:/m.test(clean) && /^exporters:/m.test(clean) && /pipelines:/.test(clean)) kind = 'otel-collector'; + // A collector config with no service block: a prometheus receiver's + // scrape jobs, or its exporters alone. + else if (/^receivers:/m.test(clean) && (/^exporters:/m.test(clean) || /^\s+scrape_configs:/m.test(clean))) kind = 'otel-collector'; // A ConfigMap whose data key looks like an observability artefact but // can't be parsed is a genuine signal worth surfacing; ordinary @@ -2066,27 +2166,38 @@ function walkPrometheusRule(f, group, rule, recordingRules, burnRateAlerts, aler } function walkPrometheusScrapeConfig(f, scrapeJobs, evidence, summary) { - for (const obj of parseYamlDocs(f.content)) { - if (!Array.isArray(obj?.scrape_configs)) continue; - for (const cfg of obj.scrape_configs) { - const job = String(cfg?.job_name || '').trim(); - if (!job) continue; - // Promtail and Alloy use the same `scrape_configs` / `job_name` keys - // to say which LOG FILES to tail. Such an entry is not a metrics - // scrape job and never appears among a metrics store's targets; - // declaring it made it read "not live" on every platform. - if (isLogScrapeConfig(cfg, obj)) { - summary.discovered.logScrapeConfigs = (summary.discovered.logScrapeConfigs || 0) + 1; - continue; - } - addScrapeJob(scrapeJobs, evidence, summary, { - job, - metrics_path: cfg.metrics_path || '/metrics', - interval: cfg.scrape_interval || obj.global?.scrape_interval || null, - targets: scrapeTargets(cfg), - origin: f.relPath, - }); + for (const obj of parseYamlDocs(f.content)) walkScrapeConfigDoc(obj, f.relPath, scrapeJobs, evidence, summary); +} + +// One Prometheus config document's scrape jobs — a standalone prometheus.yml +// or the `config:` of an OTel collector's prometheus receiver (`collector` +// then names the receiver and the exporters its samples leave by). +function walkScrapeConfigDoc(obj, origin, scrapeJobs, evidence, summary, collector = null) { + if (!Array.isArray(obj?.scrape_configs)) return; + for (const cfg of obj.scrape_configs) { + const job = String(cfg?.job_name || '').trim(); + if (!job) continue; + // Promtail and Alloy use the same `scrape_configs` / `job_name` keys + // to say which LOG FILES to tail. Such an entry is not a metrics + // scrape job and never appears among a metrics store's targets; + // declaring it made it read "not live" on every platform. + if (isLogScrapeConfig(cfg, obj)) { + summary.discovered.logScrapeConfigs = (summary.discovered.logScrapeConfigs || 0) + 1; + continue; } + addScrapeJob(scrapeJobs, evidence, summary, { + job, + metrics_path: cfg.metrics_path || '/metrics', + interval: cfg.scrape_interval || obj.global?.scrape_interval || null, + targets: scrapeTargets(cfg), + origin, + ...(collector ? { + via: 'otel-collector', + receiver: collector.receiver, + target_count: (cfg.static_configs || []).reduce((n, sc) => n + (Array.isArray(sc?.targets) ? sc.targets.filter(Boolean).length : 0), 0), + exporters: collector.exporters, + } : {}), + }); } } @@ -2169,39 +2280,238 @@ function walkAlertmanager(f, routes, evidence, summary) { } } -function walkOtelCollector(f, pipelines, evidence, summary) { +// An OTel collector config: its pipelines (receivers, processors, the +// exporter per signal) and the scrape jobs of its prometheus receivers. A +// config with no `service:` block (a Helm chart's files/ copy, assembled +// with the pipelines elsewhere) still names its receivers and exporters: +// the prometheus receivers' jobs are read, and the one metrics exporter it +// configures is the metrics exporter. +function walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary) { for (const obj of parseYamlDocs(f.content)) { - if (!obj?.service?.pipelines) continue; - const seen = new Set(); - for (const pipeline of Object.values(obj.service.pipelines)) { - for (const recv of pipeline.receivers || []) { - if (!seen.has(`r:${recv}`)) { - seen.add(`r:${recv}`); - pipelines.receivers.push({ name: recv.split('/')[0] }); + const scrapeReceivers = collectorScrapeReceivers(obj); + const exporterCfgs = isMapping(obj?.exporters) ? obj.exporters : {}; + if (obj?.service?.pipelines) { + const seen = new Set(); + for (const pipeline of Object.values(obj.service.pipelines)) { + for (const recv of pipeline.receivers || []) { + if (!seen.has(`r:${recv}`)) { + seen.add(`r:${recv}`); + pipelines.receivers.push({ name: recv.split('/')[0] }); + } } - } - for (const proc of pipeline.processors || []) { - if (!seen.has(`p:${proc}`)) { - seen.add(`p:${proc}`); - pipelines.processors.push({ name: proc.split('/')[0] }); + for (const proc of pipeline.processors || []) { + if (!seen.has(`p:${proc}`)) { + seen.add(`p:${proc}`); + pipelines.processors.push({ name: proc.split('/')[0] }); + } + } + for (const exp of pipeline.exporters || []) { + const kind = exp.split('/')[0]; + const sig = Object.entries(obj.service.pipelines).find(([_, p]) => p === pipeline)?.[0] || ''; + const signalClass = /^metrics/i.test(sig) ? 'metrics' + : /^logs/i.test(sig) ? 'logs' + : /^traces/i.test(sig) ? 'traces' : null; + if (signalClass && !pipelines.exporters[signalClass]) { + pipelines.exporters[signalClass] = { kind: mapExporterKind(kind) }; + } } } - for (const exp of pipeline.exporters || []) { - const kind = exp.split('/')[0]; - const sig = Object.entries(obj.service.pipelines).find(([_, p]) => p === pipeline)?.[0] || ''; - const signalClass = /^metrics/i.test(sig) ? 'metrics' - : /^logs/i.test(sig) ? 'logs' - : /^traces/i.test(sig) ? 'traces' : null; - if (signalClass && !pipelines.exporters[signalClass]) { - pipelines.exporters[signalClass] = { kind: mapExporterKind(kind) }; + evidence[`PIP-${summary.discovered.pipelines + 1}`] = f.relPath; + summary.discovered.pipelines++; + // Each scrape receiver's samples leave by the exporters of the metrics + // pipelines that list it. + for (const { key, config } of scrapeReceivers) { + const names = []; + for (const [sig, p] of Object.entries(obj.service.pipelines)) { + if (!/^metrics/i.test(sig) || !Array.isArray(p?.receivers) || !p.receivers.includes(key)) continue; + for (const exp of p.exporters || []) if (!names.includes(exp)) names.push(exp); } + walkScrapeConfigDoc(config, f.relPath, scrapeJobs, evidence, summary, { + receiver: key, + exporters: names.map(n => collectorExporterDetail(n, exporterCfgs[n])), + }); } + continue; + } + + // No pipelines: the metrics exporter is the one metrics-capable exporter + // the config names (the first, with a warning, when it names several). + const candidates = Object.keys(exporterCfgs).filter(n => METRICS_EXPORTER_TYPES.has(n.split('/')[0].toLowerCase())); + const chosen = candidates[0] || null; + if (candidates.length > 1) { + summary.warnings.push(`${f.relPath} has no service pipelines and ${candidates.length} metrics-capable exporters (${candidates.join(', ')}); the metrics exporter is taken to be the first, '${chosen}'.`); + } + if (!scrapeReceivers.length && !chosen) continue; + for (const { key } of scrapeReceivers) { + const name = key.split('/')[0]; + if (!pipelines.receivers.some(r => r.name === name)) pipelines.receivers.push({ name }); + } + if (chosen && !pipelines.exporters.metrics) { + pipelines.exporters.metrics = { kind: mapExporterKind(chosen.split('/')[0]) }; + evidence['pipelines.exporters.metrics'] = f.relPath; + } + const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; + for (const { key, config } of scrapeReceivers) { + walkScrapeConfigDoc(config, f.relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); } evidence[`PIP-${summary.discovered.pipelines + 1}`] = f.relPath; summary.discovered.pipelines++; } } +// Collector exporters that carry metrics, by component type. +const METRICS_EXPORTER_TYPES = new Set([ + 'kafka', 'prometheusremotewrite', 'prometheus', 'otlp', 'otlphttp', + 'googlemanagedprometheus', 'awsemf', 'datadog', 'influxdb', 'clickhouse', + 'signalfx', 'carbon', 'azuremonitor', 'splunk_hec', 'sumologic', +]); + +// A collector's prometheus receivers (`prometheus`, `prometheus/`) +// that state scrape jobs: [{ key, config }]. +function collectorScrapeReceivers(obj) { + if (!isMapping(obj?.receivers)) return []; + return Object.entries(obj.receivers) + .filter(([key, r]) => /^prometheus(\/|$)/.test(key) && isMapping(r?.config) && Array.isArray(r.config.scrape_configs)) + .map(([key, r]) => ({ key, config: r.config })); +} + +// An OTel collector config by shape: receivers, exporters and pipelines (a +// full config), or a receivers mapping with a prometheus receiver's scrape +// jobs or with exporters (one with no service block). Alertmanager's +// `receivers` is a list, never a mapping. +function looksLikeOtelCollector(obj) { + if (obj.receivers && obj.exporters && obj.service?.pipelines) return true; + if (!isMapping(obj.receivers)) return false; + return collectorScrapeReceivers(obj).length > 0 || isMapping(obj.exporters); +} + +// What a collector exporter is and where it sends: a kafka exporter's +// metrics topic, any other's endpoint — reduced by safeSourceUrl, so no +// userinfo, query or fragment is recorded. +function collectorExporterDetail(name, cfg) { + const type = String(name).split('/')[0]; + const out = { name: String(name), kind: mapExporterKind(type) }; + if (/^kafka$/i.test(type)) { + const topic = cfg?.metrics?.topic ?? cfg?.topic; + if (typeof topic === 'string' && topic.trim()) out.topic = topic.trim(); + } else { + const endpoint = cfg?.endpoint ?? cfg?.metrics_endpoint; + if (typeof endpoint === 'string' && endpoint.trim()) out.endpoint = safeSourceUrl(endpoint); + } + return out; +} + +// The scrape jobs or exporter evidence a file carries, as one line for the +// environment-scope warning (` (scrape jobs: a, b)`), or null. +function collectionEvidenceOf(relPath, content, catalogue) { + if (typeof content !== 'string') return null; + let kind; + try { kind = detectArtefactKind(relPath, content, { exporterCatalogue: catalogue }); } catch { return null; } + const jobsOf = (docs) => docs.flatMap(d => (Array.isArray(d?.scrape_configs) ? d.scrape_configs : [])) + .map(c => String(c?.job_name || '').trim()).filter(Boolean); + try { + if (kind === 'prometheus-scrape-config') { + const jobs = jobsOf(parseYamlDocs(content)); + return jobs.length ? `${relPath} (scrape jobs: ${jobs.join(', ')})` : null; + } + if (kind === 'otel-collector') { + const jobs = jobsOf(parseYamlDocs(content).flatMap(d => collectorScrapeReceivers(d).map(r => r.config))); + return jobs.length ? `${relPath} (collector scrape jobs: ${jobs.join(', ')})` : null; + } + if (kind === 'exporter-config') { + const entry = matchExporterConfig(parseYaml(content), catalogue); + return `${relPath} (exporter config: ${entry?.id ?? 'unknown'})`; + } + } catch { /* unreadable: nothing to report */ } + return null; +} + +// ---------- prebuilt exporters ---------- + +// A metrics manifest: each entry is a declared metric of the exporter. +function walkMetricsManifest(f, metricDefinitions, exporters, summary, evidence) { + const read = readMetricsManifest(f.content); + if (!read.ok) { + summary.warnings.push(`${f.relPath} is not a metrics manifest (${read.error}); it is not read.`); + return; + } + const m = read.manifest; + for (const metric of m.metrics) { + addMetricDefinition(metricDefinitions, evidence, summary, { + name: metric.name, + type: metric.type, + help: metric.help, + labels: metric.labels, + service: m.exporter, + origin: f.relPath, + originKind: 'exporter-manifest', + confidence: 'declared', + }); + } + if (read.skipped) { + summary.warnings.push(`${f.relPath}: ${read.skipped} metric entr${read.skipped === 1 ? 'y is' : 'ies are'} not read (each needs a metric name not listed before and an exposition type: ${['counter', 'gauge', 'histogram', 'summary', 'untyped'].join(', ')}, …).`); + } + recordExporter(exporters, m.exporter, { + origin_kind: 'exporter-manifest', + file: f.relPath, + metrics: m.metrics.length, + ...(m.version ? { version: m.version } : {}), + ...(m.source ? { source: m.source } : {}), + ...(m.generatedAt ? { generated_at: m.generatedAt } : {}), + }, summary); +} + +// A saved exposition: the exporter is named by the file (`ibm-mq.prom` → +// ibm-mq). +function walkExpositionSnapshot(f, metricDefinitions, exporters, summary, evidence) { + const { families } = parseExposition(f.content); + const exporter = basenameOf(f.relPath).replace(/\.(prom|metrics)$/i, '') || 'exporter'; + for (const fam of families) { + addMetricDefinition(metricDefinitions, evidence, summary, { + name: fam.name, + type: fam.type, + help: fam.help, + labels: fam.labels, + service: exporter, + origin: f.relPath, + originKind: 'exporter-snapshot', + confidence: 'declared', + }); + } + recordExporter(exporters, exporter, { origin_kind: 'exporter-snapshot', file: f.relPath, metrics: families.length }, summary); +} + +// An exporter's own config, read against the catalogue: the families it +// switches on, as candidates. +function walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence) { + for (const obj of parseYamlDocs(f.content)) { + const entry = matchExporterConfig(obj, catalogue); + if (!entry) continue; + const inferred = inferExporterFamilies(obj, entry); + for (const fam of inferred.families) { + addMetricDefinition(metricDefinitions, evidence, summary, { + name: fam.name, + type: fam.type, + help: fam.help, + labels: fam.labels, + service: entry.id, + origin: f.relPath, + originKind: 'exporter-config', + confidence: 'inferred', + candidateOnly: true, + }); + } + recordExporter(exporters, entry.id, { + origin_kind: 'exporter-config', + file: f.relPath, + metrics: inferred.families.length, + namespace: inferred.namespace, + gates: inferred.gates, + scopes: inferred.scopes, + }, summary); + } +} + function mapExporterKind(name) { // The spec's exporter kind taxonomy is small; map common collector // exporters to it. diff --git a/tools/lib/exporter-catalogue.mjs b/tools/lib/exporter-catalogue.mjs new file mode 100644 index 00000000..d9776b59 --- /dev/null +++ b/tools/lib/exporter-catalogue.mjs @@ -0,0 +1,74 @@ +// ============================================================ +// exporter-catalogue.mjs — the built-in exporter family catalogue. +// +// DATA, not engine code: the crawler (tools/lib/crawler.mjs, through +// tools/lib/exporter-metrics.mjs) reads a prebuilt exporter's config file +// against these entries and emits the metric families the config switches +// on, as candidates (`originKind: 'exporter-config'`, `confidence: +// 'inferred'`). A downstream carries a larger catalogue as a JSON file of +// the same shape (`packc x-ray --exporter-catalogue `, or +// `crawlFiles(files, { exporterCatalogue })`): its entries replace a +// built-in entry with the same `id` and add the rest, so the engine is +// never diffed. docs/DOWNSTREAM.md §18 has the format. +// +// An entry: +// id the exporter's name in crawler.discovered.exporters +// match.keys top-level mapping keys a config file must all carry +// namespace { path, default } — the metric name prefix the config sets +// gates { : { path, default } } — boolean switches; a family +// with `gate` is emitted only when that switch is on +// scopes { : { path } } — object selections; a family with +// `scope` is emitted only when that selection is non-empty +// families [{ name, type, help, labels, gate?, scope? }] — `name` is +// joined to the namespace with `_` +// +// The IBM entry is a small subset of what mq-metric-samples' Prometheus +// collector exposes, enough to name the queue manager, queue and channel +// families a config enables. The names are inferred, not observed: publish +// the exporter's real list with `packc emit --exporter ` and the +// manifest outranks every name here. +// ============================================================ + +export const EXPORTER_CATALOGUE = { + version: 1, + exporters: [ + { + id: 'ibm-mq', + title: 'IBM MQ — mq-metric-samples (Prometheus collector)', + source: 'https://github.com/ibm-messaging/mq-metric-samples', + match: { keys: ['global', 'connection', 'objects', 'prometheus'] }, + namespace: { path: 'prometheus.namespace', default: 'ibmmq' }, + gates: { + objectStatus: { path: 'global.useObjectStatus', default: true }, + publications: { path: 'global.usePublications', default: true }, + }, + scopes: { + queues: { path: 'objects.queues' }, + channels: { path: 'objects.channels' }, + }, + families: [ + { name: 'qmgr_status', type: 'gauge', help: 'Queue manager status', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_uptime', type: 'gauge', help: 'Queue manager uptime in seconds', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_connection_count', type: 'gauge', help: 'Connections to the queue manager', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_channel_initiator_status', type: 'gauge', help: 'Channel initiator status', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_command_server_status', type: 'gauge', help: 'Command server status', labels: ['qmgr', 'platform'], gate: 'objectStatus' }, + { name: 'qmgr_user_cpu_time_percentage', type: 'gauge', help: 'User CPU time of the queue manager', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_system_cpu_time_percentage', type: 'gauge', help: 'System CPU time of the queue manager', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_log_in_use_bytes', type: 'gauge', help: 'Recovery log in use', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_log_write_latency_seconds', type: 'gauge', help: 'Recovery log write latency', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'qmgr_mqconn_mqconnx_count', type: 'counter', help: 'MQCONN and MQCONNX calls', labels: ['qmgr', 'platform'], gate: 'publications' }, + { name: 'queue_depth', type: 'gauge', help: 'Messages on the queue', labels: ['qmgr', 'queue'], gate: 'publications', scope: 'queues' }, + { name: 'queue_mqput_mqput1_count', type: 'counter', help: 'MQPUT and MQPUT1 calls on the queue', labels: ['qmgr', 'queue'], gate: 'publications', scope: 'queues' }, + { name: 'queue_mqget_count', type: 'counter', help: 'MQGET calls on the queue', labels: ['qmgr', 'queue'], gate: 'publications', scope: 'queues' }, + { name: 'queue_oldest_message_age', type: 'gauge', help: 'Age of the oldest message on the queue in seconds', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'queue_uncommitted_messages', type: 'gauge', help: 'Uncommitted messages on the queue', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'queue_time_since_put', type: 'gauge', help: 'Seconds since the last put', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'queue_time_since_get', type: 'gauge', help: 'Seconds since the last get', labels: ['qmgr', 'queue'], gate: 'objectStatus', scope: 'queues' }, + { name: 'channel_status', type: 'gauge', help: 'Channel status', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + { name: 'channel_messages', type: 'counter', help: 'Messages sent over the channel', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + { name: 'channel_bytes_sent', type: 'counter', help: 'Bytes sent over the channel', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + { name: 'channel_bytes_rcvd', type: 'counter', help: 'Bytes received over the channel', labels: ['qmgr', 'channel', 'type', 'connname', 'jobname', 'rqmname'], gate: 'objectStatus', scope: 'channels' }, + ], + }, + ], +}; diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs new file mode 100644 index 00000000..2025d85a --- /dev/null +++ b/tools/lib/exporter-metrics.mjs @@ -0,0 +1,366 @@ +// ============================================================ +// exporter-metrics.mjs — what a prebuilt exporter exposes, read three ways. +// +// A service that runs a prebuilt exporter (IBM's mq-metric-samples, a +// node_exporter, a vendor's sidecar) has no client-library call or PromQL +// that names the exporter's metrics: its repository is a Dockerfile and a +// config file. The crawler (tools/lib/crawler.mjs) reads the metric names +// from, strongest first: +// +// 1. a metrics manifest — `metrics-manifest.json` / `*.metrics-manifest.json`, +// { exporter, version?, source?, generatedAt?, metrics: [{ name, type, +// help?, labels? }] }; `packc emit --exporter ` writes one from a +// live /metrics endpoint (originKind 'exporter-manifest', declared); +// 2. an exposition snapshot — a saved /metrics response (`.prom`, +// `.metrics`), its # HELP / # TYPE lines and the label names of its +// samples (originKind 'exporter-snapshot', declared); +// 3. the exporter's own config file, matched against a family catalogue +// (tools/lib/exporter-catalogue.mjs, or a downstream's JSON file of the +// same shape): the families the config switches on, as candidates +// (originKind 'exporter-config', confidence 'inferred'). +// +// Pure: no fs, no network — the crawler, the CLI (tools/emit-manifest.mjs) +// and the browser share it. Label VALUES are never kept (a channel's +// connname is an address); only label names, types and help text. +// ============================================================ + +import { EXPORTER_CATALOGUE as BUILTIN_CATALOGUE } from './exporter-catalogue.mjs'; + +/** The exposition types a manifest entry or a # TYPE line may state (text format 0.0.4 and OpenMetrics). */ +export const EXPOSITION_TYPES = Object.freeze(['counter', 'gauge', 'histogram', 'summary', 'untyped', 'unknown', 'gaugehistogram', 'info', 'stateset']); +const TYPE_SET = new Set(EXPOSITION_TYPES); +const NAME_RE = /^[A-Za-z_:][A-Za-z0-9_:]*$/; +const LABEL_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; +const LABEL_CAP = 32; + +// A sample of a family is named the family plus one of these suffixes. +const FAMILY_SUFFIXES = { + counter: ['_total', '_created'], + histogram: ['_bucket', '_sum', '_count', '_created'], + summary: ['_sum', '_count', '_created'], + gaugehistogram: ['_bucket', '_gsum', '_gcount'], + info: ['_info'], +}; +// Labels the exposition adds to a family's samples, not the family's own. +const SAMPLE_ONLY_LABELS = { histogram: ['le'], gaugehistogram: ['le'], summary: ['quantile'] }; + +/** `metrics-manifest.json` or `.metrics-manifest.json`, any folder. */ +export function isMetricsManifestPath(relPath) { + return /(^|[/\\.])metrics-manifest\.json$/i.test(String(relPath ?? '')); +} + +/** A saved exposition: `.prom` or `.metrics`. */ +export function isExpositionSnapshotPath(relPath) { + return /\.(prom|metrics)$/i.test(String(relPath ?? '')); +} + +/** + * A URL as it may be recorded: scheme, host, port and path — never the + * userinfo, query or fragment, where a credential rides. A value that is not + * a `scheme://` URL (`prometheus:9090`, `${env:ENDPOINT}`) loses anything up + * to an `@` and from a `?` or `#`. + */ +export function safeSourceUrl(raw) { + const s = String(raw ?? '').trim(); + if (!s) return ''; + if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(s)) { + try { + const u = new URL(s); + // The path as written: `http://h:9157` stays without the `/` URL adds. + const path = /^[^:]+:\/\/[^/?#]*\//.test(s) ? u.pathname : ''; + return `${u.protocol}//${u.host}${path}`; + } catch { /* fall through to the textual rule */ } + } + return s.replace(/^([A-Za-z][A-Za-z0-9+.-]*:\/\/)?[^\s/@]*@/, '$1').replace(/[?#].*$/, ''); +} + +function parseSampleLine(line) { + const m = /^[A-Za-z_:][A-Za-z0-9_:]*/.exec(line); + if (!m) return null; + const name = m[0]; + const labels = []; + let i = name.length; + // A sample ends in its value: a number, NaN or ±Inf (then an optional + // timestamp). A line that does not is not a sample. + const valued = (at) => (/^[ \t]+[-+]?(?:\d|\.\d|Inf|NaN)/i.test(line.slice(at)) ? { name, labels } : null); + while (line[i] === ' ' || line[i] === '\t') i++; + if (line[i] !== '{') return valued(name.length); + i++; + for (;;) { + while (line[i] === ' ' || line[i] === '\t' || line[i] === ',') i++; + if (i >= line.length) return null; + if (line[i] === '}') return valued(i + 1); + const lm = /^[A-Za-z_][A-Za-z0-9_]*/.exec(line.slice(i)); + if (!lm) return null; + labels.push(lm[0]); + i += lm[0].length; + while (line[i] === ' ' || line[i] === '\t') i++; + if (line[i] !== '=') return null; + i++; + while (line[i] === ' ' || line[i] === '\t') i++; + if (line[i] !== '"') return null; + i++; + while (i < line.length && line[i] !== '"') i += line[i] === '\\' ? 2 : 1; + if (i >= line.length) return null; + i++; + } +} + +function unescapeHelp(s) { + return s.replace(/\\(\\|n)/g, (_, c) => (c === 'n' ? '\n' : '\\')); +} + +/** + * Parse a Prometheus / OpenMetrics text exposition into its metric families: + * `{ families: [{ name, type, help, labels }], samples }`, families sorted by + * name, labels in first-seen order. A sample with no # TYPE for it (or for + * the family its suffix names) is its own `untyped` family. + */ +export function parseExposition(text) { + const families = new Map(); + const family = (name) => { + if (!families.has(name)) families.set(name, { name, type: 'untyped', help: '', labels: [], typed: false }); + return families.get(name); + }; + let samples = 0; + for (const raw of String(text ?? '').replace(/\r\n?/g, '\n').split('\n')) { + const line = raw.trim(); + if (!line) continue; + if (line.startsWith('#')) { + const meta = /^#\s*(HELP|TYPE)\s+([A-Za-z_:][A-Za-z0-9_:]*)(?:\s+(.*))?$/.exec(line); + if (!meta) continue; + const f = family(meta[2]); + if (meta[1] === 'HELP') f.help = unescapeHelp(meta[3] ?? ''); + else { + const t = String(meta[3] ?? '').trim().toLowerCase(); + f.type = TYPE_SET.has(t) ? t : 'untyped'; + f.typed = true; + } + continue; + } + const sample = parseSampleLine(line); + if (!sample) continue; + samples++; + let f = families.get(sample.name); + if (!f || (!f.typed && !f.help)) { + for (const [type, suffixes] of Object.entries(FAMILY_SUFFIXES)) { + const suffix = suffixes.find(s => sample.name.endsWith(s) && sample.name.length > s.length); + const base = suffix ? families.get(sample.name.slice(0, -suffix.length)) : null; + if (base && base.type === type) { f = base; break; } + } + } + if (!f) f = family(sample.name); + const skip = SAMPLE_ONLY_LABELS[f.type] || []; + for (const l of sample.labels) { + if (!skip.includes(l) && !f.labels.includes(l) && f.labels.length < LABEL_CAP) f.labels.push(l); + } + } + const out = [...families.values()] + .map(({ name, type, help, labels }) => ({ name, type, help, labels })) + .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); + return { families: out, samples }; +} + +/** + * A metrics manifest built from an exposition (what `packc emit --exporter` + * writes): `{ exporter, version?, source?, generatedAt?, metrics }`, the + * metrics sorted by name, `source` reduced by safeSourceUrl. + */ +export function manifestFromExposition(text, { exporter, version, source, generatedAt } = {}) { + const { families } = parseExposition(text); + const manifest = { exporter: String(exporter ?? '').trim() }; + if (version) manifest.version = String(version); + const src = safeSourceUrl(source); + if (src) manifest.source = src; + if (generatedAt) manifest.generatedAt = String(generatedAt); + manifest.metrics = families.map(f => ({ + name: f.name, + type: f.type, + ...(f.help ? { help: f.help } : {}), + ...(f.labels.length ? { labels: f.labels } : {}), + })); + return manifest; +} + +/** + * Read a metrics manifest's text. `{ ok: true, manifest, skipped }` with the + * entries that are not `{ name: , type: }` + * counted in `skipped` (a repeated name keeps its first entry), or + * `{ ok: false, error }` when the text is not a manifest at all. + */ +export function readMetricsManifest(text) { + let obj; + try { obj = JSON.parse(String(text ?? '')); } catch (e) { return { ok: false, error: `not JSON (${e.message})` }; } + if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return { ok: false, error: 'not a JSON object' }; + const exporter = typeof obj.exporter === 'string' ? obj.exporter.trim() : ''; + if (!exporter) return { ok: false, error: 'no `exporter` name' }; + if (!Array.isArray(obj.metrics)) return { ok: false, error: 'no `metrics` array' }; + const metrics = []; + const seen = new Set(); + let skipped = 0; + for (const m of obj.metrics) { + const name = typeof m?.name === 'string' ? m.name.trim() : ''; + const type = typeof m?.type === 'string' ? m.type.trim().toLowerCase() : ''; + if (!NAME_RE.test(name) || !TYPE_SET.has(type) || seen.has(name)) { skipped++; continue; } + seen.add(name); + metrics.push({ + name, + type, + help: typeof m.help === 'string' ? m.help : '', + labels: Array.isArray(m.labels) ? [...new Set(m.labels.filter(l => typeof l === 'string' && LABEL_RE.test(l)))].slice(0, LABEL_CAP) : [], + }); + } + const str = (v) => (typeof v === 'string' && v.trim() ? v.trim() : ''); + return { + ok: true, + skipped, + manifest: { + exporter, + version: str(obj.version) || (typeof obj.version === 'number' ? String(obj.version) : ''), + source: safeSourceUrl(str(obj.source)), + generatedAt: str(obj.generatedAt), + metrics, + }, + }; +} + +// ---------- the exporter config catalogue ---------- + +const isMapping = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); + +function pathValue(obj, path) { + let cur = obj; + for (const key of String(path ?? '').split('.').filter(Boolean)) { + if (!isMapping(cur)) return undefined; + cur = cur[key]; + } + return cur; +} + +function normalizeEntry(e, i) { + const where = `exporter catalogue entry ${i + 1}${typeof e?.id === 'string' && e.id ? ` ('${e.id}')` : ''}`; + if (!isMapping(e)) return { error: `${where} is not an object` }; + if (typeof e.id !== 'string' || !/^[a-z][a-z0-9_.-]{0,62}$/.test(e.id)) return { error: `${where}: \`id\` must be a lowercase name (letters, digits, \`_\`, \`.\`, \`-\`)` }; + const keys = Array.isArray(e.match?.keys) ? e.match.keys.filter(k => typeof k === 'string' && /^[A-Za-z_][A-Za-z0-9_-]*$/.test(k)) : []; + if (!keys.length || keys.length !== e.match.keys.length) return { error: `${where}: \`match.keys\` must list the top-level keys a config carries` }; + const switches = (o, kind) => { + if (o === undefined) return {}; + if (!isMapping(o) || Object.values(o).some(v => !isMapping(v) || typeof v.path !== 'string')) throw new Error(`${where}: every ${kind} needs a \`path\``); + return Object.fromEntries(Object.entries(o).map(([k, v]) => [k, { path: v.path, default: v.default !== false }])); + }; + let gates, scopes; + try { gates = switches(e.gates, 'gate'); scopes = switches(e.scopes, 'scope'); } + catch (err) { return { error: err.message }; } + const ns = isMapping(e.namespace) ? e.namespace : {}; + if (ns.default !== undefined && ns.default !== '' && !(typeof ns.default === 'string' && NAME_RE.test(ns.default))) { + return { error: `${where}: \`namespace.default\` must be a metric name prefix` }; + } + if (!Array.isArray(e.families) || !e.families.length) return { error: `${where}: \`families\` must list at least one family` }; + const families = []; + for (const f of e.families) { + const bad = !isMapping(f) || typeof f.name !== 'string' || !/^[A-Za-z0-9_:]+$/.test(f.name) + || !TYPE_SET.has(String(f.type ?? '').toLowerCase()) + || (f.gate !== undefined && !gates[f.gate]) || (f.scope !== undefined && !scopes[f.scope]); + if (bad) return { error: `${where}: family ${JSON.stringify(f?.name ?? f)} needs a name, an exposition type, and only the gates and scopes the entry defines` }; + families.push({ + name: f.name, + type: String(f.type).toLowerCase(), + help: typeof f.help === 'string' ? f.help : '', + labels: Array.isArray(f.labels) ? f.labels.filter(l => typeof l === 'string' && LABEL_RE.test(l)).slice(0, LABEL_CAP) : [], + ...(f.gate !== undefined ? { gate: f.gate } : {}), + ...(f.scope !== undefined ? { scope: f.scope } : {}), + }); + } + return { + entry: { + id: e.id, + title: typeof e.title === 'string' ? e.title : e.id, + match: { keys }, + namespace: { path: typeof ns.path === 'string' ? ns.path : '', default: typeof ns.default === 'string' ? ns.default : '' }, + gates, + scopes, + families, + }, + }; +} + +/** + * The catalogue a crawl reads: the built-in entries, then a caller's (an + * object `{ exporters: [...] }` or the array itself) — an entry replaces the + * built-in one with the same id, the rest are added. An entry that is not + * well-formed is dropped with a warning naming what it lacks; a catalogue + * that is not an object or array is ignored with a warning. + * → `{ exporters, warnings }`. + */ +export function resolveExporterCatalogue(custom) { + const warnings = []; + const byId = new Map(); + for (const [i, e] of BUILTIN_CATALOGUE.exporters.entries()) { + const { entry } = normalizeEntry(e, i); + if (entry) byId.set(entry.id, entry); + } + if (custom !== undefined && custom !== null) { + const list = Array.isArray(custom) ? custom : (isMapping(custom) && Array.isArray(custom.exporters) ? custom.exporters : null); + if (!list) warnings.push('the exporter catalogue is neither `{ exporters: [...] }` nor an array; the built-in catalogue is used.'); + else { + list.forEach((e, i) => { + const { entry, error } = normalizeEntry(e, i); + if (error) warnings.push(`${error}; dropped.`); + else byId.set(entry.id, entry); + }); + } + } + return { exporters: [...byId.values()], warnings }; +} + +/** + * Whether a YAML text can be an exporter config of the catalogue — every + * key of some entry's `match.keys` starts a line — before the text is + * parsed. Cheap; `matchExporterConfig` decides. + */ +export function mayBeExporterConfig(content, catalogue) { + const text = String(content ?? ''); + return catalogue.exporters.some(e => e.match.keys.every(k => new RegExp(`^${k}\\s*:`, 'm').test(text))); +} + +/** The first catalogue entry whose `match.keys` the parsed config all carries, or null. */ +export function matchExporterConfig(obj, catalogue) { + if (!isMapping(obj)) return null; + return catalogue.exporters.find(e => e.match.keys.every(k => Object.prototype.hasOwnProperty.call(obj, k))) || null; +} + +function switchOn(value, dflt) { + if (typeof value === 'boolean') return value; + if (typeof value === 'number') return value !== 0; + if (typeof value === 'string') { + const v = value.trim().toLowerCase(); + if (v === 'true' || v === 'yes' || v === 'on') return true; + if (v === 'false' || v === 'no' || v === 'off') return false; + } + return dflt; +} + +function selectionNonEmpty(value) { + if (Array.isArray(value)) return value.some(v => String(v ?? '').trim() !== ''); + if (typeof value === 'string') return value.trim() !== ''; + if (isMapping(value)) return Object.keys(value).length > 0; + return false; +} + +/** + * The families an exporter config switches on: `{ namespace, gates, scopes, + * families: [{ name, type, help, labels }] }` — each family's name joined to + * the namespace the config sets (else the entry's default), a family kept + * only when its gate is on and its scope selects something. + */ +export function inferExporterFamilies(obj, entry) { + const nsRaw = entry.namespace.path ? pathValue(obj, entry.namespace.path) : undefined; + const namespace = typeof nsRaw === 'string' && NAME_RE.test(nsRaw.trim()) ? nsRaw.trim() : entry.namespace.default; + const gates = Object.fromEntries(Object.entries(entry.gates).map(([k, g]) => [k, switchOn(pathValue(obj, g.path), g.default)])); + const scopes = Object.fromEntries(Object.entries(entry.scopes).map(([k, s]) => [k, selectionNonEmpty(pathValue(obj, s.path))])); + const families = entry.families + .filter(f => (f.gate === undefined || gates[f.gate]) && (f.scope === undefined || scopes[f.scope])) + .map(f => ({ name: namespace ? `${namespace}_${f.name}` : f.name, type: f.type, help: f.help, labels: f.labels })) + .filter(f => NAME_RE.test(f.name)); + return { namespace, gates, scopes, families }; +} diff --git a/tools/lib/journey.mjs b/tools/lib/journey.mjs index 35f44a91..b2841a43 100644 --- a/tools/lib/journey.mjs +++ b/tools/lib/journey.mjs @@ -517,7 +517,7 @@ function loadPackFile(path, baseDir, scope = null) { // Minimal repo walk for the crawl source — mirrors tools/crawl-repo.mjs's // filters (that script runs main() on import, so it can't be imported). const SKIP_DIRS = new Set(['.git', 'node_modules', 'dist', 'build', 'out', '.observogram', '.tomograph', 'coverage', 'vendor']); -const SCAN_EXT = /\.(ya?ml|json|cs|go|java|py|ts|tsx|js|mjs|rs|kt)$/i; +const SCAN_EXT = /\.(ya?ml|json|cs|go|java|py|ts|tsx|js|mjs|rs|kt|prom|metrics)$/i; const MAX_FILE_BYTES = 1024 * 1024; const MAX_TOTAL_BYTES = 16 * 1024 * 1024; diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs new file mode 100644 index 00000000..35f7603e --- /dev/null +++ b/tools/test-crawl-exporters.mjs @@ -0,0 +1,441 @@ +#!/usr/bin/env node +/** + * tools/test-crawl-exporters.mjs + * + * The crawler learns metrics from prebuilt exporters and from an OTel + * collector's scrape jobs (tools/lib/exporter-metrics.mjs, the exporter + * kinds and the collector walker in tools/lib/crawler.mjs): + * A. the fixture repository (tools/fixtures/crawl/exporters/) — an + * mq-metric-samples config, its metrics manifest and a collector config + * under a chart's files/ — crawls into a valid pack holding the + * manifest's metrics, the collector's `qmgrs` job and the kafka + * exporter, and says which files the prod scope left out; + * B. a manifest's metric outranks the same name inferred from the config; + * C. the config's gates, scopes and namespace decide the families; + * D. a saved exposition (.prom / .metrics) is read, never from a test path; + * E. a downstream catalogue replaces and extends the built-in one; + * F. a collector with pipelines: the scrape job's exporters are the + * metrics pipelines' that list the receiver, endpoints without secrets; + * G. a malformed manifest is warned about, not dropped silently; + * H. a crawl with no exporter evidence carries none of the new keys; + * I. `packc emit --exporter` writes a manifest the crawler reads, and + * refuses what it cannot honestly record. + * Exit 0 = pass. + */ + +import { readFileSync, readdirSync, statSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { join, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { spawnSync, spawn } from 'node:child_process'; +import { createServer } from 'node:http'; +import { crawlFiles, detectArtefactKind, scanReadsPath } from './lib/crawler.mjs'; +import { + parseExposition, manifestFromExposition, readMetricsManifest, safeSourceUrl, resolveExporterCatalogue, +} from './lib/exporter-metrics.mjs'; +import { EXPORTER_CATALOGUE as BUILTIN } from './lib/exporter-catalogue.mjs'; +import { validateCanonical, SPEC_SCHEMA_PATH } from './lib/validator.mjs'; +import { createHarness } from './lib/harness.mjs'; + +const { assert, report } = createHarness({ truncate: 600 }); +const SCHEMA = JSON.parse(readFileSync(new URL(`../${SPEC_SCHEMA_PATH}`, import.meta.url), 'utf8')); +const NOW = '2026-10-15T00:00:00.000Z'; +const EMIT = fileURLToPath(new URL('./emit-manifest.mjs', import.meta.url)); +const CLI = fileURLToPath(new URL('./cli.mjs', import.meta.url)); + +const ROOT = fileURLToPath(new URL('./fixtures/crawl/exporters/', import.meta.url)); +const FILES = {}; +(function walk(dir) { + for (const name of readdirSync(dir)) { + const p = join(dir, name); + if (statSync(p).isDirectory()) walk(p); + else if (scanReadsPath(name)) FILES[relative(ROOT, p).replace(/\\/g, '/')] = readFileSync(p, 'utf8'); + } +})(ROOT); + +const ann = (r) => r.canonical.metadata.annotations; +const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); + +// ---------- A. the acceptance fixture, the default (prod) scope ---------- +{ + assert(!('mq-exporter/Dockerfile' in FILES) && 'mq-exporter/config.yaml' in FILES && 'mq-exporter/metrics-manifest.json' in FILES + && 'charts/otel-collector/files/collector.yaml' in FILES, + 'the fixture: the exporter config and manifest, the chart\'s collector config; the Dockerfile is not scanned', Object.keys(FILES)); + assert(detectArtefactKind('mq-exporter/config.yaml', FILES['mq-exporter/config.yaml']) === 'exporter-config', 'an mq-metric-samples config is an exporter config'); + assert(detectArtefactKind('mq-exporter/metrics-manifest.json', FILES['mq-exporter/metrics-manifest.json']) === 'metrics-manifest', 'metrics-manifest.json is a metrics manifest'); + assert(detectArtefactKind('x/ibm-mq.metrics-manifest.json', '{}') === 'metrics-manifest', '*.metrics-manifest.json is a metrics manifest'); + assert(detectArtefactKind('charts/otel-collector/files/collector.yaml', FILES['charts/otel-collector/files/collector.yaml']) === 'otel-collector', + 'a collector config with no service block is an OTel collector'); + + const r = crawlFiles(FILES, { repoName: 'mq-platform', now: NOW }); + assert(validateCanonical(r.canonical, SCHEMA).length === 0, 'the crawl validates against the vendored schema', validateCanonical(r.canonical, SCHEMA).slice(0, 3)); + const names = json(r, 'crawler.discovered.metric_names'); + const manifest = JSON.parse(FILES['mq-exporter/metrics-manifest.json']); + assert(manifest.metrics.every(m => names.includes(m.name)), 'every manifest metric is in crawler.discovered.metric_names', names); + const origins = json(r, 'crawler.discovered.metric_origins'); + assert(manifest.metrics.every(m => origins[m.name].origin_kind === 'exporter-manifest' && origins[m.name].confidence === 'declared' + && origins[m.name].candidate === false && origins[m.name].service === 'ibm-mq' && origins[m.name].file === 'mq-exporter/metrics-manifest.json'), + 'each is declared by the manifest, service ibm-mq', origins.ibmmq_queue_depth); + assert(JSON.stringify(origins.ibmmq_queue_depth.labels) === '["qmgr","queue","usage","description"]' && origins.ibmmq_queue_depth.type === 'gauge', + 'with the manifest\'s type and labels', origins.ibmmq_queue_depth); + + const jobs = json(r, 'crawler.discovered.scrape_job_origins'); + assert(JSON.stringify(json(r, 'crawler.discovered.scrape_jobs')) === '["qmgrs"]', 'the collector\'s qmgrs job is a discovered scrape job', json(r, 'crawler.discovered.scrape_jobs')); + assert(JSON.stringify(jobs.qmgrs) === JSON.stringify({ + file: 'charts/otel-collector/files/collector.yaml', metrics_path: '/metrics', interval: '30s', + targets: ['mq-exp-qm1:9157', 'mq-exp-qm2:9157', 'mq-exp-qm3:9157'], + via: 'otel-collector', receiver: 'prometheus/mq', target_count: 3, + exporters: [{ name: 'kafka', kind: 'kafka', topic: 'mq-metrics' }], + }), 'its interval, path, target count, receiver and the kafka topic it leaves by', jobs.qmgrs); + assert(r.canonical.spec.pipelines.exporters.metrics.kind === 'kafka' && JSON.stringify(r.canonical.spec.pipelines.exporters.metrics) === '{"kind":"kafka"}', + 'pipelines.exporters.metrics is the kafka exporter, in the shape every exporter has', r.canonical.spec.pipelines.exporters.metrics); + assert(!('crawler.scaffold.pipelines.exporters.metrics' in ann(r)), 'and is no longer the prometheusremotewrite scaffold'); + assert(r.evidence['pipelines.exporters.metrics'] === 'charts/otel-collector/files/collector.yaml', 'its evidence is the collector config'); + assert(JSON.stringify(r.canonical.spec.pipelines.receivers) === '[{"name":"prometheus"}]' && !('crawler.scaffold.pipelines.receivers[0]' in ann(r)), + 'the prometheus receiver is the pipeline\'s receiver', r.canonical.spec.pipelines.receivers); + assert(!r.summary.warnings.some(w => /metrics-capable/.test(w)), 'debug is not a metrics exporter: one candidate, no ambiguity warning'); + + const exporters = json(r, 'crawler.discovered.exporters'); + assert(JSON.stringify(exporters) === JSON.stringify({ 'ibm-mq': { sources: [{ + origin_kind: 'exporter-manifest', file: 'mq-exporter/metrics-manifest.json', metrics: 6, version: '5.6.0', + source: 'http://mq-exporter:9157/metrics', generated_at: '2026-10-01T00:00:00.000Z' }] } }) + && ann(r)['crawler.discovered.exporters_count'] === '1', + 'crawler.discovered.exporters names ibm-mq and its manifest', exporters); + + const scoped = r.summary.warnings.find(w => /carry scrape jobs or exporter evidence/.test(w)); + assert(scoped === 'Environment scope "prod" excluded 2 file(s) that carry scrape jobs or exporter evidence: mq-exporter/config.yaml (exporter config: ibm-mq); preprod/otel-collector.yaml (collector scrape jobs: qmgrs-preprod). An environment that names no deployment profile (e.g. `all`) reads every file.', + 'the prod scope says which collection evidence it left out, and how to read it', scoped); + assert(r.summary.environment.excluded.includes('mq-exporter/config.yaml') && !r.summary.environment.excluded.includes('mq-exporter/metrics-manifest.json'), + 'the manifest is read in every environment; the exporter config is scoped like any config', r.summary.environment.excluded); + + // The way out, followed literally. + const all = crawlFiles(FILES, { repoName: 'mq-platform', environment: 'all', now: NOW }); + assert(!all.summary.warnings.some(w => /Environment scope/.test(w)) && all.summary.files.excludedByEnvironment === 0, + '`all` reads every file: no scope, no warning'); + assert(JSON.stringify(json(all, 'crawler.discovered.scrape_jobs')) === '["qmgrs","qmgrs-preprod"]', 'and the preprod job is read', json(all, 'crawler.discovered.scrape_jobs')); + assert(validateCanonical(all.canonical, SCHEMA).length === 0, 'that crawl validates too'); + + // ---------- B. the manifest outranks the config ---------- + const o = json(all, 'crawler.discovered.metric_origins'); + assert(o.ibmmq_queue_depth.origin_kind === 'exporter-manifest' && o.ibmmq_queue_depth.confidence === 'declared' && o.ibmmq_queue_depth.candidate === false + && o.ibmmq_queue_depth.help === 'Current depth of the queue', + 'a name both the manifest and the config give reads as the manifest\'s', o.ibmmq_queue_depth); + assert(o.ibmmq_channel_bytes_sent.origin_kind === 'exporter-config' && o.ibmmq_channel_bytes_sent.confidence === 'inferred' + && o.ibmmq_channel_bytes_sent.candidate === true && o.ibmmq_channel_bytes_sent.file === 'mq-exporter/config.yaml', + 'a name only the config gives is an inferred candidate', o.ibmmq_channel_bytes_sent); + assert(all.summary.discovered.metricCandidatesUnreferenced > 0, 'curation counts the inferred names among the unreferenced candidates', all.summary.discovered.metricCandidatesUnreferenced); + const sources = json(all, 'crawler.discovered.exporters')['ibm-mq'].sources; + assert(sources.length === 2 && sources[0].origin_kind === 'exporter-config' && sources[0].metrics === BUILTIN.exporters[0].families.length + && JSON.stringify(sources[0].gates) === '{"objectStatus":true,"publications":true}' && sources[0].namespace === 'ibmmq', + 'ibm-mq lists its config (every built-in family on) beside its manifest', sources); +} + +// A config-inferred name a dashboard reads stays an inferred candidate (the +// config says the exporter runs, not that the name is right). +{ + const dash = JSON.stringify({ uid: 'mq', title: 'MQ', schemaVersion: 39, panels: [{ type: 'timeseries', title: 'depth', targets: [{ expr: 'max(ibmmq_queue_time_since_get)' }] }] }); + const r = crawlFiles({ 'config.yaml': FILES['mq-exporter/config.yaml'], 'grafana/mq.json': dash }, { repoName: 'mq', now: NOW }); + const o = json(r, 'crawler.discovered.metric_origins').ibmmq_queue_time_since_get; + assert(o.origin_kind === 'promql-reference' && o.used_by.includes('dashboard:mq/depth'), 'a name the dashboard reads is primarily the reference', o); +} + +// ---------- C. gates, scopes, namespace ---------- +{ + const cfg = (g, objects, ns) => `global:\n${g}\nconnection:\n queueManager: QM1\nobjects:\n${objects}\nprometheus:\n port: 9157\n${ns}`; + const fams = (text) => Object.keys(json(crawlFiles({ 'config.yaml': text }, { repoName: 'mq', now: NOW }), 'crawler.discovered.metric_origins') || {}).sort(); + const noStatus = fams(cfg(' useObjectStatus: false', ' queues: ["APP.*"]\n channels: ["TO.*"]', '')); + assert(noStatus.length && noStatus.every(n => !/status|uptime|connection_count|oldest|uncommitted|time_since|channel_/.test(n)), + 'useObjectStatus: false drops the status families', noStatus); + const noPubs = fams(cfg(' usePublications: "false"', ' queues: ["APP.*"]\n channels: ["TO.*"]', '')); + assert(noPubs.length && !noPubs.some(n => /cpu|log_|mqconn|queue_depth|mqput|mqget/.test(n)), 'usePublications: "false" drops the published families', noPubs); + const noObjects = fams(cfg(' logLevel: INFO', ' queues:\n channels: []', '')); + assert(noObjects.length && noObjects.every(n => /^ibmmq_qmgr_/.test(n)), 'with no queues or channels selected only the queue manager families remain', noObjects); + const ns = fams(cfg(' logLevel: INFO', ' queues: APP.*', ' namespace: mqprod')); + assert(ns.includes('mqprod_queue_depth') && ns.every(n => n.startsWith('mqprod_')) && !ns.some(n => /^mqprod_channel_/.test(n)), + 'prometheus.namespace prefixes every family; a string selection counts', ns); + const bothOff = crawlFiles({ 'config.yaml': cfg(' useObjectStatus: off\n usePublications: no', ' queues: ["A"]', '') }, { repoName: 'mq', now: NOW }); + assert(!('crawler.discovered.metric_origins' in ann(bothOff)) && json(bothOff, 'crawler.discovered.exporters')['ibm-mq'].sources[0].metrics === 0, + 'both switches off: no family, the exporter still recorded with 0', ann(bothOff)['crawler.discovered.exporters']); + assert(detectArtefactKind('mq-collector.yaml', FILES['mq-exporter/config.yaml']) === 'exporter-config', + 'a config named like a collector is read by its shape first'); +} + +// ---------- D. exposition snapshots ---------- +{ + const PROM = [ + '# HELP ibmmq_queue_depth Current depth', + '# TYPE ibmmq_queue_depth gauge', + 'ibmmq_queue_depth{qmgr="QM1",queue="APP.IN",description="a, \\"quoted\\" } value"} 4', + 'ibmmq_queue_depth{qmgr="QM1",queue="APP.OUT"} 0', + '# HELP http_request_duration_seconds Latency\\nsecond line', + '# TYPE http_request_duration_seconds histogram', + 'http_request_duration_seconds_bucket{le="0.1",route="/a"} 1', + 'http_request_duration_seconds_sum{route="/a"} 0.2', + 'http_request_duration_seconds_count{route="/a"} 2', + '# TYPE rpc_latency summary', + 'rpc_latency{quantile="0.5",method="get"} 1', + 'rpc_latency_count{method="get"} 3', + '# TYPE jobs counter', + 'jobs_total{kind="x"} 7', + 'process_up 1', + '# EOF', + ].join('\n'); + const { families, samples } = parseExposition(PROM); + const by = Object.fromEntries(families.map(f => [f.name, f])); + assert(samples === 9 && families.map(f => f.name).join() === 'http_request_duration_seconds,ibmmq_queue_depth,jobs,process_up,rpc_latency', + 'families by name; suffixed samples fold into their family; an undeclared sample is its own', families.map(f => f.name)); + assert(JSON.stringify(by.ibmmq_queue_depth.labels) === '["qmgr","queue","description"]', 'label names only, a quoted value with `,` `}` and `\\"` parsed past', by.ibmmq_queue_depth); + assert(JSON.stringify(by.http_request_duration_seconds.labels) === '["route"]' && by.http_request_duration_seconds.help === 'Latency\nsecond line', + 'a histogram family drops `le`; HELP unescaped', by.http_request_duration_seconds); + assert(JSON.stringify(by.rpc_latency.labels) === '["method"]' && by.jobs.type === 'counter' && JSON.stringify(by.jobs.labels) === '["kind"]' + && by.process_up.type === 'untyped', 'a summary drops `quantile`; `_total` folds into an OpenMetrics counter; no TYPE is untyped', [by.rpc_latency, by.jobs, by.process_up]); + assert(!JSON.stringify(families).includes('APP.IN'), 'no label value is kept'); + + assert(scanReadsPath('exporter/ibm-mq.prom') && scanReadsPath('snapshots/node.metrics'), 'a scan reads .prom and .metrics files'); + assert(detectArtefactKind('snapshots/ibm-mq.prom', PROM) === 'exposition-snapshot', 'a .prom file with families is an exposition snapshot'); + assert(detectArtefactKind('test/fixtures/ibm-mq.prom', PROM) === 'unknown' && detectArtefactKind('src/x.test.metrics', PROM) === 'unknown' + && detectArtefactKind('testdata/a.prom', PROM) === 'unknown', 'never from a test or fixture path'); + assert(detectArtefactKind('notes.metrics', 'nothing to see here\n') === 'unknown', 'a .metrics file with no family is not a snapshot'); + + const r = crawlFiles({ 'snapshots/ibm-mq.prom': PROM, 'test/ibm-mq.prom': '# TYPE test_only gauge\ntest_only 1\n' }, { repoName: 'snap', now: NOW }); + const o = json(r, 'crawler.discovered.metric_origins'); + assert(o.ibmmq_queue_depth.origin_kind === 'exporter-snapshot' && o.ibmmq_queue_depth.confidence === 'declared' && o.ibmmq_queue_depth.service === 'ibm-mq' + && !('test_only' in o), 'a crawl declares the snapshot\'s families, named by the file; the test path is not read', o.ibmmq_queue_depth); + assert(JSON.stringify(json(r, 'crawler.discovered.exporters')) === '{"ibm-mq":{"sources":[{"origin_kind":"exporter-snapshot","file":"snapshots/ibm-mq.prom","metrics":5}]}}', + 'the snapshot is a source of the exporter', ann(r)['crawler.discovered.exporters']); + // Rank: manifest > snapshot > source code. + const ranked = crawlFiles({ + 'snapshots/ibm-mq.prom': PROM, + 'metrics-manifest.json': JSON.stringify({ exporter: 'ibm-mq', metrics: [{ name: 'ibmmq_queue_depth', type: 'gauge', help: 'from the manifest' }] }), + 'src/metrics.ts': "import { Gauge } from 'prom-client';\nexport const j = new Gauge({ name: 'jobs', help: 'from code' });\n", + }, { repoName: 'snap', now: NOW }); + const ro = json(ranked, 'crawler.discovered.metric_origins'); + assert(ro.ibmmq_queue_depth.help === 'from the manifest' && ro.ibmmq_queue_depth.origin_kind === 'exporter-manifest' + && ro.jobs.origin_kind === 'exporter-snapshot', 'the manifest outranks the snapshot, the snapshot outranks source code', [ro.ibmmq_queue_depth, ro.jobs]); +} + +// ---------- E. a downstream catalogue ---------- +{ + const custom = { + exporters: [ + { id: 'ibm-mq', match: { keys: ['global', 'connection', 'objects', 'prometheus'] }, namespace: { path: 'prometheus.namespace', default: 'ibmmq' }, + families: [{ name: 'qmgr_custom_only', type: 'gauge', labels: ['qmgr'] }] }, + { id: 'acme-gw', match: { keys: ['acme', 'listen'] }, namespace: { default: 'acme' }, + gates: { tls: { path: 'acme.tls', default: false } }, + families: [{ name: 'requests_total', type: 'counter' }, { name: 'tls_handshakes_total', type: 'counter', gate: 'tls' }] }, + { id: 'Bad Id', match: { keys: ['x'] }, families: [{ name: 'a', type: 'gauge' }] }, + { id: 'nofam', match: { keys: ['x'] }, families: [] }, + { id: 'badgate', match: { keys: ['x'] }, families: [{ name: 'a', type: 'gauge', gate: 'nope' }] }, + ], + }; + const read = resolveExporterCatalogue(custom); + assert(read.exporters.map(e => e.id).join() === 'ibm-mq,acme-gw' && read.warnings.length === 3 + && read.warnings.every(w => /; dropped\.$/.test(w)), 'an entry replaces the built-in one with its id, a new one is added, malformed ones dropped with a warning', read.warnings); + assert(BUILTIN.exporters[0].families.length > 1 && resolveExporterCatalogue().exporters[0].families.length === BUILTIN.exporters[0].families.length, + 'the built-in catalogue is untouched by a resolution'); + const r = crawlFiles({ + 'mq/config.yaml': FILES['mq-exporter/config.yaml'], + 'gw/acme.yaml': 'acme:\n tls: true\nlisten: ":8080"\n', + }, { repoName: 'cat', now: NOW, exporterCatalogue: custom }); + const o = json(r, 'crawler.discovered.metric_origins'); + assert(Object.keys(o).sort().join() === 'acme_requests_total,acme_tls_handshakes_total,ibmmq_qmgr_custom_only', + 'the crawl reads both configs with the downstream catalogue', Object.keys(o)); + assert(r.summary.warnings.filter(w => /exporter catalogue entry/.test(w)).length === 3, 'the dropped entries are crawl warnings'); + assert(detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n') === 'unknown', 'without the catalogue the gateway config is nothing'); + const junk = crawlFiles({ 'a.yaml': 'a: 1\n' }, { repoName: 'cat', now: NOW, exporterCatalogue: 'not a catalogue' }); + assert(junk.summary.warnings.some(w => /neither `\{ exporters: \[\.\.\.\] \}` nor an array/.test(w)), 'a catalogue of the wrong shape is warned about'); +} + +// ---------- F. a collector with pipelines ---------- +{ + const COL = `receivers: + otlp: + protocols: { grpc: {} } + prometheus: + config: + scrape_configs: + - job_name: node + kubernetes_sd_configs: [{ role: pod }] + - job_name: qmgrs + scrape_interval: 15s + static_configs: + - targets: ["a:9157", "b:9157"] + - targets: ["c:9157"] +exporters: + prometheusremotewrite: + endpoint: https://writer:s3cret@mimir.example:443/api/v1/push?token=abc#frag + otlphttp/tempo: + endpoint: http://tempo:4318 + kafka/metrics: + topic: legacy-topic +service: + pipelines: + metrics: + receivers: [otlp, prometheus] + exporters: [prometheusremotewrite] + metrics/archive: + receivers: [prometheus] + exporters: [kafka/metrics] + traces: + receivers: [otlp] + exporters: [otlphttp/tempo] +`; + const r = crawlFiles({ 'deploy/otel-collector.yaml': COL }, { repoName: 'col', now: NOW }); + const jobs = json(r, 'crawler.discovered.scrape_job_origins'); + const leave = [{ name: 'prometheusremotewrite', kind: 'prometheusremotewrite', endpoint: 'https://mimir.example/api/v1/push' }, { name: 'kafka/metrics', kind: 'kafka', topic: 'legacy-topic' }]; + assert(JSON.stringify(jobs.qmgrs.exporters) === JSON.stringify(leave) && jobs.qmgrs.target_count === 3 && jobs.qmgrs.interval === '15s', + 'the job leaves by every metrics pipeline that lists its receiver; the endpoint loses userinfo, query and fragment', jobs.qmgrs); + assert(jobs.node.target_count === 0 && JSON.stringify(jobs.node.targets) === '["kubernetes:pod"]', 'a discovered-target job counts no static target', jobs.node); + assert(!ann(r)['crawler.discovered.scrape_job_origins'].includes('s3cret') && !ann(r)['crawler.discovered.scrape_job_origins'].includes('token'), + 'no credential reaches an annotation'); + const pe = r.canonical.spec.pipelines.exporters; + assert(JSON.stringify([pe.metrics, pe.traces, pe.logs]) === '[{"kind":"prometheusremotewrite"},{"kind":"otlp"},{"kind":"elasticsearch"}]', + 'the pipelines read as before', r.canonical.spec.pipelines.exporters); + assert(safeSourceUrl('prometheus:9090') === 'prometheus:9090' && safeSourceUrl('${env:ENDPOINT}') === '${env:ENDPOINT}' + && safeSourceUrl('u:p@host:4317?x=1') === 'host:4317' && safeSourceUrl('http://h:9157') === 'http://h:9157' && safeSourceUrl('http://h:9157/') === 'http://h:9157/', + 'safeSourceUrl: a non-URL keeps its spelling less userinfo and query'); + + // Two metrics-capable exporters and no pipelines: the first, with a warning. + const two = crawlFiles({ 'files/collector.yaml': 'receivers:\n otlp: {}\nexporters:\n otlp:\n endpoint: http://gw:4317\n kafka:\n topic: t\n' }, { repoName: 'col', now: NOW }); + assert(two.canonical.spec.pipelines.exporters.metrics.kind === 'otlp' + && two.summary.warnings.includes("files/collector.yaml has no service pipelines and 2 metrics-capable exporters (otlp, kafka); the metrics exporter is taken to be the first, 'otlp'."), + 'two candidates: the first is taken and the warning says so', two.summary.warnings); + // An Alertmanager config (receivers is a list) is still Alertmanager. + assert(detectArtefactKind('am.yaml', 'route:\n receiver: x\nreceivers:\n - name: x\n') === 'alertmanager', 'Alertmanager receivers are not a collector\'s'); + + // A collector embedded in a Helm ConfigMap with no service block. + const TPL = `apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "c.fullname" . }} +data: + relay.yaml: | + receivers: + prometheus: + config: + scrape_configs: + - job_name: qmgrs + static_configs: + - targets: ["q:9157"] + exporters: + kafka: + topic: mq +`; + const h = crawlFiles({ 'charts/c/templates/cm.yaml': TPL, 'charts/c/Chart.yaml': 'apiVersion: v2\nname: c\nversion: 0.1.0\n' }, { repoName: 'col', now: NOW }); + assert(JSON.stringify(json(h, 'crawler.discovered.scrape_jobs')) === '["qmgrs"]' && h.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', + 'a Helm ConfigMap\'s collector with no service block: its job and its kafka exporter', ann(h)['crawler.discovered.scrape_jobs']); +} + +// ---------- G. a malformed manifest ---------- +{ + const r = crawlFiles({ + 'a/metrics-manifest.json': '{ "metrics": [] }', + 'b/metrics-manifest.json': JSON.stringify({ exporter: 'x', metrics: [{ name: 'ok_metric', type: 'Gauge' }, { name: '9bad', type: 'gauge' }, { name: 'ok_metric', type: 'counter' }, { name: 'no_type' }] }), + 'c/metrics-manifest.json': 'not json', + }, { repoName: 'bad', now: NOW }); + const w = r.summary.warnings; + assert(w.includes('a/metrics-manifest.json is not a metrics manifest (no `exporter` name); it is not read.') + && w.some(x => x.startsWith('c/metrics-manifest.json is not a metrics manifest (not JSON')), + 'a manifest that is not one is named in a warning', w); + assert(w.some(x => x.startsWith('b/metrics-manifest.json: 3 metric entries are not read')), 'skipped entries are counted', w); + assert(JSON.stringify(json(r, 'crawler.discovered.metric_names')) === '["ok_metric"]', 'the valid entry is read (the type case-insensitive)'); + const m = readMetricsManifest(JSON.stringify({ exporter: 'x', source: 'https://u:p@h/metrics?t=1', metrics: [] })); + assert(m.ok && m.manifest.source === 'https://h/metrics', 'a manifest\'s source is recorded without userinfo or query', m.manifest); +} + +// ---------- H. no exporter evidence: none of the new keys ---------- +{ + const r = crawlFiles({ + 'otel-collector.yaml': 'receivers:\n otlp: {}\nexporters:\n otlp:\n endpoint: http://x:4317\nservice:\n pipelines:\n metrics: { receivers: [otlp], exporters: [otlp] }\n', + 'src/metrics.ts': "import { Counter } from 'prom-client';\nexport const c = new Counter({ name: 'reqs_total', help: 'r' });\n", + }, { repoName: 'plain', now: NOW }); + const keys = Object.keys(ann(r)); + assert(!keys.some(k => /^crawler\.discovered\.(exporters|scrape_job)/.test(k)) && !('exporters' in r.summary.discovered) + && JSON.stringify(r.canonical.spec.pipelines.exporters.metrics) === '{"kind":"otlp"}', + 'a crawl with no exporter or collector scrape evidence carries no new key', keys.filter(k => /discovered/.test(k))); +} + +// ---------- I. packc emit --exporter ---------- +const EXPOSITION = '# HELP ibmmq_qmgr_status Status\n# TYPE ibmmq_qmgr_status gauge\nibmmq_qmgr_status{qmgr="QM1",platform="UNIX"} 1\n' + + '# TYPE ibmmq_queue_depth gauge\nibmmq_queue_depth{qmgr="QM1",queue="APP.IN"} 3\n'; +const server = createServer((req, res) => { + const path = new URL(req.url, 'http://x').pathname; + if (path === '/metrics') { res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' }); res.end(EXPOSITION); return; } + if (path === '/moved') { res.writeHead(302, { location: '/metrics?token=abc' }); res.end(); return; } + if (path === '/empty') { res.writeHead(200); res.end('\n'); return; } + res.writeHead(500); res.end('boom'); +}); +await new Promise(r => server.listen(0, '127.0.0.1', r)); +const base = `http://127.0.0.1:${server.address().port}`; +// The child must not block the event loop the server answers on. +const run = (args, script = EMIT) => new Promise((resolveRun) => { + const child = spawn(process.execPath, [script, ...args], { cwd: tmp, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = '', stderr = ''; + child.stdout.on('data', d => { stdout += d; }); + child.stderr.on('data', d => { stderr += d; }); + child.on('close', status => resolveRun({ status, stdout, stderr })); +}); +const tmp = mkdtempSync(join(tmpdir(), 'emit-manifest-')); +try { + const ok = await run(['--exporter', `${base}/metrics?token=abc`, '--name', 'ibm-mq', '--version', '5.6.0', '--generated-at', NOW]); + assert(ok.status === 0 && ok.stdout === 'wrote metrics-manifest.json: 2 metric families of ibm-mq\n', 'emit writes metrics-manifest.json', ok); + const written = JSON.parse(readFileSync(join(tmp, 'metrics-manifest.json'), 'utf8')); + assert(JSON.stringify(written) === JSON.stringify({ + exporter: 'ibm-mq', version: '5.6.0', source: `${base}/metrics`, generatedAt: NOW, + metrics: [ + { name: 'ibmmq_qmgr_status', type: 'gauge', help: 'Status', labels: ['qmgr', 'platform'] }, + { name: 'ibmmq_queue_depth', type: 'gauge', labels: ['qmgr', 'queue'] }, + ], + }), 'the manifest: names, types, help, label names; the source without its query', written); + assert(!readFileSync(join(tmp, 'metrics-manifest.json'), 'utf8').includes('APP.IN'), 'no label value is written'); + const crawled = crawlFiles({ 'metrics-manifest.json': readFileSync(join(tmp, 'metrics-manifest.json'), 'utf8') }, { repoName: 'emitted', now: NOW }); + assert(JSON.stringify(json(crawled, 'crawler.discovered.metric_names')) === '["ibmmq_qmgr_status","ibmmq_queue_depth"]', 'the crawler reads what emit writes'); + assert(JSON.stringify(manifestFromExposition(EXPOSITION, { exporter: 'ibm-mq', version: '5.6.0', source: `${base}/metrics`, generatedAt: NOW })) === JSON.stringify(written), + 'the CLI writes exactly manifestFromExposition\'s manifest'); + + const viaPackc = await run(['emit', '--exporter', `${base}/metrics`, '--out', '-', '--no-timestamp'], CLI); + const piped = JSON.parse(viaPackc.stdout); + assert(viaPackc.status === 0 && piped.exporter === '127.0.0.1' && !('generatedAt' in piped) && piped.metrics.length === 2, + '`packc emit` dispatches; --out - writes to stdout; the host names the exporter; --no-timestamp leaves the stamp out', viaPackc); + + writeFileSync(join(tmp, 'qm1.prom'), EXPOSITION); + const from = await run(['--from', 'qm1.prom', '--out', '-', '--no-timestamp']); + assert(from.status === 0 && JSON.parse(from.stdout).exporter === 'qm1' && !('source' in JSON.parse(from.stdout)), '--from reads a saved exposition, named by the file', from); + + const refusals = [ + [['--exporter', `${base}/moved`], 1, `packc emit: ${base}/moved answered 302, a redirect to ${base}/metrics; it is not followed — pass the exporter's own metrics URL\n`], + [['--exporter', `${base}/broken`], 1, `packc emit: ${base}/broken answered 500; a metrics endpoint answers 200\n`], + [['--exporter', `${base}/empty`], 1, `packc emit: ${base}/empty holds no metric family (no # TYPE, # HELP or sample line)\n`], + [['--exporter', 'http://u:p@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], + [['--exporter', 'file:///etc/passwd'], 2, 'packc emit: --exporter must be an http or https URL, not file:\n'], + [['--exporter', `${base}/metrics`, '--from', 'qm1.prom'], 2, 'packc emit: pass --exporter or --from, not both\n'], + [['--exporter', `${base}/metrics`, '--bogus'], 2, null], + [[], 2, null], + ]; + for (const [args, code, message] of refusals) { + const out = await run(args); + assert(out.status === code && out.stdout === '' && (message === null || out.stderr === message), + `emit ${args.join(' ') || '(no args)'} → exit ${code}${message ? ', the reason named' : ''}`, out, { status: code, message }); + } + // The refusal's way out works: save it yourself, pass --from. + assert(from.status === 0, 'the credentials refusal\'s way out (--from) is the one that worked above'); +} finally { + server.close(); + rmSync(tmp, { recursive: true, force: true }); +} + +// The crawl CLI reads a downstream catalogue file; an unreadable one is a usage error. +{ + const dir = mkdtempSync(join(tmpdir(), 'crawl-cat-')); + try { + writeFileSync(join(dir, 'acme.yaml'), 'acme:\n tls: true\nlisten: ":8080"\n'); + writeFileSync(join(dir, 'cat.json'), JSON.stringify({ exporters: [{ id: 'acme-gw', match: { keys: ['acme', 'listen'] }, namespace: { default: 'acme' }, families: [{ name: 'up', type: 'gauge' }] }] })); + const crawl = fileURLToPath(new URL('./crawl-repo.mjs', import.meta.url)); + const okRun = spawnSync(process.execPath, [crawl, dir, '--name', 'gw', '--exporter-catalogue', join(dir, 'cat.json')], { encoding: 'utf8' }); + assert(okRun.status === 0 && okRun.stdout.includes('acme_up') && okRun.stderr.includes('# exporters : 1'), 'x-ray --exporter-catalogue reads the file', okRun.stderr.slice(-400)); + const bad = spawnSync(process.execPath, [crawl, dir, '--name', 'gw', '--exporter-catalogue', join(dir, 'missing.json')], { encoding: 'utf8' }); + assert(bad.status === 2 && bad.stdout === '' && /--exporter-catalogue .*missing\.json.*ENOENT/.test(bad.stderr), 'a missing catalogue file is a usage error', bad.stderr); + } finally { rmSync(dir, { recursive: true, force: true }); } +} + +report('crawl-exporters'); diff --git a/tools/test-vendor-manifest.mjs b/tools/test-vendor-manifest.mjs index 18024511..846c186d 100644 --- a/tools/test-vendor-manifest.mjs +++ b/tools/test-vendor-manifest.mjs @@ -36,7 +36,8 @@ const EXPECTED_MODULES = [ 'tools/lib/backend-products.mjs', 'tools/lib/blast-radius.mjs', 'tools/lib/brand.mjs', 'tools/lib/burn-rules.mjs', 'tools/lib/chain-history.mjs', 'tools/lib/compile.mjs', 'tools/lib/conformance.mjs', 'tools/lib/contracts/mcp-capabilities.mjs', 'tools/lib/contracts/response-shapes.mjs', 'tools/lib/contracts/stack-self-metrics.mjs', 'tools/lib/crawler.mjs', - 'tools/lib/dashboards/generic.mjs', 'tools/lib/dashboards/lib.mjs', 'tools/lib/diff.mjs', 'tools/lib/good-when.mjs', 'tools/lib/identity-modes.mjs', + 'tools/lib/dashboards/generic.mjs', 'tools/lib/dashboards/lib.mjs', 'tools/lib/diff.mjs', 'tools/lib/exporter-catalogue.mjs', 'tools/lib/exporter-metrics.mjs', + 'tools/lib/good-when.mjs', 'tools/lib/identity-modes.mjs', 'tools/lib/inventory-coverage.mjs', 'tools/lib/journey-notify.mjs', 'tools/lib/l2x.mjs', 'tools/lib/legacy.mjs', 'tools/lib/library.mjs', 'tools/lib/live-fetch.mjs', 'tools/lib/mcp-client.mjs', 'tools/lib/mcp-server-settings.mjs', 'tools/lib/mcp-url-safety.mjs', 'tools/lib/mini-yaml.mjs', 'tools/lib/neuron-model.mjs', 'tools/lib/pack-conformance.mjs', 'tools/lib/profiles.mjs', 'tools/lib/promql-canon.mjs', 'tools/lib/promql-lezer.mjs', 'tools/lib/promql.mjs', From 3cd373ecf3c82859099f9c90d9634f4bd07835d6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:25:19 +0000 Subject: [PATCH 02/17] test-platform: pin the exporter-metrics URL-path .pathname uses Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- docs/CHANGELOG.md | 2 +- tools/test-platform.mjs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 56338cd7..f999c2c2 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,7 +11,7 @@ - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. -- No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. +- No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the two new `.pathname` uses, both URL paths: an endpoint's path in `safeSourceUrl` and the fake exporter's request path. ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. diff --git a/tools/test-platform.mjs b/tools/test-platform.mjs index 22cd0089..20066b0f 100644 --- a/tools/test-platform.mjs +++ b/tools/test-platform.mjs @@ -65,8 +65,10 @@ const PATHNAME_ALLOWED = Object.freeze({ 'server/routes/mcp-settings.mjs': 2, // an MCP server's settings URL path, for a refusal and the audit row 'server/test-auth-oidc.mjs': 4, // the fake IdP's HTTP request paths 'tools/build-studio-bundle.mjs': 1, // a --pack-url (http) path, for an id + 'tools/lib/exporter-metrics.mjs': 1, // an exporter or collector endpoint's path, recorded without userinfo or query 'tools/lib/journey.mjs': 1, // a notify hook URL's path and query, redacted from a delivery error 'tools/record-mcp-fixtures.mjs': 1, // an MCP URL, printed without its query + 'tools/test-crawl-exporters.mjs': 1, // the fake exporter's HTTP request path }); // P6: the fixture's two call forms and what a well-formed argument is. const SKIP_CALL_RE = /\b(win32Skip|skipOnWin32)\(([^)]*)\)/g; From 3288a79540d14b844a6b8c851fb00483b91baada Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:31:49 +0000 Subject: [PATCH 03/17] exporter-metrics: never record or echo a credential safeSourceUrl was a URL parse with a first-@ textual fallback: a password holding '@', '/' or '?' (user:p@ss@h, https://u:pa/ss@h/v1, //u:p@h, https://u:12/ss@h) reached crawler.discovered.scrape_job_origins and the manifest source. It is now textual: everything before the last '@', then from the first '?' or '#', is dropped. packc emit echoed an unparseable --exporter verbatim, and an unparseable redirect Location threw with the full request URL (query token included) on stderr. The echo is safeSourceUrl's form; the redirect refusal names the target with the query as keys only, so passing it with the values works. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- docs/CHANGELOG.md | 3 ++- docs/DOWNSTREAM.md | 17 ++++++++++------- tools/emit-manifest.mjs | 26 +++++++++++++++++++------- tools/lib/exporter-metrics.mjs | 27 ++++++++++++++------------- tools/test-crawl-exporters.mjs | 29 +++++++++++++++++++++++++---- tools/test-platform.mjs | 1 - 7 files changed, 72 insertions(+), 35 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index c46e113e..21def82e 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -523,8 +523,8 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "6a9cff6f68206ae8a43e0f6b1066cc3f595a0df128e4bf8d2bfdea6dd6e49ba5", - "bytes": 16534, + "sha256": "1132b63bf23514ef288a4fdc94650a6a11a692485651e8a50ce0056db1e7e1ce", + "bytes": 16694, "exports": [ "EXPOSITION_TYPES", "inferExporterFamilies", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f999c2c2..813d77b6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -11,7 +11,8 @@ - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. -- No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the two new `.pathname` uses, both URL paths: an endpoint's path in `safeSourceUrl` and the fake exporter's request path. +- No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). +- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops everything before the last `@`, then everything from the first `?` or `#`, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 7eed3b95..d050267b 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1873,7 +1873,9 @@ exposition type (`counter`, `gauge`, `histogram`, `summary`, `untyped`, `unknown`, `gaugehistogram`, `info`, `stateset`). An entry that has neither, or repeats a name, is not read and the crawl says how many; a file that is not a manifest is named in a warning. `source` is recorded without userinfo, -query or fragment. Label values are never part of a manifest. +query or fragment: everything before the last `@` and from the first `?` or +`#` is dropped (`safeSourceUrl`, textual — a password may hold `/` or `@`). +Label values are never part of a manifest. `packc emit --exporter ` (`tools/emit-manifest.mjs`) writes one from a live endpoint — run it in the exporter's build pipeline against the image it @@ -1885,10 +1887,11 @@ packc emit --from saved.prom --name ibm-mq --out - # a saved exposition, to ``` It scrapes once and records each family's name, type, help and label names. -A redirect is refused, not followed (pass the URL it names); a URL with -credentials is refused (save the exposition with your own client and pass it -with `--from`); a status other than 200 or a response with no metric family -exits 1. +A redirect is refused, not followed: the refusal names its target with the +query as keys only (`?token=…`) — pass that URL with the values the exporter +expects. A URL with credentials is refused (save the exposition with your own +client and pass it with `--from`). A status other than 200 or a response +with no metric family exits 1. ### 18.2 The catalogue @@ -1949,8 +1952,8 @@ without a `service:` block (a Helm chart's `files/` copy has none). its samples leave by: the exporters of the metrics pipelines that list the receiver, or, with no pipelines, the one metrics exporter the config names. Each exporter is given as `{ name, kind, topic | endpoint }`: a kafka - exporter's metrics topic, any other's endpoint without userinfo, query or - fragment. + exporter's metrics topic, any other's endpoint reduced by `safeSourceUrl` + (no userinfo, query or fragment). - That exporter's kind is `spec.pipelines.exporters.metrics`, in place of the `prometheusremotewrite` scaffold. When a config with no pipelines names several metrics exporters, the first is taken and the crawl warns. diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs index 21e8448b..f96fb774 100644 --- a/tools/emit-manifest.mjs +++ b/tools/emit-manifest.mjs @@ -26,8 +26,10 @@ * exit 1 the scrape failed: unreachable, a redirect, a status other than 200, no metric families * exit 2 usage: no source, both sources, an unknown flag, a URL that is not http(s) or carries credentials * - * The recorded `source` is the URL without userinfo, query or fragment. A - * redirect is refused, not followed: pass the URL it names. + * The recorded `source` is the URL without userinfo, query or fragment + * (tools/lib/exporter-metrics.mjs safeSourceUrl). A redirect is refused, not + * followed: the refusal names its target, the query as keys only — pass that + * URL with the values the exporter expects. */ import { readFileSync, writeFileSync } from 'node:fs'; @@ -56,6 +58,19 @@ function parseArgs(argv) { return out; } +// A redirect is named, never followed: its target without userinfo, and its +// query as keys only (`?format=…`) — a value may be a token — so the URL to +// pass is the one shown with the values the exporter expects. +function redirectRefusal(url, status, location) { + let target = null; + try { target = location ? new URL(location, url) : null; } catch { /* not a URL: named as such */ } + const keys = target ? [...new Set([...target.searchParams.keys()])] : []; + const to = !location ? '' + : !target ? ' to a location that is not a URL' + : ` to ${safeSourceUrl(target.href)}${keys.length ? `?${keys.map(k => `${encodeURIComponent(k)}=…`).join('&')} (query values not shown)` : ''}`; + return `${safeSourceUrl(url)} answered ${status}, a redirect${to}; it is not followed — pass the URL the exporter serves its metrics at${keys.length ? ', with the query values it expects' : ''}`; +} + async function scrape(url, timeoutSeconds) { let res; try { @@ -68,10 +83,7 @@ async function scrape(url, timeoutSeconds) { const why = e?.name === 'TimeoutError' ? `no answer within ${timeoutSeconds}s (raise --timeout)` : (e?.cause?.message || e?.message || String(e)); fail(1, `could not scrape ${safeSourceUrl(url)}: ${why}`); } - if (res.status >= 300 && res.status < 400) { - const to = res.headers.get('location'); - fail(1, `${safeSourceUrl(url)} answered ${res.status}, a redirect${to ? ` to ${safeSourceUrl(new URL(to, url).href)}` : ''}; it is not followed — pass the exporter's own metrics URL`); - } + if (res.status >= 300 && res.status < 400) fail(1, redirectRefusal(url, res.status, res.headers.get('location'))); if (res.status !== 200) fail(1, `${safeSourceUrl(url)} answered ${res.status}; a metrics endpoint answers 200`); return res.text(); } @@ -92,7 +104,7 @@ if (args.timestamp) { let text, source, defaultName; if (args.exporter) { let url; - try { url = new URL(args.exporter); } catch { fail(2, `--exporter '${args.exporter}' is not a URL (e.g. http://mq-exporter:9157/metrics)`); } + try { url = new URL(args.exporter); } catch { fail(2, `--exporter '${safeSourceUrl(args.exporter)}' is not a URL (e.g. http://mq-exporter:9157/metrics); shown without anything before an @ or after a ? or #`); } if (url.protocol !== 'http:' && url.protocol !== 'https:') fail(2, `--exporter must be an http or https URL, not ${url.protocol}`); if (url.username || url.password) fail(2, 'the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from'); text = await scrape(url.href, timeout); diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index 2025d85a..8864c06c 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -55,23 +55,24 @@ export function isExpositionSnapshotPath(relPath) { } /** - * A URL as it may be recorded: scheme, host, port and path — never the - * userinfo, query or fragment, where a credential rides. A value that is not - * a `scheme://` URL (`prometheus:9090`, `${env:ENDPOINT}`) loses anything up - * to an `@` and from a `?` or `#`. + * A URL as it may be recorded: never the userinfo, query or fragment, where a + * credential rides. Textual, never a URL parse: a password may hold `/`, `?` + * or `@`, which end a parsed authority early and leave the rest of the secret + * in the "host" or "path". So everything before the LAST `@` goes (a value + * whose path or query holds an `@` keeps only what follows it, which loses + * the host rather than risk a credential), then everything from the first + * `?` or `#`. A leading `scheme://` or `//` is kept; a value that is not a + * URL at all (`prometheus:9090`, `${env:ENDPOINT}`) passes through the same + * rule. */ export function safeSourceUrl(raw) { const s = String(raw ?? '').trim(); if (!s) return ''; - if (/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(s)) { - try { - const u = new URL(s); - // The path as written: `http://h:9157` stays without the `/` URL adds. - const path = /^[^:]+:\/\/[^/?#]*\//.test(s) ? u.pathname : ''; - return `${u.protocol}//${u.host}${path}`; - } catch { /* fall through to the textual rule */ } - } - return s.replace(/^([A-Za-z][A-Za-z0-9+.-]*:\/\/)?[^\s/@]*@/, '$1').replace(/[?#].*$/, ''); + const prefix = /^(?:[A-Za-z][A-Za-z0-9+.-]*:)?\/\//.exec(s)?.[0] ?? ''; + let rest = s.slice(prefix.length); + const at = rest.lastIndexOf('@'); + if (at >= 0) rest = rest.slice(at + 1); + return prefix + rest.replace(/[?#].*$/, ''); } function parseSampleLine(line) { diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 35f7603e..39721efc 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -282,7 +282,7 @@ service: `; const r = crawlFiles({ 'deploy/otel-collector.yaml': COL }, { repoName: 'col', now: NOW }); const jobs = json(r, 'crawler.discovered.scrape_job_origins'); - const leave = [{ name: 'prometheusremotewrite', kind: 'prometheusremotewrite', endpoint: 'https://mimir.example/api/v1/push' }, { name: 'kafka/metrics', kind: 'kafka', topic: 'legacy-topic' }]; + const leave = [{ name: 'prometheusremotewrite', kind: 'prometheusremotewrite', endpoint: 'https://mimir.example:443/api/v1/push' }, { name: 'kafka/metrics', kind: 'kafka', topic: 'legacy-topic' }]; assert(JSON.stringify(jobs.qmgrs.exporters) === JSON.stringify(leave) && jobs.qmgrs.target_count === 3 && jobs.qmgrs.interval === '15s', 'the job leaves by every metrics pipeline that lists its receiver; the endpoint loses userinfo, query and fragment', jobs.qmgrs); assert(jobs.node.target_count === 0 && JSON.stringify(jobs.node.targets) === '["kubernetes:pod"]', 'a discovered-target job counts no static target', jobs.node); @@ -291,6 +291,23 @@ service: const pe = r.canonical.spec.pipelines.exporters; assert(JSON.stringify([pe.metrics, pe.traces, pe.logs]) === '[{"kind":"prometheusremotewrite"},{"kind":"otlp"},{"kind":"elasticsearch"}]', 'the pipelines read as before', r.canonical.spec.pipelines.exporters); + // Every spelling of a credential: a password holding `@`, `/` or `?`, no + // scheme, a scheme-relative URL, one a URL parser reads as host:port/path. + const SECRETS = [ + ['user:p@ss@otel:4317', 'otel:4317'], + ['user:pa/ss@otel:4317', 'otel:4317'], + ['https://user:pa/ss@otel:4317/v1', 'https://otel:4317/v1'], + ['https://user:12/ss@otel/v1', 'https://otel/v1'], + ['https://u:pa?ss@h/x', 'https://h/x'], + ['//u:p@h/metrics', '//h/metrics'], + ['HTTPS://U:P@H:1/m?t=1#f', 'HTTPS://H:1/m'], + ['svc:Zx9/Qk+SECRET==@otel-gw:4317', 'otel-gw:4317'], + ]; + assert(SECRETS.every(([raw, want]) => safeSourceUrl(raw) === want), 'safeSourceUrl drops everything before the last @ and from the first ? or #', + SECRETS.map(([raw]) => safeSourceUrl(raw)), SECRETS.map(([, want]) => want)); + const leak = crawlFiles({ 'otel/collector.yaml': `receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp:\n endpoint: "svc:Zx9/Qk+SECRETPW@otel-gw:4317"\n` }, { repoName: 'col', now: NOW }); + assert(!JSON.stringify(leak.canonical).includes('SECRET') && json(leak, 'crawler.discovered.scrape_job_origins').mq.exporters[0].endpoint === 'otel-gw:4317', + 'a collector endpoint whose password holds `/` reaches the pack without it', json(leak, 'crawler.discovered.scrape_job_origins').mq); assert(safeSourceUrl('prometheus:9090') === 'prometheus:9090' && safeSourceUrl('${env:ENDPOINT}') === '${env:ENDPOINT}' && safeSourceUrl('u:p@host:4317?x=1') === 'host:4317' && safeSourceUrl('http://h:9157') === 'http://h:9157' && safeSourceUrl('http://h:9157/') === 'http://h:9157/', 'safeSourceUrl: a non-URL keeps its spelling less userinfo and query'); @@ -363,6 +380,7 @@ const server = createServer((req, res) => { if (path === '/metrics') { res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' }); res.end(EXPOSITION); return; } if (path === '/moved') { res.writeHead(302, { location: '/metrics?token=abc' }); res.end(); return; } if (path === '/empty') { res.writeHead(200); res.end('\n'); return; } + if (path === '/badloc') { res.writeHead(302, { location: 'http://[oops/metrics' }); res.end(); return; } res.writeHead(500); res.end('boom'); }); await new Promise(r => server.listen(0, '127.0.0.1', r)); @@ -403,7 +421,9 @@ try { assert(from.status === 0 && JSON.parse(from.stdout).exporter === 'qm1' && !('source' in JSON.parse(from.stdout)), '--from reads a saved exposition, named by the file', from); const refusals = [ - [['--exporter', `${base}/moved`], 1, `packc emit: ${base}/moved answered 302, a redirect to ${base}/metrics; it is not followed — pass the exporter's own metrics URL\n`], + [['--exporter', `${base}/moved`], 1, `packc emit: ${base}/moved answered 302, a redirect to ${base}/metrics?token=… (query values not shown); it is not followed — pass the URL the exporter serves its metrics at, with the query values it expects\n`], + [['--exporter', `${base}/badloc?token=SECRET3`], 1, `packc emit: ${base}/badloc answered 302, a redirect to a location that is not a URL; it is not followed — pass the URL the exporter serves its metrics at\n`], + [['--exporter', '//admin:SECRET1@127.0.0.1:1/metrics'], 2, "packc emit: --exporter '//127.0.0.1:1/metrics' is not a URL (e.g. http://mq-exporter:9157/metrics); shown without anything before an @ or after a ? or #\n"], [['--exporter', `${base}/broken`], 1, `packc emit: ${base}/broken answered 500; a metrics endpoint answers 200\n`], [['--exporter', `${base}/empty`], 1, `packc emit: ${base}/empty holds no metric family (no # TYPE, # HELP or sample line)\n`], [['--exporter', 'http://u:p@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], @@ -417,8 +437,9 @@ try { assert(out.status === code && out.stdout === '' && (message === null || out.stderr === message), `emit ${args.join(' ') || '(no args)'} → exit ${code}${message ? ', the reason named' : ''}`, out, { status: code, message }); } - // The refusal's way out works: save it yourself, pass --from. - assert(from.status === 0, 'the credentials refusal\'s way out (--from) is the one that worked above'); + // The refusals' ways out work: save it yourself and pass --from (above); + // the redirect's target with the query values it expects (the first run). + assert(from.status === 0 && ok.status === 0, 'the credentials refusal\'s --from and the redirect refusal\'s URL with its query values are the runs that worked above'); } finally { server.close(); rmSync(tmp, { recursive: true, force: true }); diff --git a/tools/test-platform.mjs b/tools/test-platform.mjs index 20066b0f..cf6576b3 100644 --- a/tools/test-platform.mjs +++ b/tools/test-platform.mjs @@ -65,7 +65,6 @@ const PATHNAME_ALLOWED = Object.freeze({ 'server/routes/mcp-settings.mjs': 2, // an MCP server's settings URL path, for a refusal and the audit row 'server/test-auth-oidc.mjs': 4, // the fake IdP's HTTP request paths 'tools/build-studio-bundle.mjs': 1, // a --pack-url (http) path, for an id - 'tools/lib/exporter-metrics.mjs': 1, // an exporter or collector endpoint's path, recorded without userinfo or query 'tools/lib/journey.mjs': 1, // a notify hook URL's path and query, redacted from a delivery error 'tools/record-mcp-fixtures.mjs': 1, // an MCP URL, printed without its query 'tools/test-crawl-exporters.mjs': 1, // the fake exporter's HTTP request path From 7d5469a856cac4f13fbc65a6661d3869078dbebd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:32:13 +0000 Subject: [PATCH 04/17] exporter-metrics: quote no manifest text in a warning; read a BOM-prefixed manifest or config A metrics-manifest.json that was not JSON put the parser's message, which quotes the file's first characters, into summary.warnings. A leading BOM hid an mq-metric-samples config from the key precheck (the file read as unknown and left the environment-scope warning) and failed a manifest's JSON.parse; one U+FEFF is now stripped, as mini-yaml does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- docs/CHANGELOG.md | 2 +- tools/lib/exporter-metrics.mjs | 8 ++++++-- tools/test-crawl-exporters.mjs | 12 +++++++++--- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 21def82e..a08eb99e 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -523,8 +523,8 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "1132b63bf23514ef288a4fdc94650a6a11a692485651e8a50ce0056db1e7e1ce", - "bytes": 16694, + "sha256": "9fb2b0456f9df4d0e54d886b9fbc2cd4e7b58a4436f5d3a067910870c68d5dd7", + "bytes": 16936, "exports": [ "EXPOSITION_TYPES", "inferExporterFamilies", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 813d77b6..4277f9df 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,7 +3,7 @@ ## Unreleased ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs -- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest, or entries without a name and an exposition type, are named in a warning. +- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. - **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches them, manifests, `config.yaml` and a chart's `files/*.yaml`). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. It is never read from a test or fixture path. - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest. A malformed entry is dropped with a warning. `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index 8864c06c..a190b127 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -191,7 +191,9 @@ export function manifestFromExposition(text, { exporter, version, source, genera */ export function readMetricsManifest(text) { let obj; - try { obj = JSON.parse(String(text ?? '')); } catch (e) { return { ok: false, error: `not JSON (${e.message})` }; } + // Never the parser's message: it quotes the start of the text, which in a + // file that is not a manifest may be a key. + try { obj = JSON.parse(stripBom(text)); } catch { return { ok: false, error: 'not JSON' }; } if (!obj || typeof obj !== 'object' || Array.isArray(obj)) return { ok: false, error: 'not a JSON object' }; const exporter = typeof obj.exporter === 'string' ? obj.exporter.trim() : ''; if (!exporter) return { ok: false, error: 'no `exporter` name' }; @@ -228,6 +230,8 @@ export function readMetricsManifest(text) { // ---------- the exporter config catalogue ---------- const isMapping = (v) => v !== null && typeof v === 'object' && !Array.isArray(v); +// One leading U+FEFF, as mini-yaml strips it (Windows editors write one). +const stripBom = (text) => String(text ?? '').replace(/^\uFEFF/, ''); function pathValue(obj, path) { let cur = obj; @@ -320,7 +324,7 @@ export function resolveExporterCatalogue(custom) { * parsed. Cheap; `matchExporterConfig` decides. */ export function mayBeExporterConfig(content, catalogue) { - const text = String(content ?? ''); + const text = stripBom(content); return catalogue.exporters.some(e => e.match.keys.every(k => new RegExp(`^${k}\\s*:`, 'm').test(text))); } diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 39721efc..d737b901 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -348,12 +348,18 @@ data: const r = crawlFiles({ 'a/metrics-manifest.json': '{ "metrics": [] }', 'b/metrics-manifest.json': JSON.stringify({ exporter: 'x', metrics: [{ name: 'ok_metric', type: 'Gauge' }, { name: '9bad', type: 'gauge' }, { name: 'ok_metric', type: 'counter' }, { name: 'no_type' }] }), - 'c/metrics-manifest.json': 'not json', + 'c/metrics-manifest.json': 'ghp_SECRETxyz: not json', }, { repoName: 'bad', now: NOW }); const w = r.summary.warnings; assert(w.includes('a/metrics-manifest.json is not a metrics manifest (no `exporter` name); it is not read.') - && w.some(x => x.startsWith('c/metrics-manifest.json is not a metrics manifest (not JSON')), - 'a manifest that is not one is named in a warning', w); + && w.includes('c/metrics-manifest.json is not a metrics manifest (not JSON); it is not read.') && !JSON.stringify(r.summary).includes('ghp_'), + 'a manifest that is not one is named in a warning, none of its text quoted', w); + const bom = crawlFiles({ + 'metrics-manifest.json': `\uFEFF${JSON.stringify({ exporter: 'x', metrics: [{ name: 'bom_metric', type: 'gauge' }] })}`, + 'mq/config.yaml': `\uFEFF${FILES['mq-exporter/config.yaml']}`, + }, { repoName: 'bom', now: NOW }); + assert(json(bom, 'crawler.discovered.metric_names').includes('bom_metric') && bom.summary.files.byKind['exporter-config'] === 1, + 'a manifest and an exporter config saved with a BOM are read', bom.summary.files.byKind); assert(w.some(x => x.startsWith('b/metrics-manifest.json: 3 metric entries are not read')), 'skipped entries are counted', w); assert(JSON.stringify(json(r, 'crawler.discovered.metric_names')) === '["ok_metric"]', 'the valid entry is read (the type case-insensitive)'); const m = readMetricsManifest(JSON.stringify({ exporter: 'x', source: 'https://u:p@h/metrics?t=1', metrics: [] })); From 55272a6344f8c0ee4d5e7ca1a63ca27e32eb13c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:33:10 +0000 Subject: [PATCH 05/17] crawler: a pipeline-less collector's metrics exporter is a guess a declared pipeline overrides A collector config split across --config files (components in one, service in another) took its metrics exporter from whichever file sorted first: the components file's first metrics-capable exporter (otlp/tempo) beat the service file's declared metrics pipeline (prometheusremotewrite), with a guess warning and a second pipeline counted. The guess is now applied after every file is walked, and only when no pipeline declares one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 +- docs/CHANGELOG.md | 2 +- docs/DOWNSTREAM.md | 9 +- tools/lib/crawler.mjs | 152 ++++++++++++++++++--------------- tools/test-crawl-exporters.mjs | 12 +++ 5 files changed, 106 insertions(+), 73 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index a08eb99e..4ef637eb 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "c4f564f2c0968120500defba9f61f0776e34342b2536011883a12553a7724e1f", - "bytes": 125803, + "sha256": "34004a75cde40b59a18a8b2c5d3f3b68ed0e7075d1fc5365e331d142233ea0f7", + "bytes": 126702, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4277f9df..d965ad9c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,7 +7,7 @@ - **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches them, manifests, `config.yaml` and a chart's `files/*.yaml`). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. It is never read from a test or fixture path. - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest. A malformed entry is dropped with a warning. `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. -- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the one metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold, and its scrape receivers become `pipelines.receivers`. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index d050267b..26cd1223 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1950,10 +1950,13 @@ without a `service:` block (a Helm chart's `files/` copy has none). `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the `receiver`, the `target_count` of its static targets, and the `exporters` its samples leave by: the exporters of the metrics pipelines that list the - receiver, or, with no pipelines, the one metrics exporter the config names. + receiver, or, with no pipelines, the metrics exporter the config names. Each exporter is given as `{ name, kind, topic | endpoint }`: a kafka exporter's metrics topic, any other's endpoint reduced by `safeSourceUrl` (no userinfo, query or fragment). - That exporter's kind is `spec.pipelines.exporters.metrics`, in place of the - `prometheusremotewrite` scaffold. When a config with no pipelines names - several metrics exporters, the first is taken and the crawl warns. + `prometheusremotewrite` scaffold. A config with no pipelines only proposes + one: it applies after every file is walked, and only when no pipeline in + any file declares a metrics exporter (a config split across `--config` + files declares it in another). When it names several, the first is taken + and the crawl warns. diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 8e9dee42..a7e108bf 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -582,6 +582,11 @@ export function crawlFiles(filesInput, opts = {}) { const metricDefinitions = []; const scrapeJobs = []; const exporters = new Map(); // exporter name → the files that say what it exposes + // A collector config with no pipelines names its metrics exporter only by + // guess; a guess is applied after every file is walked, and only when no + // pipeline declared one (a config split across --config files declares it + // in another file). + const collectorGuesses = []; const dashboards = []; const alertingRoutes = []; const pipelines = { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }; @@ -634,12 +639,12 @@ export function crawlFiles(filesInput, opts = {}) { case 'actuator-metrics-config': walkActuatorMetricsConfig(f, scrapeJobs, evidence, summary, repoName); break; case 'metric-source-code': walkMetricSourceCode(f, metricDefinitions, evidence, summary, repoName); break; case 'alertmanager': walkAlertmanager(f, alertingRoutes, evidence, summary); break; - case 'otel-collector': walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary); break; + case 'otel-collector': walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, collectorGuesses); break; case 'metrics-manifest': walkMetricsManifest(f, metricDefinitions, exporters, summary, evidence); break; case 'exposition-snapshot': walkExpositionSnapshot(f, metricDefinitions, exporters, summary, evidence); break; case 'exporter-config': walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence); break; case 'grafana-dashboard':walkGrafanaDashboard(f, dashboards, metricDefinitions, evidence, summary); break; - case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, repoName }, evidence, summary); break; + case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorGuesses, repoName }, evidence, summary); break; case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents); break; case 'k8s-workload': walkK8sWorkload(f, backends, pipelines, evidence, summary); break; // Chart.yaml itself carries no observability contracts — the payloads @@ -652,6 +657,19 @@ export function crawlFiles(filesInput, opts = {}) { } } + if (!pipelines.exporters.metrics && collectorGuesses.length) { + const g = collectorGuesses[0]; + pipelines.exporters.metrics = { kind: mapExporterKind(g.candidates[0].split('/')[0]) }; + evidence['pipelines.exporters.metrics'] = g.relPath; + if (g.candidates.length > 1) { + summary.warnings.push(`${g.relPath} has no service pipelines and ${g.candidates.length} metrics-capable exporters (${g.candidates.join(', ')}); the metrics exporter is taken to be the first, '${g.candidates[0]}'.`); + } + if (!g.counted) { + evidence[`PIP-${summary.discovered.pipelines + 1}`] = g.relPath; + summary.discovered.pipelines++; + } + } + // ----- infer SLIs/SLOs (decompiler ⇄ compiler symmetry) ----- // PRIMARY source: recording rules. compile.mjs materialises every SLI in // spec.slis as `::` recording rules, so reading those names @@ -1917,14 +1935,14 @@ function walkK8sWorkload(f, backends, pipelines, evidence, summary) { // like a rule shipped as a standalone file. If the template embeds nothing, we // strip the scaffolding and treat the document itself as a single manifest. function walkHelmTemplate(f, buckets, evidence, summary) { - const { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, repoName } = buckets; + const { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorGuesses, repoName } = buckets; const route = (kind, sub) => { switch (kind) { case 'prometheus-rules': walkPrometheusRules(sub, recordingRules, burnRateAlerts, alertRules, metricDefinitions, evidence, summary); return true; case 'prometheus-scrape-config': walkPrometheusScrapeConfig(sub, scrapeJobs, evidence, summary); return true; case 'actuator-metrics-config': walkActuatorMetricsConfig(sub, scrapeJobs, evidence, summary, repoName); return true; case 'alertmanager': walkAlertmanager(sub, alertingRoutes, evidence, summary); return true; - case 'otel-collector': walkOtelCollector(sub, pipelines, scrapeJobs, evidence, summary); return true; + case 'otel-collector': walkOtelCollector(sub, pipelines, scrapeJobs, evidence, summary, collectorGuesses); return true; case 'grafana-dashboard':walkGrafanaDashboard(sub, dashboards, metricDefinitions, evidence, summary); return true; case 'docker-compose': walkDockerCompose(sub, backends, pipelines, evidence, summary); return true; case 'k8s-workload': walkK8sWorkload(sub, backends, pipelines, evidence, summary); return true; @@ -2284,79 +2302,79 @@ function walkAlertmanager(f, routes, evidence, summary) { // exporter per signal) and the scrape jobs of its prometheus receivers. A // config with no `service:` block (a Helm chart's files/ copy, assembled // with the pipelines elsewhere) still names its receivers and exporters: -// the prometheus receivers' jobs are read, and the one metrics exporter it -// configures is the metrics exporter. -function walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary) { - for (const obj of parseYamlDocs(f.content)) { - const scrapeReceivers = collectorScrapeReceivers(obj); - const exporterCfgs = isMapping(obj?.exporters) ? obj.exporters : {}; - if (obj?.service?.pipelines) { - const seen = new Set(); - for (const pipeline of Object.values(obj.service.pipelines)) { - for (const recv of pipeline.receivers || []) { - if (!seen.has(`r:${recv}`)) { - seen.add(`r:${recv}`); - pipelines.receivers.push({ name: recv.split('/')[0] }); - } +// the prometheus receivers' jobs are read, and the metrics exporter it +// configures is a guess (`guesses`), applied by crawlFiles only when no +// pipeline in any file declares one. +function walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, guesses = []) { + for (const obj of parseYamlDocs(f.content)) walkOtelCollectorDoc(obj, f.relPath, pipelines, scrapeJobs, evidence, summary, guesses); +} + +function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, summary, guesses) { + const scrapeReceivers = collectorScrapeReceivers(obj); + const exporterCfgs = isMapping(obj?.exporters) ? obj.exporters : {}; + if (obj?.service?.pipelines) { + const seen = new Set(); + for (const pipeline of Object.values(obj.service.pipelines)) { + for (const recv of pipeline.receivers || []) { + if (!seen.has(`r:${recv}`)) { + seen.add(`r:${recv}`); + pipelines.receivers.push({ name: recv.split('/')[0] }); } - for (const proc of pipeline.processors || []) { - if (!seen.has(`p:${proc}`)) { - seen.add(`p:${proc}`); - pipelines.processors.push({ name: proc.split('/')[0] }); - } - } - for (const exp of pipeline.exporters || []) { - const kind = exp.split('/')[0]; - const sig = Object.entries(obj.service.pipelines).find(([_, p]) => p === pipeline)?.[0] || ''; - const signalClass = /^metrics/i.test(sig) ? 'metrics' - : /^logs/i.test(sig) ? 'logs' - : /^traces/i.test(sig) ? 'traces' : null; - if (signalClass && !pipelines.exporters[signalClass]) { - pipelines.exporters[signalClass] = { kind: mapExporterKind(kind) }; - } + } + for (const proc of pipeline.processors || []) { + if (!seen.has(`p:${proc}`)) { + seen.add(`p:${proc}`); + pipelines.processors.push({ name: proc.split('/')[0] }); } } - evidence[`PIP-${summary.discovered.pipelines + 1}`] = f.relPath; - summary.discovered.pipelines++; - // Each scrape receiver's samples leave by the exporters of the metrics - // pipelines that list it. - for (const { key, config } of scrapeReceivers) { - const names = []; - for (const [sig, p] of Object.entries(obj.service.pipelines)) { - if (!/^metrics/i.test(sig) || !Array.isArray(p?.receivers) || !p.receivers.includes(key)) continue; - for (const exp of p.exporters || []) if (!names.includes(exp)) names.push(exp); + for (const exp of pipeline.exporters || []) { + const kind = exp.split('/')[0]; + const sig = Object.entries(obj.service.pipelines).find(([_, p]) => p === pipeline)?.[0] || ''; + const signalClass = /^metrics/i.test(sig) ? 'metrics' + : /^logs/i.test(sig) ? 'logs' + : /^traces/i.test(sig) ? 'traces' : null; + if (signalClass && !pipelines.exporters[signalClass]) { + pipelines.exporters[signalClass] = { kind: mapExporterKind(kind) }; } - walkScrapeConfigDoc(config, f.relPath, scrapeJobs, evidence, summary, { - receiver: key, - exporters: names.map(n => collectorExporterDetail(n, exporterCfgs[n])), - }); } - continue; } - - // No pipelines: the metrics exporter is the one metrics-capable exporter - // the config names (the first, with a warning, when it names several). - const candidates = Object.keys(exporterCfgs).filter(n => METRICS_EXPORTER_TYPES.has(n.split('/')[0].toLowerCase())); - const chosen = candidates[0] || null; - if (candidates.length > 1) { - summary.warnings.push(`${f.relPath} has no service pipelines and ${candidates.length} metrics-capable exporters (${candidates.join(', ')}); the metrics exporter is taken to be the first, '${chosen}'.`); - } - if (!scrapeReceivers.length && !chosen) continue; - for (const { key } of scrapeReceivers) { - const name = key.split('/')[0]; - if (!pipelines.receivers.some(r => r.name === name)) pipelines.receivers.push({ name }); - } - if (chosen && !pipelines.exporters.metrics) { - pipelines.exporters.metrics = { kind: mapExporterKind(chosen.split('/')[0]) }; - evidence['pipelines.exporters.metrics'] = f.relPath; - } - const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; + evidence[`PIP-${summary.discovered.pipelines + 1}`] = relPath; + summary.discovered.pipelines++; + // Each scrape receiver's samples leave by the exporters of the metrics + // pipelines that list it. for (const { key, config } of scrapeReceivers) { - walkScrapeConfigDoc(config, f.relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + const names = []; + for (const [sig, p] of Object.entries(obj.service.pipelines)) { + if (!/^metrics/i.test(sig) || !Array.isArray(p?.receivers) || !p.receivers.includes(key)) continue; + for (const exp of p.exporters || []) if (!names.includes(exp)) names.push(exp); + } + walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { + receiver: key, + exporters: names.map(n => collectorExporterDetail(n, exporterCfgs[n])), + }); } - evidence[`PIP-${summary.discovered.pipelines + 1}`] = f.relPath; + return; + } + + // No pipelines: the metrics exporter is the metrics-capable exporter the + // config names (the first when it names several) — a guess, applied by + // crawlFiles after the walk, and only when no pipeline declares one. + const candidates = Object.keys(exporterCfgs).filter(n => METRICS_EXPORTER_TYPES.has(n.split('/')[0].toLowerCase())); + const chosen = candidates[0] || null; + if (!scrapeReceivers.length && !chosen) return; + for (const { key } of scrapeReceivers) { + const name = key.split('/')[0]; + if (!pipelines.receivers.some(r => r.name === name)) pipelines.receivers.push({ name }); + } + const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; + for (const { key, config } of scrapeReceivers) { + walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + } + if (scrapeReceivers.length) { + evidence[`PIP-${summary.discovered.pipelines + 1}`] = relPath; summary.discovered.pipelines++; } + if (chosen) guesses.push({ relPath, candidates, counted: scrapeReceivers.length > 0 }); } // Collector exporters that carry metrics, by component type. diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index d737b901..bc8f80eb 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -317,6 +317,18 @@ service: assert(two.canonical.spec.pipelines.exporters.metrics.kind === 'otlp' && two.summary.warnings.includes("files/collector.yaml has no service pipelines and 2 metrics-capable exporters (otlp, kafka); the metrics exporter is taken to be the first, 'otlp'."), 'two candidates: the first is taken and the warning says so', two.summary.warnings); + // A config split across --config files: the components in one file, the + // service in another. The declared metrics pipeline wins over the guess, + // whichever file is walked first, and the guess leaves no trace. + for (const [a, b] of [['otel/a-components.yaml', 'otel/b-service.yaml'], ['otel/b-components.yaml', 'otel/a-service.yaml']]) { + const split = crawlFiles({ + [a]: 'receivers:\n otlp: {}\nexporters:\n otlp/tempo:\n endpoint: http://tempo:4317\n prometheusremotewrite:\n endpoint: http://mimir:9009/api/v1/push\n', + [b]: 'receivers:\n otlp: {}\nexporters:\n otlp/tempo: {}\n prometheusremotewrite: {}\nservice:\n pipelines:\n traces: { receivers: [otlp], exporters: [otlp/tempo] }\n metrics: { receivers: [otlp], exporters: [prometheusremotewrite] }\n', + }, { repoName: 'col', now: NOW }); + assert(split.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && split.summary.discovered.pipelines === 1 + && !split.summary.warnings.some(w => /metrics-capable/.test(w)) && split.evidence['pipelines.exporters.metrics'] === undefined, + `a split config (${a} first): the declared metrics exporter, one pipeline, no guess warning`, [split.canonical.spec.pipelines.exporters.metrics, split.summary.discovered.pipelines]); + } // An Alertmanager config (receivers is a list) is still Alertmanager. assert(detectArtefactKind('am.yaml', 'route:\n receiver: x\nreceivers:\n - name: x\n') === 'alertmanager', 'Alertmanager receivers are not a collector\'s'); From e9fd4265a9a28b2fd53c65109341095330048e4a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:34:03 +0000 Subject: [PATCH 06/17] exporter-metrics: no snapshot from a fixture or mock folder; OpenMetrics families named as stored A .prom under __fixtures__/, __mocks__/, test-data/ or test-fixtures/ was read as an exporter snapshot: the test-path rule knew only tests/fixtures/ testdata. isTestOrFixturePath covers the usual spellings. An OpenMetrics counter 'jobs' and info 'build' were declared as 'jobs' and 'build', names Prometheus never stores (jobs_total, build_info), so a rule reading jobs_total matched nothing the snapshot declared. A family is now named by its sample's stored name, or with the suffix OpenMetrics implies. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 9 +++++---- docs/CHANGELOG.md | 4 ++-- docs/DOWNSTREAM.md | 2 +- tools/lib/crawler.mjs | 4 ++-- tools/lib/exporter-metrics.mjs | 36 ++++++++++++++++++++++++++++++---- tools/test-crawl-exporters.mjs | 22 ++++++++++++++------- 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 4ef637eb..f8272257 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "34004a75cde40b59a18a8b2c5d3f3b68ed0e7075d1fc5365e331d142233ea0f7", - "bytes": 126702, + "sha256": "6d287a79c23694feec56e229503c364aaa0be74991ac41a458ccd2895e0e307e", + "bytes": 126732, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", @@ -523,13 +523,14 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "9fb2b0456f9df4d0e54d886b9fbc2cd4e7b58a4436f5d3a067910870c68d5dd7", - "bytes": 16936, + "sha256": "a791bd00c83849b2a774b64d18ed035555ff3cdde93da73f1122dcbf5136d488", + "bytes": 18546, "exports": [ "EXPOSITION_TYPES", "inferExporterFamilies", "isExpositionSnapshotPath", "isMetricsManifestPath", + "isTestOrFixturePath", "manifestFromExposition", "matchExporterConfig", "mayBeExporterConfig", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index d965ad9c..42a15902 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,8 +3,8 @@ ## Unreleased ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs -- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. -- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches them, manifests, `config.yaml` and a chart's `files/*.yaml`). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. It is never read from a test or fixture path. +- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches them, manifests, `config.yaml` and a chart's `files/*.yaml`). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest. A malformed entry is dropped with a warning. `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. - **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 26cd1223..b472346f 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1842,7 +1842,7 @@ pure): | Source | Files | `origin_kind` | Confidence | |---|---|---|---| | a metrics manifest | `metrics-manifest.json`, `*.metrics-manifest.json` | `exporter-manifest` | `declared` | -| a saved exposition | `*.prom`, `*.metrics` (never under a test or fixture path) | `exporter-snapshot` | `declared` | +| a saved exposition | `*.prom`, `*.metrics` (never under a test, fixture or mock folder; an OpenMetrics counter or info named as stored, `_total` / `_info`) | `exporter-snapshot` | `declared` | | the exporter's config, read against the catalogue | any YAML whose top-level keys match a catalogue entry | `exporter-config` | `inferred` (a candidate) | A metric named in several files reads as the strongest: manifest, snapshot, diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index a7e108bf..0ff0bdc9 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -34,7 +34,7 @@ import { BACKEND_PATTERNS } from './backend-products.mjs'; import { PROMQL_KEYWORDS, extractPromqlMetricNames } from './promql.mjs'; import { symbolSlug as slug, packSlug } from './slug.mjs'; import { - isMetricsManifestPath, isExpositionSnapshotPath, parseExposition, readMetricsManifest, safeSourceUrl, + isMetricsManifestPath, isExpositionSnapshotPath, isTestOrFixturePath, parseExposition, readMetricsManifest, safeSourceUrl, resolveExporterCatalogue, mayBeExporterConfig, matchExporterConfig, inferExporterFamilies, } from './exporter-metrics.mjs'; @@ -219,7 +219,7 @@ export function detectArtefactKind(relPath, content, { exporterCatalogue = BUILT // A saved /metrics response — not from a test or fixture path, where an // exposition is sample input, not what a deployed exporter serves. if (isExpositionSnapshotPath(lower)) { - return !isTestPath(lower) && parseExposition(content).families.length ? 'exposition-snapshot' : 'unknown'; + return !isTestOrFixturePath(lower) && parseExposition(content).families.length ? 'exposition-snapshot' : 'unknown'; } // JSON first — Grafana dashboards are the only json we care about. if (lower.endsWith('.json')) { diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index a190b127..7687c327 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -49,6 +49,19 @@ export function isMetricsManifestPath(relPath) { return /(^|[/\\.])metrics-manifest\.json$/i.test(String(relPath ?? '')); } +/** + * A path a saved exposition is never read from: a test, spec, fixture, mock + * or test-data folder in any of the usual spellings (`test/`, `__tests__/`, + * `fixtures/`, `__fixtures__/`, `__mocks__/`, `testdata/`, `test-data/`, + * `test_fixtures/`, …) or a `*.test.*` / `*.spec.*` / `*_test.*` file — there + * an exposition is sample input, not what a deployed exporter serves. + */ +export function isTestOrFixturePath(relPath) { + const p = String(relPath ?? '').replace(/\\/g, '/').toLowerCase(); + return /(^|\/)(?:__)?(?:tests?|specs?|fixtures?|mocks?|testdata|test[-_]?(?:data|fixtures?))(?:__)?(\/|$)/.test(p) + || /\.(?:test|spec)\.[a-z]+$|_test\.[a-z]+$/.test(p); +} + /** A saved exposition: `.prom` or `.metrics`. */ export function isExpositionSnapshotPath(relPath) { return /\.(prom|metrics)$/i.test(String(relPath ?? '')); @@ -114,13 +127,15 @@ function unescapeHelp(s) { /** * Parse a Prometheus / OpenMetrics text exposition into its metric families: * `{ families: [{ name, type, help, labels }], samples }`, families sorted by - * name, labels in first-seen order. A sample with no # TYPE for it (or for + * name, labels in first-seen order. A family's name is the one Prometheus + * stores: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is + * `build_info`. A sample with no # TYPE for it (or for * the family its suffix names) is its own `untyped` family. */ export function parseExposition(text) { const families = new Map(); const family = (name) => { - if (!families.has(name)) families.set(name, { name, type: 'untyped', help: '', labels: [], typed: false }); + if (!families.has(name)) families.set(name, { name, type: 'untyped', help: '', labels: [], typed: false, stored: '' }); return families.get(name); }; let samples = 0; @@ -147,7 +162,13 @@ export function parseExposition(text) { for (const [type, suffixes] of Object.entries(FAMILY_SUFFIXES)) { const suffix = suffixes.find(s => sample.name.endsWith(s) && sample.name.length > s.length); const base = suffix ? families.get(sample.name.slice(0, -suffix.length)) : null; - if (base && base.type === type) { f = base; break; } + if (base && base.type === type) { + f = base; + // An OpenMetrics counter or info family is stored (and queried) + // under its sample's name: `jobs` → jobs_total, `build` → build_info. + if ((suffix === '_total' || suffix === '_info') && !f.stored) f.stored = sample.name; + break; + } } } if (!f) f = family(sample.name); @@ -156,8 +177,15 @@ export function parseExposition(text) { if (!skip.includes(l) && !f.labels.includes(l) && f.labels.length < LABEL_CAP) f.labels.push(l); } } + // OpenMetrics (`# EOF`) names a counter family without `_total` and an + // info family without `_info`; with no sample to say so, the stored name + // is the family's plus that suffix. + const openMetrics = /^#\s*EOF\s*$/m.test(String(text ?? '')); + const storedName = (f) => f.stored + || (openMetrics && f.type === 'counter' && !f.name.endsWith('_total') ? `${f.name}_total` + : openMetrics && f.type === 'info' && !f.name.endsWith('_info') ? `${f.name}_info` : f.name); const out = [...families.values()] - .map(({ name, type, help, labels }) => ({ name, type, help, labels })) + .map((f) => ({ name: storedName(f), type: f.type, help: f.help, labels: f.labels })) .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)); return { families: out, samples }; } diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index bc8f80eb..92636ece 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -178,40 +178,48 @@ const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); '# TYPE jobs counter', 'jobs_total{kind="x"} 7', 'process_up 1', - '# EOF', ].join('\n'); const { families, samples } = parseExposition(PROM); const by = Object.fromEntries(families.map(f => [f.name, f])); - assert(samples === 9 && families.map(f => f.name).join() === 'http_request_duration_seconds,ibmmq_queue_depth,jobs,process_up,rpc_latency', + assert(samples === 9 && families.map(f => f.name).join() === 'http_request_duration_seconds,ibmmq_queue_depth,jobs_total,process_up,rpc_latency', 'families by name; suffixed samples fold into their family; an undeclared sample is its own', families.map(f => f.name)); assert(JSON.stringify(by.ibmmq_queue_depth.labels) === '["qmgr","queue","description"]', 'label names only, a quoted value with `,` `}` and `\\"` parsed past', by.ibmmq_queue_depth); assert(JSON.stringify(by.http_request_duration_seconds.labels) === '["route"]' && by.http_request_duration_seconds.help === 'Latency\nsecond line', 'a histogram family drops `le`; HELP unescaped', by.http_request_duration_seconds); - assert(JSON.stringify(by.rpc_latency.labels) === '["method"]' && by.jobs.type === 'counter' && JSON.stringify(by.jobs.labels) === '["kind"]' - && by.process_up.type === 'untyped', 'a summary drops `quantile`; `_total` folds into an OpenMetrics counter; no TYPE is untyped', [by.rpc_latency, by.jobs, by.process_up]); + assert(JSON.stringify(by.rpc_latency.labels) === '["method"]' && by.jobs_total.type === 'counter' && JSON.stringify(by.jobs_total.labels) === '["kind"]' + && by.process_up.type === 'untyped', 'a summary drops `quantile`; `_total` folds into a counter, named as stored; no TYPE is untyped', [by.rpc_latency, by.jobs_total, by.process_up]); + // OpenMetrics: a counter or info family is named without its suffix and + // stored with it; the declared names are the ones rules and dashboards read. + const om = parseExposition('# TYPE jobs counter\njobs_total{k="x"} 7\njobs_created{k="x"} 1\n# TYPE build info\nbuild_info{version="1"} 1\n# TYPE idle counter\n# TYPE g gauge\ng 1\n# EOF\n'); + assert(om.families.map(f => f.name).join() === 'build_info,g,idle_total,jobs_total', + 'OpenMetrics counters and infos are named as Prometheus stores them, with or without a sample', om.families.map(f => f.name)); + assert(parseExposition('# TYPE jobs_total counter\njobs_total 1\n').families[0].name === 'jobs_total', 'a 0.0.4 counter keeps its name'); assert(!JSON.stringify(families).includes('APP.IN'), 'no label value is kept'); assert(scanReadsPath('exporter/ibm-mq.prom') && scanReadsPath('snapshots/node.metrics'), 'a scan reads .prom and .metrics files'); assert(detectArtefactKind('snapshots/ibm-mq.prom', PROM) === 'exposition-snapshot', 'a .prom file with families is an exposition snapshot'); assert(detectArtefactKind('test/fixtures/ibm-mq.prom', PROM) === 'unknown' && detectArtefactKind('src/x.test.metrics', PROM) === 'unknown' - && detectArtefactKind('testdata/a.prom', PROM) === 'unknown', 'never from a test or fixture path'); + && detectArtefactKind('testdata/a.prom', PROM) === 'unknown' + && ['src/__fixtures__/node.prom', 'test-data/node.prom', 'test_fixtures/a.prom', 'x/__mocks__/a.metrics', 'mocks/a.prom', 'spec/a.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), + 'never from a test, fixture or mock path, in any of the usual spellings'); assert(detectArtefactKind('notes.metrics', 'nothing to see here\n') === 'unknown', 'a .metrics file with no family is not a snapshot'); const r = crawlFiles({ 'snapshots/ibm-mq.prom': PROM, 'test/ibm-mq.prom': '# TYPE test_only gauge\ntest_only 1\n' }, { repoName: 'snap', now: NOW }); const o = json(r, 'crawler.discovered.metric_origins'); assert(o.ibmmq_queue_depth.origin_kind === 'exporter-snapshot' && o.ibmmq_queue_depth.confidence === 'declared' && o.ibmmq_queue_depth.service === 'ibm-mq' && !('test_only' in o), 'a crawl declares the snapshot\'s families, named by the file; the test path is not read', o.ibmmq_queue_depth); + assert(o.jobs_total.origin_kind === 'exporter-snapshot', 'the counter is declared under its stored name', Object.keys(o)); assert(JSON.stringify(json(r, 'crawler.discovered.exporters')) === '{"ibm-mq":{"sources":[{"origin_kind":"exporter-snapshot","file":"snapshots/ibm-mq.prom","metrics":5}]}}', 'the snapshot is a source of the exporter', ann(r)['crawler.discovered.exporters']); // Rank: manifest > snapshot > source code. const ranked = crawlFiles({ 'snapshots/ibm-mq.prom': PROM, 'metrics-manifest.json': JSON.stringify({ exporter: 'ibm-mq', metrics: [{ name: 'ibmmq_queue_depth', type: 'gauge', help: 'from the manifest' }] }), - 'src/metrics.ts': "import { Gauge } from 'prom-client';\nexport const j = new Gauge({ name: 'jobs', help: 'from code' });\n", + 'src/metrics.ts': "import { Counter } from 'prom-client';\nexport const j = new Counter({ name: 'jobs_total', help: 'from code' });\n", }, { repoName: 'snap', now: NOW }); const ro = json(ranked, 'crawler.discovered.metric_origins'); assert(ro.ibmmq_queue_depth.help === 'from the manifest' && ro.ibmmq_queue_depth.origin_kind === 'exporter-manifest' - && ro.jobs.origin_kind === 'exporter-snapshot', 'the manifest outranks the snapshot, the snapshot outranks source code', [ro.ibmmq_queue_depth, ro.jobs]); + && ro.jobs_total.origin_kind === 'exporter-snapshot', 'the manifest outranks the snapshot, the snapshot outranks source code', [ro.ibmmq_queue_depth, ro.jobs_total]); } // ---------- E. a downstream catalogue ---------- From a0cb20774394fb0c6fb0877b3e23cdc4664f8790 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:34:46 +0000 Subject: [PATCH 07/17] github crawl: the broad exporter patterns fill only what the established files leave isCrawlerFile accepted every config.yaml, files/**/*.yaml and .prom, and the route keeps the first 200 matches in tree order: a monorepo with 210 apps/*/config.yaml fetched none of the alertmanager, prometheus, rules and collector files develop fetched. isCrawlerFile is develop's rule plus the metrics manifest; the broad names are isExporterCandidateFile's, and selectCrawlerFiles fetches them only after every established file. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- docs/CHANGELOG.md | 2 +- server/github-crawl.mjs | 44 ++++++++++++++++++++++++++++------ server/index.mjs | 10 ++++---- tools/test-crawl-exporters.mjs | 22 +++++++++++++++++ 4 files changed, 64 insertions(+), 14 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 42a15902..5709291f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,7 +4,7 @@ ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs - **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. -- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches them, manifests, `config.yaml` and a chart's `files/*.yaml`). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest. A malformed entry is dropped with a warning. `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. - **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. diff --git a/server/github-crawl.mjs b/server/github-crawl.mjs index d30f3a51..5debea0f 100644 --- a/server/github-crawl.mjs +++ b/server/github-crawl.mjs @@ -1,10 +1,13 @@ // server/github-crawl.mjs — helpers for the public-GitHub crawl route. // // Server-only (ghFetch reads GITHUB_TOKEN from the environment, which the -// browser-safe tools/lib layer is not allowed to do). parseGithubUrl and -// isCrawlerFile are pure; ghFetch is the single authenticated wrapper +// browser-safe tools/lib layer is not allowed to do). parseGithubUrl, +// isCrawlerFile, isExporterCandidateFile and selectCrawlerFiles are pure; +// ghFetch is the single authenticated wrapper // around the GitHub REST API. +import { isTestOrFixturePath } from '../tools/lib/exporter-metrics.mjs'; + export function parseGithubUrl(input) { if (typeof input !== 'string' || !input.trim()) return null; const cleaned = input.trim().replace(/\.git$/, '').replace(/\/$/, ''); @@ -30,12 +33,9 @@ export function isCrawlerFile(path) { return ( sourceMetricCandidate || /(^|\/)(application|bootstrap)[\w.-]*\.ya?ml$/.test(p) || - // A prebuilt exporter: its manifest, a saved /metrics, its config; a - // collector config a Helm chart keeps under files/. + // A prebuilt exporter's metrics manifest (named precisely; the broad + // exporter patterns are isExporterCandidateFile's, fetched last). /(^|[/.])metrics-manifest\.json$/.test(p) || - (/\.(prom|metrics)$/.test(p) && !/(^|\/)(tests?|__tests__|specs?|fixtures?|testdata)\//.test(p)) || - /(^|\/)config\.ya?ml$/.test(p) || - /(^|\/)files\/.*\.ya?ml$/.test(p) || /(^|\/)docker[-_]compose[a-z0-9._-]*\.(ya?ml)$/.test(p) || /(^|\/)compose[a-z0-9._-]*\.(ya?ml)$/.test(p) || /\.rules\.(ya?ml)$/.test(p) || @@ -55,6 +55,36 @@ export function isCrawlerFile(path) { ); } +// Files that MAY say what a prebuilt exporter exposes or how a collector +// scrapes it, by names too common to fetch first: a saved /metrics (`.prom`, +// `.metrics`, never from a test or fixture folder), any `config.yaml` (an +// mq-metric-samples config), a Helm chart's `files/*.yaml` (a collector +// config). Never one isCrawlerFile already takes. +export function isExporterCandidateFile(path) { + if (typeof path !== 'string' || isCrawlerFile(path)) return false; + const p = path.toLowerCase(); + if (p.includes('node_modules/') || p.startsWith('.git/') || p.includes('/.git/')) return false; + return (/\.(prom|metrics)$/.test(p) && !isTestOrFixturePath(p)) + || /(^|\/)config\.ya?ml$/.test(p) + || /(^|\/)files\/.*\.ya?ml$/.test(p); +} + +/** + * The blobs of a GitHub tree a crawl downloads, at most `maxFiles` of at most + * `fileCapBytes` each: every isCrawlerFile blob first, in tree order, then + * the exporter candidates — so a monorepo's many `config.yaml` files never + * push out a rule file, an Alertmanager config or a collector config. + */ +export function selectCrawlerFiles(tree, { maxFiles = 200, fileCapBytes = Infinity } = {}) { + const blobs = (Array.isArray(tree) ? tree : []).filter(node => node?.type === 'blob' && typeof node.path === 'string'); + return [ + ...blobs.filter(node => isCrawlerFile(node.path)), + ...blobs.filter(node => isExporterCandidateFile(node.path)), + ] + .filter(node => !node.size || node.size <= fileCapBytes) + .slice(0, maxFiles); +} + export async function ghFetch(path, init = {}) { const headers = { 'Accept': 'application/vnd.github+json', diff --git a/server/index.mjs b/server/index.mjs index 0bfd22ca..37bf7eee 100644 --- a/server/index.mjs +++ b/server/index.mjs @@ -68,7 +68,7 @@ import { describeProxyAuth } from './auth-proxy.mjs'; import { stripMcpUrl, mcpUrlOrigin, droppedNote } from './mcp-url.mjs'; import { mcpCallerOf, mcpRefusalBody, redactTarget } from './mcp-target-policy.mjs'; import { captureNotifyRefusal, hooksAllTaken, notifyBootReport, notifyHookOf, notifyHooksOf, notifyNamesRefusal, notifyPostureRefusal, resolveJourneyNotify } from './notify-target-policy.mjs'; -import { parseGithubUrl, isCrawlerFile, ghFetch } from './github-crawl.mjs'; +import { parseGithubUrl, selectCrawlerFiles, ghFetch } from './github-crawl.mjs'; import { deployRoutes } from './routes/deploy.mjs'; import { auditAfter, actorForRecord, bounded, finite } from './audit-after.mjs'; import { identityRoutes } from './routes/identity.mjs'; @@ -1880,7 +1880,7 @@ app.post('/api/crawl', authorize('POST /api/crawl'), (req, res) => { // // Bandwidth guards: max 200 files, max 16 MB total, max 1 MB per file. // ---------------------------------------------------------------- -// parseGithubUrl / isCrawlerFile / ghFetch live in server/github-crawl.mjs. +// parseGithubUrl / selectCrawlerFiles / ghFetch live in server/github-crawl.mjs. app.post('/api/crawl-github', authorize('POST /api/crawl-github'), async (req, res) => { const body = req.body || {}; @@ -1914,10 +1914,8 @@ app.post('/api/crawl-github', authorize('POST /api/crawl-github'), async (req, r const FILE_CAP_BYTES = 1 * 1024 * 1024; // 1 MB / file const TOTAL_CAP_BYTES = 16 * 1024 * 1024; // 16 MB total const MAX_FILES = 200; - const candidates = (treeResp.tree || []) - .filter(node => node.type === 'blob' && isCrawlerFile(node.path)) - .filter(node => !node.size || node.size <= FILE_CAP_BYTES) - .slice(0, MAX_FILES); + // isCrawlerFile's blobs first, the exporter candidates after them. + const candidates = selectCrawlerFiles(treeResp.tree, { maxFiles: MAX_FILES, fileCapBytes: FILE_CAP_BYTES }); if (candidates.length === 0) { return res.json({ diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 92636ece..d7359977 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -35,6 +35,7 @@ import { } from './lib/exporter-metrics.mjs'; import { EXPORTER_CATALOGUE as BUILTIN } from './lib/exporter-catalogue.mjs'; import { validateCanonical, SPEC_SCHEMA_PATH } from './lib/validator.mjs'; +import { isCrawlerFile, isExporterCandidateFile, selectCrawlerFiles } from '../server/github-crawl.mjs'; import { createHarness } from './lib/harness.mjs'; const { assert, report } = createHarness({ truncate: 600 }); @@ -398,6 +399,27 @@ data: 'a crawl with no exporter or collector scrape evidence carries no new key', keys.filter(k => /discovered/.test(k))); } +// ---------- the GitHub crawl's file budget ---------- +// The exporter patterns match names too common to fetch first: a monorepo's +// 210 config.yaml files never push out its rules, Alertmanager or collector. +{ + const tree = [ + ...Array.from({ length: 210 }, (_, i) => ({ type: 'blob', path: `apps/svc-${String(i).padStart(3, '0')}/config.yaml`, size: 100 })), + { type: 'blob', path: 'monitoring/alertmanager.yml' }, { type: 'blob', path: 'monitoring/slo.rules.yaml' }, + { type: 'blob', path: 'otel/otel-collector-config.yaml' }, { type: 'blob', path: 'mq/metrics-manifest.json' }, + { type: 'blob', path: 'snap/ibm-mq.prom' }, { type: 'blob', path: 'test/fixtures/x.prom' }, { type: 'blob', path: 'charts/c/files/collector.yaml' }, + { type: 'tree', path: 'apps' }, { type: 'blob', path: 'big/config.yaml', size: 2e6 }, + ]; + const picked = selectCrawlerFiles(tree, { maxFiles: 200, fileCapBytes: 1e6 }).map(n => n.path); + assert(picked.length === 200 && picked.slice(0, 4).join() === 'monitoring/alertmanager.yml,monitoring/slo.rules.yaml,otel/otel-collector-config.yaml,mq/metrics-manifest.json', + 'the established files and the manifest come first; the exporter candidates fill what is left', picked.slice(0, 5)); + const all = selectCrawlerFiles(tree, { maxFiles: 1000, fileCapBytes: 1e6 }).map(n => n.path); + assert(all.includes('snap/ibm-mq.prom') && all.includes('charts/c/files/collector.yaml') && !all.includes('test/fixtures/x.prom') && !all.includes('big/config.yaml') && !all.includes('apps'), + 'with room, a snapshot and a chart\'s files/ collector are fetched; a fixture snapshot, an oversized file and a tree never'); + assert(isCrawlerFile('mq/metrics-manifest.json') && !isCrawlerFile('apps/x/config.yaml') && isExporterCandidateFile('apps/x/config.yaml') + && !isExporterCandidateFile('monitoring/prometheus.yml'), 'isCrawlerFile is develop\'s rule plus the manifest; the broad names are candidates only'); +} + // ---------- I. packc emit --exporter ---------- const EXPOSITION = '# HELP ibmmq_qmgr_status Status\n# TYPE ibmmq_qmgr_status gauge\nibmmq_qmgr_status{qmgr="QM1",platform="UNIX"} 1\n' + '# TYPE ibmmq_queue_depth gauge\nibmmq_queue_depth{qmgr="QM1",queue="APP.IN"} 3\n'; From 71479392f51d38946c089120454465a0e2df1891 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:35:27 +0000 Subject: [PATCH 08/17] exporter-metrics: detectArtefactKind merges a downstream catalogue; gates and scopes are own names only detectArtefactKind used a catalogue it was given as already resolved: the built-in ibm-mq entry was gone (an mq config read as unknown), and the array form or an entry without match.keys threw, though the CHANGELOG says it merges like crawlFiles. asResolvedCatalogue resolves anything not already resolved. A family gated on 'toString' or scoped on 'constructor' passed validation through the prototype and was always emitted; gates, scopes and config paths are read as own properties. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 9 +++++---- docs/CHANGELOG.md | 4 ++-- tools/lib/crawler.mjs | 17 +++++++++-------- tools/lib/exporter-metrics.mjs | 24 ++++++++++++++++++++---- tools/test-crawl-exporters.mjs | 15 +++++++++++++++ 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index f8272257..ab6b72e7 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "6d287a79c23694feec56e229503c364aaa0be74991ac41a458ccd2895e0e307e", - "bytes": 126732, + "sha256": "af729dd71bc8e1f64f2d82add27facee42563cc1e69e94e117669268c0ca2408", + "bytes": 126787, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", @@ -523,10 +523,11 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "a791bd00c83849b2a774b64d18ed035555ff3cdde93da73f1122dcbf5136d488", - "bytes": 18546, + "sha256": "2ea5fb2293e3c3247ac7aaca2b9dce7abc1c94ced7d93a60fd103d10a286c4a9", + "bytes": 19273, "exports": [ "EXPOSITION_TYPES", + "asResolvedCatalogue", "inferExporterFamilies", "isExpositionSnapshotPath", "isMetricsManifestPath", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 5709291f..e2193c78 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -3,9 +3,9 @@ ## Unreleased ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs -- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. +- **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `asResolvedCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. - **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). -- **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest. A malformed entry is dropped with a warning. `docs/DOWNSTREAM.md` §18 has the format. +- **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. - **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 0ff0bdc9..6cfc8f2b 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -35,7 +35,7 @@ import { PROMQL_KEYWORDS, extractPromqlMetricNames } from './promql.mjs'; import { symbolSlug as slug, packSlug } from './slug.mjs'; import { isMetricsManifestPath, isExpositionSnapshotPath, isTestOrFixturePath, parseExposition, readMetricsManifest, safeSourceUrl, - resolveExporterCatalogue, mayBeExporterConfig, matchExporterConfig, inferExporterFamilies, + resolveExporterCatalogue, asResolvedCatalogue, mayBeExporterConfig, matchExporterConfig, inferExporterFamilies, } from './exporter-metrics.mjs'; // The spec's closed vocabularies, mirrored by hand (a browser-safe module @@ -203,8 +203,9 @@ function registerMetricsExporterFromBackend(pattern, pipelines, relPath, evidenc * @param {string} relPath - repo-relative path * @param {string} content - file content * @param {object} [opts] - * @param {object} [opts.exporterCatalogue] - a resolved exporter catalogue - * (resolveExporterCatalogue); the built-in one when omitted + * @param {object|Array} [opts.exporterCatalogue] - a downstream's exporter + * catalogue ({ exporters: [...] } or the array), merged with the built-in + * entries; the built-in one when omitted * @returns {string} kind: 'prometheus-rules' | 'alertmanager' | 'grafana-dashboard' | * 'otel-collector' | 'docker-compose' | 'helm-chart' | * 'helm-template' | 'metrics-manifest' | 'exposition-snapshot' | @@ -212,6 +213,7 @@ function registerMetricsExporterFromBackend(pattern, pipelines, relPath, evidenc */ export function detectArtefactKind(relPath, content, { exporterCatalogue = BUILTIN_EXPORTER_CATALOGUE } = {}) { const lower = relPath.toLowerCase(); + const catalogue = asResolvedCatalogue(exporterCatalogue); // A prebuilt exporter's metric list, as `packc emit --exporter` writes it. // Named by its file, so a malformed one is walked and warned about, never // silently skipped. @@ -259,10 +261,10 @@ export function detectArtefactKind(relPath, content, { exporterCatalogue = BUILT // A prebuilt exporter's own config (mq-metric-samples' config.yaml), by // the top-level keys a catalogue entry names — before the filename // heuristics, which would read `mq-collector.yaml` as an OTel collector. - if (mayBeExporterConfig(content, exporterCatalogue)) { + if (mayBeExporterConfig(content, catalogue)) { let cfg = null; try { cfg = parseYaml(content); } catch { /* not YAML: fall through */ } - if (matchExporterConfig(cfg, exporterCatalogue)) return 'exporter-config'; + if (matchExporterConfig(cfg, catalogue)) return 'exporter-config'; } // Filename heuristics first — fastest. @@ -503,9 +505,8 @@ export function crawlFiles(filesInput, opts = {}) { // Profile detection reads the environment as given (it lowercases itself). const envScope = scopeFilesForEnvironment(rawFiles, envRaw); const files = envScope.files; - const catalogueRead = resolveExporterCatalogue(opts.exporterCatalogue); - const catalogue = { exporters: catalogueRead.exporters }; - const inputWarnings = [...catalogueRead.warnings]; + const catalogue = resolveExporterCatalogue(opts.exporterCatalogue); + const inputWarnings = [...catalogue.warnings]; const normalized = { name: null, environment: null, owners: [] }; if (repoName !== String(nameRaw)) { normalized.name = { from: String(nameRaw), to: repoName }; diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index 7687c327..6ddb28ec 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -264,7 +264,7 @@ const stripBom = (text) => String(text ?? '').replace(/^\uFEFF/, ''); function pathValue(obj, path) { let cur = obj; for (const key of String(path ?? '').split('.').filter(Boolean)) { - if (!isMapping(cur)) return undefined; + if (!isMapping(cur) || !Object.hasOwn(cur, key)) return undefined; cur = cur[key]; } return cur; @@ -293,7 +293,7 @@ function normalizeEntry(e, i) { for (const f of e.families) { const bad = !isMapping(f) || typeof f.name !== 'string' || !/^[A-Za-z0-9_:]+$/.test(f.name) || !TYPE_SET.has(String(f.type ?? '').toLowerCase()) - || (f.gate !== undefined && !gates[f.gate]) || (f.scope !== undefined && !scopes[f.scope]); + || (f.gate !== undefined && !Object.hasOwn(gates, f.gate)) || (f.scope !== undefined && !Object.hasOwn(scopes, f.scope)); if (bad) return { error: `${where}: family ${JSON.stringify(f?.name ?? f)} needs a name, an exposition type, and only the gates and scopes the entry defines` }; families.push({ name: f.name, @@ -317,6 +317,10 @@ function normalizeEntry(e, i) { }; } +// The catalogues resolveExporterCatalogue made, so a resolved one is never +// resolved again (asResolvedCatalogue). +const RESOLVED = new WeakSet(); + /** * The catalogue a crawl reads: the built-in entries, then a caller's (an * object `{ exporters: [...] }` or the array itself) — an entry replaces the @@ -343,7 +347,18 @@ export function resolveExporterCatalogue(custom) { }); } } - return { exporters: [...byId.values()], warnings }; + const resolved = { exporters: [...byId.values()], warnings }; + RESOLVED.add(resolved); + return resolved; +} + +/** + * A catalogue as a caller hands it over — resolveExporterCatalogue's result, + * or a downstream's `{ exporters: [...] }` / array (resolved here: merged + * with the built-in entries, a malformed entry dropped) — ready to read. + */ +export function asResolvedCatalogue(catalogue) { + return catalogue && RESOLVED.has(catalogue) ? catalogue : resolveExporterCatalogue(catalogue); } /** @@ -392,7 +407,8 @@ export function inferExporterFamilies(obj, entry) { const gates = Object.fromEntries(Object.entries(entry.gates).map(([k, g]) => [k, switchOn(pathValue(obj, g.path), g.default)])); const scopes = Object.fromEntries(Object.entries(entry.scopes).map(([k, s]) => [k, selectionNonEmpty(pathValue(obj, s.path))])); const families = entry.families - .filter(f => (f.gate === undefined || gates[f.gate]) && (f.scope === undefined || scopes[f.scope])) + .filter(f => (f.gate === undefined || (Object.hasOwn(gates, f.gate) && gates[f.gate])) + && (f.scope === undefined || (Object.hasOwn(scopes, f.scope) && scopes[f.scope]))) .map(f => ({ name: namespace ? `${namespace}_${f.name}` : f.name, type: f.type, help: f.help, labels: f.labels })) .filter(f => NAME_RE.test(f.name)); return { namespace, gates, scopes, families }; diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index d7359977..1191a164 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -251,6 +251,21 @@ const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); 'the crawl reads both configs with the downstream catalogue', Object.keys(o)); assert(r.summary.warnings.filter(w => /exporter catalogue entry/.test(w)).length === 3, 'the dropped entries are crawl warnings'); assert(detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n') === 'unknown', 'without the catalogue the gateway config is nothing'); + // detectArtefactKind takes the same downstream catalogue: merged with the + // built-in entries, in either form, a malformed entry dropped, never thrown. + const acmeOnly = [custom.exporters[1]]; + assert(detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n', { exporterCatalogue: { exporters: acmeOnly } }) === 'exporter-config' + && detectArtefactKind('gw/acme.yaml', 'acme:\n tls: true\nlisten: ":8080"\n', { exporterCatalogue: acmeOnly }) === 'exporter-config' + && detectArtefactKind('mq/config.yaml', FILES['mq-exporter/config.yaml'], { exporterCatalogue: acmeOnly }) === 'exporter-config' + && detectArtefactKind('a.yaml', 'acme: 1\n', { exporterCatalogue: { exporters: [{ id: 'nokeys' }] } }) === 'unknown', + 'detectArtefactKind merges a downstream catalogue (object or array) with the built-in one and never throws on a malformed entry'); + // A gate or scope the entry never defines is refused, whatever its name + // (toString and constructor are on every object's prototype). + const proto = resolveExporterCatalogue({ exporters: [ + { id: 'p1', match: { keys: ['p'] }, families: [{ name: 'a', type: 'gauge', gate: 'toString' }] }, + { id: 'p2', match: { keys: ['p'] }, families: [{ name: 'b', type: 'gauge', scope: 'constructor' }] }, + ] }); + assert(proto.warnings.length === 2 && !proto.exporters.some(e => /^p\d$/.test(e.id)), 'a family gated or scoped on a prototype name is dropped', proto.warnings); const junk = crawlFiles({ 'a.yaml': 'a: 1\n' }, { repoName: 'cat', now: NOW, exporterCatalogue: 'not a catalogue' }); assert(junk.summary.warnings.some(w => /neither `\{ exporters: \[\.\.\.\] \}` nor an array/.test(w)), 'a catalogue of the wrong shape is warned about'); } From 0bf96d6b195e67907b5857c9f4e1da2ea9e455e4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:36:30 +0000 Subject: [PATCH 09/17] crawler: read a collector in a chart's values; name excluded templates and values with scrape jobs A collector config in a chart's values.yaml (top level, or under config: as the opentelemetry-collector chart writes it) was read as helm-values only: no scrape job, no kafka exporter, even with --env all. It is walked as a collector, before the values' images. The environment-scope warning named excluded scrape-config, collector and exporter-config files but not a Helm template or values file carrying the same jobs, which were dropped silently; collectionEvidenceOf now reads them as their walkers do. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- docs/CHANGELOG.md | 4 ++-- docs/DOWNSTREAM.md | 4 +++- tools/lib/crawler.mjs | 41 +++++++++++++++++++++++++++++++--- tools/test-crawl-exporters.mjs | 23 +++++++++++++++++++ 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index ab6b72e7..d323974b 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "af729dd71bc8e1f64f2d82add27facee42563cc1e69e94e117669268c0ca2408", - "bytes": 126787, + "sha256": "7f7dab04a206997c86cb2e296679f4da8856219cabc8752e3dcecce77a4bf477", + "bytes": 128822, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e2193c78..f90e4f5c 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,9 +7,9 @@ - **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. -- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level or under `config:` (the opentelemetry-collector chart's idiom), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. -- **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. +- **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template or values file that carries either) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. - No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). - **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops everything before the last `@`, then everything from the first `?` or `#`, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index b472346f..a2c113aa 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1944,7 +1944,9 @@ with `packc emit` and the manifest outranks it. An OTel collector's prometheus receivers (`prometheus`, `prometheus/`) are read for their `config.scrape_configs`, with or -without a `service:` block (a Helm chart's `files/` copy has none). +without a `service:` block (a Helm chart's `files/` copy has none), in a +config file, a Helm ConfigMap, or a chart's `values*.yaml` (at its top level +or under `config:`). - Each job is in `crawler.discovered.scrape_jobs`. Its entry in `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 6cfc8f2b..d95a2721 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -646,7 +646,7 @@ export function crawlFiles(filesInput, opts = {}) { case 'exporter-config': walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence); break; case 'grafana-dashboard':walkGrafanaDashboard(f, dashboards, metricDefinitions, evidence, summary); break; case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorGuesses, repoName }, evidence, summary); break; - case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents); break; + case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents, { scrapeJobs, collectorGuesses }); break; case 'k8s-workload': walkK8sWorkload(f, backends, pipelines, evidence, summary); break; // Chart.yaml itself carries no observability contracts — the payloads // live in the templated manifests (handled as 'helm-template'). We @@ -1874,7 +1874,7 @@ function collectDisabledComponents(valuesFiles) { return disabled; } -function walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents = new Set()) { +function walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents = new Set(), { scrapeJobs = [], collectorGuesses = [] } = {}) { const visit = (node, path = []) => { if (!node) return; // Prune the subtree of any top-level component the environment disables @@ -1905,7 +1905,21 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo } }; - for (const obj of parseYamlDocs(f.content)) visit(obj); + for (const obj of parseYamlDocs(f.content)) { + // A collector chart's values carry the collector config: at the top + // level, or under `config:` (the opentelemetry-collector chart's idiom). + // Read first, so a pipeline it declares beats an image's inference. + const collector = valuesCollectorConfig(obj); + if (collector) walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorGuesses); + visit(obj); + } +} + +// The collector config a Helm values document carries, or null. +function valuesCollectorConfig(obj) { + if (isMapping(obj) && looksLikeOtelCollector(obj)) return obj; + if (isMapping(obj?.config) && looksLikeOtelCollector(obj.config)) return obj.config; + return null; } function workloadPodSpec(obj) { @@ -2441,6 +2455,27 @@ function collectionEvidenceOf(relPath, content, catalogue) { const entry = matchExporterConfig(parseYaml(content), catalogue); return `${relPath} (exporter config: ${entry?.id ?? 'unknown'})`; } + if (kind === 'helm-values') { + const jobs = jobsOf(parseYamlDocs(content).map(valuesCollectorConfig).filter(Boolean) + .flatMap(c => collectorScrapeReceivers(c).map(r => r.config))); + return jobs.length ? `${relPath} (collector scrape jobs: ${jobs.join(', ')})` : null; + } + if (kind === 'helm-template') { + // The jobs the template's walker would read (its ConfigMap values or + // the whole template), walked into throwaway buckets. + const buckets = { + backends: [], recordingRules: [], burnRateAlerts: [], alertRules: [], metricDefinitions: [], scrapeJobs: [], + alertingRoutes: [], dashboards: [], collectorGuesses: [], repoName: null, + pipelines: { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }, + }; + const scratch = { + discovered: {}, warnings: [], invented: { endpoints: [], channels: [] }, + omitted: { syntheticRecordingRules: [], unresolvedChannels: [], recordingRules: [], ruleIntervals: [] }, + }; + walkHelmTemplate({ relPath, content }, buckets, {}, scratch); + const jobs = [...new Set(buckets.scrapeJobs.map(j => j.job))]; + return jobs.length ? `${relPath} (template scrape jobs: ${jobs.join(', ')})` : null; + } } catch { /* unreadable: nothing to report */ } return null; } diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 1191a164..3c1bf009 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -379,6 +379,29 @@ data: 'a Helm ConfigMap\'s collector with no service block: its job and its kafka exporter', ann(h)['crawler.discovered.scrape_jobs']); } +// A collector config in a chart's values: at the top level, or under +// `config:` (the opentelemetry-collector chart's idiom). +{ + const COL = 'receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: qmgrs\n static_configs:\n - targets: ["a:1", "b:1"]\nexporters:\n kafka:\n topic: mq\n'; + const nested = 'mode: deployment\nimage:\n repository: otel/opentelemetry-collector-contrib\nconfig:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); + for (const [label, text] of [['top level', COL], ['under config:', nested]]) { + const r = crawlFiles({ 'charts/otel-collector/values.yaml': text }, { repoName: 'vals', now: NOW }); + const job = json(r, 'crawler.discovered.scrape_job_origins')?.qmgrs; + assert(job?.target_count === 2 && job.exporters[0].topic === 'mq' && r.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', + `a collector in a chart's values.yaml (${label}): its job and its kafka exporter`, job); + } + // The environment scope names an excluded values file or Helm template + // that carries scrape jobs, as it names a plain collector config. + const TPL = 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: {{ include "c.fullname" . }}\ndata:\n relay.yaml: |\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); + const scoped = crawlFiles({ + 'charts/c/templates/cm.yaml': TPL, 'charts/c/Chart.yaml': 'apiVersion: v2\nname: c\nversion: 0.1.0\n', + 'charts/v/values.yaml': nested, 'docker-compose.yml': 'services:\n p:\n image: prom/prometheus:v2.51.0\n', + }, { repoName: 'vals', environment: 'local-docker', now: NOW }); + const w = scoped.summary.warnings.find(x => /carry scrape jobs/.test(x)) || ''; + assert(w.includes('charts/c/templates/cm.yaml (template scrape jobs: qmgrs)') && w.includes('charts/v/values.yaml (collector scrape jobs: qmgrs)'), + 'an excluded Helm template or values file with scrape jobs is named in the scope warning', w); +} + // ---------- G. a malformed manifest ---------- { const r = crawlFiles({ From 9024999410614dc371140d09371bcd36459b7a29 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:50:46 +0000 Subject: [PATCH 10/17] exporter-metrics: a line break hides no query; packc emit reads --flag=value and echoes no URL-like argument safeSourceUrl's query cut stopped at a line break (no s flag), so 'http://mq/metrics?token=X\nx' kept its token in the manifest source and a collector endpoint; a URL parser drops the break and sends the query. Tabs and line breaks go first, and the cut runs to the end. packc emit printed '--exporter=' whole as an unknown argument; it now reads --flag=value and never echoes a URL-like unknown argument. A body that ended early threw an uncaught stack; it is a refusal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- docs/CHANGELOG.md | 2 +- docs/DOWNSTREAM.md | 5 +++-- tools/emit-manifest.mjs | 16 +++++++++++++--- tools/lib/exporter-metrics.mjs | 6 ++++-- tools/test-crawl-exporters.mjs | 14 +++++++++++--- 6 files changed, 34 insertions(+), 13 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index d323974b..e65d8e45 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -523,8 +523,8 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "2ea5fb2293e3c3247ac7aaca2b9dce7abc1c94ced7d93a60fd103d10a286c4a9", - "bytes": 19273, + "sha256": "633a536d2c488869627ae74a38ff44a9f9c53f4b612550ed63469949884d8b5a", + "bytes": 19410, "exports": [ "EXPOSITION_TYPES", "asResolvedCatalogue", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f90e4f5c..e96e875e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -12,7 +12,7 @@ - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template or values file that carries either) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. - No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). -- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops everything before the last `@`, then everything from the first `?` or `#`, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. +- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument that looks like a URL. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index a2c113aa..8cf7815c 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1873,8 +1873,9 @@ exposition type (`counter`, `gauge`, `histogram`, `summary`, `untyped`, `unknown`, `gaugehistogram`, `info`, `stateset`). An entry that has neither, or repeats a name, is not read and the crawl says how many; a file that is not a manifest is named in a warning. `source` is recorded without userinfo, -query or fragment: everything before the last `@` and from the first `?` or -`#` is dropped (`safeSourceUrl`, textual — a password may hold `/` or `@`). +query or fragment: tabs and line breaks, everything before the last `@`, and +everything from the first `?` or `#` are dropped (`safeSourceUrl`, textual — +a password may hold `/` or `@`). Label values are never part of a manifest. `packc emit --exporter ` (`tools/emit-manifest.mjs`) writes one from a diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs index f96fb774..36f63086 100644 --- a/tools/emit-manifest.mjs +++ b/tools/emit-manifest.mjs @@ -47,10 +47,16 @@ function parseArgs(argv) { const out = { out: 'metrics-manifest.json', timeout: 15, timestamp: true }; const takes = new Set(['--exporter', '--from', '--name', '--version', '--out', '--generated-at', '--timeout']); for (let i = 0; i < argv.length; i++) { - const a = argv[i]; + let a = argv[i]; if (a === '-h' || a === '--help') { out.help = true; continue; } if (a === '--no-timestamp') { out.timestamp = false; continue; } - if (!takes.has(a)) fail(2, `unknown argument '${a}'\n${USAGE}`); + // `--flag=value` as well as `--flag value`. + const eq = a.startsWith('--') ? a.indexOf('=') : -1; + let inline; + if (eq > 0) { inline = a.slice(eq + 1); a = a.slice(0, eq); } + // An unknown argument may be a URL with its credentials: never echoed. + if (!takes.has(a)) fail(2, `unknown argument ${/:\/\/|@/.test(a) ? '(a URL-like value, not shown)' : `'${a}'`}\n${USAGE}`); + if (inline !== undefined) { out[a.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = inline; continue; } const v = argv[++i]; if (v === undefined || (v.startsWith('--') && v !== '-')) fail(2, `${a} needs a value\n${USAGE}`); out[a.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = v; @@ -85,7 +91,11 @@ async function scrape(url, timeoutSeconds) { } if (res.status >= 300 && res.status < 400) fail(1, redirectRefusal(url, res.status, res.headers.get('location'))); if (res.status !== 200) fail(1, `${safeSourceUrl(url)} answered ${res.status}; a metrics endpoint answers 200`); - return res.text(); + try { + return await res.text(); + } catch (e) { + fail(1, `could not read the response of ${safeSourceUrl(url)}: ${e?.name === 'TimeoutError' ? `not complete within ${timeoutSeconds}s (raise --timeout)` : 'the connection ended before the body did'}`); + } } const args = parseArgs(process.argv.slice(2)); diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index 6ddb28ec..e4b76b9b 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -79,13 +79,15 @@ export function isExpositionSnapshotPath(relPath) { * rule. */ export function safeSourceUrl(raw) { - const s = String(raw ?? '').trim(); + // Tab and line breaks go first, as a URL parser drops them (the value + // would still send its query). + const s = String(raw ?? '').replace(/[\t\n\r]/g, '').trim(); if (!s) return ''; const prefix = /^(?:[A-Za-z][A-Za-z0-9+.-]*:)?\/\//.exec(s)?.[0] ?? ''; let rest = s.slice(prefix.length); const at = rest.lastIndexOf('@'); if (at >= 0) rest = rest.slice(at + 1); - return prefix + rest.replace(/[?#].*$/, ''); + return prefix + rest.replace(/[?#][\s\S]*$/, ''); } function parseSampleLine(line) { diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 3c1bf009..82eb31e9 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -326,6 +326,10 @@ service: ['//u:p@h/metrics', '//h/metrics'], ['HTTPS://U:P@H:1/m?t=1#f', 'HTTPS://H:1/m'], ['svc:Zx9/Qk+SECRET==@otel-gw:4317', 'otel-gw:4317'], + // A line break or tab, which a URL parser drops (the value still sends its query). + ['http://mq:9157/metrics?token=SECRET1\nx', 'http://mq:9157/metrics'], + ['https://i/v1?k=SECRET\u2028x', 'https://i/v1'], + ['https://u:p\t@h/v1#frag\r\nx', 'https://h/v1'], ]; assert(SECRETS.every(([raw, want]) => safeSourceUrl(raw) === want), 'safeSourceUrl drops everything before the last @ and from the first ? or #', SECRETS.map(([raw]) => safeSourceUrl(raw)), SECRETS.map(([, want]) => want)); @@ -466,6 +470,7 @@ const server = createServer((req, res) => { if (path === '/metrics') { res.writeHead(200, { 'content-type': 'text/plain; version=0.0.4' }); res.end(EXPOSITION); return; } if (path === '/moved') { res.writeHead(302, { location: '/metrics?token=abc' }); res.end(); return; } if (path === '/empty') { res.writeHead(200); res.end('\n'); return; } + if (path === '/cut') { res.writeHead(200, { 'content-length': '1000' }); res.write('# TYPE x gauge\n'); setTimeout(() => res.destroy(), 50); return; } if (path === '/badloc') { res.writeHead(302, { location: 'http://[oops/metrics' }); res.end(); return; } res.writeHead(500); res.end('boom'); }); @@ -497,10 +502,10 @@ try { assert(JSON.stringify(manifestFromExposition(EXPOSITION, { exporter: 'ibm-mq', version: '5.6.0', source: `${base}/metrics`, generatedAt: NOW })) === JSON.stringify(written), 'the CLI writes exactly manifestFromExposition\'s manifest'); - const viaPackc = await run(['emit', '--exporter', `${base}/metrics`, '--out', '-', '--no-timestamp'], CLI); + const viaPackc = await run(['emit', `--exporter=${base}/metrics`, '--out=-', '--no-timestamp'], CLI); const piped = JSON.parse(viaPackc.stdout); assert(viaPackc.status === 0 && piped.exporter === '127.0.0.1' && !('generatedAt' in piped) && piped.metrics.length === 2, - '`packc emit` dispatches; --out - writes to stdout; the host names the exporter; --no-timestamp leaves the stamp out', viaPackc); + '`packc emit` dispatches; `--flag=value` is read; --out - writes to stdout; the host names the exporter; --no-timestamp leaves the stamp out', viaPackc); writeFileSync(join(tmp, 'qm1.prom'), EXPOSITION); const from = await run(['--from', 'qm1.prom', '--out', '-', '--no-timestamp']); @@ -516,11 +521,14 @@ try { [['--exporter', 'file:///etc/passwd'], 2, 'packc emit: --exporter must be an http or https URL, not file:\n'], [['--exporter', `${base}/metrics`, '--from', 'qm1.prom'], 2, 'packc emit: pass --exporter or --from, not both\n'], [['--exporter', `${base}/metrics`, '--bogus'], 2, null], + [['http://user:SECRET4@127.0.0.1:1/metrics?token=SECRET5'], 2, null], + [['--exporter=http://user:SECRET6@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], + [['--exporter', `${base}/cut?token=SECRET7`], 1, `packc emit: could not read the response of ${base}/cut: the connection ended before the body did\n`], [[], 2, null], ]; for (const [args, code, message] of refusals) { const out = await run(args); - assert(out.status === code && out.stdout === '' && (message === null || out.stderr === message), + assert(out.status === code && out.stdout === '' && (message === null || out.stderr === message) && !/SECRET/.test(out.stderr), `emit ${args.join(' ') || '(no args)'} → exit ${code}${message ? ', the reason named' : ''}`, out, { status: code, message }); } // The refusals' ways out work: save it yourself and pass --from (above); From 2b7dbcc37f926b65ea2daa2e0419678152d90c89 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:51:48 +0000 Subject: [PATCH 11/17] crawler: a collector config in values never costs the file its images; umbrella values; null pipelines walkHelmValues walked the collector config before the images with no guard: the chart's own 'pipelines.metrics: null', a non-array list or a non-string name threw, the values file failed to parse, and backends develop found from it were lost. The walk is guarded (a warning, then the images), and a null pipeline is skipped in any collector config. An umbrella chart's subchart .config is read and named in the scope warning. detectArtefactKind takes null options again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- docs/CHANGELOG.md | 4 ++-- docs/DOWNSTREAM.md | 4 ++-- tools/lib/crawler.mjs | 34 ++++++++++++++++++++++------------ tools/test-crawl-exporters.mjs | 16 ++++++++++++++++ 5 files changed, 44 insertions(+), 18 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index e65d8e45..4b974470 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "7f7dab04a206997c86cb2e296679f4da8856219cabc8752e3dcecce77a4bf477", - "bytes": 128822, + "sha256": "f80b54640ce25be723e3702115168998d26c6c5246aac7b6c2758decf06e8442", + "bytes": 129559, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e96e875e..846cba5f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,9 +7,9 @@ - **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. -- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level or under `config:` (the opentelemetry-collector chart's idiom), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. -- **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template or values file that carries either) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. +- **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template that carries either, a values file that carries a collector config) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. - No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). - **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument that looks like a URL. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 8cf7815c..0264ef76 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1946,8 +1946,8 @@ with `packc emit` and the manifest outranks it. An OTel collector's prometheus receivers (`prometheus`, `prometheus/`) are read for their `config.scrape_configs`, with or without a `service:` block (a Helm chart's `files/` copy has none), in a -config file, a Helm ConfigMap, or a chart's `values*.yaml` (at its top level -or under `config:`). +config file, a Helm ConfigMap, or a chart's `values*.yaml` (at its top level, +under `config:`, or under a subchart's `.config:`). - Each job is in `crawler.discovered.scrape_jobs`. Its entry in `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index d95a2721..12b06965 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -211,7 +211,8 @@ function registerMetricsExporterFromBackend(pattern, pipelines, relPath, evidenc * 'helm-template' | 'metrics-manifest' | 'exposition-snapshot' | * 'exporter-config' | 'unknown' */ -export function detectArtefactKind(relPath, content, { exporterCatalogue = BUILTIN_EXPORTER_CATALOGUE } = {}) { +export function detectArtefactKind(relPath, content, opts) { + const { exporterCatalogue = BUILTIN_EXPORTER_CATALOGUE } = opts ?? {}; const lower = relPath.toLowerCase(); const catalogue = asResolvedCatalogue(exporterCatalogue); // A prebuilt exporter's metric list, as `packc emit --exporter` writes it. @@ -1907,19 +1908,26 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo for (const obj of parseYamlDocs(f.content)) { // A collector chart's values carry the collector config: at the top - // level, or under `config:` (the opentelemetry-collector chart's idiom). - // Read first, so a pipeline it declares beats an image's inference. - const collector = valuesCollectorConfig(obj); - if (collector) walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorGuesses); + // level, under `config:` (the opentelemetry-collector chart's idiom), or + // under a subchart's key in an umbrella chart. Read first, so a pipeline + // it declares beats an image's inference; a config this walk cannot read + // (the chart's `pipelines.metrics: null` drops a pipeline) is warned + // about and never costs the values file its images. + for (const collector of valuesCollectorConfigs(obj)) { + try { walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorGuesses); } + catch (e) { summary.warnings.push(`The collector config in ${f.relPath} could not be read (${e.message}); its images are read as before.`); } + } visit(obj); } } -// The collector config a Helm values document carries, or null. -function valuesCollectorConfig(obj) { - if (isMapping(obj) && looksLikeOtelCollector(obj)) return obj; - if (isMapping(obj?.config) && looksLikeOtelCollector(obj.config)) return obj.config; - return null; +// The collector configs a Helm values document carries: the document +// itself, its `config:`, or a subchart's `.config:` (an umbrella chart). +function valuesCollectorConfigs(obj) { + if (!isMapping(obj)) return []; + if (looksLikeOtelCollector(obj)) return [obj]; + if (isMapping(obj.config) && looksLikeOtelCollector(obj.config)) return [obj.config]; + return Object.values(obj).filter(v => isMapping(v?.config) && looksLikeOtelCollector(v.config)).map(v => v.config); } function workloadPodSpec(obj) { @@ -2227,7 +2235,7 @@ function walkScrapeConfigDoc(obj, origin, scrapeJobs, evidence, summary, collect ...(collector ? { via: 'otel-collector', receiver: collector.receiver, - target_count: (cfg.static_configs || []).reduce((n, sc) => n + (Array.isArray(sc?.targets) ? sc.targets.filter(Boolean).length : 0), 0), + target_count: (Array.isArray(cfg.static_configs) ? cfg.static_configs : []).reduce((n, sc) => n + (Array.isArray(sc?.targets) ? sc.targets.filter(Boolean).length : 0), 0), exporters: collector.exporters, } : {}), }); @@ -2330,6 +2338,8 @@ function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, sum if (obj?.service?.pipelines) { const seen = new Set(); for (const pipeline of Object.values(obj.service.pipelines)) { + // `metrics: null` removes a pipeline (the Helm chart's idiom). + if (!isMapping(pipeline)) continue; for (const recv of pipeline.receivers || []) { if (!seen.has(`r:${recv}`)) { seen.add(`r:${recv}`); @@ -2456,7 +2466,7 @@ function collectionEvidenceOf(relPath, content, catalogue) { return `${relPath} (exporter config: ${entry?.id ?? 'unknown'})`; } if (kind === 'helm-values') { - const jobs = jobsOf(parseYamlDocs(content).map(valuesCollectorConfig).filter(Boolean) + const jobs = jobsOf(parseYamlDocs(content).flatMap(valuesCollectorConfigs) .flatMap(c => collectorScrapeReceivers(c).map(r => r.config))); return jobs.length ? `${relPath} (collector scrape jobs: ${jobs.join(', ')})` : null; } diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 82eb31e9..eb05d6de 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -394,6 +394,22 @@ data: assert(job?.target_count === 2 && job.exporters[0].topic === 'mq' && r.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', `a collector in a chart's values.yaml (${label}): its job and its kafka exporter`, job); } + // An umbrella chart: the collector under its subchart's key. + const umbrella = crawlFiles({ 'charts/u/values.yaml': 'opentelemetry-collector:\n config:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n') }, { repoName: 'vals', now: NOW }); + assert(json(umbrella, 'crawler.discovered.scrape_jobs')?.includes('qmgrs') && umbrella.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', + 'an umbrella chart\'s subchart collector config is read', ann(umbrella)['crawler.discovered.scrape_jobs']); + // A collector config this walk cannot read never costs the values file + // its images; `pipelines.metrics: null` (the chart's way to drop a + // pipeline) is skipped, in values and in a plain collector file alike. + const LOKI = 'loki:\n image:\n repository: grafana/loki\n tag: 3.0.0\n'; + const nullPipe = crawlFiles({ 'charts/c/values.yaml': `${LOKI}config:\n receivers:\n otlp: {}\n exporters:\n kafka: {topic: t}\n service:\n pipelines:\n metrics: null\n traces: { receivers: [otlp], exporters: [kafka] }\n` }, { repoName: 'vals', now: NOW }); + assert(nullPipe.canonical.spec.telemetry.backends.some(b => b.product === 'loki') && nullPipe.canonical.spec.pipelines.exporters.traces.kind === 'kafka' + && !nullPipe.summary.warnings.some(w => /could not be read|Failed to parse/.test(w)), 'a null pipeline is skipped; the images are read', nullPipe.summary.warnings); + const broken = crawlFiles({ 'charts/c/values.yaml': `${LOKI}config:\n receivers:\n otlp: {}\n exporters:\n kafka: {topic: t}\n service:\n pipelines:\n metrics: { receivers: [1], exporters: [kafka] }\n` }, { repoName: 'vals', now: NOW }); + assert(broken.canonical.spec.telemetry.backends.some(b => b.product === 'loki') + && broken.summary.warnings.some(w => /^The collector config in charts\/c\/values\.yaml could not be read \(.*\); its images are read as before\.$/.test(w)), + 'a collector config the walk cannot read is warned about and the values file keeps its images', broken.summary.warnings); + assert(detectArtefactKind('a.yaml', 'a: 1\n', null) === 'unknown', 'detectArtefactKind takes null options, as before'); // The environment scope names an excluded values file or Helm template // that carries scrape jobs, as it names a plain collector config. const TPL = 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: {{ include "c.fullname" . }}\ndata:\n relay.yaml: |\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); From dd28c7a0ce471c0f5ce00d3ed3a42fb5cb0f852a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:52:13 +0000 Subject: [PATCH 12/17] exporter-metrics: no snapshot from e2e/, integration-tests/, testing/, __snapshots__/ or *_spec.* The fixture-path rule listed single words: e2e/, integration-tests/, e2e-tests/, unit_tests/, testing/, __snapshots__/ and mq_spec.prom were still read as exporter snapshots (and fetched by the GitHub crawl). A folder is a test folder when its name is such a word, alone or joined to others by - or _; a longer word that only contains one (contest/, latest/) is not. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- docs/CHANGELOG.md | 2 +- tools/lib/exporter-metrics.mjs | 18 +++++++++++------- tools/test-crawl-exporters.mjs | 5 ++++- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 4b974470..5865a547 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -523,8 +523,8 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "633a536d2c488869627ae74a38ff44a9f9c53f4b612550ed63469949884d8b5a", - "bytes": 19410, + "sha256": "ac1cfcb52ee2c7ef081b64c1ec4dac02652ce3f50eec17e53b582cdbb82953d2", + "bytes": 19663, "exports": [ "EXPOSITION_TYPES", "asResolvedCatalogue", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 846cba5f..06434a4d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,7 +4,7 @@ ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs - **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `asResolvedCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. -- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a test, spec, fixture, mock or test-data folder in any of the usual spellings (`__fixtures__/`, `__mocks__/`, `test-data/`, `test_fixtures/`, …) or from a `*.test.*` file (`isTestOrFixturePath`). +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `__mocks__/`, `e2e/`, `integration-tests/`, `unit_tests/`, `testing/`, `test-data/`, `__snapshots__/`, …), nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. - **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index e4b76b9b..3196b85c 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -50,16 +50,20 @@ export function isMetricsManifestPath(relPath) { } /** - * A path a saved exposition is never read from: a test, spec, fixture, mock - * or test-data folder in any of the usual spellings (`test/`, `__tests__/`, - * `fixtures/`, `__fixtures__/`, `__mocks__/`, `testdata/`, `test-data/`, - * `test_fixtures/`, …) or a `*.test.*` / `*.spec.*` / `*_test.*` file — there - * an exposition is sample input, not what a deployed exporter serves. + * A path a saved exposition is never read from: a folder whose name is a + * test, spec, fixture, mock or test-data word, alone or joined to others by + * `-` / `_` and wrapped in `__` or not (`test/`, `__tests__/`, `e2e/`, + * `integration-tests/`, `unit_tests/`, `testing/`, `__fixtures__/`, + * `__mocks__/`, `testdata/`, `test-data/`, `__snapshots__/`, …), or a + * `*.test.*` / `*.spec.*` / `*_test.*` / `*_spec.*` file — there an + * exposition is sample input, not what a deployed exporter serves. */ export function isTestOrFixturePath(relPath) { const p = String(relPath ?? '').replace(/\\/g, '/').toLowerCase(); - return /(^|\/)(?:__)?(?:tests?|specs?|fixtures?|mocks?|testdata|test[-_]?(?:data|fixtures?))(?:__)?(\/|$)/.test(p) - || /\.(?:test|spec)\.[a-z]+$|_test\.[a-z]+$/.test(p); + const word = '(?:tests?|specs?|fixtures?|mocks?|testdata|testing|e2e|test[-_]?data)'; + const folder = new RegExp(`^(?:__)?(?:[a-z0-9]+[-_])*${word}(?:[-_][a-z0-9]+)*(?:__)?$|^__snapshots__$`); + return p.split('/').slice(0, -1).some(seg => folder.test(seg)) + || /[._](?:test|spec)\.[a-z]+$/.test(p); } /** A saved exposition: `.prom` or `.metrics`. */ diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index eb05d6de..0dcd4577 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -201,8 +201,11 @@ const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); assert(detectArtefactKind('snapshots/ibm-mq.prom', PROM) === 'exposition-snapshot', 'a .prom file with families is an exposition snapshot'); assert(detectArtefactKind('test/fixtures/ibm-mq.prom', PROM) === 'unknown' && detectArtefactKind('src/x.test.metrics', PROM) === 'unknown' && detectArtefactKind('testdata/a.prom', PROM) === 'unknown' - && ['src/__fixtures__/node.prom', 'test-data/node.prom', 'test_fixtures/a.prom', 'x/__mocks__/a.metrics', 'mocks/a.prom', 'spec/a.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), + && ['src/__fixtures__/node.prom', 'test-data/node.prom', 'test_fixtures/a.prom', 'x/__mocks__/a.metrics', 'mocks/a.prom', 'spec/a.prom', + 'e2e/mq.prom', 'integration-tests/mq.prom', 'e2e-tests/mq.prom', 'unit_tests/mq.prom', 'testing/mq.prom', 'src/__snapshots__/mq.prom', 'exp/mq_spec.prom', 'exp/mq.spec.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), 'never from a test, fixture or mock path, in any of the usual spellings'); + assert(['snapshots/ibm-mq.prom', 'exporters/latest/mq.prom', 'contest/mq.prom', 'attestation/mq.prom'].every(p => detectArtefactKind(p, PROM) === 'exposition-snapshot'), + 'a folder that only contains such a word in a longer one is not a test folder'); assert(detectArtefactKind('notes.metrics', 'nothing to see here\n') === 'unknown', 'a .metrics file with no family is not a snapshot'); const r = crawlFiles({ 'snapshots/ibm-mq.prom': PROM, 'test/ibm-mq.prom': '# TYPE test_only gauge\ntest_only 1\n' }, { repoName: 'snap', now: NOW }); From 3f7e8069b4a24434143b1cd8be30352de2a93592 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 18:53:43 +0000 Subject: [PATCH 13/17] crawler: a pipeline-less collector's scrape job leaves by the declared pipeline that lists its receiver The guess was deferred for spec.pipelines but not for the scrape job: in a split config the job still named the guessed otlp exporter while the other file's metrics pipeline listed its receiver with prometheusremotewrite, the first-of-several warning was dropped, the receiver was pushed twice and the config counted as a second pipeline. settleCollectorGuesses now settles the jobs, receivers and the metrics exporter after the walk; the docs say a metrics backend image keeps precedence over a guess, as over a pipeline. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 +- docs/CHANGELOG.md | 2 +- docs/DOWNSTREAM.md | 11 +-- tools/lib/crawler.mjs | 121 ++++++++++++++++++++++----------- tools/test-crawl-exporters.mjs | 26 +++++++ 5 files changed, 117 insertions(+), 47 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 5865a547..bd9128f5 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "f80b54640ce25be723e3702115168998d26c6c5246aac7b6c2758decf06e8442", - "bytes": 129559, + "sha256": "f6a3465fae6207259a5f9419d3afeb0495c2291860712ca5a7403ad3e41fbf67", + "bytes": 131876, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 06434a4d..a469cb0f 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -7,7 +7,7 @@ - **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `__mocks__/`, `e2e/`, `integration-tests/`, `unit_tests/`, `testing/`, `test-data/`, `__snapshots__/`, …), nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. -- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess. After every file is walked, and only when no pipeline in any file declares a metrics exporter (a config split across `--config` files declares it in another), it sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold. The config's scrape receivers become `pipelines.receivers`. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess, settled after every file is walked. A scrape job leaves by the exporters of the declared metrics pipelines, in any file, that list its receiver (a config split across `--config` files), else by the guess. The guess sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold only when nothing else set it: no declared pipeline, and no metrics backend image (whose `prometheusremotewrite` reading the walk already gives precedence over a declared pipeline). The config's scrape receivers become `pipelines.receivers`, once. A first-of-several guess that stands is warned about. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template that carries either, a values file that carries a collector config) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 0264ef76..30b5a25e 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1953,13 +1953,14 @@ under `config:`, or under a subchart's `.config:`). `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the `receiver`, the `target_count` of its static targets, and the `exporters` its samples leave by: the exporters of the metrics pipelines that list the - receiver, or, with no pipelines, the metrics exporter the config names. + receiver, in the same file or another (a config split across `--config` + files), or, when none does, the metrics exporter the config names. Each exporter is given as `{ name, kind, topic | endpoint }`: a kafka exporter's metrics topic, any other's endpoint reduced by `safeSourceUrl` (no userinfo, query or fragment). - That exporter's kind is `spec.pipelines.exporters.metrics`, in place of the `prometheusremotewrite` scaffold. A config with no pipelines only proposes - one: it applies after every file is walked, and only when no pipeline in - any file declares a metrics exporter (a config split across `--config` - files declares it in another). When it names several, the first is taken - and the crawl warns. + one: it applies after every file is walked, and only when nothing else set + the metrics exporter — no pipeline in any file declares one, and no metrics + backend image implies one. When it names several, the first is taken and + the crawl warns. diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 12b06965..35c97e8c 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -585,10 +585,10 @@ export function crawlFiles(filesInput, opts = {}) { const scrapeJobs = []; const exporters = new Map(); // exporter name → the files that say what it exposes // A collector config with no pipelines names its metrics exporter only by - // guess; a guess is applied after every file is walked, and only when no - // pipeline declared one (a config split across --config files declares it - // in another file). - const collectorGuesses = []; + // guess; the guesses are settled after every file is walked against the + // metrics pipelines the walk found (a config split across --config files + // declares them in another file) — settleCollectorGuesses. + const collectorState = newCollectorState(); const dashboards = []; const alertingRoutes = []; const pipelines = { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }; @@ -641,13 +641,13 @@ export function crawlFiles(filesInput, opts = {}) { case 'actuator-metrics-config': walkActuatorMetricsConfig(f, scrapeJobs, evidence, summary, repoName); break; case 'metric-source-code': walkMetricSourceCode(f, metricDefinitions, evidence, summary, repoName); break; case 'alertmanager': walkAlertmanager(f, alertingRoutes, evidence, summary); break; - case 'otel-collector': walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, collectorGuesses); break; + case 'otel-collector': walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, collectorState); break; case 'metrics-manifest': walkMetricsManifest(f, metricDefinitions, exporters, summary, evidence); break; case 'exposition-snapshot': walkExpositionSnapshot(f, metricDefinitions, exporters, summary, evidence); break; case 'exporter-config': walkExporterConfig(f, catalogue, metricDefinitions, exporters, summary, evidence); break; case 'grafana-dashboard':walkGrafanaDashboard(f, dashboards, metricDefinitions, evidence, summary); break; - case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorGuesses, repoName }, evidence, summary); break; - case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents, { scrapeJobs, collectorGuesses }); break; + case 'helm-template': walkHelmTemplate(f, { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorState, repoName }, evidence, summary); break; + case 'helm-values': walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents, { scrapeJobs, collectorState }); break; case 'k8s-workload': walkK8sWorkload(f, backends, pipelines, evidence, summary); break; // Chart.yaml itself carries no observability contracts — the payloads // live in the templated manifests (handled as 'helm-template'). We @@ -659,18 +659,7 @@ export function crawlFiles(filesInput, opts = {}) { } } - if (!pipelines.exporters.metrics && collectorGuesses.length) { - const g = collectorGuesses[0]; - pipelines.exporters.metrics = { kind: mapExporterKind(g.candidates[0].split('/')[0]) }; - evidence['pipelines.exporters.metrics'] = g.relPath; - if (g.candidates.length > 1) { - summary.warnings.push(`${g.relPath} has no service pipelines and ${g.candidates.length} metrics-capable exporters (${g.candidates.join(', ')}); the metrics exporter is taken to be the first, '${g.candidates[0]}'.`); - } - if (!g.counted) { - evidence[`PIP-${summary.discovered.pipelines + 1}`] = g.relPath; - summary.discovered.pipelines++; - } - } + settleCollectorGuesses(collectorState, pipelines, evidence, summary); // ----- infer SLIs/SLOs (decompiler ⇄ compiler symmetry) ----- // PRIMARY source: recording rules. compile.mjs materialises every SLI in @@ -1875,7 +1864,7 @@ function collectDisabledComponents(valuesFiles) { return disabled; } -function walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents = new Set(), { scrapeJobs = [], collectorGuesses = [] } = {}) { +function walkHelmValues(f, backends, pipelines, evidence, summary, disabledComponents = new Set(), { scrapeJobs = [], collectorState = newCollectorState() } = {}) { const visit = (node, path = []) => { if (!node) return; // Prune the subtree of any top-level component the environment disables @@ -1914,7 +1903,7 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo // (the chart's `pipelines.metrics: null` drops a pipeline) is warned // about and never costs the values file its images. for (const collector of valuesCollectorConfigs(obj)) { - try { walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorGuesses); } + try { walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorState); } catch (e) { summary.warnings.push(`The collector config in ${f.relPath} could not be read (${e.message}); its images are read as before.`); } } visit(obj); @@ -1958,14 +1947,14 @@ function walkK8sWorkload(f, backends, pipelines, evidence, summary) { // like a rule shipped as a standalone file. If the template embeds nothing, we // strip the scaffolding and treat the document itself as a single manifest. function walkHelmTemplate(f, buckets, evidence, summary) { - const { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorGuesses, repoName } = buckets; + const { backends, recordingRules, burnRateAlerts, alertRules, metricDefinitions, scrapeJobs, alertingRoutes, dashboards, pipelines, collectorState, repoName } = buckets; const route = (kind, sub) => { switch (kind) { case 'prometheus-rules': walkPrometheusRules(sub, recordingRules, burnRateAlerts, alertRules, metricDefinitions, evidence, summary); return true; case 'prometheus-scrape-config': walkPrometheusScrapeConfig(sub, scrapeJobs, evidence, summary); return true; case 'actuator-metrics-config': walkActuatorMetricsConfig(sub, scrapeJobs, evidence, summary, repoName); return true; case 'alertmanager': walkAlertmanager(sub, alertingRoutes, evidence, summary); return true; - case 'otel-collector': walkOtelCollector(sub, pipelines, scrapeJobs, evidence, summary, collectorGuesses); return true; + case 'otel-collector': walkOtelCollector(sub, pipelines, scrapeJobs, evidence, summary, collectorState); return true; case 'grafana-dashboard':walkGrafanaDashboard(sub, dashboards, metricDefinitions, evidence, summary); return true; case 'docker-compose': walkDockerCompose(sub, backends, pipelines, evidence, summary); return true; case 'k8s-workload': walkK8sWorkload(sub, backends, pipelines, evidence, summary); return true; @@ -2326,13 +2315,19 @@ function walkAlertmanager(f, routes, evidence, summary) { // config with no `service:` block (a Helm chart's files/ copy, assembled // with the pipelines elsewhere) still names its receivers and exporters: // the prometheus receivers' jobs are read, and the metrics exporter it -// configures is a guess (`guesses`), applied by crawlFiles only when no -// pipeline in any file declares one. -function walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, guesses = []) { - for (const obj of parseYamlDocs(f.content)) walkOtelCollectorDoc(obj, f.relPath, pipelines, scrapeJobs, evidence, summary, guesses); +// configures is a guess, settled after the walk (settleCollectorGuesses). +function walkOtelCollector(f, pipelines, scrapeJobs, evidence, summary, state = newCollectorState()) { + for (const obj of parseYamlDocs(f.content)) walkOtelCollectorDoc(obj, f.relPath, pipelines, scrapeJobs, evidence, summary, state); +} + +// What the walk learns about collector configs across files: each metrics +// pipeline declared (its receivers and exporters, with the exporters' +// configs), and each pipeline-less config's guess. +function newCollectorState() { + return { declared: [], guesses: [] }; } -function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, summary, guesses) { +function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, summary, state) { const scrapeReceivers = collectorScrapeReceivers(obj); const exporterCfgs = isMapping(obj?.exporters) ? obj.exporters : {}; if (obj?.service?.pipelines) { @@ -2365,6 +2360,11 @@ function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, sum } evidence[`PIP-${summary.discovered.pipelines + 1}`] = relPath; summary.discovered.pipelines++; + for (const [sig, p] of Object.entries(obj.service.pipelines)) { + if (/^metrics/i.test(sig) && isMapping(p) && Array.isArray(p.receivers)) { + state.declared.push({ receivers: p.receivers, exporters: Array.isArray(p.exporters) ? p.exporters : [], cfgs: exporterCfgs }); + } + } // Each scrape receiver's samples leave by the exporters of the metrics // pipelines that list it. for (const { key, config } of scrapeReceivers) { @@ -2382,24 +2382,67 @@ function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, sum } // No pipelines: the metrics exporter is the metrics-capable exporter the - // config names (the first when it names several) — a guess, applied by - // crawlFiles after the walk, and only when no pipeline declares one. + // config names (the first when it names several) — a guess. Its scrape + // jobs carry it for now; settleCollectorGuesses replaces it with a + // declared pipeline's exporters when one lists the receiver, and applies + // it to spec.pipelines only when nothing else set the metrics exporter. const candidates = Object.keys(exporterCfgs).filter(n => METRICS_EXPORTER_TYPES.has(n.split('/')[0].toLowerCase())); const chosen = candidates[0] || null; if (!scrapeReceivers.length && !chosen) return; - for (const { key } of scrapeReceivers) { - const name = key.split('/')[0]; - if (!pipelines.receivers.some(r => r.name === name)) pipelines.receivers.push({ name }); - } - const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; + const groups = []; for (const { key, config } of scrapeReceivers) { + // One array per receiver, shared with its jobs' entries (patched in place). + const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; + groups.push({ receiver: key, exporters }); walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); } - if (scrapeReceivers.length) { - evidence[`PIP-${summary.discovered.pipelines + 1}`] = relPath; + state.guesses.push({ relPath, candidates, cfgs: exporterCfgs, groups, receivers: scrapeReceivers.map(({ key }) => key.split('/')[0]) }); +} + +// After every file is walked: a pipeline-less config's scrape jobs leave by +// the exporters of the declared metrics pipelines that list their receiver +// (a config split across --config files), else by its guess; its receivers +// join spec.pipelines; the first guess sets spec.pipelines.exporters.metrics +// only when nothing else set it (no declared pipeline, no metrics backend +// image). A first-of-several guess left standing is warned about. +function settleCollectorGuesses(state, pipelines, evidence, summary) { + const warned = new Set(); + const warnFirstOfSeveral = (g) => { + if (g.candidates.length < 2 || warned.has(g.relPath)) return; + warned.add(g.relPath); + summary.warnings.push(`${g.relPath} has no service pipelines and ${g.candidates.length} metrics-capable exporters (${g.candidates.join(', ')}); the metrics exporter is taken to be the first, '${g.candidates[0]}'.`); + }; + for (const g of state.guesses) { + for (const group of g.groups) { + const names = []; + const cfgOf = {}; + for (const d of state.declared) { + if (!d.receivers.includes(group.receiver)) continue; + for (const n of d.exporters) { + if (typeof n !== 'string') continue; + if (!names.includes(n)) names.push(n); + if (isMapping(d.cfgs?.[n]) && Object.keys(d.cfgs[n]).length && !cfgOf[n]) cfgOf[n] = d.cfgs[n]; + } + } + if (names.length) { + group.exporters.splice(0, group.exporters.length, + ...names.map(n => collectorExporterDetail(n, isMapping(g.cfgs[n]) && Object.keys(g.cfgs[n]).length ? g.cfgs[n] : cfgOf[n]))); + } else if (group.exporters.length) { + warnFirstOfSeveral(g); + } + } + for (const name of g.receivers) { + if (!pipelines.receivers.some(r => r.name === name)) pipelines.receivers.push({ name }); + } + } + const first = state.guesses.find(g => g.candidates.length); + if (!pipelines.exporters.metrics && first) { + pipelines.exporters.metrics = { kind: mapExporterKind(first.candidates[0].split('/')[0]) }; + evidence['pipelines.exporters.metrics'] = first.relPath; + warnFirstOfSeveral(first); + evidence[`PIP-${summary.discovered.pipelines + 1}`] = first.relPath; summary.discovered.pipelines++; } - if (chosen) guesses.push({ relPath, candidates, counted: scrapeReceivers.length > 0 }); } // Collector exporters that carry metrics, by component type. @@ -2475,7 +2518,7 @@ function collectionEvidenceOf(relPath, content, catalogue) { // the whole template), walked into throwaway buckets. const buckets = { backends: [], recordingRules: [], burnRateAlerts: [], alertRules: [], metricDefinitions: [], scrapeJobs: [], - alertingRoutes: [], dashboards: [], collectorGuesses: [], repoName: null, + alertingRoutes: [], dashboards: [], collectorState: newCollectorState(), repoName: null, pipelines: { receivers: [], processors: [], exporters: { metrics: null, logs: null, traces: null } }, }; const scratch = { diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 0dcd4577..17b3469d 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -360,6 +360,32 @@ service: && !split.summary.warnings.some(w => /metrics-capable/.test(w)) && split.evidence['pipelines.exporters.metrics'] === undefined, `a split config (${a} first): the declared metrics exporter, one pipeline, no guess warning`, [split.canonical.spec.pipelines.exporters.metrics, split.summary.discovered.pipelines]); } + // The same split with a scrape receiver: the job leaves by the declared + // pipeline that lists its receiver (endpoint from the file that configures + // the exporter), not by the guess; one receiver, one pipeline, no warning. + const A = 'receivers:\n prometheus/mq:\n config:\n scrape_configs:\n - job_name: qmgrs\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp:\n endpoint: http://gw:4317\n prometheusremotewrite:\n endpoint: https://w:SECRET@mimir:9009/api/v1/push\n'; + const B = 'receivers:\n prometheus/mq: {}\nexporters:\n otlp: {}\n prometheusremotewrite: {}\nservice:\n pipelines:\n metrics: { receivers: [prometheus/mq], exporters: [prometheusremotewrite] }\n'; + for (const [a, b] of [['otel/a.yaml', 'otel/b.yaml'], ['otel/b.yaml', 'otel/a.yaml']]) { + const split = crawlFiles({ [a]: A, [b]: B }, { repoName: 'col', now: NOW }); + const job = json(split, 'crawler.discovered.scrape_job_origins').qmgrs; + assert(JSON.stringify(job.exporters) === '[{"name":"prometheusremotewrite","kind":"prometheusremotewrite","endpoint":"https://mimir:9009/api/v1/push"}]' + && split.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && split.summary.discovered.pipelines === 1 + && JSON.stringify(split.canonical.spec.pipelines.receivers) === '[{"name":"prometheus"}]' && !split.summary.warnings.some(w => /metrics-capable/.test(w)), + `a split config with a scrape receiver (${a} first): the job leaves by the declared pipeline`, [job, split.canonical.spec.pipelines, split.summary.discovered.pipelines]); + } + // Alone, the same config's job carries the guess, and the warning says so. + const alone = crawlFiles({ 'otel/a.yaml': A }, { repoName: 'col', now: NOW }); + assert(json(alone, 'crawler.discovered.scrape_job_origins').qmgrs.exporters[0].name === 'otlp' && alone.summary.warnings.some(w => /taken to be the first, 'otlp'/.test(w)), + 'alone, a first-of-several guess stands on the job and is warned about'); + // A metrics backend image already set the metrics exporter: the guess does + // not override it (the walk's precedence, as for a declared pipeline); the + // job still names the exporter its collector configures. + const img = crawlFiles({ + 'charts/vm/values.yaml': 'vm:\n image:\n repository: victoriametrics/victoria-metrics\n tag: v1.99.0\n', + 'charts/c/files/collector.yaml': FILES['charts/otel-collector/files/collector.yaml'], + }, { repoName: 'col', now: NOW }); + assert(img.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && json(img, 'crawler.discovered.scrape_job_origins').qmgrs.exporters[0].kind === 'kafka', + 'a metrics backend image keeps the metrics exporter it implies; the job names the kafka exporter', img.canonical.spec.pipelines.exporters.metrics); // An Alertmanager config (receivers is a list) is still Alertmanager. assert(detectArtefactKind('am.yaml', 'route:\n receiver: x\nreceivers:\n - name: x\n') === 'alertmanager', 'Alertmanager receivers are not a collector\'s'); From 283e0e1f032cee0a10f56f3319c4106fbdcac73f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 19:07:24 +0000 Subject: [PATCH 14/17] crawler: settle unlisted collector receivers; skip disabled subcharts; a malformed collector job costs only itself - a scrape receiver no metrics pipeline in its own file lists (traces only, metrics: null) got no exporters, though another file's metrics pipeline listed it; it is settled after the walk like a pipeline-less config's - an umbrella chart's disabled subchart (enabled: false, in the file or the environment's overlay) still had its collector config read - one malformed job in a collector config threw and lost the file's other jobs and documents (develop read the rest); it is warned about and skipped - the first-of-several warning fired for a guess no job carried - the CHANGELOG's precedence sentence: image and declared pipeline fill the metrics exporter in walk order, as before Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 +- docs/CHANGELOG.md | 6 +-- docs/DOWNSTREAM.md | 9 ++-- tools/lib/crawler.mjs | 88 ++++++++++++++++++++-------------- tools/test-crawl-exporters.mjs | 37 +++++++++++++- 5 files changed, 99 insertions(+), 45 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index bd9128f5..10a56d39 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "f6a3465fae6207259a5f9419d3afeb0495c2291860712ca5a7403ad3e41fbf67", - "bytes": 131876, + "sha256": "7c011c2b4c8e2307a2ffc53273916061979fa1d2ad2f41316e0025633c1a7a79", + "bytes": 133095, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index a469cb0f..2c96ec09 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,15 +4,15 @@ ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs - **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `asResolvedCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. -- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `__mocks__/`, `e2e/`, `integration-tests/`, `unit_tests/`, `testing/`, `test-data/`, `__snapshots__/`, …), nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `e2e/`, `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, `testing/`, `test-data/`, `__snapshots__/`, …), nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. -- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess, settled after every file is walked. A scrape job leaves by the exporters of the declared metrics pipelines, in any file, that list its receiver (a config split across `--config` files), else by the guess. The guess sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold only when nothing else set it: no declared pipeline, and no metrics backend image (whose `prometheusremotewrite` reading the walk already gives precedence over a declared pipeline). The config's scrape receivers become `pipelines.receivers`, once. A first-of-several guess that stands is warned about. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart, unless the subchart is disabled: `enabled: false` in the file or the environment's overlay), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess, settled after every file is walked. A scrape job leaves by the exporters of its own file's metrics pipelines that list its receiver, else by those of the declared metrics pipelines in any other file that do (a config split across `--config` files), else by the guess. A collector job that cannot be read is warned about and skipped; its siblings and the rest of the config are read. The guess sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold only when nothing else set it: no declared pipeline and no metrics backend image (those two fill the metrics exporter from whichever file the walk reads first, as before; a guess comes after both). The config's scrape receivers become `pipelines.receivers`, once. A first-of-several guess that stands is warned about. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template that carries either, a values file that carries a collector config) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. - No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). -- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument that looks like a URL. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. +- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument or a `--from` value that looks like a URL (`://`, `@`, `?`, or `host:port`); `--from` takes a file, and a URL there is refused. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 30b5a25e..062d6c83 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1947,14 +1947,17 @@ An OTel collector's prometheus receivers (`prometheus`, `prometheus/`) are read for their `config.scrape_configs`, with or without a `service:` block (a Helm chart's `files/` copy has none), in a config file, a Helm ConfigMap, or a chart's `values*.yaml` (at its top level, -under `config:`, or under a subchart's `.config:`). +under `config:`, or under a subchart's `.config:` unless the subchart +is disabled). - Each job is in `crawler.discovered.scrape_jobs`. Its entry in `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the `receiver`, the `target_count` of its static targets, and the `exporters` its samples leave by: the exporters of the metrics pipelines that list the - receiver, in the same file or another (a config split across `--config` - files), or, when none does, the metrics exporter the config names. + receiver, in the same file or, when none there does, another (a config + split across `--config` files), or, when none does, the metrics exporter a + pipeline-less config names. A job that cannot be read is warned about and + skipped; the rest of the config is read. Each exporter is given as `{ name, kind, topic | endpoint }`: a kafka exporter's metrics topic, any other's endpoint reduced by `safeSourceUrl` (no userinfo, query or fragment). diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index 35c97e8c..d6a0dd5a 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -1902,7 +1902,7 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo // it declares beats an image's inference; a config this walk cannot read // (the chart's `pipelines.metrics: null` drops a pipeline) is warned // about and never costs the values file its images. - for (const collector of valuesCollectorConfigs(obj)) { + for (const collector of valuesCollectorConfigs(obj, disabledComponents)) { try { walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorState); } catch (e) { summary.warnings.push(`The collector config in ${f.relPath} could not be read (${e.message}); its images are read as before.`); } } @@ -1911,12 +1911,16 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo } // The collector configs a Helm values document carries: the document -// itself, its `config:`, or a subchart's `.config:` (an umbrella chart). -function valuesCollectorConfigs(obj) { +// itself, its `config:`, or a subchart's `.config:` (an umbrella +// chart) — never a subchart the environment disables (`disabled`, the +// merged `enabled: false` toggles) or this document sets `enabled: false`. +function valuesCollectorConfigs(obj, disabled = new Set()) { if (!isMapping(obj)) return []; if (looksLikeOtelCollector(obj)) return [obj]; if (isMapping(obj.config) && looksLikeOtelCollector(obj.config)) return [obj.config]; - return Object.values(obj).filter(v => isMapping(v?.config) && looksLikeOtelCollector(v.config)).map(v => v.config); + return Object.entries(obj) + .filter(([key, v]) => !disabled.has(key) && v?.enabled !== false && isMapping(v?.config) && looksLikeOtelCollector(v.config)) + .map(([, v]) => v.config); } function workloadPodSpec(obj) { @@ -2202,33 +2206,43 @@ function walkPrometheusScrapeConfig(f, scrapeJobs, evidence, summary) { // One Prometheus config document's scrape jobs — a standalone prometheus.yml // or the `config:` of an OTel collector's prometheus receiver (`collector` // then names the receiver and the exporters its samples leave by). +// Answers how many jobs it added. In a collector config a job this walk +// cannot read is warned about and skipped (its siblings and the rest of the +// config are still read); a standalone prometheus.yml reads as before. function walkScrapeConfigDoc(obj, origin, scrapeJobs, evidence, summary, collector = null) { - if (!Array.isArray(obj?.scrape_configs)) return; + if (!Array.isArray(obj?.scrape_configs)) return 0; + let added = 0; for (const cfg of obj.scrape_configs) { const job = String(cfg?.job_name || '').trim(); if (!job) continue; - // Promtail and Alloy use the same `scrape_configs` / `job_name` keys - // to say which LOG FILES to tail. Such an entry is not a metrics - // scrape job and never appears among a metrics store's targets; - // declaring it made it read "not live" on every platform. - if (isLogScrapeConfig(cfg, obj)) { - summary.discovered.logScrapeConfigs = (summary.discovered.logScrapeConfigs || 0) + 1; - continue; - } - addScrapeJob(scrapeJobs, evidence, summary, { - job, - metrics_path: cfg.metrics_path || '/metrics', - interval: cfg.scrape_interval || obj.global?.scrape_interval || null, - targets: scrapeTargets(cfg), - origin, - ...(collector ? { - via: 'otel-collector', - receiver: collector.receiver, - target_count: (Array.isArray(cfg.static_configs) ? cfg.static_configs : []).reduce((n, sc) => n + (Array.isArray(sc?.targets) ? sc.targets.filter(Boolean).length : 0), 0), - exporters: collector.exporters, - } : {}), - }); + const read = () => { + // Promtail and Alloy use the same `scrape_configs` / `job_name` keys + // to say which LOG FILES to tail. Such an entry is not a metrics + // scrape job and never appears among a metrics store's targets; + // declaring it made it read "not live" on every platform. + if (isLogScrapeConfig(cfg, obj)) { + summary.discovered.logScrapeConfigs = (summary.discovered.logScrapeConfigs || 0) + 1; + return; + } + if (addScrapeJob(scrapeJobs, evidence, summary, { + job, + metrics_path: cfg.metrics_path || '/metrics', + interval: cfg.scrape_interval || obj.global?.scrape_interval || null, + targets: scrapeTargets(cfg), + origin, + ...(collector ? { + via: 'otel-collector', + receiver: collector.receiver, + target_count: (Array.isArray(cfg.static_configs) ? cfg.static_configs : []).reduce((n, sc) => n + (Array.isArray(sc?.targets) ? sc.targets.filter(Boolean).length : 0), 0), + exporters: collector.exporters, + } : {}), + })) added++; + }; + if (!collector) { read(); continue; } + try { read(); } + catch (e) { summary.warnings.push(`Scrape job '${job}' of receiver ${collector.receiver} in ${origin} could not be read (${e.message}); the rest of the config is read.`); } } + return added; } // A log-collection scrape config (Promtail, Alloy's loki.source): it has @@ -2258,11 +2272,12 @@ function walkActuatorMetricsConfig(f, scrapeJobs, evidence, summary, repoName = } function addScrapeJob(scrapeJobs, evidence, summary, item) { - if (!item?.job) return; - if (scrapeJobs.some(j => j.job === item.job && j.origin === item.origin)) return; + if (!item?.job) return false; + if (scrapeJobs.some(j => j.job === item.job && j.origin === item.origin)) return false; scrapeJobs.push(item); if (!evidence[`scrape.${item.job}`]) evidence[`scrape.${item.job}`] = item.origin; summary.discovered.scrapeJobs++; + return true; } function scrapeTargets(cfg) { @@ -2373,10 +2388,11 @@ function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, sum if (!/^metrics/i.test(sig) || !Array.isArray(p?.receivers) || !p.receivers.includes(key)) continue; for (const exp of p.exporters || []) if (!names.includes(exp)) names.push(exp); } - walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { - receiver: key, - exporters: names.map(n => collectorExporterDetail(n, exporterCfgs[n])), - }); + const exporters = names.map(n => collectorExporterDetail(n, exporterCfgs[n])); + const jobs = walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + // No metrics pipeline here lists the receiver: another file's may (a + // config split across --config files), settled after the walk. + if (!names.length) state.guesses.push({ relPath, candidates: [], cfgs: exporterCfgs, groups: [{ receiver: key, exporters, jobs }], receivers: [] }); } return; } @@ -2393,8 +2409,8 @@ function walkOtelCollectorDoc(obj, relPath, pipelines, scrapeJobs, evidence, sum for (const { key, config } of scrapeReceivers) { // One array per receiver, shared with its jobs' entries (patched in place). const exporters = chosen ? [collectorExporterDetail(chosen, exporterCfgs[chosen])] : []; - groups.push({ receiver: key, exporters }); - walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + const jobs = walkScrapeConfigDoc(config, relPath, scrapeJobs, evidence, summary, { receiver: key, exporters }); + groups.push({ receiver: key, exporters, jobs }); } state.guesses.push({ relPath, candidates, cfgs: exporterCfgs, groups, receivers: scrapeReceivers.map(({ key }) => key.split('/')[0]) }); } @@ -2427,7 +2443,7 @@ function settleCollectorGuesses(state, pipelines, evidence, summary) { if (names.length) { group.exporters.splice(0, group.exporters.length, ...names.map(n => collectorExporterDetail(n, isMapping(g.cfgs[n]) && Object.keys(g.cfgs[n]).length ? g.cfgs[n] : cfgOf[n]))); - } else if (group.exporters.length) { + } else if (group.exporters.length && group.jobs) { warnFirstOfSeveral(g); } } @@ -2509,7 +2525,7 @@ function collectionEvidenceOf(relPath, content, catalogue) { return `${relPath} (exporter config: ${entry?.id ?? 'unknown'})`; } if (kind === 'helm-values') { - const jobs = jobsOf(parseYamlDocs(content).flatMap(valuesCollectorConfigs) + const jobs = jobsOf(parseYamlDocs(content).flatMap(d => valuesCollectorConfigs(d)) .flatMap(c => collectorScrapeReceivers(c).map(r => r.config))); return jobs.length ? `${relPath} (collector scrape jobs: ${jobs.join(', ')})` : null; } diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 17b3469d..4b5afea7 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -202,7 +202,7 @@ const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); assert(detectArtefactKind('test/fixtures/ibm-mq.prom', PROM) === 'unknown' && detectArtefactKind('src/x.test.metrics', PROM) === 'unknown' && detectArtefactKind('testdata/a.prom', PROM) === 'unknown' && ['src/__fixtures__/node.prom', 'test-data/node.prom', 'test_fixtures/a.prom', 'x/__mocks__/a.metrics', 'mocks/a.prom', 'spec/a.prom', - 'e2e/mq.prom', 'integration-tests/mq.prom', 'e2e-tests/mq.prom', 'unit_tests/mq.prom', 'testing/mq.prom', 'src/__snapshots__/mq.prom', 'exp/mq_spec.prom', 'exp/mq.spec.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), + 'e2e/mq.prom', 'integration-tests/mq.prom', 'src/testFixtures/resources/node.prom', 'testfixtures/a.prom', 'src/integrationTest/resources/mq.prom', 'unittests/a.prom', 'e2etests/a.prom', 'e2e-tests/mq.prom', 'unit_tests/mq.prom', 'testing/mq.prom', 'src/__snapshots__/mq.prom', 'exp/mq_spec.prom', 'exp/mq.spec.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), 'never from a test, fixture or mock path, in any of the usual spellings'); assert(['snapshots/ibm-mq.prom', 'exporters/latest/mq.prom', 'contest/mq.prom', 'attestation/mq.prom'].every(p => detectArtefactKind(p, PROM) === 'exposition-snapshot'), 'a folder that only contains such a word in a longer one is not a test folder'); @@ -373,6 +373,30 @@ service: && JSON.stringify(split.canonical.spec.pipelines.receivers) === '[{"name":"prometheus"}]' && !split.summary.warnings.some(w => /metrics-capable/.test(w)), `a split config with a scrape receiver (${a} first): the job leaves by the declared pipeline`, [job, split.canonical.spec.pipelines, split.summary.discovered.pipelines]); } + // A config WITH pipelines whose receiver no metrics pipeline lists (only + // traces here, or `metrics: null`): another file's metrics pipeline that + // lists it says where its jobs go. + for (const pipes of [' traces: { receivers: [otlp], exporters: [otlp] }\n', ' metrics: null\n traces: { receivers: [otlp], exporters: [otlp] }\n']) { + const tracesOnly = 'receivers:\n otlp: {}\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp: {}\nservice:\n pipelines:\n' + pipes; + const metricsFile = 'receivers:\n prometheus: {}\nexporters:\n kafka:\n topic: mq-metrics\nservice:\n pipelines:\n metrics: { receivers: [prometheus], exporters: [kafka] }\n'; + const r = crawlFiles({ 'otel/a.yaml': tracesOnly, 'otel/b.yaml': metricsFile }, { repoName: 'col', now: NOW }); + assert(JSON.stringify(json(r, 'crawler.discovered.scrape_job_origins').mq.exporters) === '[{"name":"kafka","kind":"kafka","topic":"mq-metrics"}]', + 'a receiver its own file\'s pipelines do not list for metrics leaves by the other file\'s metrics pipeline', json(r, 'crawler.discovered.scrape_job_origins').mq); + } + // A guess no job carries (its receiver's jobs are settled elsewhere, or it + // has none) is not warned about. + const spur = crawlFiles({ + 'otel/a.yaml': A.replace('exporters:\n', 'receivers2: {}\nexporters:\n').replace('receivers:\n prometheus/mq:', 'receivers:\n prometheus/empty:\n config:\n scrape_configs: []\n prometheus/mq:'), + 'otel/b.yaml': B, + }, { repoName: 'col', now: NOW }); + assert(!spur.summary.warnings.some(w => /metrics-capable/.test(w)) && spur.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite', + 'no first-of-several warning for a guess no job and no spec field carries', spur.summary.warnings); + // A malformed job in a collector config is warned about and skipped; its + // sibling and the rest of the file (a second document) are read. + const mal = crawlFiles({ 'otel/otel-collector.yaml': 'receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: bad\n static_configs: { targets: [x] }\n - job_name: good\n static_configs: [{ targets: ["g:1"] }]\nexporters:\n kafka: { topic: t }\nservice:\n pipelines:\n metrics: { receivers: [prometheus], exporters: [kafka] }\n---\nreceivers:\n otlp: {}\nexporters:\n otlp: {}\nservice:\n pipelines:\n traces: { receivers: [otlp], exporters: [otlp] }\n' }, { repoName: 'col', now: NOW }); + assert(JSON.stringify(json(mal, 'crawler.discovered.scrape_jobs')) === '["good"]' && mal.canonical.spec.pipelines.exporters.traces.kind === 'otlp' && mal.summary.discovered.pipelines === 2 + && mal.summary.warnings.some(w => /^Scrape job 'bad' of receiver prometheus in otel\/otel-collector\.yaml could not be read \(.*\); the rest of the config is read\.$/.test(w)) + && !mal.summary.warnings.some(w => /Failed to parse/.test(w)), 'a malformed collector job costs only itself', mal.summary.warnings); // Alone, the same config's job carries the guess, and the warning says so. const alone = crawlFiles({ 'otel/a.yaml': A }, { repoName: 'col', now: NOW }); assert(json(alone, 'crawler.discovered.scrape_job_origins').qmgrs.exporters[0].name === 'otlp' && alone.summary.warnings.some(w => /taken to be the first, 'otlp'/.test(w)), @@ -423,6 +447,14 @@ data: assert(job?.target_count === 2 && job.exporters[0].topic === 'mq' && r.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', `a collector in a chart's values.yaml (${label}): its job and its kafka exporter`, job); } + // A disabled subchart's collector config is not read: in the same file, or + // by the environment's overlay. + const sub = 'opentelemetry-collector:\n enabled: ENABLED\n config:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); + const off = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'false') }, { repoName: 'vals', now: NOW }); + const overlay = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'true'), 'charts/u/values-prod.yaml': 'opentelemetry-collector:\n enabled: false\n' }, { repoName: 'vals', environment: 'prod', now: NOW }); + assert(!('crawler.discovered.scrape_jobs' in ann(off)) && !('crawler.discovered.scrape_jobs' in ann(overlay)) + && off.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && overlay.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite', + 'a disabled umbrella subchart\'s collector is not read (same file, prod overlay)', [ann(off)['crawler.discovered.scrape_jobs'], ann(overlay)['crawler.discovered.scrape_jobs']]); // An umbrella chart: the collector under its subchart's key. const umbrella = crawlFiles({ 'charts/u/values.yaml': 'opentelemetry-collector:\n config:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n') }, { repoName: 'vals', now: NOW }); assert(json(umbrella, 'crawler.discovered.scrape_jobs')?.includes('qmgrs') && umbrella.canonical.spec.pipelines.exporters.metrics.kind === 'kafka', @@ -567,6 +599,9 @@ try { [['--exporter', `${base}/metrics`, '--from', 'qm1.prom'], 2, 'packc emit: pass --exporter or --from, not both\n'], [['--exporter', `${base}/metrics`, '--bogus'], 2, null], [['http://user:SECRET4@127.0.0.1:1/metrics?token=SECRET5'], 2, null], + [['mq:9157/metrics?token=SECRET8'], 2, null], + [['--from', 'http://svc:SECRET9@mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], + [['--from=mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], [['--exporter=http://user:SECRET6@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], [['--exporter', `${base}/cut?token=SECRET7`], 1, `packc emit: could not read the response of ${base}/cut: the connection ended before the body did\n`], [[], 2, null], From d409d6810b091a86c672b966c25b0b2cd156840f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 19:07:25 +0000 Subject: [PATCH 15/17] exporter-metrics: testFixtures/, integrationTest/, unittests/ are test folders; packc emit never echoes a URL given to --from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The folder rule dropped test[-_]?fixtures? when it was reworked (Gradle's src/testFixtures/ read as snapshots) and never had integrationTest/, unittests/ or e2etests/. packc emit echoed '--from ' in its cannot-read error and a schemeless host:port?token=… unknown argument whole; a URL-like value is never shown, and --from refuses one. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 4 ++-- tools/emit-manifest.mjs | 6 +++++- tools/lib/exporter-metrics.mjs | 7 ++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index 10a56d39..f440fba3 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -523,8 +523,8 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "ac1cfcb52ee2c7ef081b64c1ec4dac02652ce3f50eec17e53b582cdbb82953d2", - "bytes": 19663, + "sha256": "5d002100de4a32c9c548429f92a3c55fe912d722772514ccf2c44cdb41ab0cde", + "bytes": 19780, "exports": [ "EXPOSITION_TYPES", "asResolvedCatalogue", diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs index 36f63086..e0bb2286 100644 --- a/tools/emit-manifest.mjs +++ b/tools/emit-manifest.mjs @@ -36,6 +36,9 @@ import { readFileSync, writeFileSync } from 'node:fs'; import { resolve, basename } from 'node:path'; import { manifestFromExposition, safeSourceUrl } from './lib/exporter-metrics.mjs'; +// A value that may be a URL with its credentials or token: never echoed. +const urlLike = (v) => /:\/\/|@|\?|^[^/\\\s]+:\d+(?:[/?#]|$)/.test(String(v ?? '')); + const USAGE = 'usage: packc emit --exporter | --from [--name ] [--version ] [--out |-] [--generated-at | --no-timestamp] [--timeout ]'; function fail(code, message) { @@ -55,7 +58,7 @@ function parseArgs(argv) { let inline; if (eq > 0) { inline = a.slice(eq + 1); a = a.slice(0, eq); } // An unknown argument may be a URL with its credentials: never echoed. - if (!takes.has(a)) fail(2, `unknown argument ${/:\/\/|@/.test(a) ? '(a URL-like value, not shown)' : `'${a}'`}\n${USAGE}`); + if (!takes.has(a)) fail(2, `unknown argument ${urlLike(a) ? '(a URL-like value, not shown)' : `'${a}'`}\n${USAGE}`); if (inline !== undefined) { out[a.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase())] = inline; continue; } const v = argv[++i]; if (v === undefined || (v.startsWith('--') && v !== '-')) fail(2, `${a} needs a value\n${USAGE}`); @@ -121,6 +124,7 @@ if (args.exporter) { source = url.href; defaultName = url.hostname; } else { + if (urlLike(args.from)) fail(2, '--from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter'); try { text = readFileSync(resolve(args.from), 'utf8'); } catch (e) { fail(2, `cannot read --from ${args.from}: ${e.code || e.message}`); } defaultName = basename(args.from).replace(/\.[^.]+$/, ''); } diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index 3196b85c..3bf7e127 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -53,14 +53,15 @@ export function isMetricsManifestPath(relPath) { * A path a saved exposition is never read from: a folder whose name is a * test, spec, fixture, mock or test-data word, alone or joined to others by * `-` / `_` and wrapped in `__` or not (`test/`, `__tests__/`, `e2e/`, - * `integration-tests/`, `unit_tests/`, `testing/`, `__fixtures__/`, - * `__mocks__/`, `testdata/`, `test-data/`, `__snapshots__/`, …), or a + * `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, + * `testing/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `testdata/`, + * `test-data/`, `__snapshots__/`, …), or a * `*.test.*` / `*.spec.*` / `*_test.*` / `*_spec.*` file — there an * exposition is sample input, not what a deployed exporter serves. */ export function isTestOrFixturePath(relPath) { const p = String(relPath ?? '').replace(/\\/g, '/').toLowerCase(); - const word = '(?:tests?|specs?|fixtures?|mocks?|testdata|testing|e2e|test[-_]?data)'; + const word = '(?:tests?|specs?|fixtures?|mocks?|testing|e2e|test[-_]?(?:data|fixtures?)|(?:integration|unit|e2e|functional|acceptance)[-_]?tests?)'; const folder = new RegExp(`^(?:__)?(?:[a-z0-9]+[-_])*${word}(?:[-_][a-z0-9]+)*(?:__)?$|^__snapshots__$`); return p.split('/').slice(0, -1).some(seg => folder.test(seg)) || /[._](?:test|spec)\.[a-z]+$/.test(p); From ff78d6d0939b3eb5feffafbf78354649ec49ceee Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 19:17:04 +0000 Subject: [PATCH 16/17] exporter-metrics: an @ in a query hides no secret; overlays re-enable subcharts; camelCase test source sets - safeSourceUrl dropped everything before the last @ before cutting the query, so 'https://h/push?user=ops@corp&api_key=X' kept 'corp&api_key=X' (endpoint, manifest source, emit's stderr). A '?' or '#' before the last '@' now records nothing; a value with nothing past its scheme is omitted. - a subchart disabled in values.yaml and re-enabled by values-prod.yaml kept its collector skipped; a crawl reads the merged toggles (overlay wins). - componentTest/, smokeTest/, intTest/, testUtils/, src/it/ are test folders. - packc emit echoed --generated-at and threw a stack for an unwritable --out; neither does now, and a --from file named with an @ is read. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- VENDOR-MANIFEST.json | 8 ++++---- docs/CHANGELOG.md | 6 +++--- docs/DOWNSTREAM.md | 6 ++++-- tools/emit-manifest.mjs | 29 ++++++++++++++++++----------- tools/lib/crawler.mjs | 18 +++++++++++------- tools/lib/exporter-metrics.mjs | 32 +++++++++++++++++++++++--------- tools/test-crawl-exporters.mjs | 28 +++++++++++++++++++++++++--- 7 files changed, 88 insertions(+), 39 deletions(-) diff --git a/VENDOR-MANIFEST.json b/VENDOR-MANIFEST.json index f440fba3..fe5c4a01 100644 --- a/VENDOR-MANIFEST.json +++ b/VENDOR-MANIFEST.json @@ -378,8 +378,8 @@ "changedSinceRelease": false }, "tools/lib/crawler.mjs": { - "sha256": "7c011c2b4c8e2307a2ffc53273916061979fa1d2ad2f41316e0025633c1a7a79", - "bytes": 133095, + "sha256": "d0387b62b132f273fc622aaa7ca0411c3e90dc907db4bd07e4a98c89f552455e", + "bytes": 133296, "exports": [ "SCAN_EXT", "SCAN_IGNORE_DIRS", @@ -523,8 +523,8 @@ "releasedExports": null }, "tools/lib/exporter-metrics.mjs": { - "sha256": "5d002100de4a32c9c548429f92a3c55fe912d722772514ccf2c44cdb41ab0cde", - "bytes": 19780, + "sha256": "57be2804bd81a9c9c4666839540d0c22d00486f292a85d54b852226099a88471", + "bytes": 20774, "exports": [ "EXPOSITION_TYPES", "asResolvedCatalogue", diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 2c96ec09..941fd65d 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -4,15 +4,15 @@ ### Follow-up `exporter-metrics`: the crawler learns metrics from prebuilt exporters and from an OTel collector's scrape jobs - **A metrics manifest is an artefact** (`tools/lib/exporter-metrics.mjs`, new and listed: `readMetricsManifest`, `manifestFromExposition`, `parseExposition`, `safeSourceUrl`, `resolveExporterCatalogue`, `asResolvedCatalogue`, `matchExporterConfig`, `mayBeExporterConfig`, `inferExporterFamilies`, `isMetricsManifestPath`, `isExpositionSnapshotPath`, `isTestOrFixturePath`, `EXPOSITION_TYPES`). `metrics-manifest.json` and `*.metrics-manifest.json` are the kind `metrics-manifest`. Each entry is a metric with `origin_kind: exporter-manifest`, `confidence: declared`, and the exporter as its service. A file that is not a manifest (none of its text quoted: the JSON parser's message would quote its start), or entries without a name and an exposition type, are named in a warning. A manifest or exporter config saved with a leading BOM is read. -- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `e2e/`, `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, `testing/`, `test-data/`, `__snapshots__/`, …), nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). +- **A saved exposition is read** (**a declared default change**: `SCAN_EXT` adds `.prom` and `.metrics`, so the CLI walk, the studio's folder picker and a journey's crawl read them; `server/github-crawl.mjs` fetches a manifest with the files it fetched before, and a saved exposition, any `config.yaml` and a chart's `files/*.yaml` only after them (`isExporterCandidateFile`, `selectCrawlerFiles`): a monorepo's many `config.yaml` files never use up the 200-file budget its rule, Alertmanager or collector files had). The kind is `exposition-snapshot`: `# HELP` / `# TYPE` and the label names of the samples, label values never kept, `origin_kind: exporter-snapshot`. A family is named as Prometheus stores it: an OpenMetrics counter `jobs` is `jobs_total`, an info `build` is `build_info`. It is never read from a folder named by a test, spec, fixture, mock or test-data word, alone or joined to others by `-` / `_` (`test/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `e2e/`, `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, `testing/`, `test-data/`, `__snapshots__/`, …), a camelCase test source set (`componentTest/`, `smokeTest/`, `testUtils/`) or Maven's `src/it/`, nor from a `*.test.*`, `*.spec.*`, `*_test.*` or `*_spec.*` file (`isTestOrFixturePath`). - **An exporter's config is read against a family catalogue** (`tools/lib/exporter-catalogue.mjs`, new and listed: `EXPORTER_CATALOGUE`, data only, a small IBM mq-metric-samples entry). The kind is `exporter-config`, recognised by the top-level keys an entry names (`global`, `connection`, `objects`, `prometheus`) before the filename heuristics. It emits the families the config's gates (`useObjectStatus`, `usePublications`) and scopes (`objects.queues`, `objects.channels`) switch on, under its `prometheus.namespace`, as candidates: `origin_kind: exporter-config`, `confidence: inferred` (curation keeps it). A downstream's catalogue (`packc x-ray --exporter-catalogue `, `crawlFiles(files, { exporterCatalogue })`, `detectArtefactKind(path, text, { exporterCatalogue })`) replaces a built-in entry with its id and adds the rest, in either form (`{ exporters: [...] }` or the array). A malformed entry, or a family gated or scoped on a name its entry does not define (`toString` included), is dropped, with a crawl warning from `crawlFiles` and the CLI (`asResolvedCatalogue` resolves a catalogue once). `docs/DOWNSTREAM.md` §18 has the format. - **The strongest origin speaks for a metric**: `metricOriginRank` reads exporter-manifest, exporter-snapshot, source code, source fragment, recording-rule output, PromQL reference, then exporter-config. The relative order of the existing kinds is unchanged. -- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart, unless the subchart is disabled: `enabled: false` in the file or the environment's overlay), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess, settled after every file is walked. A scrape job leaves by the exporters of its own file's metrics pipelines that list its receiver, else by those of the declared metrics pipelines in any other file that do (a config split across `--config` files), else by the guess. A collector job that cannot be read is warned about and skipped; its siblings and the rest of the config are read. The guess sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold only when nothing else set it: no declared pipeline and no metrics backend image (those two fill the metrics exporter from whichever file the walk reads first, as before; a guess comes after both). The config's scrape receivers become `pipelines.receivers`, once. A first-of-several guess that stands is warned about. +- **An OTel collector's scrape jobs are read** (**a declared default change**). A collector config's prometheus receivers (`prometheus`, `prometheus/`) give their `config.scrape_configs` jobs, with or without `service.pipelines`. A config with no service block is classified by its receivers mapping (with scrape jobs or exporters), as a file, a Helm ConfigMap value or a whole template. A chart's `values*.yaml` that carries a collector config, at its top level, under `config:` (the opentelemetry-collector chart's idiom) or under a subchart's `.config:` (an umbrella chart, unless the subchart is disabled: its `enabled` toggles merged across the in-scope values files, an overlay winning as with `helm -f`, so an overlay can disable or re-enable it), is walked as one too, receivers, processors, pipelines and all, before its images (**a declared default change** for such a values file). A collector config there that the walk cannot read is warned about and never costs the file its images. A pipeline set to `null` (the chart's way to drop one) is skipped, in a values file and in a plain collector config (**a declared default change**: such a config was `Failed to parse` and read nothing). A collector job's `crawler.discovered.scrape_job_origins` entry adds `via`, `receiver`, `target_count` and `exporters` (`{ name, kind, topic | endpoint }`, an endpoint without userinfo, query or fragment). A job from a standalone `prometheus.yml` keeps its four fields. With no pipelines, the metrics-capable exporter the config names (kafka, prometheusremotewrite, otlp, …; the first, with a warning, if several) is a guess, settled after every file is walked. A scrape job leaves by the exporters of its own file's metrics pipelines that list its receiver, else by those of the declared metrics pipelines in any other file that do (a config split across `--config` files), else by the guess. A collector job that cannot be read is warned about and skipped; its siblings and the rest of the config are read. The guess sets `spec.pipelines.exporters.metrics` (still `{ kind }`) in place of the `prometheusremotewrite` scaffold only when nothing else set it: no declared pipeline and no metrics backend image (those two fill the metrics exporter from whichever file the walk reads first, as before; a guess comes after both). The config's scrape receivers become `pipelines.receivers`, once. A first-of-several guess that stands is warned about. - **`crawler.discovered.exporters`** (+ `_count`, `summary.discovered.exporters`) lists each exporter the crawl learned from and its sources (`{ origin_kind, file, metrics, version?, source?, generated_at?, namespace?, gates?, scopes? }`). It appears only when there is one. - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template that carries either, a values file that carries a collector config) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. - No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). -- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument or a `--from` value that looks like a URL (`://`, `@`, `?`, or `host:port`); `--from` takes a file, and a URL there is refused. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. +- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value with a `?` or `#` before its last `@` is recorded as nothing: the `@` may be in a query (`?user=ops@corp&api_key=…`) or a `?` in a password, and text cannot tell which. An endpoint or source that leaves nothing past its scheme is left out. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument, a `--from`, `--out` or `--generated-at` value that looks like a URL (`://`, `@`, `?`, or `host:port`). `--from` takes a file (a file whose name holds `@` is read); a URL-like value that is no file is refused. An `--out` it cannot write is refused, not left as a stack trace. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 062d6c83..8cf1baa5 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1875,7 +1875,8 @@ or repeats a name, is not read and the crawl says how many; a file that is not a manifest is named in a warning. `source` is recorded without userinfo, query or fragment: tabs and line breaks, everything before the last `@`, and everything from the first `?` or `#` are dropped (`safeSourceUrl`, textual — -a password may hold `/` or `@`). +a password may hold `/` or `@`). A value with a `?` or `#` before its last +`@` is not recorded at all. Label values are never part of a manifest. `packc emit --exporter ` (`tools/emit-manifest.mjs`) writes one from a @@ -1948,7 +1949,8 @@ An OTel collector's prometheus receivers (`prometheus`, without a `service:` block (a Helm chart's `files/` copy has none), in a config file, a Helm ConfigMap, or a chart's `values*.yaml` (at its top level, under `config:`, or under a subchart's `.config:` unless the subchart -is disabled). +is disabled — its `enabled` toggles merged across the in-scope values files, +an overlay winning). - Each job is in `crawler.discovered.scrape_jobs`. Its entry in `crawler.discovered.scrape_job_origins` adds `via: otel-collector`, the diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs index e0bb2286..dc523841 100644 --- a/tools/emit-manifest.mjs +++ b/tools/emit-manifest.mjs @@ -32,10 +32,14 @@ * URL with the values the exporter expects. */ -import { readFileSync, writeFileSync } from 'node:fs'; +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; import { resolve, basename } from 'node:path'; import { manifestFromExposition, safeSourceUrl } from './lib/exporter-metrics.mjs'; +// A URL as a message may name it: safeSourceUrl's form, or a stand-in when +// that leaves nothing (the URL is then not shown at all). +const shown = (url) => safeSourceUrl(url) || 'the exporter URL (not shown)'; + // A value that may be a URL with its credentials or token: never echoed. const urlLike = (v) => /:\/\/|@|\?|^[^/\\\s]+:\d+(?:[/?#]|$)/.test(String(v ?? '')); @@ -76,8 +80,8 @@ function redirectRefusal(url, status, location) { const keys = target ? [...new Set([...target.searchParams.keys()])] : []; const to = !location ? '' : !target ? ' to a location that is not a URL' - : ` to ${safeSourceUrl(target.href)}${keys.length ? `?${keys.map(k => `${encodeURIComponent(k)}=…`).join('&')} (query values not shown)` : ''}`; - return `${safeSourceUrl(url)} answered ${status}, a redirect${to}; it is not followed — pass the URL the exporter serves its metrics at${keys.length ? ', with the query values it expects' : ''}`; + : ` to ${shown(target.href)}${keys.length ? `?${keys.map(k => `${encodeURIComponent(k)}=…`).join('&')} (query values not shown)` : ''}`; + return `${shown(url)} answered ${status}, a redirect${to}; it is not followed — pass the URL the exporter serves its metrics at${keys.length ? ', with the query values it expects' : ''}`; } async function scrape(url, timeoutSeconds) { @@ -90,14 +94,14 @@ async function scrape(url, timeoutSeconds) { }); } catch (e) { const why = e?.name === 'TimeoutError' ? `no answer within ${timeoutSeconds}s (raise --timeout)` : (e?.cause?.message || e?.message || String(e)); - fail(1, `could not scrape ${safeSourceUrl(url)}: ${why}`); + fail(1, `could not scrape ${shown(url)}: ${why}`); } if (res.status >= 300 && res.status < 400) fail(1, redirectRefusal(url, res.status, res.headers.get('location'))); - if (res.status !== 200) fail(1, `${safeSourceUrl(url)} answered ${res.status}; a metrics endpoint answers 200`); + if (res.status !== 200) fail(1, `${shown(url)} answered ${res.status}; a metrics endpoint answers 200`); try { return await res.text(); } catch (e) { - fail(1, `could not read the response of ${safeSourceUrl(url)}: ${e?.name === 'TimeoutError' ? `not complete within ${timeoutSeconds}s (raise --timeout)` : 'the connection ended before the body did'}`); + fail(1, `could not read the response of ${shown(url)}: ${e?.name === 'TimeoutError' ? `not complete within ${timeoutSeconds}s (raise --timeout)` : 'the connection ended before the body did'}`); } } @@ -110,21 +114,23 @@ if (!Number.isFinite(timeout) || timeout <= 0 || timeout > 600) fail(2, '--timeo let generatedAt; if (args.timestamp) { const at = args.generatedAt === undefined ? new Date() : new Date(args.generatedAt); - if (Number.isNaN(at.getTime())) fail(2, `--generated-at '${args.generatedAt}' is not a date`); + if (Number.isNaN(at.getTime())) fail(2, `--generated-at ${urlLike(args.generatedAt) ? '(a URL-like value, not shown)' : `'${args.generatedAt}'`} is not a date`); generatedAt = at.toISOString(); } else if (args.generatedAt !== undefined) fail(2, 'pass --generated-at or --no-timestamp, not both'); let text, source, defaultName; if (args.exporter) { let url; - try { url = new URL(args.exporter); } catch { fail(2, `--exporter '${safeSourceUrl(args.exporter)}' is not a URL (e.g. http://mq-exporter:9157/metrics); shown without anything before an @ or after a ? or #`); } + try { url = new URL(args.exporter); } catch { fail(2, `--exporter '${shown(args.exporter)}' is not a URL (e.g. http://mq-exporter:9157/metrics); shown without anything before an @ or after a ? or #`); } if (url.protocol !== 'http:' && url.protocol !== 'https:') fail(2, `--exporter must be an http or https URL, not ${url.protocol}`); if (url.username || url.password) fail(2, 'the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from'); text = await scrape(url.href, timeout); source = url.href; defaultName = url.hostname; } else { - if (urlLike(args.from)) fail(2, '--from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter'); + // A file whose name holds `@` or `?` is read; a URL-like value that is not + // a file is refused unshown. + if (urlLike(args.from) && !existsSync(resolve(args.from))) fail(2, '--from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter'); try { text = readFileSync(resolve(args.from), 'utf8'); } catch (e) { fail(2, `cannot read --from ${args.from}: ${e.code || e.message}`); } defaultName = basename(args.from).replace(/\.[^.]+$/, ''); } @@ -132,12 +138,13 @@ if (args.exporter) { const name = String(args.name ?? defaultName ?? '').trim(); if (!name) fail(2, 'the exporter has no name; pass --name'); const manifest = manifestFromExposition(text, { exporter: name, version: args.version, source, generatedAt }); -if (!manifest.metrics.length) fail(1, `${source ? safeSourceUrl(source) : args.from} holds no metric family (no # TYPE, # HELP or sample line)`); +if (!manifest.metrics.length) fail(1, `${source ? shown(source) : args.from} holds no metric family (no # TYPE, # HELP or sample line)`); const json = `${JSON.stringify(manifest, null, 2)}\n`; if (args.out === '-') { process.stdout.write(json, () => process.exit(0)); } else { - writeFileSync(resolve(args.out), json); + try { writeFileSync(resolve(args.out), json); } + catch (e) { fail(2, `cannot write --out ${urlLike(args.out) ? '(a URL-like value, not shown)' : args.out}: ${e.code || 'write failed'}`); } process.stdout.write(`wrote ${args.out}: ${manifest.metrics.length} metric famil${manifest.metrics.length === 1 ? 'y' : 'ies'} of ${name}\n`, () => process.exit(0)); } diff --git a/tools/lib/crawler.mjs b/tools/lib/crawler.mjs index d6a0dd5a..a54e3c35 100644 --- a/tools/lib/crawler.mjs +++ b/tools/lib/crawler.mjs @@ -222,7 +222,7 @@ export function detectArtefactKind(relPath, content, opts) { // A saved /metrics response — not from a test or fixture path, where an // exposition is sample input, not what a deployed exporter serves. if (isExpositionSnapshotPath(lower)) { - return !isTestOrFixturePath(lower) && parseExposition(content).families.length ? 'exposition-snapshot' : 'unknown'; + return !isTestOrFixturePath(relPath) && parseExposition(content).families.length ? 'exposition-snapshot' : 'unknown'; } // JSON first — Grafana dashboards are the only json we care about. if (lower.endsWith('.json')) { @@ -1902,7 +1902,7 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo // it declares beats an image's inference; a config this walk cannot read // (the chart's `pipelines.metrics: null` drops a pipeline) is warned // about and never costs the values file its images. - for (const collector of valuesCollectorConfigs(obj, disabledComponents)) { + for (const collector of valuesCollectorConfigs(obj, { disabled: disabledComponents })) { try { walkOtelCollectorDoc(collector, f.relPath, pipelines, scrapeJobs, evidence, summary, collectorState); } catch (e) { summary.warnings.push(`The collector config in ${f.relPath} could not be read (${e.message}); its images are read as before.`); } } @@ -1912,14 +1912,17 @@ function walkHelmValues(f, backends, pipelines, evidence, summary, disabledCompo // The collector configs a Helm values document carries: the document // itself, its `config:`, or a subchart's `.config:` (an umbrella -// chart) — never a subchart the environment disables (`disabled`, the -// merged `enabled: false` toggles) or this document sets `enabled: false`. -function valuesCollectorConfigs(obj, disabled = new Set()) { +// chart) — never a subchart that is disabled. A crawl knows that from +// `disabled`, the `enabled` toggles merged across the in-scope values files +// (an overlay wins, as with `helm -f`); a file read alone (the scope warning) +// from its own `enabled: false`. +function valuesCollectorConfigs(obj, { disabled = null } = {}) { if (!isMapping(obj)) return []; if (looksLikeOtelCollector(obj)) return [obj]; if (isMapping(obj.config) && looksLikeOtelCollector(obj.config)) return [obj.config]; + const off = (key, v) => (disabled ? disabled.has(key) : v?.enabled === false); return Object.entries(obj) - .filter(([key, v]) => !disabled.has(key) && v?.enabled !== false && isMapping(v?.config) && looksLikeOtelCollector(v.config)) + .filter(([key, v]) => !off(key, v) && isMapping(v?.config) && looksLikeOtelCollector(v.config)) .map(([, v]) => v.config); } @@ -2498,7 +2501,8 @@ function collectorExporterDetail(name, cfg) { if (typeof topic === 'string' && topic.trim()) out.topic = topic.trim(); } else { const endpoint = cfg?.endpoint ?? cfg?.metrics_endpoint; - if (typeof endpoint === 'string' && endpoint.trim()) out.endpoint = safeSourceUrl(endpoint); + const safe = typeof endpoint === 'string' ? safeSourceUrl(endpoint) : ''; + if (safe) out.endpoint = safe; } return out; } diff --git a/tools/lib/exporter-metrics.mjs b/tools/lib/exporter-metrics.mjs index 3bf7e127..194adaac 100644 --- a/tools/lib/exporter-metrics.mjs +++ b/tools/lib/exporter-metrics.mjs @@ -55,16 +55,21 @@ export function isMetricsManifestPath(relPath) { * `-` / `_` and wrapped in `__` or not (`test/`, `__tests__/`, `e2e/`, * `integration-tests/`, `integrationTest/`, `unit_tests/`, `unittests/`, * `testing/`, `__fixtures__/`, `testFixtures/`, `__mocks__/`, `testdata/`, - * `test-data/`, `__snapshots__/`, …), or a + * `test-data/`, `__snapshots__/`, …), a camelCase test source set + * (`componentTest/`, `smokeTest/`, `testUtils/`), Maven's `src/it/`, or a * `*.test.*` / `*.spec.*` / `*_test.*` / `*_spec.*` file — there an * exposition is sample input, not what a deployed exporter serves. */ export function isTestOrFixturePath(relPath) { - const p = String(relPath ?? '').replace(/\\/g, '/').toLowerCase(); - const word = '(?:tests?|specs?|fixtures?|mocks?|testing|e2e|test[-_]?(?:data|fixtures?)|(?:integration|unit|e2e|functional|acceptance)[-_]?tests?)'; + const segs = String(relPath ?? '').replace(/\\/g, '/').split('/'); + const word = '(?:tests?|specs?|fixtures?|mocks?|testing|e2e|test[-_]?(?:data|fixtures?|resources|utils?|support|helpers?)|(?:integration|unit|e2e|functional|acceptance)[-_]?tests?)'; const folder = new RegExp(`^(?:__)?(?:[a-z0-9]+[-_])*${word}(?:[-_][a-z0-9]+)*(?:__)?$|^__snapshots__$`); - return p.split('/').slice(0, -1).some(seg => folder.test(seg)) - || /[._](?:test|spec)\.[a-z]+$/.test(p); + // A camelCase test source set (Gradle: componentTest, smokeTest, intTest, + // testFixtures, testUtils) and Maven's src/it. + const camel = /^[a-z][a-zA-Z0-9]*Tests?$|^tests?[A-Z][a-zA-Z0-9]*$/; + return segs.slice(0, -1).some((seg, i) => folder.test(seg.toLowerCase()) || camel.test(seg) + || (/^(?:it|int)$/i.test(seg) && segs[i - 1]?.toLowerCase() === 'src')) + || /[._](?:test|spec)\.[a-z]+$/i.test(segs[segs.length - 1] || ''); } /** A saved exposition: `.prom` or `.metrics`. */ @@ -77,9 +82,11 @@ export function isExpositionSnapshotPath(relPath) { * credential rides. Textual, never a URL parse: a password may hold `/`, `?` * or `@`, which end a parsed authority early and leave the rest of the secret * in the "host" or "path". So everything before the LAST `@` goes (a value - * whose path or query holds an `@` keeps only what follows it, which loses - * the host rather than risk a credential), then everything from the first - * `?` or `#`. A leading `scheme://` or `//` is kept; a value that is not a + * whose path holds an `@` keeps only what follows it, which loses the host + * rather than risk a credential), then everything from the first `?` or + * `#`. A value with a `?` or `#` before its last `@` (the `@` in a query, or + * a `?` in a password: text cannot tell which) is recorded as nothing (''); + * so is one with nothing left past its scheme. A leading `scheme://` or `//` is kept; a value that is not a * URL at all (`prometheus:9090`, `${env:ENDPOINT}`) passes through the same * rule. */ @@ -91,8 +98,15 @@ export function safeSourceUrl(raw) { const prefix = /^(?:[A-Za-z][A-Za-z0-9+.-]*:)?\/\//.exec(s)?.[0] ?? ''; let rest = s.slice(prefix.length); const at = rest.lastIndexOf('@'); + const query = rest.search(/[?#]/); + // A `?` or `#` before the last `@`: the `@` is in a query or fragment, or + // a password holds the `?` — text cannot tell which, so nothing after the + // scheme is kept. + if (at >= 0 && query >= 0 && query < at) return ''; if (at >= 0) rest = rest.slice(at + 1); - return prefix + rest.replace(/[?#][\s\S]*$/, ''); + rest = rest.replace(/[?#][\s\S]*$/, ''); + // Nothing past the scheme: nothing to record. + return rest ? prefix + rest : ''; } function parseSampleLine(line) { diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 4b5afea7..473800f2 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -202,9 +202,10 @@ const json = (r, key) => JSON.parse(ann(r)[key] ?? 'null'); assert(detectArtefactKind('test/fixtures/ibm-mq.prom', PROM) === 'unknown' && detectArtefactKind('src/x.test.metrics', PROM) === 'unknown' && detectArtefactKind('testdata/a.prom', PROM) === 'unknown' && ['src/__fixtures__/node.prom', 'test-data/node.prom', 'test_fixtures/a.prom', 'x/__mocks__/a.metrics', 'mocks/a.prom', 'spec/a.prom', - 'e2e/mq.prom', 'integration-tests/mq.prom', 'src/testFixtures/resources/node.prom', 'testfixtures/a.prom', 'src/integrationTest/resources/mq.prom', 'unittests/a.prom', 'e2etests/a.prom', 'e2e-tests/mq.prom', 'unit_tests/mq.prom', 'testing/mq.prom', 'src/__snapshots__/mq.prom', 'exp/mq_spec.prom', 'exp/mq.spec.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), + 'e2e/mq.prom', 'integration-tests/mq.prom', 'src/testFixtures/resources/node.prom', 'testfixtures/a.prom', 'src/integrationTest/resources/mq.prom', 'unittests/a.prom', 'e2etests/a.prom', + 'src/componentTest/resources/a.prom', 'src/smokeTest/a.prom', 'src/intTest/a.prom', 'src/systemTest/a.prom', 'src/testUtils/a.prom', 'src/it/resources/a.prom', 'module/testResources/a.prom', 'e2e-tests/mq.prom', 'unit_tests/mq.prom', 'testing/mq.prom', 'src/__snapshots__/mq.prom', 'exp/mq_spec.prom', 'exp/mq.spec.prom'].every(p => detectArtefactKind(p, PROM) === 'unknown'), 'never from a test, fixture or mock path, in any of the usual spellings'); - assert(['snapshots/ibm-mq.prom', 'exporters/latest/mq.prom', 'contest/mq.prom', 'attestation/mq.prom'].every(p => detectArtefactKind(p, PROM) === 'exposition-snapshot'), + assert(['snapshots/ibm-mq.prom', 'exporters/latest/mq.prom', 'contest/mq.prom', 'attestation/mq.prom', 'i18n/it/mq.prom', 'prod/mq.prom', 'Latest/mq.prom'].every(p => detectArtefactKind(p, PROM) === 'exposition-snapshot'), 'a folder that only contains such a word in a longer one is not a test folder'); assert(detectArtefactKind('notes.metrics', 'nothing to see here\n') === 'unknown', 'a .metrics file with no family is not a snapshot'); @@ -325,7 +326,12 @@ service: ['user:pa/ss@otel:4317', 'otel:4317'], ['https://user:pa/ss@otel:4317/v1', 'https://otel:4317/v1'], ['https://user:12/ss@otel/v1', 'https://otel/v1'], - ['https://u:pa?ss@h/x', 'https://h/x'], + // A `?` or `#` before the last `@`: a query holding an `@`, or a password + // holding a `?` — nothing is kept. + ['https://u:pa?ss@h/x', ''], + ['https://mimir/api/v1/push?user=ops@corp.example&api_key=SECRET', ''], + ['https://h/push#frag@SECRET', ''], + ['https://', ''], ['//u:p@h/metrics', '//h/metrics'], ['HTTPS://U:P@H:1/m?t=1#f', 'HTTPS://H:1/m'], ['svc:Zx9/Qk+SECRET==@otel-gw:4317', 'otel-gw:4317'], @@ -336,6 +342,13 @@ service: ]; assert(SECRETS.every(([raw, want]) => safeSourceUrl(raw) === want), 'safeSourceUrl drops everything before the last @ and from the first ? or #', SECRETS.map(([raw]) => safeSourceUrl(raw)), SECRETS.map(([, want]) => want)); + const queryAt = crawlFiles({ + 'otel/collector.yaml': 'receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlphttp:\n endpoint: "https://mimir/push?user=ops@corp.example&api_key=SECRETQ"\n', + 'metrics-manifest.json': JSON.stringify({ exporter: 'x', source: 'http://mq/metrics?u=a@b&token=SECRETR', metrics: [{ name: 'm', type: 'gauge' }] }), + }, { repoName: 'col', now: NOW }); + assert(!JSON.stringify(queryAt).includes('SECRET') && !('endpoint' in json(queryAt, 'crawler.discovered.scrape_job_origins').mq.exporters[0]) + && !('source' in json(queryAt, 'crawler.discovered.exporters').x.sources[0]), + 'an endpoint or source whose query holds an @ is not recorded at all', [json(queryAt, 'crawler.discovered.scrape_job_origins').mq, json(queryAt, 'crawler.discovered.exporters')]); const leak = crawlFiles({ 'otel/collector.yaml': `receivers:\n prometheus:\n config:\n scrape_configs:\n - job_name: mq\n static_configs: [{ targets: ["a:1"] }]\nexporters:\n otlp:\n endpoint: "svc:Zx9/Qk+SECRETPW@otel-gw:4317"\n` }, { repoName: 'col', now: NOW }); assert(!JSON.stringify(leak.canonical).includes('SECRET') && json(leak, 'crawler.discovered.scrape_job_origins').mq.exporters[0].endpoint === 'otel-gw:4317', 'a collector endpoint whose password holds `/` reaches the pack without it', json(leak, 'crawler.discovered.scrape_job_origins').mq); @@ -452,6 +465,9 @@ data: const sub = 'opentelemetry-collector:\n enabled: ENABLED\n config:\n' + COL.split('\n').map(l => (l ? ` ${l}` : l)).join('\n'); const off = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'false') }, { repoName: 'vals', now: NOW }); const overlay = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'true'), 'charts/u/values-prod.yaml': 'opentelemetry-collector:\n enabled: false\n' }, { repoName: 'vals', environment: 'prod', now: NOW }); + // The overlay wins both ways: re-enabled by values-prod.yaml, it is read. + const reenabled = crawlFiles({ 'charts/u/values.yaml': sub.replace('ENABLED', 'false'), 'charts/u/values-prod.yaml': 'opentelemetry-collector:\n enabled: true\n' }, { repoName: 'vals', environment: 'prod', now: NOW }); + assert(JSON.stringify(json(reenabled, 'crawler.discovered.scrape_jobs')) === '["qmgrs"]', 'a subchart an overlay re-enables has its collector read', ann(reenabled)['crawler.discovered.scrape_jobs']); assert(!('crawler.discovered.scrape_jobs' in ann(off)) && !('crawler.discovered.scrape_jobs' in ann(overlay)) && off.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite' && overlay.canonical.spec.pipelines.exporters.metrics.kind === 'prometheusremotewrite', 'a disabled umbrella subchart\'s collector is not read (same file, prod overlay)', [ann(off)['crawler.discovered.scrape_jobs'], ann(overlay)['crawler.discovered.scrape_jobs']]); @@ -585,6 +601,9 @@ try { '`packc emit` dispatches; `--flag=value` is read; --out - writes to stdout; the host names the exporter; --no-timestamp leaves the stamp out', viaPackc); writeFileSync(join(tmp, 'qm1.prom'), EXPOSITION); + writeFileSync(join(tmp, 'node@1.0.prom'), EXPOSITION); + const atName = await run(['--from', 'node@1.0.prom', '--out', '-', '--no-timestamp']); + assert(atName.status === 0 && JSON.parse(atName.stdout).exporter === 'node@1.0', 'a --from file whose name holds an @ is read', atName); const from = await run(['--from', 'qm1.prom', '--out', '-', '--no-timestamp']); assert(from.status === 0 && JSON.parse(from.stdout).exporter === 'qm1' && !('source' in JSON.parse(from.stdout)), '--from reads a saved exposition, named by the file', from); @@ -601,6 +620,9 @@ try { [['http://user:SECRET4@127.0.0.1:1/metrics?token=SECRET5'], 2, null], [['mq:9157/metrics?token=SECRET8'], 2, null], [['--from', 'http://svc:SECRET9@mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], + [['--from', 'x.prom', '--generated-at', 'http://u:SECRETG@h'], 2, 'packc emit: --generated-at (a URL-like value, not shown) is not a date\n'], + [['--from', 'qm1.prom', '--out', 'http://u:SECRETO@h/x/y'], 2, 'packc emit: cannot write --out (a URL-like value, not shown): ENOENT\n'], + [['--from', 'qm1.prom', '--out', 'no/such/dir/m.json'], 2, 'packc emit: cannot write --out no/such/dir/m.json: ENOENT\n'], [['--from=mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], [['--exporter=http://user:SECRET6@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'], [['--exporter', `${base}/cut?token=SECRET7`], 1, `packc emit: could not read the response of ${base}/cut: the connection ended before the body did\n`], From 46f854f3a10c89c2ce09cdef17841c6bc87dadeb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 19:59:53 +0000 Subject: [PATCH 17/17] github crawl and packc emit: the last echo and test-folder gaps - isExporterCandidateFile lowercased the path before the test-folder rule, so componentTest/, smokeTest/ and intTest/ snapshots were fetched (and could use up the 200-file budget) though the crawler then ignored them. - packc emit named a URL-like --from or --out in three messages (no metric family, cannot read, the success line), and a redirect's bare ? as a query key; the query is named by its key=value keys only. - CHANGELOG: only a path '@' keeps what follows it (a query '@' records nothing since ff78d6d). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UhPGnqVJ24goSVtbKpdkLJ --- docs/CHANGELOG.md | 2 +- docs/DOWNSTREAM.md | 4 ++-- server/github-crawl.mjs | 3 ++- tools/emit-manifest.mjs | 26 +++++++++++++++++--------- tools/test-crawl-exporters.mjs | 15 +++++++++++++-- 5 files changed, 35 insertions(+), 15 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 941fd65d..96c133c6 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -12,7 +12,7 @@ - **An environment scope says which collection evidence it leaves out** (**a declared default change**: a warning, emitted only when such a file is excluded). A manifest and a snapshot are read in every environment. A file the scope excludes that carries scrape jobs (a `scrape_configs` file, a collector's prometheus receiver, a Helm template that carries either, a values file that carries a collector config) or an exporter config is named in a warning, with the way to read it: an environment that names no deployment profile, e.g. `--env all`. - **`packc emit --exporter `** (`tools/emit-manifest.mjs`, `tools/cli.mjs`) scrapes an exporter's /metrics once and writes `metrics-manifest.json` (`--name`, `--version`, `--out |-`, `--generated-at` / `--no-timestamp`, `--timeout`). `--from ` reads a saved exposition. A redirect, a URL with credentials, a status other than 200, and a response with no family are refused, each naming its way out. - No golden changed: the golden crawl, compile, diff, snapshot and audit-report outputs are byte-identical, and a crawl with no exporter evidence and no collector scrape job carries none of the new keys. `tools/test-crawl-exporters.mjs` is new: the fixture repository `tools/fixtures/crawl/exporters/` (an mq-metric-samples config, its manifest, and a chart collector with a `qmgrs` job and a kafka exporter) yields the manifest's metrics, the job and the kafka exporter; the manifest outranks the config. `tools/test-platform.mjs` P4 pins the one new `.pathname` use, the fake exporter's request path (a URL path). -- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value with a `?` or `#` before its last `@` is recorded as nothing: the `@` may be in a query (`?user=ops@corp&api_key=…`) or a `?` in a password, and text cannot tell which. An endpoint or source that leaves nothing past its scheme is left out. A value whose path or query holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument, a `--from`, `--out` or `--generated-at` value that looks like a URL (`://`, `@`, `?`, or `host:port`). `--from` takes a file (a file whose name holds `@` is read); a URL-like value that is no file is refused. An `--out` it cannot write is refused, not left as a stack trace. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as keys only (`?token=…`), and a `Location` that is not a URL is named as such. Neither one is followed. +- **Nothing secret is recorded or echoed.** `safeSourceUrl` is textual, never a URL parse, because a password may hold `/`, `?` or `@`, which end a parsed authority early and leave the secret in the "host". It drops tabs and line breaks (a URL parser drops them, so the value would still send its query), then everything before the last `@`, then everything from the first `?` or `#` to the end, and keeps a leading `scheme://` or `//`. A value with a `?` or `#` before its last `@` is recorded as nothing: the `@` may be in a query (`?user=ops@corp&api_key=…`) or a `?` in a password, and text cannot tell which. An endpoint or source that leaves nothing past its scheme is left out. A value whose path holds an `@` keeps only what follows it, losing the host rather than risking a credential. `packc emit` echoes an `--exporter` it cannot parse only in that form. It reads `--flag=value` as well as `--flag value`, and never echoes an unknown argument, a `--from`, `--out` or `--generated-at` value that looks like a URL (`://`, `@`, `?`, or `host:port`). `--from` takes a file (a file whose name holds `@` is read); a URL-like value that is no file is refused. An `--out` it cannot write is refused, not left as a stack trace. A response body that ends early or stalls is refused like a failed scrape, not left as a stack trace. A redirect is named by its target with the query as the keys of its `key=value` parts only (`?token=…`; a bare `?` is named as "a query part"), and a `Location` that is not a URL is named as such. Neither one is followed. ### Follow-up `windows-ci-races`: races the Windows leg lost under load, and the same races beside them - **`tools/test-bundle-compare.mjs` C6: a multipart request to the child server leaves no tail unread** (a test fix; no runtime, `tools/lib`, golden or output change). C6 failed the `windows (node 22)` job of run 38737724941 (`not ok 810 - C6 retrofeed in the shim`, `fetch failed` at the `serverFetch` call in `same()`) on the tree that had passed at 7950b03. Its `multipart/form-data is not parsed` case POSTs a `FormData`, which fetch streams in parts: the headers and the first part in one write, the closing boundary in a later one. The server parses no multipart, so retrofeed answers its 404 from the headers alone, and with `connection: close` (which `serverFetch` sets since the C3 keep-alive fix) Node's server closes the socket right after the answer, the closing boundary unread: its TCP stack sends an RST behind the 404. Linux still hands the client the 404; Windows discards bytes the client had not read when the RST lands (`WSAECONNRESET`), and fetch, with no status line yet, fails. `serverFetch`, in this file and in `tools/test-bundle-compare-pairs.mjs` (the same text, C3 pins it), now builds the request through `new Request()` and hands fetch its body as one `ArrayBuffer`, so the body leaves in the headers' write and the server reads all of it before it closes; the bytes and the Content-Type, boundary included, are the ones fetch would send, the shim still receives the original `FormData`, and a GET sends no body. On Linux (node 22.22.2), 40 multipart requests to a real child server made the server's kernel send 26–29 RSTs before and 0 after; through a relay that discards unread data on an RST as Winsock does, `TypeError: fetch failed` (cause `ECONNRESET`) came back 1–5 times in 40 before and 0 after, and C6's other request kinds 0 either way. Under four busy loops, ten runs of the 40: 6–16 RSTs and 3–10 failures in each before, 0 and 0 in each after. C3 now also pins it: a `FormData` goes to fetch as one buffer, with the bytes fetch serializes it to, `connection: close`, and a GET with a null body — an assertion in an existing test (it fails against the previous `serverFetch`), so Suite total 1519 → 1519. diff --git a/docs/DOWNSTREAM.md b/docs/DOWNSTREAM.md index 8cf1baa5..6d5b333a 100644 --- a/docs/DOWNSTREAM.md +++ b/docs/DOWNSTREAM.md @@ -1890,8 +1890,8 @@ packc emit --from saved.prom --name ibm-mq --out - # a saved exposition, to It scrapes once and records each family's name, type, help and label names. A redirect is refused, not followed: the refusal names its target with the -query as keys only (`?token=…`) — pass that URL with the values the exporter -expects. A URL with credentials is refused (save the exposition with your own +query as the keys of its `key=value` parts only (`?token=…`) — pass that URL +with the values the exporter expects. A URL with credentials is refused (save the exposition with your own client and pass it with `--from`). A status other than 200 or a response with no metric family exits 1. diff --git a/server/github-crawl.mjs b/server/github-crawl.mjs index 5debea0f..b8f2935c 100644 --- a/server/github-crawl.mjs +++ b/server/github-crawl.mjs @@ -64,7 +64,8 @@ export function isExporterCandidateFile(path) { if (typeof path !== 'string' || isCrawlerFile(path)) return false; const p = path.toLowerCase(); if (p.includes('node_modules/') || p.startsWith('.git/') || p.includes('/.git/')) return false; - return (/\.(prom|metrics)$/.test(p) && !isTestOrFixturePath(p)) + // The test-folder rule reads the path as written (camelCase source sets). + return (/\.(prom|metrics)$/.test(p) && !isTestOrFixturePath(path)) || /(^|\/)config\.ya?ml$/.test(p) || /(^|\/)files\/.*\.ya?ml$/.test(p); } diff --git a/tools/emit-manifest.mjs b/tools/emit-manifest.mjs index dc523841..2049996c 100644 --- a/tools/emit-manifest.mjs +++ b/tools/emit-manifest.mjs @@ -28,7 +28,7 @@ * * The recorded `source` is the URL without userinfo, query or fragment * (tools/lib/exporter-metrics.mjs safeSourceUrl). A redirect is refused, not - * followed: the refusal names its target, the query as keys only — pass that + * followed: the refusal names its target, the query as `key=value` keys only — pass that * URL with the values the exporter expects. */ @@ -42,6 +42,8 @@ const shown = (url) => safeSourceUrl(url) || 'the exporter URL (not shown)'; // A value that may be a URL with its credentials or token: never echoed. const urlLike = (v) => /:\/\/|@|\?|^[^/\\\s]+:\d+(?:[/?#]|$)/.test(String(v ?? '')); +// A path argument as a message may name it. +const showArg = (v) => (urlLike(v) ? '(a URL-like value, not shown)' : v); const USAGE = 'usage: packc emit --exporter | --from [--name ] [--version ] [--out |-] [--generated-at | --no-timestamp] [--timeout ]'; @@ -72,16 +74,22 @@ function parseArgs(argv) { } // A redirect is named, never followed: its target without userinfo, and its -// query as keys only (`?format=…`) — a value may be a token — so the URL to +// query as the keys of `key=value` parts only (`?format=…`; a bare `?` is +// a value) — a value may be a token — so the URL to // pass is the one shown with the values the exporter expects. function redirectRefusal(url, status, location) { let target = null; try { target = location ? new URL(location, url) : null; } catch { /* not a URL: named as such */ } - const keys = target ? [...new Set([...target.searchParams.keys()])] : []; + // Keys of `key=value` parts only: a bare `?` is a value, not a key. + const parts = target?.search ? target.search.slice(1).split('&').filter(Boolean) : []; + const keys = [...new Set(parts.filter(p => p.includes('=')).map(p => p.slice(0, p.indexOf('='))))]; + const bare = parts.some(p => !p.includes('=')); + const query = !parts.length ? '' + : `${keys.length ? `?${keys.map(k => `${k}=…`).join('&')}` : ''}${bare ? `${keys.length ? ' and' : ' with'} a query part` : ''} (query values not shown)`; const to = !location ? '' : !target ? ' to a location that is not a URL' - : ` to ${shown(target.href)}${keys.length ? `?${keys.map(k => `${encodeURIComponent(k)}=…`).join('&')} (query values not shown)` : ''}`; - return `${shown(url)} answered ${status}, a redirect${to}; it is not followed — pass the URL the exporter serves its metrics at${keys.length ? ', with the query values it expects' : ''}`; + : ` to ${shown(target.href)}${query}`; + return `${shown(url)} answered ${status}, a redirect${to}; it is not followed — pass the URL the exporter serves its metrics at${parts.length ? ', with the query values it expects' : ''}`; } async function scrape(url, timeoutSeconds) { @@ -131,20 +139,20 @@ if (args.exporter) { // A file whose name holds `@` or `?` is read; a URL-like value that is not // a file is refused unshown. if (urlLike(args.from) && !existsSync(resolve(args.from))) fail(2, '--from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter'); - try { text = readFileSync(resolve(args.from), 'utf8'); } catch (e) { fail(2, `cannot read --from ${args.from}: ${e.code || e.message}`); } + try { text = readFileSync(resolve(args.from), 'utf8'); } catch (e) { fail(2, `cannot read --from ${showArg(args.from)}: ${e.code || 'read failed'}`); } defaultName = basename(args.from).replace(/\.[^.]+$/, ''); } const name = String(args.name ?? defaultName ?? '').trim(); if (!name) fail(2, 'the exporter has no name; pass --name'); const manifest = manifestFromExposition(text, { exporter: name, version: args.version, source, generatedAt }); -if (!manifest.metrics.length) fail(1, `${source ? shown(source) : args.from} holds no metric family (no # TYPE, # HELP or sample line)`); +if (!manifest.metrics.length) fail(1, `${source ? shown(source) : showArg(args.from)} holds no metric family (no # TYPE, # HELP or sample line)`); const json = `${JSON.stringify(manifest, null, 2)}\n`; if (args.out === '-') { process.stdout.write(json, () => process.exit(0)); } else { try { writeFileSync(resolve(args.out), json); } - catch (e) { fail(2, `cannot write --out ${urlLike(args.out) ? '(a URL-like value, not shown)' : args.out}: ${e.code || 'write failed'}`); } - process.stdout.write(`wrote ${args.out}: ${manifest.metrics.length} metric famil${manifest.metrics.length === 1 ? 'y' : 'ies'} of ${name}\n`, () => process.exit(0)); + catch (e) { fail(2, `cannot write --out ${showArg(args.out)}: ${e.code || 'write failed'}`); } + process.stdout.write(`wrote ${showArg(args.out)}: ${manifest.metrics.length} metric famil${manifest.metrics.length === 1 ? 'y' : 'ies'} of ${name}\n`, () => process.exit(0)); } diff --git a/tools/test-crawl-exporters.mjs b/tools/test-crawl-exporters.mjs index 473800f2..cb02ee65 100644 --- a/tools/test-crawl-exporters.mjs +++ b/tools/test-crawl-exporters.mjs @@ -23,7 +23,7 @@ * Exit 0 = pass. */ -import { readFileSync, readdirSync, statSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { readFileSync, readdirSync, statSync, mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { join, relative } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; @@ -551,6 +551,8 @@ data: const all = selectCrawlerFiles(tree, { maxFiles: 1000, fileCapBytes: 1e6 }).map(n => n.path); assert(all.includes('snap/ibm-mq.prom') && all.includes('charts/c/files/collector.yaml') && !all.includes('test/fixtures/x.prom') && !all.includes('big/config.yaml') && !all.includes('apps'), 'with room, a snapshot and a chart\'s files/ collector are fetched; a fixture snapshot, an oversized file and a tree never'); + assert(!isExporterCandidateFile('src/componentTest/resources/node.prom') && !isExporterCandidateFile('src/smokeTest/a.prom') && isExporterCandidateFile('snap/a.prom'), + 'the GitHub crawl skips a camelCase test source set\'s snapshot, as the crawler does'); assert(isCrawlerFile('mq/metrics-manifest.json') && !isCrawlerFile('apps/x/config.yaml') && isExporterCandidateFile('apps/x/config.yaml') && !isExporterCandidateFile('monitoring/prometheus.yml'), 'isCrawlerFile is develop\'s rule plus the manifest; the broad names are candidates only'); } @@ -564,6 +566,7 @@ const server = createServer((req, res) => { if (path === '/moved') { res.writeHead(302, { location: '/metrics?token=abc' }); res.end(); return; } if (path === '/empty') { res.writeHead(200); res.end('\n'); return; } if (path === '/cut') { res.writeHead(200, { 'content-length': '1000' }); res.write('# TYPE x gauge\n'); setTimeout(() => res.destroy(), 50); return; } + if (path === '/bare') { res.writeHead(302, { location: '/m?SECRETBARE&format=prom' }); res.end(); return; } if (path === '/badloc') { res.writeHead(302, { location: 'http://[oops/metrics' }); res.end(); return; } res.writeHead(500); res.end('boom'); }); @@ -602,6 +605,10 @@ try { writeFileSync(join(tmp, 'qm1.prom'), EXPOSITION); writeFileSync(join(tmp, 'node@1.0.prom'), EXPOSITION); + writeFileSync(join(tmp, 'svcSECRETF@h.prom'), '\n'); + mkdirSync(join(tmp, 'svcSECRETD@h')); + const named = await run(['--from', 'qm1.prom', '--out', 'outSECRETO@h.json', '--no-timestamp']); + assert(named.status === 0 && named.stdout === 'wrote (a URL-like value, not shown): 2 metric families of qm1\n', 'the success line names no URL-like --out', named); const atName = await run(['--from', 'node@1.0.prom', '--out', '-', '--no-timestamp']); assert(atName.status === 0 && JSON.parse(atName.stdout).exporter === 'node@1.0', 'a --from file whose name holds an @ is read', atName); const from = await run(['--from', 'qm1.prom', '--out', '-', '--no-timestamp']); @@ -621,7 +628,11 @@ try { [['mq:9157/metrics?token=SECRET8'], 2, null], [['--from', 'http://svc:SECRET9@mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], [['--from', 'x.prom', '--generated-at', 'http://u:SECRETG@h'], 2, 'packc emit: --generated-at (a URL-like value, not shown) is not a date\n'], - [['--from', 'qm1.prom', '--out', 'http://u:SECRETO@h/x/y'], 2, 'packc emit: cannot write --out (a URL-like value, not shown): ENOENT\n'], + // The error code is the platform's (a ':' in a Windows path is invalid): only the exit and the unshown value are pinned. + [['--from', 'qm1.prom', '--out', 'http://u:SECRETO@h/x/y'], 2, null], + [['--exporter', `${base}/bare`], 1, `packc emit: ${base}/bare answered 302, a redirect to ${base}/m?format=… and a query part (query values not shown); it is not followed — pass the URL the exporter serves its metrics at, with the query values it expects\n`], + [['--from', 'svcSECRETF@h.prom', '--out', '-'], 1, 'packc emit: (a URL-like value, not shown) holds no metric family (no # TYPE, # HELP or sample line)\n'], + [['--from', 'svcSECRETD@h'], 2, null], // a directory: the code is the platform's [['--from', 'qm1.prom', '--out', 'no/such/dir/m.json'], 2, 'packc emit: cannot write --out no/such/dir/m.json: ENOENT\n'], [['--from=mq:9157/metrics'], 2, 'packc emit: --from takes a saved exposition file, not a URL (the value is not shown); scrape a URL with --exporter\n'], [['--exporter=http://user:SECRET6@127.0.0.1:1/metrics'], 2, 'packc emit: the --exporter URL carries credentials, which are neither sent nor recorded; save the exposition with your own client and pass it with --from\n'],