From 571c8e7c1569079bd9074eadded64c55a89d4f83 Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Sun, 15 Nov 2026 10:43:49 +0530 Subject: [PATCH 1/2] Ingester: store TestClient traffic as environment test, forward the running commit A FastAPI/Starlette test suite that runs with an exporter and no environment sent its errors as production. A resourceSpans entry with no deployment.environment(.name) now stores a request's trace as `test` when a SERVER span of it carries the exact user agent `testclient` (read before the identifying drop, which adds `autter.test_client: true` and still drops the user agent) or the exact host `testserver`. Per trace, never per process: both are client-controlled headers. An explicit environment is never rewritten. /v1/logs records follow the same rule from their own attributes. Occurrences now carry `commit` (deployment.commit, else vcs.ref.head.revision; event, then span or record, then resource), only when 7-40 hex, lower-cased, and copy it into the attributes as deployment.commit when absent. The sink emits it only when valid. Fixtures recorded from the FastAPI recipe app under TestClient (test-fixtures/python/record_fastapi_testclient.py). Co-Authored-By: Claude Opus 5.5 --- packages/otlp-ingester/README.md | 46 ++ .../otlp-ingester/src/identifying.test.ts | 108 +++++ packages/otlp-ingester/src/identifying.ts | 62 ++- packages/otlp-ingester/src/logs.test.ts | 60 +++ packages/otlp-ingester/src/logs.ts | 29 +- packages/otlp-ingester/src/normalize-otlp.ts | 92 +++- packages/otlp-ingester/src/normalize.test.ts | 152 +++++++ .../otlp-ingester/src/python-otel.test.ts | 74 ++++ .../otlp-ingester/src/server.sink.test.ts | 45 ++ packages/otlp-ingester/src/sink.test.ts | 10 + packages/otlp-ingester/src/sink.ts | 22 +- packages/otlp-ingester/src/types.ts | 37 ++ .../fastapi-testclient-staging.otlp.json | 417 ++++++++++++++++++ .../python/fastapi-testclient-staging.otlp.pb | Bin 0 -> 5763 bytes .../python/fastapi-testclient.otlp.json | 411 +++++++++++++++++ .../python/fastapi-testclient.otlp.pb | Bin 0 -> 5726 bytes .../python/record_fastapi_testclient.py | 99 +++++ 17 files changed, 1631 insertions(+), 33 deletions(-) create mode 100644 packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json create mode 100644 packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.pb create mode 100644 packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.json create mode 100644 packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.pb create mode 100644 packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 0c5b887..96165ce 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -227,6 +227,52 @@ hashing every IPv4 address) and the ingester warns at startup. A hashed IP is still pseudonymous personal data in some readings of privacy law: it is off by default, and the salt should be kept and rotated like any secret. +## Test-client traffic (unreleased) + +A test suite that drives a FastAPI or Starlette app through its in-process +`TestClient`, with an exporter configured and no environment set, used to +send its errors as `production`. The ingester now stores them as +environment `test`, which the platform leaves out of its default views, +alerts and incidents: + +- Only when the resource names no environment (`deployment.environment.name` + or `deployment.environment`). An explicit environment, `staging` or even + `production`, is never changed. +- Only on strong evidence, on a SERVER span: the exact user agent + `testclient` or the exact host `testserver` (`server.address`, + `net.host.name`, `http.host` or `http.server_name`), which is what + Starlette's `TestClient` sends. A user agent that merely contains + "testclient", or a host such as `testserver.example.com`, does not count. +- Per request: every span and error of that request's trace is stored as + `test`; the process's other requests in the same export are not. The user + agent and the host are headers any client can send, so a request that + claims to be a test client only reclassifies itself. + +The raw user agent is read before the identifying attributes are dropped: +the server span gets `autter.test_client: true` and the user agent itself is +still dropped. The Python adapter (adapters/python) drops the user agent +before export since Runtime 1.19.0, so for it the host is the evidence. A `/v1/logs` record is +classified the same way only from its own attributes (the exact host +`testserver`, or `autter.test_client: true`), and only when its resource +names no environment. + +Known limit: the same process's `/v1/metrics` (the request-duration +histogram that counts requests) carries nothing per request to tell a test +client apart, so its request counts still default to `production`. Request +counts and error rates for that process are exact only once the SDK sets +`deployment.environment=test` itself under a test runner. + +## Running commit (unreleased) + +Each occurrence forwarded to the sink carries `commit` when the app reports +the commit it runs: `deployment.commit`, else `vcs.ref.head.revision`, read +from the exception event, then the span (or log record), then the resource. +Only a value of 7 to 40 hexadecimal characters is forwarded, lower-cased; +anything else (a repository URL, a branch name) is dropped, and nothing else +of the resource is forwarded. When the occurrence's attributes have no +`deployment.commit`, the SHA is also stored there. The field is optional on +sink payload version 1: a platform that does not know it ignores it. + ## Rate limits and key lookups (1.12.0) In the order a request meets them: diff --git a/packages/otlp-ingester/src/identifying.test.ts b/packages/otlp-ingester/src/identifying.test.ts index f6f686d..be3232c 100644 --- a/packages/otlp-ingester/src/identifying.test.ts +++ b/packages/otlp-ingester/src/identifying.test.ts @@ -13,6 +13,7 @@ import { IdentifyingAttributeFilter, identifyingKeysToDrop, keepIdentifyingEnv, + clientIpHasher, } from "./identifying.js"; import { createIngesterApp } from "./server.js"; @@ -117,6 +118,49 @@ test("keep (true or a list) and extra drops; env parsing", () => { // --------------------------------------------------------------------------- // Through the ingester: a real Python OpenTelemetry export (protobuf, Flask // instrumentation: net.peer.ip, net.peer.port, http.user_agent) and a plain +test("test client: a SERVER span whose raw user agent is exactly `testclient` gets autter.test_client; the user agent is still dropped", () => { + const filter = new IdentifyingAttributeFilter(); + const span = (kind: number | string, attributes: Record) => ({ kind, name: "GET /", attributes: kv(attributes) }); + const request = { + resourceSpans: [{ + scopeSpans: [{ + spans: [ + span(2, { "http.user_agent": "testclient", "http.host": "testserver" }), + span("SPAN_KIND_SERVER", { "user_agent.original": "testclient" }), + span(1, { "http.user_agent": "testclient" }), + span(3, { "user_agent.original": "testclient" }), + span(2, { "user_agent.original": "Mozilla/5.0 testclient" }), + span(2, { "http.user_agent": "TestClient" }), + ], + }], + }], + }; + const before = JSON.stringify(request); + const spans = filter.traces(request).resourceSpans[0]!.scopeSpans[0]!.spans; + assert.equal(JSON.stringify(request), before, "input not mutated"); + assert.deepEqual(spans.map((s) => keysOf(s.attributes)), [ + ["http.host", "autter.test_client"], + ["autter.test_client"], + [], [], [], [], + ]); + assert.deepEqual((spans[0]!.attributes as Array<{ value: unknown }>)[1]!.value, { boolValue: true }); + + // Already marked (an SDK did it): not added twice. With the client IP hash too. + const hashing = new IdentifyingAttributeFilter(identifyingKeysToDrop(), clientIpHasher(true, "s".repeat(16))); + const marked = hashing.traces({ resourceSpans: [{ scopeSpans: [{ spans: [ + { kind: 2, attributes: [...kv({ "http.user_agent": "testclient", "client.address": "203.0.113.9" }), { key: "autter.test_client", value: { boolValue: true } }] }, + { kind: 2, attributes: kv({ "http.user_agent": "testclient", "client.address": "203.0.113.9" }) }, + ] }] }] }).resourceSpans[0]!.scopeSpans[0]!.spans; + assert.deepEqual(marked.map((s) => keysOf(s.attributes)), [ + ["autter.test_client", "client.address_hash"], + ["client.address_hash", "autter.test_client"], + ]); + + // Log records are never marked. + const logs = filter.logs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ attributes: kv({ "user_agent.original": "testclient" }) }] }] }] }); + assert.deepEqual(keysOf(logs.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes), []); +}); + // OTLP/JSON logs export, against a fake ClickHouse. const listen = async (server: Server) => { @@ -206,3 +250,67 @@ test("keepIdentifyingAttributes (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) stores them assert.ok(!storedKeys(rows, "runtime_spans").has("net.peer.ip")); }); }); + +// FastAPI's TestClient, recorded (test-fixtures/python/record_fastapi_testclient.py). +const TESTCLIENT = readFileSync(new URL("../test-fixtures/python/fastapi-testclient.otlp.pb", import.meta.url)); +const TESTCLIENT_STAGING = readFileSync(new URL("../test-fixtures/python/fastapi-testclient-staging.otlp.pb", import.meta.url)); +/** Waits (up to 2 s) for the inserts the ingester makes after answering + * (SDK sightings, ingester info), so closing the fake ClickHouse does not + * reset them mid-request. Quiets the log only: nothing is asserted on them. */ +async function backgroundInserts(rows: Record) { + const deadline = Date.now() + 2000; + while (!(rows.runtime_sdk_versions && rows.runtime_ingester_info) && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)); +} +const attributesOf = (row: Record) => JSON.parse(String(row.attributes ?? "{}")) as Record; + +test("TestClient traffic with no environment is stored as `test`: the server spans carry autter.test_client and no user agent", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200); + const spans = rows.runtime_spans ?? []; + assert.equal(spans.length, 6); + assert.deepEqual(new Set(spans.map((row) => row.environment)), new Set(["test"])); + const servers = spans.filter((row) => row.kind === "server"); + assert.equal(servers.length, 2); + for (const row of servers) { + const attributes = attributesOf(row); + assert.equal(attributes["autter.test_client"], true); + assert.equal(attributes["http.user_agent"], undefined, "the user agent is still dropped"); + assert.equal(attributes["user_agent.original"], undefined); + assert.equal(attributes["http.host"], "testserver", "the host is kept"); + } + const occurrences = rows.runtime_error_occurrences ?? []; + assert.deepEqual(occurrences.map((row) => row.environment), ["test", "test"]); + for (const row of occurrences) { + assert.equal(attributesOf(row)["autter.test_client"], true); + assert.equal(attributesOf(row)["http.user_agent"], undefined); + } + for (const table of ["runtime_spans", "runtime_error_occurrences"]) + for (const key of vectors.drop) assert.ok(!storedKeys(rows, table).has(key), `${key} stored in ${table}`); + await backgroundInserts(rows); + }); +}); + +test("the same TestClient export with deployment.environment=staging stays `staging`", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", TESTCLIENT_STAGING, "application/x-protobuf"), 200); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["staging"])); + assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["staging", "staging"]); + // The request still came from a test client, and says so. + for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) + assert.equal(attributesOf(row)["autter.test_client"], true); + await backgroundInserts(rows); + }); +}); + +test("with the user agent kept (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) TestClient traffic is still `test`", async () => { + await withIngester({ keepIdentifyingAttributes: true }, async (post, rows) => { + assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["test"])); + for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) { + assert.equal(attributesOf(row)["http.user_agent"], "testclient"); + assert.equal(attributesOf(row)["autter.test_client"], true); + } + await backgroundInserts(rows); + }); +}); diff --git a/packages/otlp-ingester/src/identifying.ts b/packages/otlp-ingester/src/identifying.ts index f0e3d23..21ebf2b 100644 --- a/packages/otlp-ingester/src/identifying.ts +++ b/packages/otlp-ingester/src/identifying.ts @@ -121,17 +121,25 @@ export class IdentifyingAttributeFilter { private readonly hasher: ClientIpHasher | null = null, ) {} - /** The attributes of a span or log record: the client IP hash added (from the raw list), then the drop. */ + /** + * The attributes of a span or log record: what is read from the raw list + * added first (the client IP hash; on a SERVER span, the test-client + * marker), then the drop. + */ private recordAttributes = (obj: unknown, serverSpan: boolean): unknown => { + if (!isObject(obj)) return this.attributesOf(obj); + const added: Json[] = []; const hasher = this.hasher; - if (!hasher || !isObject(obj) || hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) return this.attributesOf(obj); - const ip = clientIpOfList(obj.attributes, serverSpan); + if (hasher && !hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) { + const ip = clientIpOfList(obj.attributes, serverSpan); + if (ip) added.push({ key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }); + } + if (serverSpan && testClientUserAgentIn(obj.attributes) && !hasKey(obj.attributes, TEST_CLIENT_ATTRIBUTE)) { + added.push({ key: TEST_CLIENT_ATTRIBUTE, value: { boolValue: true } }); + } const filtered = this.attributesOf(obj); - if (!ip || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered; - return { - ...filtered, - attributes: [...filtered.attributes, { key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }], - }; + if (!added.length || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered; + return { ...filtered, attributes: [...filtered.attributes, ...added] }; }; /** An OTLP key/value list without the dropped keys; the same array when none is present. */ @@ -149,6 +157,8 @@ export class IdentifyingAttributeFilter { /** A traces request without identifying attributes (resource, spans, events, links). */ traces(request: T): T { + // Nothing dropped: the raw user agent stays, and the normaliser reads + // it itself (isTestClientRequest). if (!this.dropped.size && !this.hasher) return request; const span = (item: unknown) => eachIn( @@ -262,3 +272,39 @@ export function clientIpOfList(list: unknown, serverSpan: boolean): string | nul /** OTLP SERVER span kind, as a number or the enum name. */ export const isServerKind = (kind: unknown) => kind === 2 || kind === "SPAN_KIND_SERVER"; + +// --------------------------------------------------------------------------- +// Test-client traffic + +/** + * On a SERVER span: the request came from a framework's in-process test + * client. Added by the ingester from the raw user agent before the drop + * (the user agent itself is still dropped), or sent by an SDK. + */ +export const TEST_CLIENT_ATTRIBUTE = "autter.test_client"; +/** The user agent Starlette's TestClient (and FastAPI's, which is Starlette's) sends, exactly. */ +export const TEST_CLIENT_USER_AGENT = "testclient"; +/** The host Starlette's TestClient requests (`http://testserver`), exactly. */ +export const TEST_CLIENT_HOST = "testserver"; +const USER_AGENT_KEYS: readonly string[] = ["user_agent.original", "http.user_agent"]; +/** Host attributes, none of them identifying (they name the app's own host). */ +export const TEST_CLIENT_HOST_KEYS: readonly string[] = ["server.address", "net.host.name", "http.host", "http.server_name"]; + +/** The raw OTLP attribute list names the test client's user agent, exactly. */ +function testClientUserAgentIn(list: unknown): boolean { + return USER_AGENT_KEYS.some((key) => stringAttribute(list, key) === TEST_CLIENT_USER_AGENT); +} + +/** + * Strong evidence, on one span or log record, that it is a test client's + * request: the marker, the exact test-client user agent (when it was kept), + * or the exact test-client host. Exact values only: a real user agent that + * merely contains "testclient", or a host such as `testserver.example.com`, + * is not evidence. + */ +export function isTestClientRequest(get: (key: string) => unknown): boolean { + const marker = get(TEST_CLIENT_ATTRIBUTE); + if (marker === true || marker === "true") return true; + if (USER_AGENT_KEYS.some((key) => get(key) === TEST_CLIENT_USER_AGENT)) return true; + return TEST_CLIENT_HOST_KEYS.some((key) => get(key) === TEST_CLIENT_HOST); +} diff --git a/packages/otlp-ingester/src/logs.test.ts b/packages/otlp-ingester/src/logs.test.ts index cb74072..1d47239 100644 --- a/packages/otlp-ingester/src/logs.test.ts +++ b/packages/otlp-ingester/src/logs.test.ts @@ -230,3 +230,63 @@ test("server exceptions' autter.debug_ids locate source maps and are not kept (R assert.equal("autter.debug_ids" in row!.attributes, false); assert.equal("autter.debug_ids" in row!.occurrence!.attributes!, false); }); + +// ── Test-client records and the running commit ────────────────────────── + +function oneRecord(resource: Record, attributes: Record) { + return normalizeLogs({ + resourceLogs: [{ + resource: { attributes: Object.entries({ "service.name": "shop", ...resource }).map(([k, v]) => attr(k, v)) }, + scopeLogs: [{ + logRecords: [{ + timeUnixNano: "1760000000000000000", + severityText: "error", + body: { stringValue: "checkout failed" }, + attributes: Object.entries({ + "autter.capture.mode": "log", + "exception.type": "OutOfStock", + "exception.message": "none left", + ...attributes, + }).map(([k, v]) => attr(k, v)), + }], + }], + }], + })[0]!; +} + +test("a record naming the exact host `testserver`, with no environment, is `test` (its promoted error too)", () => { + for (const key of ["server.address", "net.host.name", "http.host", "http.server_name"]) { + const row = oneRecord({}, { [key]: "testserver" }); + assert.equal(row.environment, "test", key); + assert.equal(row.occurrence?.environment, "test", key); + } + const marked = normalizeLogs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "x" }, attributes: [{ key: "autter.test_client", value: { boolValue: true } }] }] }] }] }); + assert.equal(marked[0]!.environment, "test"); +}); + +test("an explicit environment is never rewritten; near misses and plain records stay `production`", () => { + assert.equal(oneRecord({ "deployment.environment": "staging" }, { "http.host": "testserver" }).environment, "staging"); + assert.equal(oneRecord({ "deployment.environment.name": "production" }, { "http.host": "testserver" }).environment, "production"); + for (const attributes of [{}, { "http.host": "testserver.example.com" }, { "server.address": "testserver:8000" }, { "user_agent.original": "Mozilla/5.0 testclient" }]) { + const row = oneRecord({}, attributes); + assert.equal(row.environment, "production", JSON.stringify(attributes)); + assert.equal(row.occurrence?.environment, "production"); + } +}); + +test("a promoted error carries the running commit: the record's beats the resource's; invalid or none, no field", () => { + const fromResource = oneRecord({ "deployment.commit": "ABCDEF1234567" }, {}); + assert.equal(fromResource.occurrence?.commit, "abcdef1234567"); + assert.equal(fromResource.occurrence?.attributes?.["deployment.commit"], "abcdef1234567"); + // The log row's own attributes are left as they were. + assert.equal(fromResource.attributes["deployment.commit"], undefined); + + assert.equal(oneRecord({ "deployment.commit": "1111111" }, { "deployment.commit": "2222222" }).occurrence?.commit, "2222222"); + assert.equal(oneRecord({ "vcs.ref.head.revision": "3333333" }, {}).occurrence?.commit, "3333333"); + for (const bad of ["https://github.com/acme/shop", "main", "123456"]) { + const row = oneRecord({ "deployment.commit": bad }, {}); + assert.equal("commit" in row.occurrence!, false, bad); + assert.equal(row.occurrence?.attributes?.["deployment.commit"], undefined, bad); + } + assert.equal("commit" in oneRecord({}, {}).occurrence!, false); +}); diff --git a/packages/otlp-ingester/src/logs.ts b/packages/otlp-ingester/src/logs.ts index cf41af0..a35322a 100644 --- a/packages/otlp-ingester/src/logs.ts +++ b/packages/otlp-ingester/src/logs.ts @@ -8,8 +8,9 @@ import { import { declaredLink, liftErrorFields } from "./error-fields.js"; import { fingerprintOverrideOf } from "./fingerprint-override.js"; import { normalizeRoute } from "./fingerprint.js"; +import { isTestClientRequest } from "./identifying.js"; import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js"; -import type { RuntimeOccurrenceInput } from "./types.js"; +import { commitOf, withCommitAttribute, type RuntimeOccurrenceInput } from "./types.js"; export interface RuntimeLogRecord { id: string; @@ -192,10 +193,14 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta if (typeof attributes.path === "string") attributes.path = normalizeRoute(attributes.path.split(/[?#]/)[0]!); const service = String(info["service.name"] ?? "unknown").slice(0, 200); + const explicitEnvironment = info["deployment.environment.name"] ?? info["deployment.environment"]; + // A record of a test client's request, by the record's own + // attributes (the exact host `testserver`, or the marker), is + // stored under `test`, as its trace is; only when the resource + // names no environment, which is never rewritten. const environment = String( - info["deployment.environment.name"] ?? - info["deployment.environment"] ?? - "production", + explicitEnvironment ?? + (isTestClientRequest((key) => attrs[key]) ? "test" : "production"), ).slice(0, 100); const release = String(info["service.version"] ?? "").slice(0, 200); const declared = liftErrorFields(attributes); @@ -249,7 +254,7 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta statusCode: statusCode || null, traceId: traceId || null, sessionId: null, - attributes, + ...promotedCommit(attributes, attrs, info), occurredAt, ...declared, ...fingerprintOverrideOf(attributes["autter.fingerprint"]), @@ -323,6 +328,20 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta return rows; } +/** A promoted occurrence's attributes, with the running commit (record + * attributes first, then the resource; 7–40 hex) as `commit` and, when the + * attributes carry none, as `deployment.commit`. The record's own + * attributes are left as they are. */ +function promotedCommit( + attributes: Record, + attrs: Record, + info: Record, +): { attributes: Record; commit?: string } { + const commit = commitOf((key) => attrs[key], (key) => info[key]); + if (!commit) return { attributes }; + return { attributes: withCommitAttribute(attributes, commit) ?? attributes, commit }; +} + function hasException(attributes: Record): boolean { return ["exception.type", "exception.message", "exception.stacktrace"].some( (key) => typeof attributes[key] === "string" && attributes[key] !== "", diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index 1f42fbb..e08afd8 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -9,8 +9,11 @@ import { dedupeTraceSignals, hasAutterKey, hasNextContext, type TraceSignal } fr import { requestMembership, type RequestMembership } from "./request-spans.js"; import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js"; import { histogramLatency, mergeLatency, requestLatency, type LatencySamples } from "./latency.js"; +import { isTestClientRequest, TEST_CLIENT_ATTRIBUTE } from "./identifying.js"; import { asSeverity, + commitOf, + withCommitAttribute, type RuntimeLlmCall, type RuntimeMetricPoint, type RuntimeOccurrenceInput, @@ -161,7 +164,12 @@ function spanKind(kind: number | string | undefined): string { interface ResourceInfo { service: string; environment: string; + /** `deployment.environment.name` or `deployment.environment` was set: + * the environment is never rewritten (testClientTraces). */ + explicitEnvironment: boolean; release: string | null; + /** `deployment.commit`, else `vcs.ref.head.revision`, when 7–40 hex. */ + commit: string | null; /** `autter.metrics_wired` and `autter.metrics_routes` (runtime-node * 1.7.0+): the sender's delta HTTP-server duration histograms * (`/v1/metrics`) count every request it serves, and each request span @@ -182,15 +190,15 @@ interface ResourceInfo { function resourceInfo(resource: OtlpResource | undefined): ResourceInfo { const attrs = attrMap(resource?.attributes); const service = attrs.get("service.name") ?? "unknown"; - const environment = - attrs.get("deployment.environment.name") ?? - attrs.get("deployment.environment") ?? - "production"; + const explicit = attrs.get("deployment.environment.name") ?? attrs.get("deployment.environment"); + const environment = explicit ?? "production"; const release = attrs.get("service.version") ?? null; return { service, environment, + explicitEnvironment: explicit !== undefined, release, + commit: commitOf((key) => attrs.get(key)), metricsCountRoutedRequests: attrs.get("autter.metrics_wired") === "true" && attrs.get("autter.metrics_routes") === "true", metricsWired: attrs.get("autter.metrics_wired") === "true", @@ -279,12 +287,17 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { for (const resourceSpan of request.resourceSpans ?? []) { const resource = resourceInfo(resourceSpan.resource); + const testTraces = resource.explicitEnvironment ? NO_TRACES : testClientTraces(resourceSpan.scopeSpans); for (const scopeSpan of resourceSpan.scopeSpans ?? []) { for (const span of scopeSpan.spans ?? []) { if (spanCount >= MAX_SPANS_PER_REQUEST) break; spanCount += 1; const attrs = attrMap(span.attributes); + // A test client's request (testClientTraces): every span and + // error of its trace is stored under environment `test`. + const testClient = testTraces.has(span.traceId ?? ""); + const environment = testClient ? TEST_ENVIRONMENT : resource.environment; const route = routeOf(attrs); const statusCode = statusCodeOf(attrs); const startedAt = nanosToDate(span.startTimeUnixNano); @@ -303,7 +316,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { spans.push({ service: resource.service, - environment: resource.environment, + environment, release: resource.release, traceId: span.traceId ?? "", spanId: span.spanId ?? "", @@ -317,6 +330,10 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { attributes: { ...decodeOtlpAttributes(span.attributes), "http.request.method": (attrs.get("http.request.method") ?? attrs.get("http.method") ?? "").slice(0, 20), + // The span that made the trace a test client's says why. + ...(testClient && kind === "server" && isTestClientRequest((key) => attrs.get(key)) + ? { [TEST_CLIENT_ATTRIBUTE]: true } + : {}), }, startedAt, }); @@ -342,6 +359,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { for (const event of exceptionEvents) { const eventAttrs = attrMap(event.attributes); const handled = eventAttrs.get("autter.handled") === "true"; + const commit = commitOf((key) => eventAttrs.get(key), (key) => attrs.get(key), () => resource.commit); const eventAutter = hasAutterKey(eventAttrs); signalRollups.push(rollup); signals.push({ @@ -356,7 +374,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { source: "server", severity: severityOf(eventAttrs, attrs), service: resource.service, - environment: resource.environment, + environment, release: resource.release, errorType: eventAttrs.get("exception.type") ?? "Error", message: @@ -370,8 +388,9 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), - ...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })), + attributes: occurrenceAttributes(withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), + ...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })), commit, testClient), + ...(commit ? { commit } : {}), // Server SDKs (runtime-core errorAttributes): the debug ids of the bundles in the // stack (RF-05), which locate their source maps. ...serverDebugIds(eventAttrs.get(DEBUG_IDS_ATTRIBUTE)), @@ -391,6 +410,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { for (const event of failedOutcomes) { const outcome = attrMap(event.attributes); const name = (outcome.get("autter.outcome.name") ?? "operation").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 200); + const commit = commitOf((key) => outcome.get(key), (key) => attrs.get(key), () => resource.commit); signalRollups.push(rollup); signals.push({ traceId, @@ -402,18 +422,20 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { handled: false, occurrence: { source: "server", severity: "error", service: resource.service, - environment: resource.environment, release: resource.release, + environment, release: resource.release, errorType: "OutcomeFailure", message: `${name}: ${(outcome.get("autter.outcome.message") ?? "failed").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 1000)}`, stack: String(decodeOtlpAttributes(event.attributes)["autter.outcome.stack"] ?? "") || null, route, method: methodOf(attrs), statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), + attributes: occurrenceAttributes(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), commit, testClient), + ...(commit ? { commit } : {}), occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano), ...liftErrorFields(outcome, attrs), }, }); } if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) { + const commit = commitOf((key) => attrs.get(key), () => resource.commit); signalRollups.push(rollup); signals.push({ traceId, @@ -427,7 +449,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { source: "server", severity: severityOf(new Map(), attrs), service: resource.service, - environment: resource.environment, + environment, release: resource.release, errorType: statusCode !== null && statusCode >= 500 ? "HttpServerError" : "SpanError", message: span.status?.message || (statusCode !== null && statusCode >= 500 @@ -439,7 +461,8 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: decodeOtlpAttributes(span.attributes), + attributes: occurrenceAttributes(decodeOtlpAttributes(span.attributes), commit, testClient), + ...(commit ? { commit } : {}), occurredAt: startedAt, ...liftErrorFields(attrs), }, @@ -462,7 +485,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { durationMs, startedAt, service: resource.service, - environment: resource.environment, + environment, release: resource.release, spanKind: kind, }); @@ -588,6 +611,49 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { }; } +const TEST_ENVIRONMENT = "test"; +const NO_TRACES: ReadonlySet = new Set(); + +/** + * The traces of one resource (one process) that are a test client's + * requests: a SERVER span of the trace carries strong evidence + * (isTestClientRequest: `autter.test_client`, which the identifying filter + * adds from the exact `testclient` user agent before dropping it, or the + * exact host `testserver`, as Starlette's and FastAPI's TestClient send). + * Only called for a resource with no environment: an explicit one is never + * rewritten. + * + * Per trace, not per resource: the user agent and the host are request + * headers any client can send, so a request to a production process that + * names no environment can claim to be a test client, and only its own + * trace follows that claim; the process's other requests in the same + * export stay where they were. + */ +function testClientTraces(scopeSpans: Array<{ spans?: OtlpSpan[] }> | undefined): ReadonlySet { + let traces: Set | null = null; + for (const scopeSpan of scopeSpans ?? []) { + for (const span of scopeSpan.spans ?? []) { + if (!span.traceId || spanKind(span.kind) !== "server") continue; + const attrs = attrMap(span.attributes); + if (!isTestClientRequest((key) => attrs.get(key))) continue; + (traces ??= new Set()).add(span.traceId); + } + } + return traces ?? NO_TRACES; +} + +/** An occurrence's attributes with the running commit (when absent) and, + * in a test client's trace, the test-client marker. */ +function occurrenceAttributes( + attributes: Record | null, + commit: string | null, + testClient: boolean, +): Record | null { + const withCommit = withCommitAttribute(attributes, commit); + if (!testClient || withCommit?.[TEST_CLIENT_ATTRIBUTE] === true) return withCommit; + return { ...(withCommit ?? {}), [TEST_CLIENT_ATTRIBUTE]: true }; +} + /** * Who counts a request, decided on its outermost server span: * - "metrics": the sender's metric feed. Its data point carries the route diff --git a/packages/otlp-ingester/src/normalize.test.ts b/packages/otlp-ingester/src/normalize.test.ts index 7e543a4..98d0483 100644 --- a/packages/otlp-ingester/src/normalize.test.ts +++ b/packages/otlp-ingester/src/normalize.test.ts @@ -237,3 +237,155 @@ test("warning occurrences outside a request count nothing", () => { assert.deepEqual(metricPoints, []); }); + +// ── Test-client traffic and the running commit ─────────────────────────── + +const NOW_NANOS = NANOS; +const kv = (attributes: Record) => + Object.entries(attributes).map(([key, value]) => ({ + key, + value: typeof value === "boolean" ? { boolValue: value } : { stringValue: value }, + })); + +/** One process: a request per entry of `requests`, each a server span with + * an exception event and an internal child span, in its own trace. */ +function processExport( + resource: Record, + requests: Array<{ trace: string; attributes: Record; event?: Record }>, +) { + return { + resourceSpans: [{ + resource: { attributes: kv({ "service.name": "shop", ...resource }) }, + scopeSpans: [{ + spans: requests.flatMap(({ trace, attributes, event }) => [ + { + traceId: trace, spanId: `${trace.slice(0, 15)}1`, name: "GET /carts/{cart_id}", kind: "SPAN_KIND_SERVER", + startTimeUnixNano: NOW_NANOS, endTimeUnixNano: NOW_NANOS, + attributes: kv({ "http.route": "/carts/{cart_id}", "http.method": "GET", "http.status_code": "500", ...attributes }), + status: { code: "STATUS_CODE_ERROR" }, + events: [{ name: "exception", timeUnixNano: NOW_NANOS, attributes: kv({ "exception.type": "OutOfStock", "exception.message": "none left", ...event }) }], + }, + { + traceId: trace, spanId: `${trace.slice(0, 15)}2`, parentSpanId: `${trace.slice(0, 15)}1`, name: "db", kind: "SPAN_KIND_INTERNAL", + startTimeUnixNano: NOW_NANOS, endTimeUnixNano: NOW_NANOS, attributes: [], + }, + ]), + }], + }], + }; +} + +const TRACE_A = "a".repeat(32); +const TRACE_B = "b".repeat(32); + +test("test client: no environment and the exact host `testserver` → that request's spans, error and rollup are `test`", () => { + for (const host of ["server.address", "net.host.name", "http.host", "http.server_name"]) { + const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes: { [host]: "testserver" } }])); + assert.deepEqual(result.spans.map((s) => s.environment), ["test", "test"], host); + assert.deepEqual(result.occurrences.map((o) => o.environment), ["test"], host); + assert.deepEqual(result.metricPoints.map((p) => p.environment), ["test"], host); + // The server span says why; the child does not claim to be a request. + assert.equal(result.spans[0]!.attributes?.["autter.test_client"], true, host); + assert.equal(result.spans[1]!.attributes?.["autter.test_client"], undefined, host); + assert.equal(result.occurrences[0]!.attributes?.["autter.test_client"], true, host); + } +}); + +test("test client: the marker or the exact `testclient` user agent (kept) also classifies; an explicit environment never changes", () => { + for (const attributes of [{ "autter.test_client": true }, { "user_agent.original": "testclient" }, { "http.user_agent": "testclient" }]) { + const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes }])); + assert.deepEqual(result.occurrences.map((o) => o.environment), ["test"], JSON.stringify(attributes)); + for (const explicit of [{ "deployment.environment": "staging" }, { "deployment.environment.name": "staging" }, { "deployment.environment": "production" }]) { + const kept = normalizeTraces(processExport(explicit, [{ trace: TRACE_A, attributes: { ...attributes, "http.host": "testserver" } }])); + const name = Object.values(explicit)[0]; + assert.deepEqual(kept.spans.map((s) => s.environment), [name, name], JSON.stringify(explicit)); + assert.deepEqual(kept.occurrences.map((o) => o.environment), [name]); + assert.deepEqual(kept.metricPoints.map((p) => p.environment), [name]); + } + } +}); + +test("test client: near misses stay `production`", () => { + const nearMisses: Array> = [ + { "user_agent.original": "Mozilla/5.0 (compatible; testclient/1.0)" }, + { "http.user_agent": "testclient " }, + { "user_agent.original": "TestClient" }, + { "http.host": "testserver.example.com" }, + { "server.address": "testserver.internal" }, + { "http.host": "testserver:8000" }, + { "http.server_name": "TESTSERVER" }, + { "net.peer.name": "testserver" }, + { "autter.test_client": "yes" }, + { "http.host": "shop.example.com", "http.user_agent": "python-httpx/0.28.1" }, + ]; + for (const attributes of nearMisses) { + const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes }])); + assert.deepEqual(result.spans.map((s) => s.environment), ["production", "production"], JSON.stringify(attributes)); + assert.deepEqual(result.occurrences.map((o) => o.environment), ["production"], JSON.stringify(attributes)); + assert.notEqual(result.occurrences[0]!.attributes?.["autter.test_client"], true); + assert.notEqual(result.spans[0]!.attributes?.["autter.test_client"], true); + } +}); + +test("test client: only a SERVER span is evidence, and only its own trace follows it", () => { + // A request claiming to be a test client (its headers are the client's to + // choose) next to a real request of the same process, in one export. + const mixed = normalizeTraces(processExport({}, [ + { trace: TRACE_A, attributes: { "http.host": "testserver" } }, + { trace: TRACE_B, attributes: { "http.host": "shop.example.com" } }, + ])); + assert.deepEqual(mixed.spans.map((s) => [s.traceId[0], s.environment]), [["a", "test"], ["a", "test"], ["b", "production"], ["b", "production"]]); + assert.deepEqual(mixed.occurrences.map((o) => [o.traceId![0], o.environment]), [["a", "test"], ["b", "production"]]); + + // An internal or client span naming testserver (a call made to it) is no evidence. + const request = processExport({}, [{ trace: TRACE_A, attributes: {} }]); + request.resourceSpans[0]!.scopeSpans[0]!.spans[1]!.attributes = kv({ "server.address": "testserver" }); + assert.deepEqual(normalizeTraces(request).occurrences.map((o) => o.environment), ["production"]); + request.resourceSpans[0]!.scopeSpans[0]!.spans[1]!.kind = "SPAN_KIND_CLIENT"; + assert.deepEqual(normalizeTraces(request).occurrences.map((o) => o.environment), ["production"]); +}); + +test("commit: deployment.commit (else vcs.ref.head.revision) from the resource, lower-cased, also in the attributes", () => { + const fromResource = normalizeTraces(processExport({ "deployment.commit": "9C1E4F2A7B3D" }, [{ trace: TRACE_A, attributes: {} }])); + assert.equal(fromResource.occurrences[0]!.commit, "9c1e4f2a7b3d"); + assert.equal(fromResource.occurrences[0]!.attributes?.["deployment.commit"], "9c1e4f2a7b3d"); + + const sha = "0123456789abcdef0123456789abcdef01234567"; + const vcs = normalizeTraces(processExport({ "vcs.ref.head.revision": sha }, [{ trace: TRACE_A, attributes: {} }])); + assert.equal(vcs.occurrences[0]!.commit, sha); + const both = normalizeTraces(processExport({ "vcs.ref.head.revision": sha, "deployment.commit": "abcdef1" }, [{ trace: TRACE_A, attributes: {} }])); + assert.equal(both.occurrences[0]!.commit, "abcdef1"); +}); + +test("commit: an event attribute beats the span's and the resource's", () => { + const result = normalizeTraces(processExport( + { "deployment.commit": "1111111111111111111111111111111111111111" }, + [{ trace: TRACE_A, attributes: { "deployment.commit": "2222222" }, event: { "deployment.commit": "3333333" } }], + )); + assert.equal(result.occurrences[0]!.commit, "3333333"); + // Already in the attributes: kept as sent. + assert.equal(result.occurrences[0]!.attributes?.["deployment.commit"], "3333333"); + const spanOnly = normalizeTraces(processExport( + { "deployment.commit": "1111111111111111111111111111111111111111" }, + [{ trace: TRACE_A, attributes: { "deployment.commit": "2222222" } }], + )); + assert.equal(spanOnly.occurrences[0]!.commit, "2222222"); +}); + +test("commit: anything but 7–40 hex is dropped; none means no field", () => { + for (const bad of ["https://github.com/acme/shop", "9c1e4f", "0".repeat(41), "main", "v1.2.3", "9c1e4f2-dirty"]) { + const result = normalizeTraces(processExport({ "deployment.commit": bad, "vcs.repository.url.full": "https://github.com/acme/shop" }, [{ trace: TRACE_A, attributes: {} }])); + const [occurrence] = result.occurrences; + assert.equal("commit" in occurrence!, false, bad); + // Resource attributes never reach the occurrence's attributes. + assert.equal(occurrence!.attributes?.["deployment.commit"], undefined, bad); + assert.equal(occurrence!.attributes?.["vcs.repository.url.full"], undefined, bad); + } + // An invalid event value does not hide a valid one below it. + const fallback = normalizeTraces(processExport({ "deployment.commit": "abcdef1234" }, [{ trace: TRACE_A, attributes: {}, event: { "deployment.commit": "not-a-sha" } }])); + assert.equal(fallback.occurrences[0]!.commit, "abcdef1234"); + + const none = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes: {} }])); + assert.equal("commit" in none.occurrences[0]!, false); + assert.equal(none.occurrences[0]!.attributes?.["deployment.commit"], undefined); +}); diff --git a/packages/otlp-ingester/src/python-otel.test.ts b/packages/otlp-ingester/src/python-otel.test.ts index d704ce6..889f67c 100644 --- a/packages/otlp-ingester/src/python-otel.test.ts +++ b/packages/otlp-ingester/src/python-otel.test.ts @@ -23,6 +23,11 @@ import type { RuntimeMetricPoint, RuntimeOccurrenceInput, StackFrame } from "./t * source, not from the parser under test. * Regenerate: AUTTER_CAPTURE_FIXTURES= python -m unittest in each * examples/python/ directory. + * + * `fastapi-testclient*.otlp.*`: the FastAPI recipe app exercised in-process + * through `fastapi.testclient.TestClient`, as a test suite does, with no + * environment and with `deployment.environment=staging`; recorded by + * test-fixtures/python/record_fastapi_testclient.py (no expectations file). */ const FIXTURES = new URL("../test-fixtures/python/", import.meta.url); @@ -263,6 +268,75 @@ describe("python request counting and dedupe on real spans", () => { }); }); +// ── FastAPI's TestClient: test traffic stored as environment `test` ─────── + +type OtlpAttributeList = NonNullable[number]["resource"]>["attributes"]; + +/** `request` with `keys` removed from every span's attributes and the resource's. */ +function withoutAttributes(request: OtlpTraceRequest, keys: string[]): OtlpTraceRequest { + const strip = (list: OtlpAttributeList) => list?.filter((kv) => !keys.includes(kv.key ?? "")); + return { + resourceSpans: request.resourceSpans?.map((rs) => ({ + resource: { attributes: strip(rs.resource?.attributes) }, + scopeSpans: rs.scopeSpans?.map((ss) => ({ + spans: ss.spans?.map((span) => ({ ...span, attributes: strip(span.attributes) })), + })), + })), + }; +} + +describe("python fastapi TestClient (real OTel export)", () => { + const environments = (result: ReturnType) => ({ + spans: [...new Set(result.spans.map((s) => s.environment))], + occurrences: result.occurrences.map((o) => o.environment), + rollups: [...new Set(result.metricPoints.map((p) => p.environment))], + }); + + test("what TestClient sends: user agent `testclient`, host `testserver`, no environment", () => { + const request = loadJson("fastapi-testclient"); + const resource = request.resourceSpans!.flatMap((rs) => rs.resource?.attributes ?? []).map((kv) => kv.key); + assert.ok(!resource.includes("deployment.environment") && !resource.includes("deployment.environment.name")); + const servers = request.resourceSpans!.flatMap((rs) => rs.scopeSpans!.flatMap((ss) => ss.spans!)).filter((s) => s.kind === "SPAN_KIND_SERVER"); + assert.equal(servers.length, 2); + for (const span of servers) { + const attrs = Object.fromEntries((span.attributes ?? []).map((kv) => [kv.key, kv.value?.stringValue])); + assert.equal(attrs["http.user_agent"], "testclient"); + assert.equal(attrs["http.host"], "testserver"); + assert.equal(attrs["http.server_name"], "testserver"); + } + }); + + test("no environment: occurrences, spans and rollups are stored as `test`, from JSON and protobuf", () => { + for (const result of [normalizeTraces(loadJson("fastapi-testclient")), normalizeTraces(decodeTraceRequest(loadProto("fastapi-testclient")))]) { + assert.deepEqual(environments(result), { spans: ["test"], occurrences: ["test", "test"], rollups: ["test"] }); + assert.deepEqual(result.occurrences.map((o) => o.errorType).sort(), ["inventory.OutOfStock", "inventory.PricingTimeout"]); + for (const occurrence of result.occurrences) assert.equal(occurrence.attributes?.["autter.test_client"], true); + } + }); + + test("without the user agent (adapters/python 1.19.0 drops it before export) the host alone still classifies", () => { + const result = normalizeTraces(withoutAttributes(loadJson("fastapi-testclient"), ["http.user_agent", "user_agent.original"])); + assert.deepEqual(environments(result), { spans: ["test"], occurrences: ["test", "test"], rollups: ["test"] }); + // Neither user agent nor host: nothing says test, so production as before. + const neither = normalizeTraces(withoutAttributes(loadJson("fastapi-testclient"), ["http.user_agent", "http.host", "http.server_name"])); + assert.deepEqual(environments(neither), { spans: ["production"], occurrences: ["production", "production"], rollups: ["production"] }); + }); + + test("deployment.environment=staging: unchanged, `staging`", () => { + for (const result of [normalizeTraces(loadJson("fastapi-testclient-staging")), normalizeTraces(decodeTraceRequest(loadProto("fastapi-testclient-staging")))]) { + assert.deepEqual(environments(result), { spans: ["staging"], occurrences: ["staging", "staging"], rollups: ["staging"] }); + } + }); + + test("a normal request (uvicorn, a real client) with no environment stays `production`", () => { + const fastapi = expectations.find((e) => e.framework === "fastapi")!.scenarios; + const request = withoutAttributes(loadJson(fastapi.unhandled!.fixture), ["deployment.environment"]); + const result = normalizeTraces(request); + assert.deepEqual(environments(result), { spans: ["production"], occurrences: ["production"], rollups: ["production"] }); + assert.equal(result.occurrences[0]!.attributes?.["autter.test_client"], undefined); + }); +}); + // ── The recommended rate: 1% head sampling, 2% of requests failing ──────── /** diff --git a/packages/otlp-ingester/src/server.sink.test.ts b/packages/otlp-ingester/src/server.sink.test.ts index ec3d118..34b7c08 100644 --- a/packages/otlp-ingester/src/server.sink.test.ts +++ b/packages/otlp-ingester/src/server.sink.test.ts @@ -340,3 +340,48 @@ test("one error recorded on two spans reaches the sink once, with the same id wh assert.equal(requests, 0); assert.equal(errors, 0); }); + +test("the running commit and test-client traffic reach the sink: hex SHA only, environment `test` only without an explicit one", async () => { + const now = String(Date.now() * 1_000_000); + const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); + const process = (n: number, resource: Array>, span: Array>, event: Array> = []) => ({ + resource: { attributes: [str("service.name", "shop"), ...resource] }, + scopeSpans: [{ + spans: [{ + traceId: String(n).repeat(32), spanId: String(n).repeat(16), name: "GET /orders", kind: 2, + startTimeUnixNano: now, endTimeUnixNano: now, + attributes: [str("http.route", "/orders"), ...span], + events: [{ name: "exception", timeUnixNano: now, attributes: [str("exception.type", `Error${n}`), str("exception.message", `boom ${n}`), ...event] }], + }], + }], + }); + const sha = "9C1E4F2A7B3D9C1E4F2A7B3D9C1E4F2A7B3D9C1E"; + const res = await fetch(`${ingestUrl}/v1/traces`, { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer test-key" }, + body: JSON.stringify({ + resourceSpans: [ + process(1, [str("deployment.commit", sha), str("deployment.environment", "production")], []), + process(2, [str("deployment.commit", sha)], [], [str("deployment.commit", "abcdef1")]), + process(3, [str("deployment.commit", "https://github.com/acme/shop"), str("vcs.repository.url.full", "https://github.com/acme/shop")], []), + process(4, [], [str("http.host", "testserver")]), + process(5, [str("deployment.environment", "staging")], [str("http.host", "testserver")]), + process(6, [], [str("http.host", "testserver.example.com"), str("user_agent.original", "Mozilla/5.0 (testclient)")]), + ], + }), + }); + assert.equal(res.status, 200); + const batch = await nextSinkBatch(); + const byType = Object.fromEntries(batch.occurrences.map((o: Record) => [o.errorType, o])); + assert.equal(byType.Error1.commit, sha.toLowerCase()); + assert.equal(byType.Error1.attributes["deployment.commit"], sha.toLowerCase()); + assert.equal(byType.Error2.commit, "abcdef1", "the event's commit wins"); + for (const name of ["Error3", "Error4", "Error5", "Error6"]) assert.equal("commit" in byType[name], false, name); + assert.equal(JSON.stringify(byType.Error3).includes("github.com"), false, "nothing else of the resource is forwarded"); + assert.deepEqual( + ["Error1", "Error2", "Error3", "Error4", "Error5", "Error6"].map((name) => byType[name].environment), + ["production", "production", "production", "test", "staging", "production"], + ); + assert.equal(byType.Error4.attributes["autter.test_client"], true); + assert.equal(byType.Error6.attributes["user_agent.original"], undefined, "the user agent is dropped"); +}); diff --git a/packages/otlp-ingester/src/sink.test.ts b/packages/otlp-ingester/src/sink.test.ts index 0d3c94e..05a4caa 100644 --- a/packages/otlp-ingester/src/sink.test.ts +++ b/packages/otlp-ingester/src/sink.test.ts @@ -319,3 +319,13 @@ test("1.6.0 sink fields: frames, fingerprints and the record ride along only whe assert.equal(key in bare, false, key); } }); + +test("commit: a 7–40 hex SHA rides along lower-cased; anything else, or none, emits no field", () => { + assert.equal(sinkOccurrence(occ({ commit: "9c1e4f2a7b3d" })).commit, "9c1e4f2a7b3d"); + const full = "0123456789abcdef0123456789ABCDEF01234567"; + assert.equal(sinkOccurrence(occ({ commit: full })).commit, full.toLowerCase()); + for (const bad of ["9c1e4f", "0".repeat(41), "https://github.com/acme/shop", "v1.2.3", "main", " ", ""]) { + assert.equal("commit" in sinkOccurrence(occ({ commit: bad })), false, bad); + } + assert.equal("commit" in sinkOccurrence(occ({})), false); +}); diff --git a/packages/otlp-ingester/src/sink.ts b/packages/otlp-ingester/src/sink.ts index 967d3b7..9d6f53c 100644 --- a/packages/otlp-ingester/src/sink.ts +++ b/packages/otlp-ingester/src/sink.ts @@ -1,11 +1,12 @@ import { createHash, randomUUID } from "node:crypto"; import type { IngesterConfig } from "./config.js"; import type { OutboxRecord, SinkOutbox } from "./sink-outbox.js"; -import type { - IngestContext, - RuntimeLlmCall, - RuntimeMetricPoint, - RuntimeOccurrence, +import { + validCommit, + type IngestContext, + type RuntimeLlmCall, + type RuntimeMetricPoint, + type RuntimeOccurrence, } from "./types.js"; /** @@ -678,8 +679,11 @@ export function sinkLlmCall(c: RuntimeLlmCall): Record { * their event counts. Unreleased (RF-06): `fingerprints` may end with a * `cause-v1` candidate after the legacy one; consumers that do not know the * scheme drop it (the platform validates schemes one by one). RF-05: in-app - * frames' context lines are up to 5 on each side (3 before). Older consumers - * ignore unknown keys, so the version stays 1. + * frames' context lines are up to 5 on each side (3 before). Unreleased: + * `commit`, the commit the app runs (`deployment.commit`, else + * `vcs.ref.head.revision`), sent only when it is 7–40 hexadecimal + * characters (lower-cased); nothing else of the resource is forwarded. + * Older consumers ignore unknown keys, so the version stays 1. */ export function sinkOccurrence(o: RuntimeOccurrence): Record { const { @@ -700,8 +704,11 @@ export function sinkOccurrence(o: RuntimeOccurrence): Record { // Input-only: locates source maps, never leaves the ingester. debugIds: _debugIds, collapsedCount, + commit: rawCommit, ...rest } = o; + // Checked again here: only a hex SHA ever leaves the ingester. + const commit = validCommit(rawCommit); return { ...rest, occurredAt: o.occurredAt.toISOString(), @@ -715,6 +722,7 @@ export function sinkOccurrence(o: RuntimeOccurrence): Record { ...(link ? { link } : {}), ...(expected ? { expected: true } : {}), ...(requestId ? { requestId } : {}), + ...(commit ? { commit } : {}), // 1.6.0: steps before the error, oldest first, ≤ 30. ...(breadcrumbs?.length ? { breadcrumbs } : {}), ...(frames?.length ? { frames } : {}), diff --git a/packages/otlp-ingester/src/types.ts b/packages/otlp-ingester/src/types.ts index 9177dea..e66da11 100644 --- a/packages/otlp-ingester/src/types.ts +++ b/packages/otlp-ingester/src/types.ts @@ -29,6 +29,36 @@ export function asSeverity( : fallback; } +const COMMIT_SHA = /^[0-9a-f]{7,40}$/; + +/** A commit SHA: 7 to 40 hexadecimal characters, trimmed and lower-cased; null for anything else. */ +export function validCommit(value: unknown): string | null { + if (typeof value !== "string") return null; + const sha = value.trim().toLowerCase(); + return COMMIT_SHA.test(sha) ? sha : null; +} + +/** The first valid commit among `deployment.commit` then `vcs.ref.head.revision` in each source, in order. */ +export function commitOf(...sources: Array<(key: string) => unknown>): string | null { + for (const get of sources) { + const sha = validCommit(get("deployment.commit")) ?? validCommit(get("vcs.ref.head.revision")); + if (sha) return sha; + } + return null; +} + +/** + * `attributes` with `deployment.commit` set to `commit` when they carry + * none, so a reader of the stored attributes JSON finds it there too. + */ +export function withCommitAttribute( + attributes: Record | null, + commit: string | null, +): Record | null { + if (!commit || attributes?.["deployment.commit"] !== undefined) return attributes; + return { ...(attributes ?? {}), "deployment.commit": commit }; +} + /** A normalised error/warning event, before fingerprinting. */ export interface RuntimeOccurrenceInput { source: RuntimeSource; @@ -64,6 +94,13 @@ export interface RuntimeOccurrenceInput { expected?: boolean; /** `autter.request.id` — joins the occurrence to its request summary. */ requestId?: string; + /** + * The commit the app runs: `deployment.commit`, then + * `vcs.ref.head.revision`, from the event, the span (or log record), then + * the resource; only a 7–40 hex value (`validCommit`), lower-cased. + * Omitted when absent. + */ + commit?: string; /** * SDK fingerprint override (browser `fingerprint`, OTLP/log * `autter.fingerprint`): ≤10 scrubbed values, each ≤200 chars. Hashed into diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json new file mode 100644 index 0000000..35a44b6 --- /dev/null +++ b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json @@ -0,0 +1,417 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "161b8d5e-2838-4348-ab30-28414d647a99" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "9c1e4f2a7b3d" + } + }, + { + "key": "deployment.environment", + "value": { + "stringValue": "staging" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "shop-fastapi" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "opentelemetry.instrumentation.fastapi", + "version": "0.66b1" + }, + "spans": [ + { + "traceId": "b00d289cf99d9ec3dda016ae7e1ca0c5", + "spanId": "5ec630f3e9befc33", + "parentSpanId": "bae89fabbd89cef6", + "name": "POST /carts/{cart_id}/checkout http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122666672000", + "endTimeUnixNano": "1794719122666693000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "500" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "b00d289cf99d9ec3dda016ae7e1ca0c5", + "spanId": "2147636e71782fe8", + "parentSpanId": "bae89fabbd89cef6", + "name": "POST /carts/{cart_id}/checkout http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122666762000", + "endTimeUnixNano": "1794719122666768000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "b00d289cf99d9ec3dda016ae7e1ca0c5", + "spanId": "bae89fabbd89cef6", + "name": "POST /carts/{cart_id}/checkout", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719122664158000", + "endTimeUnixNano": "1794719122667414000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/checkout" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/checkout" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/checkout" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "500" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719122666616000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.OutOfStock" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "sku-1: 2 requested, 0 left" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/venv/lib/python3.14/site-packages/opentelemetry/instrumentation/fastapi/__init__.py\", line 360, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/telemetry/_asgi.py\", line 151, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/middleware/exceptions.py\", line 63, in __call__\n await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/middleware/asyncexitstack.py\", line 18, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/routing.py\", line 678, in __call__\n await self.middleware_stack(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 2811, in app\n await route.handle(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 1330, in handle\n await super().handle(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/routing.py\", line 284, in handle\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 166, in app\n await wrap_app_handling_exceptions(app, request)(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 152, in app\n response = await f(request)\n ^^^^^^^^^^^^^^^^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 745, in app\n raw_response = await run_endpoint_function(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n )\n ^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 372, in run_endpoint_function\n return await dependant.call(**values)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/srv/app/app.py\", line 36, in checkout\n reserve(\"sku-1\", 2) # frame: checkout\n ~~~~~~~^^^^^^^^^^^^\n File \"/srv/app/inventory.py\", line 17, in reserve\n raise OutOfStock(f\"{sku}: {quantity} requested, {STOCK.get(sku, 0)} left\") # frame: reserve\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.OutOfStock: sku-1: 2 requested, 0 left\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + } + ] + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "2d98a731d4ed701e58a0900f0d295645", + "spanId": "c3881eae62134149", + "parentSpanId": "a7a8c3faa5ea5706", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122669819000", + "endTimeUnixNano": "1794719122669832000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "2d98a731d4ed701e58a0900f0d295645", + "spanId": "d92eb23fddde26ba", + "parentSpanId": "a7a8c3faa5ea5706", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122669869000", + "endTimeUnixNano": "1794719122669873000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "2d98a731d4ed701e58a0900f0d295645", + "spanId": "a7a8c3faa5ea5706", + "name": "GET /carts/{cart_id}/quote", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719122669085000", + "endTimeUnixNano": "1794719122670223000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/quote" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/quote" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/quote" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719122669687000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.PricingTimeout" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "pricing for cart 12 timed out after 2.5s" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/app/app.py\", line 43, in quote\n return {\"cart\": cart_id, \"total\": price(cart_id)} # frame: quote\n ~~~~~^^^^^^^^^\n File \"/srv/app/inventory.py\", line 21, in price\n raise PricingTimeout(f\"pricing for cart {cart_id} timed out after 2.5s\") # frame: price\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.PricingTimeout: pricing for cart 12 timed out after 2.5s\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + }, + { + "key": "autter.handled", + "value": { + "boolValue": true + } + } + ] + } + ], + "status": {}, + "flags": 256 + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.11.0" + } + ] + } + ] +} diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.pb b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.pb new file mode 100644 index 0000000000000000000000000000000000000000..142814d2f4f183eb78a3c39a612f996a31e0c315 GIT binary patch literal 5763 zcmeHLTWB0r7;chYl4IM{X`05h)YDDVZbCA%v%A?`@K(gC>q82y7b0{G*6ZNBB zlX$kfFb~jV;#4i)b=X+yLL0#1jOEOrooHjY4tE6>31(eyq^!dX&ys!^LJAs4c*pVj z6}FO2!6`YMyFM!yi_@y~MH|}k-loi}zrXhSg^Qe7f`8 zr8nOB@VPI48_FD*I(P^xhVIiqS!7?DWu8!sSz^q&6^dsm^>9EOGut+C@~eSoPTg^- zZ{Mk1d-$M~LEU;VW67k-{G$sVk+z|&>>1f7f#*5_k--U{mbRlU{5D+CN}w68Nu-Xx zD;=d2>d`Bd5?`jeWotfxFNc^?` z;m^taVo=)K=uZ!llKxD)=EAb5*VHgSe|xhW)FZ!h`N(J6C$hmmPxE;HvP%{5-iB@w zQH1ZvVLw)Lv( zOY2aFDyyab+Fi)k451P@UMmYVSFeYXiaNQeif>B;sGp4|6(us*RRu6r72BFt1bOCVwV(tRB_kfJ0bl|p!7B>m!5FV7HC|CP z&2lWNX|lJF9>zB8P@FGjhq2{gO=H$I4Gd$fAJZ)gd|{Vm-SaYm0m~0_lvWi6j7SDo ziZH}br*lQ)q!U3=N7erZ3ga@H09z^ZmTB7LnC=s$o~X81f}GKf5-zHS8s1ICr+?HbJ%ay(K? z&5BBDRYb(U?lOvHP7fB~oqyb-e4E8Wp^mr8UYMZO&J3e4sD&7Y;@I*aB0n?^e{~Rr zo2?N~Zd?_+EG!I>Ag){=WDb&{p zKrI$IfMhb3b{3%5CIvO2A%2P z3ErEg<7nds;wW!yj3SpJypy5l;ypGA3>}&mX#-lfU$+AilaQZ*gdBP0jQY*b z-p#X z>v)3*e(=qbi1I6cKOv$V>dD|{h+v!)!MCFBNYU~VD9JGeoJPDNS}0&KKNQOhw}Rw{-Y zNxC2;506ugq4`T0mOJc8FS$FhUnIh^_m7b!RpcCb#OO0h%tH+sR$)h%T>$ZDjc8cL7Qtwh@u@A ghljOexYS;DL=?alO`LSdmaA|A(E|kzzf?K...\n )\n ^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 372, in run_endpoint_function\n return await dependant.call(**values)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/srv/app/app.py\", line 36, in checkout\n reserve(\"sku-1\", 2) # frame: checkout\n ~~~~~~~^^^^^^^^^^^^\n File \"/srv/app/inventory.py\", line 17, in reserve\n raise OutOfStock(f\"{sku}: {quantity} requested, {STOCK.get(sku, 0)} left\") # frame: reserve\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.OutOfStock: sku-1: 2 requested, 0 left\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + } + ] + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "30d47486f4f685da0daa8d24539c418e", + "spanId": "d8cff7b6824b78db", + "parentSpanId": "b882fc6e690bc358", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719121766313000", + "endTimeUnixNano": "1794719121766324000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "30d47486f4f685da0daa8d24539c418e", + "spanId": "d12781cf29bfb254", + "parentSpanId": "b882fc6e690bc358", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719121766358000", + "endTimeUnixNano": "1794719121766362000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "30d47486f4f685da0daa8d24539c418e", + "spanId": "b882fc6e690bc358", + "name": "GET /carts/{cart_id}/quote", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719121765657000", + "endTimeUnixNano": "1794719121766594000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/quote" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/quote" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/quote" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719121766204000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.PricingTimeout" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "pricing for cart 12 timed out after 2.5s" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/app/app.py\", line 43, in quote\n return {\"cart\": cart_id, \"total\": price(cart_id)} # frame: quote\n ~~~~~^^^^^^^^^\n File \"/srv/app/inventory.py\", line 21, in price\n raise PricingTimeout(f\"pricing for cart {cart_id} timed out after 2.5s\") # frame: price\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.PricingTimeout: pricing for cart 12 timed out after 2.5s\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + }, + { + "key": "autter.handled", + "value": { + "boolValue": true + } + } + ] + } + ], + "status": {}, + "flags": 256 + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.11.0" + } + ] + } + ] +} diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.pb b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.pb new file mode 100644 index 0000000000000000000000000000000000000000..4e08b8db7086c1ae0f1a6522aab3ee8be30833a8 GIT binary patch literal 5726 zcmeHLU1%Id9B-1QO{X8J+cc)vYR7A9FKM!SdzX(U_(2h?oXQe_ z&F;<4|M&lx-~NAtei%gO+fWwuGV0Jd%7UsITC=J{-02FLrb-uDFK85-6L<`605>CiYaE%;3@>U(vt&>cMa{g=Ld=Q|$~gHjH4lW^Kr={z7&S=Fb?X0(YvqXsne zJvXE(I1#YYjcB8|4OgrZ8m4DarK9gcM=67PNQE&PsEpW-5m5MYsim}UY^e4Yv4obTl{pmqU+Mg-UsxFFpMGf-{Tbt#e z9{HOWk9@v;{Os$uNPPFYkI8s%Mr&mhAwN*htI$e*S=xfuh}$z>$dq*mF&Z*nQZy)S zLhD`1RQ`+VdjV70`wo_RYRi-zGVcXSC+g5uy|klt7xFbtnF5a2%0e&H>!GBhE@7$? zILa=xgO4XPEjHNXkq9(w*cs2_ptYKYQLW8aRtr7iHqfc@>(oiiO9u1fs~ zi3tHzDpj_kEiLQ#s_O62k8fP_%!Nr2dSwkgW>TMTwoDFfs9(sU??pYf%lCi>lysoN z4wMfu&z#NZ=!SUa91TOxQrl1_oUIJ$6S#l_dZYr$qSheJV~3Vm<{eloeg?LSIm-fK zQsq=HdZl-g52xTB&dote;dv9RVv{%yc7QMXClJQ_ZHMBl76$VgV4Bw)drFhMSXA|4 zEwmXO@?i}C6KW}5(Ky8=ctxx6ie?zLYcs=8{c3g)JFr7>aWp@OZ5JB`w{93<7-Mpj z*bMl>DXYZybD;^#4+@m_JPbII^e+`*grUyVnjuIhhN6$?{|yw*Wex?l(&lW-a_CVK zP_3S=FcF>6Vk<;P1L7MH9Ak#K7DRH|h!@C3ksHLO=eqsN6TldCY)V=O){Ub$A|Z2p zKD9VRk9Xv%>`usmsmEN@`U^G`Q+AN@R|NRd}5rvzgdtiBRZ!TV*dw z&~j&nQyA1j0>kLo;vix_3;}<65Jj7<5l>;SE_Yc}7&1eGdsR`(^m;?ki$%#f(p-{p zDn1Qz{mY6YuHRV+UK#$K1*fa)*E?jcqCOi0XYh7x9ryIz=Dcvz{}eAY?m43 zifi)DCwDnWj`cE}sY@|mRn^;yqGt&2fU&s8*&7~(|4V9CT02!7laX$UTNndnQ3>2y z+${<^&>|pOc=M7Q7?>vxG-47KhX$XNCk+>A=#XFy{u10LhQZ)#oE6-&`{|Aj! zp&#QNxEz2uoxrK7PsvZxY@@}v=ZxRt`j|*AB$W5n(yVxc$8Bk@oLv9}$0zW@kqX4X zX4T^nG4M`w;n3uPyY5#(1ms{Ebf*2sMQ@r-qD>k|qP(^-id~BFPT=t8Oh%Pl=I~@T=oKxt^vb(0W)~U!tfL(WjGDO z1$89MoLt#L4FA@Fe(hMQ7;Yr#VJUe;oN5d$UJ?sg4l4_sEDyv6ah7?EIB~8Q+i`h#SUZNx?B+*AA#BmqNr!5CnjjDoYH;|atNDKb7nOCh literal 0 HcmV?d00001 diff --git a/packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py b/packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py new file mode 100644 index 0000000..f8cd641 --- /dev/null +++ b/packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py @@ -0,0 +1,99 @@ +"""Records the FastAPI TestClient fixtures of the ingester's Python tests +(``fastapi-testclient*.otlp.{json,pb}``): the FastAPI recipe app +(examples/python/fastapi) exercised in-process through +``fastapi.testclient.TestClient``, as a test suite does, exporting OTLP/HTTP +to the recipes' stub receiver (examples/python/testing/otlp_stub.py). + + cd examples/python/fastapi + python3 -m venv .venv && .venv/bin/pip install -r requirements.txt httpx + .venv/bin/python ../../../packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py + +Two captures, each in its own process (the app calls ``init_tracing()`` at +import): ``fastapi-testclient`` with no ``deployment.environment`` (a test +run that names none), and ``fastapi-testclient-staging`` with +``deployment.environment=staging``. Both keep the raw user agent +(``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES=http.user_agent,user_agent.original``), +as a plain OpenTelemetry sender or an adapter before 1.19.0 sends it; the +current adapter drops it before export, which the ingester's tests model by +removing it. Sampling ratio 0: only the always-on error path exports. + +Not run by CI (TestClient needs httpx, which the recipe does not install). +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) +RECIPE = os.path.join(REPO, "examples", "python", "fastapi") +TESTING = os.path.join(REPO, "examples", "python", "testing") +ADAPTER = os.path.join(REPO, "adapters", "python") + +CAPTURES = { + "fastapi-testclient": "service.version=9c1e4f2a7b3d", + "fastapi-testclient-staging": "service.version=9c1e4f2a7b3d,deployment.environment=staging", +} + + +def capture(name: str) -> None: + """In the child: run the app under TestClient and write the fixture.""" + sys.path[:0] = [TESTING, RECIPE, ADAPTER] + from otlp_stub import OtlpStub, package_versions, write_fixture + + stub = OtlpStub() + os.environ.update({ + "OTEL_EXPORTER_OTLP_ENDPOINT": stub.url, + "OTEL_EXPORTER_OTLP_HEADERS": "x-autter-key=autter_rt_recipe_test_key_0001", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SERVICE_NAME": "shop-fastapi", + "OTEL_RESOURCE_ATTRIBUTES": CAPTURES[name], + "OTEL_TRACES_SAMPLER": "parentbased_traceidratio", + "OTEL_TRACES_SAMPLER_ARG": "0", + "OTEL_BSP_SCHEDULE_DELAY": "200", + "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES": "http.user_agent,user_agent.original", + "AUTTER_COUNT_REQUESTS": "false", + }) + os.chdir(RECIPE) + from fastapi.testclient import TestClient + from opentelemetry import trace + + import app as recipe + + # raise_server_exceptions=False: answer 500 as the server would, instead + # of re-raising in the test, so the request span records the exception. + client = TestClient(recipe.app, raise_server_exceptions=False) + assert client.post("/carts/12/checkout").status_code == 500 + assert client.get("/carts/12/quote").status_code == 200 + trace.get_tracer_provider().force_flush() + deadline = time.monotonic() + 20 + while True: + spans = stub.spans() + servers = [s for s in spans if s["kind"] == "SERVER"] + if len(servers) >= 2: + break + if time.monotonic() > deadline: + raise AssertionError(f"timed out; got {[s['name'] for s in spans]}") + time.sleep(0.05) + for server in servers: + print(name, server["name"], {k: v for k, v in server["attributes"].items() if "host" in k or "server" in k or "agent" in k or "peer" in k}) + write_fixture(stub, name, sorted({s["trace_id"] for s in servers}), RECIPE) + print(name, package_versions("fastapi", "starlette", "httpx", "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi")) + stub.close() + + +def main() -> None: + if len(sys.argv) == 2: + capture(sys.argv[1]) + return + for name in CAPTURES: + env = {k: v for k, v in os.environ.items() if not k.startswith("OTEL_")} + env["AUTTER_CAPTURE_FIXTURES"] = HERE + subprocess.run([sys.executable, os.path.abspath(__file__), name], env=env, check=True) + + +if __name__ == "__main__": + main() From 7a42aff71681de269877818b506804c11e8f2ada Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Sun, 15 Nov 2026 10:51:57 +0530 Subject: [PATCH 2/2] Ingester: never change the environment from what a request says Remove the test-client reclassification. The user agent `testclient`, the host `testserver` and the peer address come from the caller, so on a production app that names no environment a caller could have its own errors stored as `test`, out of default views and paging. The environment is again the resource's deployment.environment(.name), else production, on traces and logs alike; test runs are to set deployment.environment=test from the process (SDK work, separate PRs). `autter.test_client: true` stays on a SERVER span whose raw user agent is exactly `testclient`, added before the user agent is dropped, as a hint nothing in the ingester acts on. Commit forwarding is unchanged. Tests now pin the safe behaviour on the recorded TestClient fixtures: no environment is production, explicit test is test, staging stays staging. Co-Authored-By: Claude Opus 5.5 --- packages/otlp-ingester/README.md | 50 ++++-------- .../otlp-ingester/src/identifying.test.ts | 40 ++++++--- packages/otlp-ingester/src/identifying.ts | 38 +++------ packages/otlp-ingester/src/logs.test.ts | 22 ++--- packages/otlp-ingester/src/logs.ts | 11 +-- packages/otlp-ingester/src/normalize-otlp.ts | 81 +++---------------- packages/otlp-ingester/src/normalize.test.ts | 70 ++++------------ .../otlp-ingester/src/python-otel.test.ts | 29 +++---- .../otlp-ingester/src/server.sink.test.ts | 5 +- 9 files changed, 105 insertions(+), 241 deletions(-) diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 96165ce..51e2133 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -227,40 +227,22 @@ hashing every IPv4 address) and the ingester warns at startup. A hashed IP is still pseudonymous personal data in some readings of privacy law: it is off by default, and the salt should be kept and rotated like any secret. -## Test-client traffic (unreleased) - -A test suite that drives a FastAPI or Starlette app through its in-process -`TestClient`, with an exporter configured and no environment set, used to -send its errors as `production`. The ingester now stores them as -environment `test`, which the platform leaves out of its default views, -alerts and incidents: - -- Only when the resource names no environment (`deployment.environment.name` - or `deployment.environment`). An explicit environment, `staging` or even - `production`, is never changed. -- Only on strong evidence, on a SERVER span: the exact user agent - `testclient` or the exact host `testserver` (`server.address`, - `net.host.name`, `http.host` or `http.server_name`), which is what - Starlette's `TestClient` sends. A user agent that merely contains - "testclient", or a host such as `testserver.example.com`, does not count. -- Per request: every span and error of that request's trace is stored as - `test`; the process's other requests in the same export are not. The user - agent and the host are headers any client can send, so a request that - claims to be a test client only reclassifies itself. - -The raw user agent is read before the identifying attributes are dropped: -the server span gets `autter.test_client: true` and the user agent itself is -still dropped. The Python adapter (adapters/python) drops the user agent -before export since Runtime 1.19.0, so for it the host is the evidence. A `/v1/logs` record is -classified the same way only from its own attributes (the exact host -`testserver`, or `autter.test_client: true`), and only when its resource -names no environment. - -Known limit: the same process's `/v1/metrics` (the request-duration -histogram that counts requests) carries nothing per request to tell a test -client apart, so its request counts still default to `production`. Request -counts and error rates for that process are exact only once the SDK sets -`deployment.environment=test` itself under a test runner. +## Test traffic (unreleased) + +Test traffic is separated by setting `deployment.environment=test` in the +test run. The Python adapter and `@autter/runtime-node` do this themselves +under a test runner from the next release; anything else sets it in +`OTEL_RESOURCE_ATTRIBUTES`. The ingester never changes the environment +because of what a request says about itself (its user agent, its host, its +peer address): those come from the caller, and a caller could use them to +have its errors stored outside production, out of default views and alerts. +The environment is the resource's `deployment.environment.name` or +`deployment.environment`, else `production`. + +A SERVER span whose raw user agent is exactly `testclient` (what Starlette's +and FastAPI's in-process `TestClient` sends) gets `autter.test_client: true` +before the user agent is dropped. It is a hint for a reader, not trusted: +anyone can send that user agent, and nothing in the ingester acts on it. ## Running commit (unreleased) diff --git a/packages/otlp-ingester/src/identifying.test.ts b/packages/otlp-ingester/src/identifying.test.ts index be3232c..bbfbbe4 100644 --- a/packages/otlp-ingester/src/identifying.test.ts +++ b/packages/otlp-ingester/src/identifying.test.ts @@ -264,12 +264,20 @@ async function backgroundInserts(rows: Record) { } const attributesOf = (row: Record) => JSON.parse(String(row.attributes ?? "{}")) as Record; -test("TestClient traffic with no environment is stored as `test`: the server spans carry autter.test_client and no user agent", async () => { +/** The recorded no-environment export, as OTLP/JSON, with `deployment.environment` set to `name`. */ +function testClientWithEnvironment(name: string): string { + const request = JSON.parse(readFileSync(new URL("../test-fixtures/python/fastapi-testclient.otlp.json", import.meta.url), "utf8")); + for (const rs of request.resourceSpans) rs.resource.attributes.push({ key: "deployment.environment", value: { stringValue: name } }); + return JSON.stringify(request); +} + +test("TestClient traffic with no environment is stored as `production`: the server spans carry the autter.test_client hint and no user agent", async () => { await withIngester({}, async (post, rows) => { assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200); const spans = rows.runtime_spans ?? []; assert.equal(spans.length, 6); - assert.deepEqual(new Set(spans.map((row) => row.environment)), new Set(["test"])); + // The user agent and host are the caller's to send: they never change the environment. + assert.deepEqual(new Set(spans.map((row) => row.environment)), new Set(["production"])); const servers = spans.filter((row) => row.kind === "server"); assert.equal(servers.length, 2); for (const row of servers) { @@ -277,39 +285,45 @@ test("TestClient traffic with no environment is stored as `test`: the server spa assert.equal(attributes["autter.test_client"], true); assert.equal(attributes["http.user_agent"], undefined, "the user agent is still dropped"); assert.equal(attributes["user_agent.original"], undefined); - assert.equal(attributes["http.host"], "testserver", "the host is kept"); } + for (const row of spans.filter((r) => r.kind !== "server")) assert.equal(attributesOf(row)["autter.test_client"], undefined); const occurrences = rows.runtime_error_occurrences ?? []; - assert.deepEqual(occurrences.map((row) => row.environment), ["test", "test"]); - for (const row of occurrences) { - assert.equal(attributesOf(row)["autter.test_client"], true); - assert.equal(attributesOf(row)["http.user_agent"], undefined); - } + assert.deepEqual(occurrences.map((row) => row.environment), ["production", "production"]); for (const table of ["runtime_spans", "runtime_error_occurrences"]) for (const key of vectors.drop) assert.ok(!storedKeys(rows, table).has(key), `${key} stored in ${table}`); await backgroundInserts(rows); }); }); -test("the same TestClient export with deployment.environment=staging stays `staging`", async () => { +test("the same TestClient export with deployment.environment=test set explicitly is stored as `test`", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", testClientWithEnvironment("test"), "application/json"), 200); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["test"])); + assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["test", "test"]); + await backgroundInserts(rows); + }); +}); + +test("the TestClient export with deployment.environment=staging stays `staging`", async () => { await withIngester({}, async (post, rows) => { assert.equal(await post("/v1/traces", TESTCLIENT_STAGING, "application/x-protobuf"), 200); assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["staging"])); assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["staging", "staging"]); - // The request still came from a test client, and says so. for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) assert.equal(attributesOf(row)["autter.test_client"], true); await backgroundInserts(rows); }); }); -test("with the user agent kept (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) TestClient traffic is still `test`", async () => { +test("with the user agent kept (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) it is stored, and the environment is still `production`", async () => { await withIngester({ keepIdentifyingAttributes: true }, async (post, rows) => { assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200); - assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["test"])); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["production"])); for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) { assert.equal(attributesOf(row)["http.user_agent"], "testclient"); - assert.equal(attributesOf(row)["autter.test_client"], true); + // Nothing is dropped, so the filter passes the request through: + // the raw user agent is stored instead of the hint. + assert.equal(attributesOf(row)["autter.test_client"], undefined); } await backgroundInserts(rows); }); diff --git a/packages/otlp-ingester/src/identifying.ts b/packages/otlp-ingester/src/identifying.ts index 21ebf2b..3d6acd8 100644 --- a/packages/otlp-ingester/src/identifying.ts +++ b/packages/otlp-ingester/src/identifying.ts @@ -124,7 +124,7 @@ export class IdentifyingAttributeFilter { /** * The attributes of a span or log record: what is read from the raw list * added first (the client IP hash; on a SERVER span, the test-client - * marker), then the drop. + * marker, a hint only), then the drop. */ private recordAttributes = (obj: unknown, serverSpan: boolean): unknown => { if (!isObject(obj)) return this.attributesOf(obj); @@ -157,8 +157,6 @@ export class IdentifyingAttributeFilter { /** A traces request without identifying attributes (resource, spans, events, links). */ traces(request: T): T { - // Nothing dropped: the raw user agent stays, and the normaliser reads - // it itself (isTestClientRequest). if (!this.dropped.size && !this.hasher) return request; const span = (item: unknown) => eachIn( @@ -274,37 +272,23 @@ export function clientIpOfList(list: unknown, serverSpan: boolean): string | nul export const isServerKind = (kind: unknown) => kind === 2 || kind === "SPAN_KIND_SERVER"; // --------------------------------------------------------------------------- -// Test-client traffic +// Test-client marker (a hint, never trusted) /** - * On a SERVER span: the request came from a framework's in-process test - * client. Added by the ingester from the raw user agent before the drop - * (the user agent itself is still dropped), or sent by an SDK. + * On a SERVER span whose raw user agent is exactly `testclient` (what + * Starlette's and FastAPI's in-process TestClient sends): added before the + * user agent is dropped, so the stored span still says so. A hint only: the + * user agent is whatever the caller sent, so anyone can produce it, and + * nothing in the ingester acts on it. The environment is never changed + * because of it: test runs set `deployment.environment=test` themselves. + * Not added when nothing is dropped (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES=true + * and no client IP hash): the request passes through with its user agent. */ export const TEST_CLIENT_ATTRIBUTE = "autter.test_client"; -/** The user agent Starlette's TestClient (and FastAPI's, which is Starlette's) sends, exactly. */ -export const TEST_CLIENT_USER_AGENT = "testclient"; -/** The host Starlette's TestClient requests (`http://testserver`), exactly. */ -export const TEST_CLIENT_HOST = "testserver"; +const TEST_CLIENT_USER_AGENT = "testclient"; const USER_AGENT_KEYS: readonly string[] = ["user_agent.original", "http.user_agent"]; -/** Host attributes, none of them identifying (they name the app's own host). */ -export const TEST_CLIENT_HOST_KEYS: readonly string[] = ["server.address", "net.host.name", "http.host", "http.server_name"]; /** The raw OTLP attribute list names the test client's user agent, exactly. */ function testClientUserAgentIn(list: unknown): boolean { return USER_AGENT_KEYS.some((key) => stringAttribute(list, key) === TEST_CLIENT_USER_AGENT); } - -/** - * Strong evidence, on one span or log record, that it is a test client's - * request: the marker, the exact test-client user agent (when it was kept), - * or the exact test-client host. Exact values only: a real user agent that - * merely contains "testclient", or a host such as `testserver.example.com`, - * is not evidence. - */ -export function isTestClientRequest(get: (key: string) => unknown): boolean { - const marker = get(TEST_CLIENT_ATTRIBUTE); - if (marker === true || marker === "true") return true; - if (USER_AGENT_KEYS.some((key) => get(key) === TEST_CLIENT_USER_AGENT)) return true; - return TEST_CLIENT_HOST_KEYS.some((key) => get(key) === TEST_CLIENT_HOST); -} diff --git a/packages/otlp-ingester/src/logs.test.ts b/packages/otlp-ingester/src/logs.test.ts index 1d47239..1b626f4 100644 --- a/packages/otlp-ingester/src/logs.test.ts +++ b/packages/otlp-ingester/src/logs.test.ts @@ -231,7 +231,7 @@ test("server exceptions' autter.debug_ids locate source maps and are not kept (R assert.equal("autter.debug_ids" in row!.occurrence!.attributes!, false); }); -// ── Test-client records and the running commit ────────────────────────── +// ── Environment from the resource only, and the running commit ────────────────────────── function oneRecord(resource: Record, attributes: Record) { return normalizeLogs({ @@ -254,24 +254,16 @@ function oneRecord(resource: Record, attributes: Record { - for (const key of ["server.address", "net.host.name", "http.host", "http.server_name"]) { - const row = oneRecord({}, { [key]: "testserver" }); - assert.equal(row.environment, "test", key); - assert.equal(row.occurrence?.environment, "test", key); - } - const marked = normalizeLogs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "x" }, attributes: [{ key: "autter.test_client", value: { boolValue: true } }] }] }] }] }); - assert.equal(marked[0]!.environment, "test"); -}); - -test("an explicit environment is never rewritten; near misses and plain records stay `production`", () => { - assert.equal(oneRecord({ "deployment.environment": "staging" }, { "http.host": "testserver" }).environment, "staging"); - assert.equal(oneRecord({ "deployment.environment.name": "production" }, { "http.host": "testserver" }).environment, "production"); - for (const attributes of [{}, { "http.host": "testserver.example.com" }, { "server.address": "testserver:8000" }, { "user_agent.original": "Mozilla/5.0 testclient" }]) { +test("a record's own attributes never change its environment", () => { + for (const attributes of [{ "http.host": "testserver" }, { "server.address": "testserver" }, { "user_agent.original": "testclient" }]) { const row = oneRecord({}, attributes); assert.equal(row.environment, "production", JSON.stringify(attributes)); assert.equal(row.occurrence?.environment, "production"); + assert.equal(oneRecord({ "deployment.environment": "test" }, attributes).environment, "test"); + assert.equal(oneRecord({ "deployment.environment": "staging" }, attributes).environment, "staging"); } + const marked = normalizeLogs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "x" }, attributes: [{ key: "autter.test_client", value: { boolValue: true } }] }] }] }] }); + assert.equal(marked[0]!.environment, "production"); }); test("a promoted error carries the running commit: the record's beats the resource's; invalid or none, no field", () => { diff --git a/packages/otlp-ingester/src/logs.ts b/packages/otlp-ingester/src/logs.ts index a35322a..bc857ec 100644 --- a/packages/otlp-ingester/src/logs.ts +++ b/packages/otlp-ingester/src/logs.ts @@ -8,7 +8,6 @@ import { import { declaredLink, liftErrorFields } from "./error-fields.js"; import { fingerprintOverrideOf } from "./fingerprint-override.js"; import { normalizeRoute } from "./fingerprint.js"; -import { isTestClientRequest } from "./identifying.js"; import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js"; import { commitOf, withCommitAttribute, type RuntimeOccurrenceInput } from "./types.js"; @@ -193,14 +192,10 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta if (typeof attributes.path === "string") attributes.path = normalizeRoute(attributes.path.split(/[?#]/)[0]!); const service = String(info["service.name"] ?? "unknown").slice(0, 200); - const explicitEnvironment = info["deployment.environment.name"] ?? info["deployment.environment"]; - // A record of a test client's request, by the record's own - // attributes (the exact host `testserver`, or the marker), is - // stored under `test`, as its trace is; only when the resource - // names no environment, which is never rewritten. const environment = String( - explicitEnvironment ?? - (isTestClientRequest((key) => attrs[key]) ? "test" : "production"), + info["deployment.environment.name"] ?? + info["deployment.environment"] ?? + "production", ).slice(0, 100); const release = String(info["service.version"] ?? "").slice(0, 200); const declared = liftErrorFields(attributes); diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index e08afd8..7e9f8f1 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -9,7 +9,6 @@ import { dedupeTraceSignals, hasAutterKey, hasNextContext, type TraceSignal } fr import { requestMembership, type RequestMembership } from "./request-spans.js"; import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js"; import { histogramLatency, mergeLatency, requestLatency, type LatencySamples } from "./latency.js"; -import { isTestClientRequest, TEST_CLIENT_ATTRIBUTE } from "./identifying.js"; import { asSeverity, commitOf, @@ -164,9 +163,6 @@ function spanKind(kind: number | string | undefined): string { interface ResourceInfo { service: string; environment: string; - /** `deployment.environment.name` or `deployment.environment` was set: - * the environment is never rewritten (testClientTraces). */ - explicitEnvironment: boolean; release: string | null; /** `deployment.commit`, else `vcs.ref.head.revision`, when 7–40 hex. */ commit: string | null; @@ -190,13 +186,14 @@ interface ResourceInfo { function resourceInfo(resource: OtlpResource | undefined): ResourceInfo { const attrs = attrMap(resource?.attributes); const service = attrs.get("service.name") ?? "unknown"; - const explicit = attrs.get("deployment.environment.name") ?? attrs.get("deployment.environment"); - const environment = explicit ?? "production"; + const environment = + attrs.get("deployment.environment.name") ?? + attrs.get("deployment.environment") ?? + "production"; const release = attrs.get("service.version") ?? null; return { service, environment, - explicitEnvironment: explicit !== undefined, release, commit: commitOf((key) => attrs.get(key)), metricsCountRoutedRequests: @@ -287,17 +284,12 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { for (const resourceSpan of request.resourceSpans ?? []) { const resource = resourceInfo(resourceSpan.resource); - const testTraces = resource.explicitEnvironment ? NO_TRACES : testClientTraces(resourceSpan.scopeSpans); for (const scopeSpan of resourceSpan.scopeSpans ?? []) { for (const span of scopeSpan.spans ?? []) { if (spanCount >= MAX_SPANS_PER_REQUEST) break; spanCount += 1; const attrs = attrMap(span.attributes); - // A test client's request (testClientTraces): every span and - // error of its trace is stored under environment `test`. - const testClient = testTraces.has(span.traceId ?? ""); - const environment = testClient ? TEST_ENVIRONMENT : resource.environment; const route = routeOf(attrs); const statusCode = statusCodeOf(attrs); const startedAt = nanosToDate(span.startTimeUnixNano); @@ -316,7 +308,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { spans.push({ service: resource.service, - environment, + environment: resource.environment, release: resource.release, traceId: span.traceId ?? "", spanId: span.spanId ?? "", @@ -330,10 +322,6 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { attributes: { ...decodeOtlpAttributes(span.attributes), "http.request.method": (attrs.get("http.request.method") ?? attrs.get("http.method") ?? "").slice(0, 20), - // The span that made the trace a test client's says why. - ...(testClient && kind === "server" && isTestClientRequest((key) => attrs.get(key)) - ? { [TEST_CLIENT_ATTRIBUTE]: true } - : {}), }, startedAt, }); @@ -374,7 +362,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { source: "server", severity: severityOf(eventAttrs, attrs), service: resource.service, - environment, + environment: resource.environment, release: resource.release, errorType: eventAttrs.get("exception.type") ?? "Error", message: @@ -388,8 +376,8 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: occurrenceAttributes(withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), - ...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })), commit, testClient), + attributes: withCommitAttribute(withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), + ...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })), commit), ...(commit ? { commit } : {}), // Server SDKs (runtime-core errorAttributes): the debug ids of the bundles in the // stack (RF-05), which locate their source maps. @@ -422,12 +410,12 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { handled: false, occurrence: { source: "server", severity: "error", service: resource.service, - environment, release: resource.release, + environment: resource.environment, release: resource.release, errorType: "OutcomeFailure", message: `${name}: ${(outcome.get("autter.outcome.message") ?? "failed").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 1000)}`, stack: String(decodeOtlpAttributes(event.attributes)["autter.outcome.stack"] ?? "") || null, route, method: methodOf(attrs), statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: occurrenceAttributes(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), commit, testClient), + attributes: withCommitAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), commit), ...(commit ? { commit } : {}), occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano), ...liftErrorFields(outcome, attrs), @@ -449,7 +437,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { source: "server", severity: severityOf(new Map(), attrs), service: resource.service, - environment, + environment: resource.environment, release: resource.release, errorType: statusCode !== null && statusCode >= 500 ? "HttpServerError" : "SpanError", message: span.status?.message || (statusCode !== null && statusCode >= 500 @@ -461,7 +449,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: occurrenceAttributes(decodeOtlpAttributes(span.attributes), commit, testClient), + attributes: withCommitAttribute(decodeOtlpAttributes(span.attributes), commit), ...(commit ? { commit } : {}), occurredAt: startedAt, ...liftErrorFields(attrs), @@ -485,7 +473,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { durationMs, startedAt, service: resource.service, - environment, + environment: resource.environment, release: resource.release, spanKind: kind, }); @@ -611,49 +599,6 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { }; } -const TEST_ENVIRONMENT = "test"; -const NO_TRACES: ReadonlySet = new Set(); - -/** - * The traces of one resource (one process) that are a test client's - * requests: a SERVER span of the trace carries strong evidence - * (isTestClientRequest: `autter.test_client`, which the identifying filter - * adds from the exact `testclient` user agent before dropping it, or the - * exact host `testserver`, as Starlette's and FastAPI's TestClient send). - * Only called for a resource with no environment: an explicit one is never - * rewritten. - * - * Per trace, not per resource: the user agent and the host are request - * headers any client can send, so a request to a production process that - * names no environment can claim to be a test client, and only its own - * trace follows that claim; the process's other requests in the same - * export stay where they were. - */ -function testClientTraces(scopeSpans: Array<{ spans?: OtlpSpan[] }> | undefined): ReadonlySet { - let traces: Set | null = null; - for (const scopeSpan of scopeSpans ?? []) { - for (const span of scopeSpan.spans ?? []) { - if (!span.traceId || spanKind(span.kind) !== "server") continue; - const attrs = attrMap(span.attributes); - if (!isTestClientRequest((key) => attrs.get(key))) continue; - (traces ??= new Set()).add(span.traceId); - } - } - return traces ?? NO_TRACES; -} - -/** An occurrence's attributes with the running commit (when absent) and, - * in a test client's trace, the test-client marker. */ -function occurrenceAttributes( - attributes: Record | null, - commit: string | null, - testClient: boolean, -): Record | null { - const withCommit = withCommitAttribute(attributes, commit); - if (!testClient || withCommit?.[TEST_CLIENT_ATTRIBUTE] === true) return withCommit; - return { ...(withCommit ?? {}), [TEST_CLIENT_ATTRIBUTE]: true }; -} - /** * Who counts a request, decided on its outermost server span: * - "metrics": the sender's metric feed. Its data point carries the route diff --git a/packages/otlp-ingester/src/normalize.test.ts b/packages/otlp-ingester/src/normalize.test.ts index 98d0483..f40276c 100644 --- a/packages/otlp-ingester/src/normalize.test.ts +++ b/packages/otlp-ingester/src/normalize.test.ts @@ -278,71 +278,29 @@ function processExport( const TRACE_A = "a".repeat(32); const TRACE_B = "b".repeat(32); -test("test client: no environment and the exact host `testserver` → that request's spans, error and rollup are `test`", () => { - for (const host of ["server.address", "net.host.name", "http.host", "http.server_name"]) { - const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes: { [host]: "testserver" } }])); - assert.deepEqual(result.spans.map((s) => s.environment), ["test", "test"], host); - assert.deepEqual(result.occurrences.map((o) => o.environment), ["test"], host); - assert.deepEqual(result.metricPoints.map((p) => p.environment), ["test"], host); - // The server span says why; the child does not claim to be a request. - assert.equal(result.spans[0]!.attributes?.["autter.test_client"], true, host); - assert.equal(result.spans[1]!.attributes?.["autter.test_client"], undefined, host); - assert.equal(result.occurrences[0]!.attributes?.["autter.test_client"], true, host); - } -}); - -test("test client: the marker or the exact `testclient` user agent (kept) also classifies; an explicit environment never changes", () => { - for (const attributes of [{ "autter.test_client": true }, { "user_agent.original": "testclient" }, { "http.user_agent": "testclient" }]) { - const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes }])); - assert.deepEqual(result.occurrences.map((o) => o.environment), ["test"], JSON.stringify(attributes)); - for (const explicit of [{ "deployment.environment": "staging" }, { "deployment.environment.name": "staging" }, { "deployment.environment": "production" }]) { - const kept = normalizeTraces(processExport(explicit, [{ trace: TRACE_A, attributes: { ...attributes, "http.host": "testserver" } }])); - const name = Object.values(explicit)[0]; - assert.deepEqual(kept.spans.map((s) => s.environment), [name, name], JSON.stringify(explicit)); - assert.deepEqual(kept.occurrences.map((o) => o.environment), [name]); - assert.deepEqual(kept.metricPoints.map((p) => p.environment), [name]); - } - } -}); - -test("test client: near misses stay `production`", () => { - const nearMisses: Array> = [ - { "user_agent.original": "Mozilla/5.0 (compatible; testclient/1.0)" }, - { "http.user_agent": "testclient " }, - { "user_agent.original": "TestClient" }, - { "http.host": "testserver.example.com" }, - { "server.address": "testserver.internal" }, - { "http.host": "testserver:8000" }, - { "http.server_name": "TESTSERVER" }, - { "net.peer.name": "testserver" }, - { "autter.test_client": "yes" }, - { "http.host": "shop.example.com", "http.user_agent": "python-httpx/0.28.1" }, +test("what a request says about itself never changes the environment: no environment is `production`, an explicit one is kept", () => { + const claims: Array> = [ + { "autter.test_client": true }, + { "user_agent.original": "testclient" }, + { "http.user_agent": "testclient" }, + { "http.host": "testserver", "http.server_name": "testserver", "server.address": "testserver", "net.host.name": "testserver" }, ]; - for (const attributes of nearMisses) { + for (const attributes of claims) { const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes }])); assert.deepEqual(result.spans.map((s) => s.environment), ["production", "production"], JSON.stringify(attributes)); assert.deepEqual(result.occurrences.map((o) => o.environment), ["production"], JSON.stringify(attributes)); - assert.notEqual(result.occurrences[0]!.attributes?.["autter.test_client"], true); - assert.notEqual(result.spans[0]!.attributes?.["autter.test_client"], true); + assert.deepEqual(result.metricPoints.map((p) => p.environment), ["production"], JSON.stringify(attributes)); + for (const name of ["test", "staging"]) { + const explicit = normalizeTraces(processExport({ "deployment.environment": name }, [{ trace: TRACE_A, attributes }])); + assert.deepEqual(explicit.occurrences.map((o) => o.environment), [name]); + } } -}); - -test("test client: only a SERVER span is evidence, and only its own trace follows it", () => { - // A request claiming to be a test client (its headers are the client's to - // choose) next to a real request of the same process, in one export. + // Two requests of one process, one claiming to be a test client: both production. const mixed = normalizeTraces(processExport({}, [ { trace: TRACE_A, attributes: { "http.host": "testserver" } }, { trace: TRACE_B, attributes: { "http.host": "shop.example.com" } }, ])); - assert.deepEqual(mixed.spans.map((s) => [s.traceId[0], s.environment]), [["a", "test"], ["a", "test"], ["b", "production"], ["b", "production"]]); - assert.deepEqual(mixed.occurrences.map((o) => [o.traceId![0], o.environment]), [["a", "test"], ["b", "production"]]); - - // An internal or client span naming testserver (a call made to it) is no evidence. - const request = processExport({}, [{ trace: TRACE_A, attributes: {} }]); - request.resourceSpans[0]!.scopeSpans[0]!.spans[1]!.attributes = kv({ "server.address": "testserver" }); - assert.deepEqual(normalizeTraces(request).occurrences.map((o) => o.environment), ["production"]); - request.resourceSpans[0]!.scopeSpans[0]!.spans[1]!.kind = "SPAN_KIND_CLIENT"; - assert.deepEqual(normalizeTraces(request).occurrences.map((o) => o.environment), ["production"]); + assert.deepEqual(mixed.occurrences.map((o) => o.environment), ["production", "production"]); }); test("commit: deployment.commit (else vcs.ref.head.revision) from the resource, lower-cased, also in the attributes", () => { diff --git a/packages/otlp-ingester/src/python-otel.test.ts b/packages/otlp-ingester/src/python-otel.test.ts index 889f67c..5a1ed97 100644 --- a/packages/otlp-ingester/src/python-otel.test.ts +++ b/packages/otlp-ingester/src/python-otel.test.ts @@ -28,6 +28,7 @@ import type { RuntimeMetricPoint, RuntimeOccurrenceInput, StackFrame } from "./t * through `fastapi.testclient.TestClient`, as a test suite does, with no * environment and with `deployment.environment=staging`; recorded by * test-fixtures/python/record_fastapi_testclient.py (no expectations file). + * The environment comes from the resource only, never from the request. */ const FIXTURES = new URL("../test-fixtures/python/", import.meta.url); @@ -268,16 +269,16 @@ describe("python request counting and dedupe on real spans", () => { }); }); -// ── FastAPI's TestClient: test traffic stored as environment `test` ─────── +// ── FastAPI's TestClient: the environment comes from the resource only ──── type OtlpAttributeList = NonNullable[number]["resource"]>["attributes"]; -/** `request` with `keys` removed from every span's attributes and the resource's. */ -function withoutAttributes(request: OtlpTraceRequest, keys: string[]): OtlpTraceRequest { +/** `request` with `keys` removed from every span's attributes and the resource's, and `add` added to the resource. */ +function withAttributes(request: OtlpTraceRequest, keys: string[], add: OtlpAttributeList = []): OtlpTraceRequest { const strip = (list: OtlpAttributeList) => list?.filter((kv) => !keys.includes(kv.key ?? "")); return { resourceSpans: request.resourceSpans?.map((rs) => ({ - resource: { attributes: strip(rs.resource?.attributes) }, + resource: { attributes: [...(strip(rs.resource?.attributes) ?? []), ...(add ?? [])] }, scopeSpans: rs.scopeSpans?.map((ss) => ({ spans: ss.spans?.map((span) => ({ ...span, attributes: strip(span.attributes) })), })), @@ -306,20 +307,16 @@ describe("python fastapi TestClient (real OTel export)", () => { } }); - test("no environment: occurrences, spans and rollups are stored as `test`, from JSON and protobuf", () => { + test("no environment: stored as `production`, whatever the request says about itself (JSON and protobuf)", () => { for (const result of [normalizeTraces(loadJson("fastapi-testclient")), normalizeTraces(decodeTraceRequest(loadProto("fastapi-testclient")))]) { - assert.deepEqual(environments(result), { spans: ["test"], occurrences: ["test", "test"], rollups: ["test"] }); + assert.deepEqual(environments(result), { spans: ["production"], occurrences: ["production", "production"], rollups: ["production"] }); assert.deepEqual(result.occurrences.map((o) => o.errorType).sort(), ["inventory.OutOfStock", "inventory.PricingTimeout"]); - for (const occurrence of result.occurrences) assert.equal(occurrence.attributes?.["autter.test_client"], true); } }); - test("without the user agent (adapters/python 1.19.0 drops it before export) the host alone still classifies", () => { - const result = normalizeTraces(withoutAttributes(loadJson("fastapi-testclient"), ["http.user_agent", "user_agent.original"])); - assert.deepEqual(environments(result), { spans: ["test"], occurrences: ["test", "test"], rollups: ["test"] }); - // Neither user agent nor host: nothing says test, so production as before. - const neither = normalizeTraces(withoutAttributes(loadJson("fastapi-testclient"), ["http.user_agent", "http.host", "http.server_name"])); - assert.deepEqual(environments(neither), { spans: ["production"], occurrences: ["production", "production"], rollups: ["production"] }); + test("deployment.environment=test set explicitly (as test runs will): stored as `test`", () => { + const request = withAttributes(loadJson("fastapi-testclient"), [], [{ key: "deployment.environment", value: { stringValue: "test" } }]); + assert.deepEqual(environments(normalizeTraces(request)), { spans: ["test"], occurrences: ["test", "test"], rollups: ["test"] }); }); test("deployment.environment=staging: unchanged, `staging`", () => { @@ -328,12 +325,10 @@ describe("python fastapi TestClient (real OTel export)", () => { } }); - test("a normal request (uvicorn, a real client) with no environment stays `production`", () => { + test("a normal request (uvicorn, a real client) with no environment is `production` too", () => { const fastapi = expectations.find((e) => e.framework === "fastapi")!.scenarios; - const request = withoutAttributes(loadJson(fastapi.unhandled!.fixture), ["deployment.environment"]); - const result = normalizeTraces(request); + const result = normalizeTraces(withAttributes(loadJson(fastapi.unhandled!.fixture), ["deployment.environment"])); assert.deepEqual(environments(result), { spans: ["production"], occurrences: ["production"], rollups: ["production"] }); - assert.equal(result.occurrences[0]!.attributes?.["autter.test_client"], undefined); }); }); diff --git a/packages/otlp-ingester/src/server.sink.test.ts b/packages/otlp-ingester/src/server.sink.test.ts index 34b7c08..16c5c95 100644 --- a/packages/otlp-ingester/src/server.sink.test.ts +++ b/packages/otlp-ingester/src/server.sink.test.ts @@ -341,7 +341,7 @@ test("one error recorded on two spans reaches the sink once, with the same id wh assert.equal(errors, 0); }); -test("the running commit and test-client traffic reach the sink: hex SHA only, environment `test` only without an explicit one", async () => { +test("the running commit reaches the sink as a hex SHA only; what a request says about itself never changes the environment", async () => { const now = String(Date.now() * 1_000_000); const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); const process = (n: number, resource: Array>, span: Array>, event: Array> = []) => ({ @@ -380,8 +380,7 @@ test("the running commit and test-client traffic reach the sink: hex SHA only, e assert.equal(JSON.stringify(byType.Error3).includes("github.com"), false, "nothing else of the resource is forwarded"); assert.deepEqual( ["Error1", "Error2", "Error3", "Error4", "Error5", "Error6"].map((name) => byType[name].environment), - ["production", "production", "production", "test", "staging", "production"], + ["production", "production", "production", "production", "staging", "production"], ); - assert.equal(byType.Error4.attributes["autter.test_client"], true); assert.equal(byType.Error6.attributes["user_agent.original"], undefined, "the user agent is dropped"); });