diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 0c5b887..51e2133 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -227,6 +227,34 @@ hashing every IPv4 address) and the ingester warns at startup. A hashed IP is still pseudonymous personal data in some readings of privacy law: it is off by default, and the salt should be kept and rotated like any secret. +## Test traffic (unreleased) + +Test traffic is separated by setting `deployment.environment=test` in the +test run. The Python adapter and `@autter/runtime-node` do this themselves +under a test runner from the next release; anything else sets it in +`OTEL_RESOURCE_ATTRIBUTES`. The ingester never changes the environment +because of what a request says about itself (its user agent, its host, its +peer address): those come from the caller, and a caller could use them to +have its errors stored outside production, out of default views and alerts. +The environment is the resource's `deployment.environment.name` or +`deployment.environment`, else `production`. + +A SERVER span whose raw user agent is exactly `testclient` (what Starlette's +and FastAPI's in-process `TestClient` sends) gets `autter.test_client: true` +before the user agent is dropped. It is a hint for a reader, not trusted: +anyone can send that user agent, and nothing in the ingester acts on it. + +## Running commit (unreleased) + +Each occurrence forwarded to the sink carries `commit` when the app reports +the commit it runs: `deployment.commit`, else `vcs.ref.head.revision`, read +from the exception event, then the span (or log record), then the resource. +Only a value of 7 to 40 hexadecimal characters is forwarded, lower-cased; +anything else (a repository URL, a branch name) is dropped, and nothing else +of the resource is forwarded. When the occurrence's attributes have no +`deployment.commit`, the SHA is also stored there. The field is optional on +sink payload version 1: a platform that does not know it ignores it. + ## Rate limits and key lookups (1.12.0) In the order a request meets them: diff --git a/packages/otlp-ingester/src/identifying.test.ts b/packages/otlp-ingester/src/identifying.test.ts index f6f686d..bbfbbe4 100644 --- a/packages/otlp-ingester/src/identifying.test.ts +++ b/packages/otlp-ingester/src/identifying.test.ts @@ -13,6 +13,7 @@ import { IdentifyingAttributeFilter, identifyingKeysToDrop, keepIdentifyingEnv, + clientIpHasher, } from "./identifying.js"; import { createIngesterApp } from "./server.js"; @@ -117,6 +118,49 @@ test("keep (true or a list) and extra drops; env parsing", () => { // --------------------------------------------------------------------------- // Through the ingester: a real Python OpenTelemetry export (protobuf, Flask // instrumentation: net.peer.ip, net.peer.port, http.user_agent) and a plain +test("test client: a SERVER span whose raw user agent is exactly `testclient` gets autter.test_client; the user agent is still dropped", () => { + const filter = new IdentifyingAttributeFilter(); + const span = (kind: number | string, attributes: Record) => ({ kind, name: "GET /", attributes: kv(attributes) }); + const request = { + resourceSpans: [{ + scopeSpans: [{ + spans: [ + span(2, { "http.user_agent": "testclient", "http.host": "testserver" }), + span("SPAN_KIND_SERVER", { "user_agent.original": "testclient" }), + span(1, { "http.user_agent": "testclient" }), + span(3, { "user_agent.original": "testclient" }), + span(2, { "user_agent.original": "Mozilla/5.0 testclient" }), + span(2, { "http.user_agent": "TestClient" }), + ], + }], + }], + }; + const before = JSON.stringify(request); + const spans = filter.traces(request).resourceSpans[0]!.scopeSpans[0]!.spans; + assert.equal(JSON.stringify(request), before, "input not mutated"); + assert.deepEqual(spans.map((s) => keysOf(s.attributes)), [ + ["http.host", "autter.test_client"], + ["autter.test_client"], + [], [], [], [], + ]); + assert.deepEqual((spans[0]!.attributes as Array<{ value: unknown }>)[1]!.value, { boolValue: true }); + + // Already marked (an SDK did it): not added twice. With the client IP hash too. + const hashing = new IdentifyingAttributeFilter(identifyingKeysToDrop(), clientIpHasher(true, "s".repeat(16))); + const marked = hashing.traces({ resourceSpans: [{ scopeSpans: [{ spans: [ + { kind: 2, attributes: [...kv({ "http.user_agent": "testclient", "client.address": "203.0.113.9" }), { key: "autter.test_client", value: { boolValue: true } }] }, + { kind: 2, attributes: kv({ "http.user_agent": "testclient", "client.address": "203.0.113.9" }) }, + ] }] }] }).resourceSpans[0]!.scopeSpans[0]!.spans; + assert.deepEqual(marked.map((s) => keysOf(s.attributes)), [ + ["autter.test_client", "client.address_hash"], + ["client.address_hash", "autter.test_client"], + ]); + + // Log records are never marked. + const logs = filter.logs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ attributes: kv({ "user_agent.original": "testclient" }) }] }] }] }); + assert.deepEqual(keysOf(logs.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes), []); +}); + // OTLP/JSON logs export, against a fake ClickHouse. const listen = async (server: Server) => { @@ -206,3 +250,81 @@ test("keepIdentifyingAttributes (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) stores them assert.ok(!storedKeys(rows, "runtime_spans").has("net.peer.ip")); }); }); + +// FastAPI's TestClient, recorded (test-fixtures/python/record_fastapi_testclient.py). +const TESTCLIENT = readFileSync(new URL("../test-fixtures/python/fastapi-testclient.otlp.pb", import.meta.url)); +const TESTCLIENT_STAGING = readFileSync(new URL("../test-fixtures/python/fastapi-testclient-staging.otlp.pb", import.meta.url)); +/** Waits (up to 2 s) for the inserts the ingester makes after answering + * (SDK sightings, ingester info), so closing the fake ClickHouse does not + * reset them mid-request. Quiets the log only: nothing is asserted on them. */ +async function backgroundInserts(rows: Record) { + const deadline = Date.now() + 2000; + while (!(rows.runtime_sdk_versions && rows.runtime_ingester_info) && Date.now() < deadline) + await new Promise((resolve) => setTimeout(resolve, 10)); +} +const attributesOf = (row: Record) => JSON.parse(String(row.attributes ?? "{}")) as Record; + +/** The recorded no-environment export, as OTLP/JSON, with `deployment.environment` set to `name`. */ +function testClientWithEnvironment(name: string): string { + const request = JSON.parse(readFileSync(new URL("../test-fixtures/python/fastapi-testclient.otlp.json", import.meta.url), "utf8")); + for (const rs of request.resourceSpans) rs.resource.attributes.push({ key: "deployment.environment", value: { stringValue: name } }); + return JSON.stringify(request); +} + +test("TestClient traffic with no environment is stored as `production`: the server spans carry the autter.test_client hint and no user agent", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200); + const spans = rows.runtime_spans ?? []; + assert.equal(spans.length, 6); + // The user agent and host are the caller's to send: they never change the environment. + assert.deepEqual(new Set(spans.map((row) => row.environment)), new Set(["production"])); + const servers = spans.filter((row) => row.kind === "server"); + assert.equal(servers.length, 2); + for (const row of servers) { + const attributes = attributesOf(row); + assert.equal(attributes["autter.test_client"], true); + assert.equal(attributes["http.user_agent"], undefined, "the user agent is still dropped"); + assert.equal(attributes["user_agent.original"], undefined); + } + for (const row of spans.filter((r) => r.kind !== "server")) assert.equal(attributesOf(row)["autter.test_client"], undefined); + const occurrences = rows.runtime_error_occurrences ?? []; + assert.deepEqual(occurrences.map((row) => row.environment), ["production", "production"]); + for (const table of ["runtime_spans", "runtime_error_occurrences"]) + for (const key of vectors.drop) assert.ok(!storedKeys(rows, table).has(key), `${key} stored in ${table}`); + await backgroundInserts(rows); + }); +}); + +test("the same TestClient export with deployment.environment=test set explicitly is stored as `test`", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", testClientWithEnvironment("test"), "application/json"), 200); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["test"])); + assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["test", "test"]); + await backgroundInserts(rows); + }); +}); + +test("the TestClient export with deployment.environment=staging stays `staging`", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", TESTCLIENT_STAGING, "application/x-protobuf"), 200); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["staging"])); + assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["staging", "staging"]); + for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) + assert.equal(attributesOf(row)["autter.test_client"], true); + await backgroundInserts(rows); + }); +}); + +test("with the user agent kept (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) it is stored, and the environment is still `production`", async () => { + await withIngester({ keepIdentifyingAttributes: true }, async (post, rows) => { + assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200); + assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["production"])); + for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) { + assert.equal(attributesOf(row)["http.user_agent"], "testclient"); + // Nothing is dropped, so the filter passes the request through: + // the raw user agent is stored instead of the hint. + assert.equal(attributesOf(row)["autter.test_client"], undefined); + } + await backgroundInserts(rows); + }); +}); diff --git a/packages/otlp-ingester/src/identifying.ts b/packages/otlp-ingester/src/identifying.ts index f0e3d23..3d6acd8 100644 --- a/packages/otlp-ingester/src/identifying.ts +++ b/packages/otlp-ingester/src/identifying.ts @@ -121,17 +121,25 @@ export class IdentifyingAttributeFilter { private readonly hasher: ClientIpHasher | null = null, ) {} - /** The attributes of a span or log record: the client IP hash added (from the raw list), then the drop. */ + /** + * The attributes of a span or log record: what is read from the raw list + * added first (the client IP hash; on a SERVER span, the test-client + * marker, a hint only), then the drop. + */ private recordAttributes = (obj: unknown, serverSpan: boolean): unknown => { + if (!isObject(obj)) return this.attributesOf(obj); + const added: Json[] = []; const hasher = this.hasher; - if (!hasher || !isObject(obj) || hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) return this.attributesOf(obj); - const ip = clientIpOfList(obj.attributes, serverSpan); + if (hasher && !hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) { + const ip = clientIpOfList(obj.attributes, serverSpan); + if (ip) added.push({ key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }); + } + if (serverSpan && testClientUserAgentIn(obj.attributes) && !hasKey(obj.attributes, TEST_CLIENT_ATTRIBUTE)) { + added.push({ key: TEST_CLIENT_ATTRIBUTE, value: { boolValue: true } }); + } const filtered = this.attributesOf(obj); - if (!ip || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered; - return { - ...filtered, - attributes: [...filtered.attributes, { key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }], - }; + if (!added.length || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered; + return { ...filtered, attributes: [...filtered.attributes, ...added] }; }; /** An OTLP key/value list without the dropped keys; the same array when none is present. */ @@ -262,3 +270,25 @@ export function clientIpOfList(list: unknown, serverSpan: boolean): string | nul /** OTLP SERVER span kind, as a number or the enum name. */ export const isServerKind = (kind: unknown) => kind === 2 || kind === "SPAN_KIND_SERVER"; + +// --------------------------------------------------------------------------- +// Test-client marker (a hint, never trusted) + +/** + * On a SERVER span whose raw user agent is exactly `testclient` (what + * Starlette's and FastAPI's in-process TestClient sends): added before the + * user agent is dropped, so the stored span still says so. A hint only: the + * user agent is whatever the caller sent, so anyone can produce it, and + * nothing in the ingester acts on it. The environment is never changed + * because of it: test runs set `deployment.environment=test` themselves. + * Not added when nothing is dropped (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES=true + * and no client IP hash): the request passes through with its user agent. + */ +export const TEST_CLIENT_ATTRIBUTE = "autter.test_client"; +const TEST_CLIENT_USER_AGENT = "testclient"; +const USER_AGENT_KEYS: readonly string[] = ["user_agent.original", "http.user_agent"]; + +/** The raw OTLP attribute list names the test client's user agent, exactly. */ +function testClientUserAgentIn(list: unknown): boolean { + return USER_AGENT_KEYS.some((key) => stringAttribute(list, key) === TEST_CLIENT_USER_AGENT); +} diff --git a/packages/otlp-ingester/src/logs.test.ts b/packages/otlp-ingester/src/logs.test.ts index cb74072..1b626f4 100644 --- a/packages/otlp-ingester/src/logs.test.ts +++ b/packages/otlp-ingester/src/logs.test.ts @@ -230,3 +230,55 @@ test("server exceptions' autter.debug_ids locate source maps and are not kept (R assert.equal("autter.debug_ids" in row!.attributes, false); assert.equal("autter.debug_ids" in row!.occurrence!.attributes!, false); }); + +// ── Environment from the resource only, and the running commit ────────────────────────── + +function oneRecord(resource: Record, attributes: Record) { + return normalizeLogs({ + resourceLogs: [{ + resource: { attributes: Object.entries({ "service.name": "shop", ...resource }).map(([k, v]) => attr(k, v)) }, + scopeLogs: [{ + logRecords: [{ + timeUnixNano: "1760000000000000000", + severityText: "error", + body: { stringValue: "checkout failed" }, + attributes: Object.entries({ + "autter.capture.mode": "log", + "exception.type": "OutOfStock", + "exception.message": "none left", + ...attributes, + }).map(([k, v]) => attr(k, v)), + }], + }], + }], + })[0]!; +} + +test("a record's own attributes never change its environment", () => { + for (const attributes of [{ "http.host": "testserver" }, { "server.address": "testserver" }, { "user_agent.original": "testclient" }]) { + const row = oneRecord({}, attributes); + assert.equal(row.environment, "production", JSON.stringify(attributes)); + assert.equal(row.occurrence?.environment, "production"); + assert.equal(oneRecord({ "deployment.environment": "test" }, attributes).environment, "test"); + assert.equal(oneRecord({ "deployment.environment": "staging" }, attributes).environment, "staging"); + } + const marked = normalizeLogs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "x" }, attributes: [{ key: "autter.test_client", value: { boolValue: true } }] }] }] }] }); + assert.equal(marked[0]!.environment, "production"); +}); + +test("a promoted error carries the running commit: the record's beats the resource's; invalid or none, no field", () => { + const fromResource = oneRecord({ "deployment.commit": "ABCDEF1234567" }, {}); + assert.equal(fromResource.occurrence?.commit, "abcdef1234567"); + assert.equal(fromResource.occurrence?.attributes?.["deployment.commit"], "abcdef1234567"); + // The log row's own attributes are left as they were. + assert.equal(fromResource.attributes["deployment.commit"], undefined); + + assert.equal(oneRecord({ "deployment.commit": "1111111" }, { "deployment.commit": "2222222" }).occurrence?.commit, "2222222"); + assert.equal(oneRecord({ "vcs.ref.head.revision": "3333333" }, {}).occurrence?.commit, "3333333"); + for (const bad of ["https://github.com/acme/shop", "main", "123456"]) { + const row = oneRecord({ "deployment.commit": bad }, {}); + assert.equal("commit" in row.occurrence!, false, bad); + assert.equal(row.occurrence?.attributes?.["deployment.commit"], undefined, bad); + } + assert.equal("commit" in oneRecord({}, {}).occurrence!, false); +}); diff --git a/packages/otlp-ingester/src/logs.ts b/packages/otlp-ingester/src/logs.ts index cf41af0..bc857ec 100644 --- a/packages/otlp-ingester/src/logs.ts +++ b/packages/otlp-ingester/src/logs.ts @@ -9,7 +9,7 @@ import { declaredLink, liftErrorFields } from "./error-fields.js"; import { fingerprintOverrideOf } from "./fingerprint-override.js"; import { normalizeRoute } from "./fingerprint.js"; import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js"; -import type { RuntimeOccurrenceInput } from "./types.js"; +import { commitOf, withCommitAttribute, type RuntimeOccurrenceInput } from "./types.js"; export interface RuntimeLogRecord { id: string; @@ -249,7 +249,7 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta statusCode: statusCode || null, traceId: traceId || null, sessionId: null, - attributes, + ...promotedCommit(attributes, attrs, info), occurredAt, ...declared, ...fingerprintOverrideOf(attributes["autter.fingerprint"]), @@ -323,6 +323,20 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta return rows; } +/** A promoted occurrence's attributes, with the running commit (record + * attributes first, then the resource; 7–40 hex) as `commit` and, when the + * attributes carry none, as `deployment.commit`. The record's own + * attributes are left as they are. */ +function promotedCommit( + attributes: Record, + attrs: Record, + info: Record, +): { attributes: Record; commit?: string } { + const commit = commitOf((key) => attrs[key], (key) => info[key]); + if (!commit) return { attributes }; + return { attributes: withCommitAttribute(attributes, commit) ?? attributes, commit }; +} + function hasException(attributes: Record): boolean { return ["exception.type", "exception.message", "exception.stacktrace"].some( (key) => typeof attributes[key] === "string" && attributes[key] !== "", diff --git a/packages/otlp-ingester/src/normalize-otlp.ts b/packages/otlp-ingester/src/normalize-otlp.ts index 1f42fbb..7e9f8f1 100644 --- a/packages/otlp-ingester/src/normalize-otlp.ts +++ b/packages/otlp-ingester/src/normalize-otlp.ts @@ -11,6 +11,8 @@ import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from ". import { histogramLatency, mergeLatency, requestLatency, type LatencySamples } from "./latency.js"; import { asSeverity, + commitOf, + withCommitAttribute, type RuntimeLlmCall, type RuntimeMetricPoint, type RuntimeOccurrenceInput, @@ -162,6 +164,8 @@ interface ResourceInfo { service: string; environment: string; release: string | null; + /** `deployment.commit`, else `vcs.ref.head.revision`, when 7–40 hex. */ + commit: string | null; /** `autter.metrics_wired` and `autter.metrics_routes` (runtime-node * 1.7.0+): the sender's delta HTTP-server duration histograms * (`/v1/metrics`) count every request it serves, and each request span @@ -191,6 +195,7 @@ function resourceInfo(resource: OtlpResource | undefined): ResourceInfo { service, environment, release, + commit: commitOf((key) => attrs.get(key)), metricsCountRoutedRequests: attrs.get("autter.metrics_wired") === "true" && attrs.get("autter.metrics_routes") === "true", metricsWired: attrs.get("autter.metrics_wired") === "true", @@ -342,6 +347,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { for (const event of exceptionEvents) { const eventAttrs = attrMap(event.attributes); const handled = eventAttrs.get("autter.handled") === "true"; + const commit = commitOf((key) => eventAttrs.get(key), (key) => attrs.get(key), () => resource.commit); const eventAutter = hasAutterKey(eventAttrs); signalRollups.push(rollup); signals.push({ @@ -370,8 +376,9 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), - ...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })), + attributes: withCommitAttribute(withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes), + ...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })), commit), + ...(commit ? { commit } : {}), // Server SDKs (runtime-core errorAttributes): the debug ids of the bundles in the // stack (RF-05), which locate their source maps. ...serverDebugIds(eventAttrs.get(DEBUG_IDS_ATTRIBUTE)), @@ -391,6 +398,7 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { for (const event of failedOutcomes) { const outcome = attrMap(event.attributes); const name = (outcome.get("autter.outcome.name") ?? "operation").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 200); + const commit = commitOf((key) => outcome.get(key), (key) => attrs.get(key), () => resource.commit); signalRollups.push(rollup); signals.push({ traceId, @@ -407,13 +415,15 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { message: `${name}: ${(outcome.get("autter.outcome.message") ?? "failed").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 1000)}`, stack: String(decodeOtlpAttributes(event.attributes)["autter.outcome.stack"] ?? "") || null, route, method: methodOf(attrs), statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), + attributes: withCommitAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }), commit), + ...(commit ? { commit } : {}), occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano), ...liftErrorFields(outcome, attrs), }, }); } if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) { + const commit = commitOf((key) => attrs.get(key), () => resource.commit); signalRollups.push(rollup); signals.push({ traceId, @@ -439,7 +449,8 @@ export function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces { statusCode, traceId: span.traceId ?? null, sessionId: null, - attributes: decodeOtlpAttributes(span.attributes), + attributes: withCommitAttribute(decodeOtlpAttributes(span.attributes), commit), + ...(commit ? { commit } : {}), occurredAt: startedAt, ...liftErrorFields(attrs), }, diff --git a/packages/otlp-ingester/src/normalize.test.ts b/packages/otlp-ingester/src/normalize.test.ts index 7e543a4..f40276c 100644 --- a/packages/otlp-ingester/src/normalize.test.ts +++ b/packages/otlp-ingester/src/normalize.test.ts @@ -237,3 +237,113 @@ test("warning occurrences outside a request count nothing", () => { assert.deepEqual(metricPoints, []); }); + +// ── Test-client traffic and the running commit ─────────────────────────── + +const NOW_NANOS = NANOS; +const kv = (attributes: Record) => + Object.entries(attributes).map(([key, value]) => ({ + key, + value: typeof value === "boolean" ? { boolValue: value } : { stringValue: value }, + })); + +/** One process: a request per entry of `requests`, each a server span with + * an exception event and an internal child span, in its own trace. */ +function processExport( + resource: Record, + requests: Array<{ trace: string; attributes: Record; event?: Record }>, +) { + return { + resourceSpans: [{ + resource: { attributes: kv({ "service.name": "shop", ...resource }) }, + scopeSpans: [{ + spans: requests.flatMap(({ trace, attributes, event }) => [ + { + traceId: trace, spanId: `${trace.slice(0, 15)}1`, name: "GET /carts/{cart_id}", kind: "SPAN_KIND_SERVER", + startTimeUnixNano: NOW_NANOS, endTimeUnixNano: NOW_NANOS, + attributes: kv({ "http.route": "/carts/{cart_id}", "http.method": "GET", "http.status_code": "500", ...attributes }), + status: { code: "STATUS_CODE_ERROR" }, + events: [{ name: "exception", timeUnixNano: NOW_NANOS, attributes: kv({ "exception.type": "OutOfStock", "exception.message": "none left", ...event }) }], + }, + { + traceId: trace, spanId: `${trace.slice(0, 15)}2`, parentSpanId: `${trace.slice(0, 15)}1`, name: "db", kind: "SPAN_KIND_INTERNAL", + startTimeUnixNano: NOW_NANOS, endTimeUnixNano: NOW_NANOS, attributes: [], + }, + ]), + }], + }], + }; +} + +const TRACE_A = "a".repeat(32); +const TRACE_B = "b".repeat(32); + +test("what a request says about itself never changes the environment: no environment is `production`, an explicit one is kept", () => { + const claims: Array> = [ + { "autter.test_client": true }, + { "user_agent.original": "testclient" }, + { "http.user_agent": "testclient" }, + { "http.host": "testserver", "http.server_name": "testserver", "server.address": "testserver", "net.host.name": "testserver" }, + ]; + for (const attributes of claims) { + const result = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes }])); + assert.deepEqual(result.spans.map((s) => s.environment), ["production", "production"], JSON.stringify(attributes)); + assert.deepEqual(result.occurrences.map((o) => o.environment), ["production"], JSON.stringify(attributes)); + assert.deepEqual(result.metricPoints.map((p) => p.environment), ["production"], JSON.stringify(attributes)); + for (const name of ["test", "staging"]) { + const explicit = normalizeTraces(processExport({ "deployment.environment": name }, [{ trace: TRACE_A, attributes }])); + assert.deepEqual(explicit.occurrences.map((o) => o.environment), [name]); + } + } + // Two requests of one process, one claiming to be a test client: both production. + const mixed = normalizeTraces(processExport({}, [ + { trace: TRACE_A, attributes: { "http.host": "testserver" } }, + { trace: TRACE_B, attributes: { "http.host": "shop.example.com" } }, + ])); + assert.deepEqual(mixed.occurrences.map((o) => o.environment), ["production", "production"]); +}); + +test("commit: deployment.commit (else vcs.ref.head.revision) from the resource, lower-cased, also in the attributes", () => { + const fromResource = normalizeTraces(processExport({ "deployment.commit": "9C1E4F2A7B3D" }, [{ trace: TRACE_A, attributes: {} }])); + assert.equal(fromResource.occurrences[0]!.commit, "9c1e4f2a7b3d"); + assert.equal(fromResource.occurrences[0]!.attributes?.["deployment.commit"], "9c1e4f2a7b3d"); + + const sha = "0123456789abcdef0123456789abcdef01234567"; + const vcs = normalizeTraces(processExport({ "vcs.ref.head.revision": sha }, [{ trace: TRACE_A, attributes: {} }])); + assert.equal(vcs.occurrences[0]!.commit, sha); + const both = normalizeTraces(processExport({ "vcs.ref.head.revision": sha, "deployment.commit": "abcdef1" }, [{ trace: TRACE_A, attributes: {} }])); + assert.equal(both.occurrences[0]!.commit, "abcdef1"); +}); + +test("commit: an event attribute beats the span's and the resource's", () => { + const result = normalizeTraces(processExport( + { "deployment.commit": "1111111111111111111111111111111111111111" }, + [{ trace: TRACE_A, attributes: { "deployment.commit": "2222222" }, event: { "deployment.commit": "3333333" } }], + )); + assert.equal(result.occurrences[0]!.commit, "3333333"); + // Already in the attributes: kept as sent. + assert.equal(result.occurrences[0]!.attributes?.["deployment.commit"], "3333333"); + const spanOnly = normalizeTraces(processExport( + { "deployment.commit": "1111111111111111111111111111111111111111" }, + [{ trace: TRACE_A, attributes: { "deployment.commit": "2222222" } }], + )); + assert.equal(spanOnly.occurrences[0]!.commit, "2222222"); +}); + +test("commit: anything but 7–40 hex is dropped; none means no field", () => { + for (const bad of ["https://github.com/acme/shop", "9c1e4f", "0".repeat(41), "main", "v1.2.3", "9c1e4f2-dirty"]) { + const result = normalizeTraces(processExport({ "deployment.commit": bad, "vcs.repository.url.full": "https://github.com/acme/shop" }, [{ trace: TRACE_A, attributes: {} }])); + const [occurrence] = result.occurrences; + assert.equal("commit" in occurrence!, false, bad); + // Resource attributes never reach the occurrence's attributes. + assert.equal(occurrence!.attributes?.["deployment.commit"], undefined, bad); + assert.equal(occurrence!.attributes?.["vcs.repository.url.full"], undefined, bad); + } + // An invalid event value does not hide a valid one below it. + const fallback = normalizeTraces(processExport({ "deployment.commit": "abcdef1234" }, [{ trace: TRACE_A, attributes: {}, event: { "deployment.commit": "not-a-sha" } }])); + assert.equal(fallback.occurrences[0]!.commit, "abcdef1234"); + + const none = normalizeTraces(processExport({}, [{ trace: TRACE_A, attributes: {} }])); + assert.equal("commit" in none.occurrences[0]!, false); + assert.equal(none.occurrences[0]!.attributes?.["deployment.commit"], undefined); +}); diff --git a/packages/otlp-ingester/src/python-otel.test.ts b/packages/otlp-ingester/src/python-otel.test.ts index d704ce6..5a1ed97 100644 --- a/packages/otlp-ingester/src/python-otel.test.ts +++ b/packages/otlp-ingester/src/python-otel.test.ts @@ -23,6 +23,12 @@ import type { RuntimeMetricPoint, RuntimeOccurrenceInput, StackFrame } from "./t * source, not from the parser under test. * Regenerate: AUTTER_CAPTURE_FIXTURES= python -m unittest in each * examples/python/ directory. + * + * `fastapi-testclient*.otlp.*`: the FastAPI recipe app exercised in-process + * through `fastapi.testclient.TestClient`, as a test suite does, with no + * environment and with `deployment.environment=staging`; recorded by + * test-fixtures/python/record_fastapi_testclient.py (no expectations file). + * The environment comes from the resource only, never from the request. */ const FIXTURES = new URL("../test-fixtures/python/", import.meta.url); @@ -263,6 +269,69 @@ describe("python request counting and dedupe on real spans", () => { }); }); +// ── FastAPI's TestClient: the environment comes from the resource only ──── + +type OtlpAttributeList = NonNullable[number]["resource"]>["attributes"]; + +/** `request` with `keys` removed from every span's attributes and the resource's, and `add` added to the resource. */ +function withAttributes(request: OtlpTraceRequest, keys: string[], add: OtlpAttributeList = []): OtlpTraceRequest { + const strip = (list: OtlpAttributeList) => list?.filter((kv) => !keys.includes(kv.key ?? "")); + return { + resourceSpans: request.resourceSpans?.map((rs) => ({ + resource: { attributes: [...(strip(rs.resource?.attributes) ?? []), ...(add ?? [])] }, + scopeSpans: rs.scopeSpans?.map((ss) => ({ + spans: ss.spans?.map((span) => ({ ...span, attributes: strip(span.attributes) })), + })), + })), + }; +} + +describe("python fastapi TestClient (real OTel export)", () => { + const environments = (result: ReturnType) => ({ + spans: [...new Set(result.spans.map((s) => s.environment))], + occurrences: result.occurrences.map((o) => o.environment), + rollups: [...new Set(result.metricPoints.map((p) => p.environment))], + }); + + test("what TestClient sends: user agent `testclient`, host `testserver`, no environment", () => { + const request = loadJson("fastapi-testclient"); + const resource = request.resourceSpans!.flatMap((rs) => rs.resource?.attributes ?? []).map((kv) => kv.key); + assert.ok(!resource.includes("deployment.environment") && !resource.includes("deployment.environment.name")); + const servers = request.resourceSpans!.flatMap((rs) => rs.scopeSpans!.flatMap((ss) => ss.spans!)).filter((s) => s.kind === "SPAN_KIND_SERVER"); + assert.equal(servers.length, 2); + for (const span of servers) { + const attrs = Object.fromEntries((span.attributes ?? []).map((kv) => [kv.key, kv.value?.stringValue])); + assert.equal(attrs["http.user_agent"], "testclient"); + assert.equal(attrs["http.host"], "testserver"); + assert.equal(attrs["http.server_name"], "testserver"); + } + }); + + test("no environment: stored as `production`, whatever the request says about itself (JSON and protobuf)", () => { + for (const result of [normalizeTraces(loadJson("fastapi-testclient")), normalizeTraces(decodeTraceRequest(loadProto("fastapi-testclient")))]) { + assert.deepEqual(environments(result), { spans: ["production"], occurrences: ["production", "production"], rollups: ["production"] }); + assert.deepEqual(result.occurrences.map((o) => o.errorType).sort(), ["inventory.OutOfStock", "inventory.PricingTimeout"]); + } + }); + + test("deployment.environment=test set explicitly (as test runs will): stored as `test`", () => { + const request = withAttributes(loadJson("fastapi-testclient"), [], [{ key: "deployment.environment", value: { stringValue: "test" } }]); + assert.deepEqual(environments(normalizeTraces(request)), { spans: ["test"], occurrences: ["test", "test"], rollups: ["test"] }); + }); + + test("deployment.environment=staging: unchanged, `staging`", () => { + for (const result of [normalizeTraces(loadJson("fastapi-testclient-staging")), normalizeTraces(decodeTraceRequest(loadProto("fastapi-testclient-staging")))]) { + assert.deepEqual(environments(result), { spans: ["staging"], occurrences: ["staging", "staging"], rollups: ["staging"] }); + } + }); + + test("a normal request (uvicorn, a real client) with no environment is `production` too", () => { + const fastapi = expectations.find((e) => e.framework === "fastapi")!.scenarios; + const result = normalizeTraces(withAttributes(loadJson(fastapi.unhandled!.fixture), ["deployment.environment"])); + assert.deepEqual(environments(result), { spans: ["production"], occurrences: ["production"], rollups: ["production"] }); + }); +}); + // ── The recommended rate: 1% head sampling, 2% of requests failing ──────── /** diff --git a/packages/otlp-ingester/src/server.sink.test.ts b/packages/otlp-ingester/src/server.sink.test.ts index ec3d118..16c5c95 100644 --- a/packages/otlp-ingester/src/server.sink.test.ts +++ b/packages/otlp-ingester/src/server.sink.test.ts @@ -340,3 +340,47 @@ test("one error recorded on two spans reaches the sink once, with the same id wh assert.equal(requests, 0); assert.equal(errors, 0); }); + +test("the running commit reaches the sink as a hex SHA only; what a request says about itself never changes the environment", async () => { + const now = String(Date.now() * 1_000_000); + const str = (key: string, value: string) => ({ key, value: { stringValue: value } }); + const process = (n: number, resource: Array>, span: Array>, event: Array> = []) => ({ + resource: { attributes: [str("service.name", "shop"), ...resource] }, + scopeSpans: [{ + spans: [{ + traceId: String(n).repeat(32), spanId: String(n).repeat(16), name: "GET /orders", kind: 2, + startTimeUnixNano: now, endTimeUnixNano: now, + attributes: [str("http.route", "/orders"), ...span], + events: [{ name: "exception", timeUnixNano: now, attributes: [str("exception.type", `Error${n}`), str("exception.message", `boom ${n}`), ...event] }], + }], + }], + }); + const sha = "9C1E4F2A7B3D9C1E4F2A7B3D9C1E4F2A7B3D9C1E"; + const res = await fetch(`${ingestUrl}/v1/traces`, { + method: "POST", + headers: { "content-type": "application/json", authorization: "Bearer test-key" }, + body: JSON.stringify({ + resourceSpans: [ + process(1, [str("deployment.commit", sha), str("deployment.environment", "production")], []), + process(2, [str("deployment.commit", sha)], [], [str("deployment.commit", "abcdef1")]), + process(3, [str("deployment.commit", "https://github.com/acme/shop"), str("vcs.repository.url.full", "https://github.com/acme/shop")], []), + process(4, [], [str("http.host", "testserver")]), + process(5, [str("deployment.environment", "staging")], [str("http.host", "testserver")]), + process(6, [], [str("http.host", "testserver.example.com"), str("user_agent.original", "Mozilla/5.0 (testclient)")]), + ], + }), + }); + assert.equal(res.status, 200); + const batch = await nextSinkBatch(); + const byType = Object.fromEntries(batch.occurrences.map((o: Record) => [o.errorType, o])); + assert.equal(byType.Error1.commit, sha.toLowerCase()); + assert.equal(byType.Error1.attributes["deployment.commit"], sha.toLowerCase()); + assert.equal(byType.Error2.commit, "abcdef1", "the event's commit wins"); + for (const name of ["Error3", "Error4", "Error5", "Error6"]) assert.equal("commit" in byType[name], false, name); + assert.equal(JSON.stringify(byType.Error3).includes("github.com"), false, "nothing else of the resource is forwarded"); + assert.deepEqual( + ["Error1", "Error2", "Error3", "Error4", "Error5", "Error6"].map((name) => byType[name].environment), + ["production", "production", "production", "production", "staging", "production"], + ); + assert.equal(byType.Error6.attributes["user_agent.original"], undefined, "the user agent is dropped"); +}); diff --git a/packages/otlp-ingester/src/sink.test.ts b/packages/otlp-ingester/src/sink.test.ts index 0d3c94e..05a4caa 100644 --- a/packages/otlp-ingester/src/sink.test.ts +++ b/packages/otlp-ingester/src/sink.test.ts @@ -319,3 +319,13 @@ test("1.6.0 sink fields: frames, fingerprints and the record ride along only whe assert.equal(key in bare, false, key); } }); + +test("commit: a 7–40 hex SHA rides along lower-cased; anything else, or none, emits no field", () => { + assert.equal(sinkOccurrence(occ({ commit: "9c1e4f2a7b3d" })).commit, "9c1e4f2a7b3d"); + const full = "0123456789abcdef0123456789ABCDEF01234567"; + assert.equal(sinkOccurrence(occ({ commit: full })).commit, full.toLowerCase()); + for (const bad of ["9c1e4f", "0".repeat(41), "https://github.com/acme/shop", "v1.2.3", "main", " ", ""]) { + assert.equal("commit" in sinkOccurrence(occ({ commit: bad })), false, bad); + } + assert.equal("commit" in sinkOccurrence(occ({})), false); +}); diff --git a/packages/otlp-ingester/src/sink.ts b/packages/otlp-ingester/src/sink.ts index 967d3b7..9d6f53c 100644 --- a/packages/otlp-ingester/src/sink.ts +++ b/packages/otlp-ingester/src/sink.ts @@ -1,11 +1,12 @@ import { createHash, randomUUID } from "node:crypto"; import type { IngesterConfig } from "./config.js"; import type { OutboxRecord, SinkOutbox } from "./sink-outbox.js"; -import type { - IngestContext, - RuntimeLlmCall, - RuntimeMetricPoint, - RuntimeOccurrence, +import { + validCommit, + type IngestContext, + type RuntimeLlmCall, + type RuntimeMetricPoint, + type RuntimeOccurrence, } from "./types.js"; /** @@ -678,8 +679,11 @@ export function sinkLlmCall(c: RuntimeLlmCall): Record { * their event counts. Unreleased (RF-06): `fingerprints` may end with a * `cause-v1` candidate after the legacy one; consumers that do not know the * scheme drop it (the platform validates schemes one by one). RF-05: in-app - * frames' context lines are up to 5 on each side (3 before). Older consumers - * ignore unknown keys, so the version stays 1. + * frames' context lines are up to 5 on each side (3 before). Unreleased: + * `commit`, the commit the app runs (`deployment.commit`, else + * `vcs.ref.head.revision`), sent only when it is 7–40 hexadecimal + * characters (lower-cased); nothing else of the resource is forwarded. + * Older consumers ignore unknown keys, so the version stays 1. */ export function sinkOccurrence(o: RuntimeOccurrence): Record { const { @@ -700,8 +704,11 @@ export function sinkOccurrence(o: RuntimeOccurrence): Record { // Input-only: locates source maps, never leaves the ingester. debugIds: _debugIds, collapsedCount, + commit: rawCommit, ...rest } = o; + // Checked again here: only a hex SHA ever leaves the ingester. + const commit = validCommit(rawCommit); return { ...rest, occurredAt: o.occurredAt.toISOString(), @@ -715,6 +722,7 @@ export function sinkOccurrence(o: RuntimeOccurrence): Record { ...(link ? { link } : {}), ...(expected ? { expected: true } : {}), ...(requestId ? { requestId } : {}), + ...(commit ? { commit } : {}), // 1.6.0: steps before the error, oldest first, ≤ 30. ...(breadcrumbs?.length ? { breadcrumbs } : {}), ...(frames?.length ? { frames } : {}), diff --git a/packages/otlp-ingester/src/types.ts b/packages/otlp-ingester/src/types.ts index 9177dea..e66da11 100644 --- a/packages/otlp-ingester/src/types.ts +++ b/packages/otlp-ingester/src/types.ts @@ -29,6 +29,36 @@ export function asSeverity( : fallback; } +const COMMIT_SHA = /^[0-9a-f]{7,40}$/; + +/** A commit SHA: 7 to 40 hexadecimal characters, trimmed and lower-cased; null for anything else. */ +export function validCommit(value: unknown): string | null { + if (typeof value !== "string") return null; + const sha = value.trim().toLowerCase(); + return COMMIT_SHA.test(sha) ? sha : null; +} + +/** The first valid commit among `deployment.commit` then `vcs.ref.head.revision` in each source, in order. */ +export function commitOf(...sources: Array<(key: string) => unknown>): string | null { + for (const get of sources) { + const sha = validCommit(get("deployment.commit")) ?? validCommit(get("vcs.ref.head.revision")); + if (sha) return sha; + } + return null; +} + +/** + * `attributes` with `deployment.commit` set to `commit` when they carry + * none, so a reader of the stored attributes JSON finds it there too. + */ +export function withCommitAttribute( + attributes: Record | null, + commit: string | null, +): Record | null { + if (!commit || attributes?.["deployment.commit"] !== undefined) return attributes; + return { ...(attributes ?? {}), "deployment.commit": commit }; +} + /** A normalised error/warning event, before fingerprinting. */ export interface RuntimeOccurrenceInput { source: RuntimeSource; @@ -64,6 +94,13 @@ export interface RuntimeOccurrenceInput { expected?: boolean; /** `autter.request.id` — joins the occurrence to its request summary. */ requestId?: string; + /** + * The commit the app runs: `deployment.commit`, then + * `vcs.ref.head.revision`, from the event, the span (or log record), then + * the resource; only a 7–40 hex value (`validCommit`), lower-cased. + * Omitted when absent. + */ + commit?: string; /** * SDK fingerprint override (browser `fingerprint`, OTLP/log * `autter.fingerprint`): ≤10 scrubbed values, each ≤200 chars. Hashed into diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json new file mode 100644 index 0000000..35a44b6 --- /dev/null +++ b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.json @@ -0,0 +1,417 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "161b8d5e-2838-4348-ab30-28414d647a99" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "9c1e4f2a7b3d" + } + }, + { + "key": "deployment.environment", + "value": { + "stringValue": "staging" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "shop-fastapi" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "opentelemetry.instrumentation.fastapi", + "version": "0.66b1" + }, + "spans": [ + { + "traceId": "b00d289cf99d9ec3dda016ae7e1ca0c5", + "spanId": "5ec630f3e9befc33", + "parentSpanId": "bae89fabbd89cef6", + "name": "POST /carts/{cart_id}/checkout http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122666672000", + "endTimeUnixNano": "1794719122666693000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "500" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "b00d289cf99d9ec3dda016ae7e1ca0c5", + "spanId": "2147636e71782fe8", + "parentSpanId": "bae89fabbd89cef6", + "name": "POST /carts/{cart_id}/checkout http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122666762000", + "endTimeUnixNano": "1794719122666768000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "b00d289cf99d9ec3dda016ae7e1ca0c5", + "spanId": "bae89fabbd89cef6", + "name": "POST /carts/{cart_id}/checkout", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719122664158000", + "endTimeUnixNano": "1794719122667414000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/checkout" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/checkout" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/checkout" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "500" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719122666616000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.OutOfStock" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "sku-1: 2 requested, 0 left" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/venv/lib/python3.14/site-packages/opentelemetry/instrumentation/fastapi/__init__.py\", line 360, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/telemetry/_asgi.py\", line 151, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/middleware/exceptions.py\", line 63, in __call__\n await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/middleware/asyncexitstack.py\", line 18, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/routing.py\", line 678, in __call__\n await self.middleware_stack(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 2811, in app\n await route.handle(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 1330, in handle\n await super().handle(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/routing.py\", line 284, in handle\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 166, in app\n await wrap_app_handling_exceptions(app, request)(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 152, in app\n response = await f(request)\n ^^^^^^^^^^^^^^^^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 745, in app\n raw_response = await run_endpoint_function(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n )\n ^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 372, in run_endpoint_function\n return await dependant.call(**values)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/srv/app/app.py\", line 36, in checkout\n reserve(\"sku-1\", 2) # frame: checkout\n ~~~~~~~^^^^^^^^^^^^\n File \"/srv/app/inventory.py\", line 17, in reserve\n raise OutOfStock(f\"{sku}: {quantity} requested, {STOCK.get(sku, 0)} left\") # frame: reserve\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.OutOfStock: sku-1: 2 requested, 0 left\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + } + ] + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "2d98a731d4ed701e58a0900f0d295645", + "spanId": "c3881eae62134149", + "parentSpanId": "a7a8c3faa5ea5706", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122669819000", + "endTimeUnixNano": "1794719122669832000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "2d98a731d4ed701e58a0900f0d295645", + "spanId": "d92eb23fddde26ba", + "parentSpanId": "a7a8c3faa5ea5706", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719122669869000", + "endTimeUnixNano": "1794719122669873000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "2d98a731d4ed701e58a0900f0d295645", + "spanId": "a7a8c3faa5ea5706", + "name": "GET /carts/{cart_id}/quote", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719122669085000", + "endTimeUnixNano": "1794719122670223000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/quote" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/quote" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/quote" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719122669687000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.PricingTimeout" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "pricing for cart 12 timed out after 2.5s" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/app/app.py\", line 43, in quote\n return {\"cart\": cart_id, \"total\": price(cart_id)} # frame: quote\n ~~~~~^^^^^^^^^\n File \"/srv/app/inventory.py\", line 21, in price\n raise PricingTimeout(f\"pricing for cart {cart_id} timed out after 2.5s\") # frame: price\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.PricingTimeout: pricing for cart 12 timed out after 2.5s\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + }, + { + "key": "autter.handled", + "value": { + "boolValue": true + } + } + ] + } + ], + "status": {}, + "flags": 256 + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.11.0" + } + ] + } + ] +} diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.pb b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.pb new file mode 100644 index 0000000..142814d Binary files /dev/null and b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient-staging.otlp.pb differ diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.json b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.json new file mode 100644 index 0000000..4e9c124 --- /dev/null +++ b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.json @@ -0,0 +1,411 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.1" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "cdfc7fa7-f26a-412c-a20e-9acdcdf13181" + } + }, + { + "key": "service.version", + "value": { + "stringValue": "9c1e4f2a7b3d" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "shop-fastapi" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "opentelemetry.instrumentation.fastapi", + "version": "0.66b1" + }, + "spans": [ + { + "traceId": "550d1dd9795574c50e2b4652ef6cf997", + "spanId": "50ff69fca9ae13bc", + "parentSpanId": "75c53a052549811b", + "name": "POST /carts/{cart_id}/checkout http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719121763844000", + "endTimeUnixNano": "1794719121763862000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "500" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "550d1dd9795574c50e2b4652ef6cf997", + "spanId": "e7f739e307d69564", + "parentSpanId": "75c53a052549811b", + "name": "POST /carts/{cart_id}/checkout http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719121763928000", + "endTimeUnixNano": "1794719121763934000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "550d1dd9795574c50e2b4652ef6cf997", + "spanId": "75c53a052549811b", + "name": "POST /carts/{cart_id}/checkout", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719121760429000", + "endTimeUnixNano": "1794719121764413000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/checkout" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/checkout" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "POST" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/checkout" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "500" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719121763792000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.OutOfStock" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "sku-1: 2 requested, 0 left" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/venv/lib/python3.14/site-packages/opentelemetry/instrumentation/fastapi/__init__.py\", line 360, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/telemetry/_asgi.py\", line 151, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/middleware/exceptions.py\", line 63, in __call__\n await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/middleware/asyncexitstack.py\", line 18, in __call__\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/routing.py\", line 678, in __call__\n await self.middleware_stack(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 2811, in app\n await route.handle(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 1330, in handle\n await super().handle(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/routing.py\", line 284, in handle\n await self.app(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 166, in app\n await wrap_app_handling_exceptions(app, request)(scope, receive, send)\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 53, in wrapped_app\n raise exc\n File \"/srv/venv/lib/python3.14/site-packages/starlette/_exception_handler.py\", line 42, in wrapped_app\n await app(scope, receive, sender)\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 152, in app\n response = await f(request)\n ^^^^^^^^^^^^^^^^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 745, in app\n raw_response = await run_endpoint_function(\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n )\n ^\n File \"/srv/venv/lib/python3.14/site-packages/fastapi/routing.py\", line 372, in run_endpoint_function\n return await dependant.call(**values)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"/srv/app/app.py\", line 36, in checkout\n reserve(\"sku-1\", 2) # frame: checkout\n ~~~~~~~^^^^^^^^^^^^\n File \"/srv/app/inventory.py\", line 17, in reserve\n raise OutOfStock(f\"{sku}: {quantity} requested, {STOCK.get(sku, 0)} left\") # frame: reserve\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.OutOfStock: sku-1: 2 requested, 0 left\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + } + ] + } + ], + "status": { + "code": "STATUS_CODE_ERROR" + }, + "flags": 256 + }, + { + "traceId": "30d47486f4f685da0daa8d24539c418e", + "spanId": "d8cff7b6824b78db", + "parentSpanId": "b882fc6e690bc358", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719121766313000", + "endTimeUnixNano": "1794719121766324000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.start" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "30d47486f4f685da0daa8d24539c418e", + "spanId": "d12781cf29bfb254", + "parentSpanId": "b882fc6e690bc358", + "name": "GET /carts/{cart_id}/quote http send", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1794719121766358000", + "endTimeUnixNano": "1794719121766362000", + "attributes": [ + { + "key": "asgi.event.type", + "value": { + "stringValue": "http.response.body" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "status": {}, + "flags": 256 + }, + { + "traceId": "30d47486f4f685da0daa8d24539c418e", + "spanId": "b882fc6e690bc358", + "name": "GET /carts/{cart_id}/quote", + "kind": "SPAN_KIND_SERVER", + "startTimeUnixNano": "1794719121765657000", + "endTimeUnixNano": "1794719121766594000", + "attributes": [ + { + "key": "http.scheme", + "value": { + "stringValue": "http" + } + }, + { + "key": "http.host", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "net.host.port", + "value": { + "intValue": "80" + } + }, + { + "key": "http.flavor", + "value": { + "stringValue": "1.1" + } + }, + { + "key": "http.target", + "value": { + "stringValue": "/carts/12/quote" + } + }, + { + "key": "http.url", + "value": { + "stringValue": "http://testserver/carts/12/quote" + } + }, + { + "key": "http.method", + "value": { + "stringValue": "GET" + } + }, + { + "key": "http.server_name", + "value": { + "stringValue": "testserver" + } + }, + { + "key": "http.user_agent", + "value": { + "stringValue": "testclient" + } + }, + { + "key": "http.route", + "value": { + "stringValue": "/carts/{cart_id}/quote" + } + }, + { + "key": "http.status_code", + "value": { + "intValue": "200" + } + }, + { + "key": "autter.tail_retained", + "value": { + "boolValue": true + } + } + ], + "events": [ + { + "timeUnixNano": "1794719121766204000", + "name": "exception", + "attributes": [ + { + "key": "exception.type", + "value": { + "stringValue": "inventory.PricingTimeout" + } + }, + { + "key": "exception.message", + "value": { + "stringValue": "pricing for cart 12 timed out after 2.5s" + } + }, + { + "key": "exception.stacktrace", + "value": { + "stringValue": "Traceback (most recent call last):\n File \"/srv/app/app.py\", line 43, in quote\n return {\"cart\": cart_id, \"total\": price(cart_id)} # frame: quote\n ~~~~~^^^^^^^^^\n File \"/srv/app/inventory.py\", line 21, in price\n raise PricingTimeout(f\"pricing for cart {cart_id} timed out after 2.5s\") # frame: price\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\ninventory.PricingTimeout: pricing for cart 12 timed out after 2.5s\n" + } + }, + { + "key": "exception.escaped", + "value": { + "stringValue": "False" + } + }, + { + "key": "autter.handled", + "value": { + "boolValue": true + } + } + ] + } + ], + "status": {}, + "flags": 256 + } + ], + "schemaUrl": "https://opentelemetry.io/schemas/1.11.0" + } + ] + } + ] +} diff --git a/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.pb b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.pb new file mode 100644 index 0000000..4e08b8d Binary files /dev/null and b/packages/otlp-ingester/test-fixtures/python/fastapi-testclient.otlp.pb differ diff --git a/packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py b/packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py new file mode 100644 index 0000000..f8cd641 --- /dev/null +++ b/packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py @@ -0,0 +1,99 @@ +"""Records the FastAPI TestClient fixtures of the ingester's Python tests +(``fastapi-testclient*.otlp.{json,pb}``): the FastAPI recipe app +(examples/python/fastapi) exercised in-process through +``fastapi.testclient.TestClient``, as a test suite does, exporting OTLP/HTTP +to the recipes' stub receiver (examples/python/testing/otlp_stub.py). + + cd examples/python/fastapi + python3 -m venv .venv && .venv/bin/pip install -r requirements.txt httpx + .venv/bin/python ../../../packages/otlp-ingester/test-fixtures/python/record_fastapi_testclient.py + +Two captures, each in its own process (the app calls ``init_tracing()`` at +import): ``fastapi-testclient`` with no ``deployment.environment`` (a test +run that names none), and ``fastapi-testclient-staging`` with +``deployment.environment=staging``. Both keep the raw user agent +(``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES=http.user_agent,user_agent.original``), +as a plain OpenTelemetry sender or an adapter before 1.19.0 sends it; the +current adapter drops it before export, which the ingester's tests model by +removing it. Sampling ratio 0: only the always-on error path exports. + +Not run by CI (TestClient needs httpx, which the recipe does not install). +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) +RECIPE = os.path.join(REPO, "examples", "python", "fastapi") +TESTING = os.path.join(REPO, "examples", "python", "testing") +ADAPTER = os.path.join(REPO, "adapters", "python") + +CAPTURES = { + "fastapi-testclient": "service.version=9c1e4f2a7b3d", + "fastapi-testclient-staging": "service.version=9c1e4f2a7b3d,deployment.environment=staging", +} + + +def capture(name: str) -> None: + """In the child: run the app under TestClient and write the fixture.""" + sys.path[:0] = [TESTING, RECIPE, ADAPTER] + from otlp_stub import OtlpStub, package_versions, write_fixture + + stub = OtlpStub() + os.environ.update({ + "OTEL_EXPORTER_OTLP_ENDPOINT": stub.url, + "OTEL_EXPORTER_OTLP_HEADERS": "x-autter-key=autter_rt_recipe_test_key_0001", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_SERVICE_NAME": "shop-fastapi", + "OTEL_RESOURCE_ATTRIBUTES": CAPTURES[name], + "OTEL_TRACES_SAMPLER": "parentbased_traceidratio", + "OTEL_TRACES_SAMPLER_ARG": "0", + "OTEL_BSP_SCHEDULE_DELAY": "200", + "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES": "http.user_agent,user_agent.original", + "AUTTER_COUNT_REQUESTS": "false", + }) + os.chdir(RECIPE) + from fastapi.testclient import TestClient + from opentelemetry import trace + + import app as recipe + + # raise_server_exceptions=False: answer 500 as the server would, instead + # of re-raising in the test, so the request span records the exception. + client = TestClient(recipe.app, raise_server_exceptions=False) + assert client.post("/carts/12/checkout").status_code == 500 + assert client.get("/carts/12/quote").status_code == 200 + trace.get_tracer_provider().force_flush() + deadline = time.monotonic() + 20 + while True: + spans = stub.spans() + servers = [s for s in spans if s["kind"] == "SERVER"] + if len(servers) >= 2: + break + if time.monotonic() > deadline: + raise AssertionError(f"timed out; got {[s['name'] for s in spans]}") + time.sleep(0.05) + for server in servers: + print(name, server["name"], {k: v for k, v in server["attributes"].items() if "host" in k or "server" in k or "agent" in k or "peer" in k}) + write_fixture(stub, name, sorted({s["trace_id"] for s in servers}), RECIPE) + print(name, package_versions("fastapi", "starlette", "httpx", "opentelemetry-sdk", "opentelemetry-instrumentation-fastapi")) + stub.close() + + +def main() -> None: + if len(sys.argv) == 2: + capture(sys.argv[1]) + return + for name in CAPTURES: + env = {k: v for k, v in os.environ.items() if not k.startswith("OTEL_")} + env["AUTTER_CAPTURE_FIXTURES"] = HERE + subprocess.run([sys.executable, os.path.abspath(__file__), name], env=env, check=True) + + +if __name__ == "__main__": + main()