Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions packages/otlp-ingester/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,34 @@ hashing every IPv4 address) and the ingester warns at startup. A hashed IP
is still pseudonymous personal data in some readings of privacy law: it is
off by default, and the salt should be kept and rotated like any secret.

## Test traffic (unreleased)

Test traffic is separated by setting `deployment.environment=test` in the
test run. The Python adapter and `@autter/runtime-node` do this themselves
under a test runner from the next release; anything else sets it in
`OTEL_RESOURCE_ATTRIBUTES`. The ingester never changes the environment
because of what a request says about itself (its user agent, its host, its
peer address): those come from the caller, and a caller could use them to
have its errors stored outside production, out of default views and alerts.
The environment is the resource's `deployment.environment.name` or
`deployment.environment`, else `production`.

A SERVER span whose raw user agent is exactly `testclient` (what Starlette's
and FastAPI's in-process `TestClient` sends) gets `autter.test_client: true`
before the user agent is dropped. It is a hint for a reader, not trusted:
anyone can send that user agent, and nothing in the ingester acts on it.

## Running commit (unreleased)

Each occurrence forwarded to the sink carries `commit` when the app reports
the commit it runs: `deployment.commit`, else `vcs.ref.head.revision`, read
from the exception event, then the span (or log record), then the resource.
Only a value of 7 to 40 hexadecimal characters is forwarded, lower-cased;
anything else (a repository URL, a branch name) is dropped, and nothing else
of the resource is forwarded. When the occurrence's attributes have no
`deployment.commit`, the SHA is also stored there. The field is optional on
sink payload version 1: a platform that does not know it ignores it.

## Rate limits and key lookups (1.12.0)

In the order a request meets them:
Expand Down
122 changes: 122 additions & 0 deletions packages/otlp-ingester/src/identifying.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
IdentifyingAttributeFilter,
identifyingKeysToDrop,
keepIdentifyingEnv,
clientIpHasher,
} from "./identifying.js";
import { createIngesterApp } from "./server.js";

Expand Down Expand Up @@ -117,6 +118,49 @@ test("keep (true or a list) and extra drops; env parsing", () => {
// ---------------------------------------------------------------------------
// Through the ingester: a real Python OpenTelemetry export (protobuf, Flask
// instrumentation: net.peer.ip, net.peer.port, http.user_agent) and a plain
test("test client: a SERVER span whose raw user agent is exactly `testclient` gets autter.test_client; the user agent is still dropped", () => {
const filter = new IdentifyingAttributeFilter();
const span = (kind: number | string, attributes: Record<string, string>) => ({ kind, name: "GET /", attributes: kv(attributes) });
const request = {
resourceSpans: [{
scopeSpans: [{
spans: [
span(2, { "http.user_agent": "testclient", "http.host": "testserver" }),
span("SPAN_KIND_SERVER", { "user_agent.original": "testclient" }),
span(1, { "http.user_agent": "testclient" }),
span(3, { "user_agent.original": "testclient" }),
span(2, { "user_agent.original": "Mozilla/5.0 testclient" }),
span(2, { "http.user_agent": "TestClient" }),
],
}],
}],
};
const before = JSON.stringify(request);
const spans = filter.traces(request).resourceSpans[0]!.scopeSpans[0]!.spans;
assert.equal(JSON.stringify(request), before, "input not mutated");
assert.deepEqual(spans.map((s) => keysOf(s.attributes)), [
["http.host", "autter.test_client"],
["autter.test_client"],
[], [], [], [],
]);
assert.deepEqual((spans[0]!.attributes as Array<{ value: unknown }>)[1]!.value, { boolValue: true });

// Already marked (an SDK did it): not added twice. With the client IP hash too.
const hashing = new IdentifyingAttributeFilter(identifyingKeysToDrop(), clientIpHasher(true, "s".repeat(16)));
const marked = hashing.traces({ resourceSpans: [{ scopeSpans: [{ spans: [
{ kind: 2, attributes: [...kv({ "http.user_agent": "testclient", "client.address": "203.0.113.9" }), { key: "autter.test_client", value: { boolValue: true } }] },
{ kind: 2, attributes: kv({ "http.user_agent": "testclient", "client.address": "203.0.113.9" }) },
] }] }] }).resourceSpans[0]!.scopeSpans[0]!.spans;
assert.deepEqual(marked.map((s) => keysOf(s.attributes)), [
["autter.test_client", "client.address_hash"],
["client.address_hash", "autter.test_client"],
]);

// Log records are never marked.
const logs = filter.logs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ attributes: kv({ "user_agent.original": "testclient" }) }] }] }] });
assert.deepEqual(keysOf(logs.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!.attributes), []);
});

// OTLP/JSON logs export, against a fake ClickHouse.

const listen = async (server: Server) => {
Expand Down Expand Up @@ -206,3 +250,81 @@ test("keepIdentifyingAttributes (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) stores them
assert.ok(!storedKeys(rows, "runtime_spans").has("net.peer.ip"));
});
});

// FastAPI's TestClient, recorded (test-fixtures/python/record_fastapi_testclient.py).
const TESTCLIENT = readFileSync(new URL("../test-fixtures/python/fastapi-testclient.otlp.pb", import.meta.url));
const TESTCLIENT_STAGING = readFileSync(new URL("../test-fixtures/python/fastapi-testclient-staging.otlp.pb", import.meta.url));
/** Waits (up to 2 s) for the inserts the ingester makes after answering
* (SDK sightings, ingester info), so closing the fake ClickHouse does not
* reset them mid-request. Quiets the log only: nothing is asserted on them. */
async function backgroundInserts(rows: Record<string, unknown[]>) {
const deadline = Date.now() + 2000;
while (!(rows.runtime_sdk_versions && rows.runtime_ingester_info) && Date.now() < deadline)
await new Promise((resolve) => setTimeout(resolve, 10));
}
const attributesOf = (row: Record<string, unknown>) => JSON.parse(String(row.attributes ?? "{}")) as Record<string, unknown>;

/** The recorded no-environment export, as OTLP/JSON, with `deployment.environment` set to `name`. */
function testClientWithEnvironment(name: string): string {
const request = JSON.parse(readFileSync(new URL("../test-fixtures/python/fastapi-testclient.otlp.json", import.meta.url), "utf8"));
for (const rs of request.resourceSpans) rs.resource.attributes.push({ key: "deployment.environment", value: { stringValue: name } });
return JSON.stringify(request);
}

test("TestClient traffic with no environment is stored as `production`: the server spans carry the autter.test_client hint and no user agent", async () => {
await withIngester({}, async (post, rows) => {
assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200);
const spans = rows.runtime_spans ?? [];
assert.equal(spans.length, 6);
// The user agent and host are the caller's to send: they never change the environment.
assert.deepEqual(new Set(spans.map((row) => row.environment)), new Set(["production"]));
const servers = spans.filter((row) => row.kind === "server");
assert.equal(servers.length, 2);
for (const row of servers) {
const attributes = attributesOf(row);
assert.equal(attributes["autter.test_client"], true);
assert.equal(attributes["http.user_agent"], undefined, "the user agent is still dropped");
assert.equal(attributes["user_agent.original"], undefined);
}
for (const row of spans.filter((r) => r.kind !== "server")) assert.equal(attributesOf(row)["autter.test_client"], undefined);
const occurrences = rows.runtime_error_occurrences ?? [];
assert.deepEqual(occurrences.map((row) => row.environment), ["production", "production"]);
for (const table of ["runtime_spans", "runtime_error_occurrences"])
for (const key of vectors.drop) assert.ok(!storedKeys(rows, table).has(key), `${key} stored in ${table}`);
await backgroundInserts(rows);
});
});

test("the same TestClient export with deployment.environment=test set explicitly is stored as `test`", async () => {
await withIngester({}, async (post, rows) => {
assert.equal(await post("/v1/traces", testClientWithEnvironment("test"), "application/json"), 200);
assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["test"]));
assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["test", "test"]);
await backgroundInserts(rows);
});
});

test("the TestClient export with deployment.environment=staging stays `staging`", async () => {
await withIngester({}, async (post, rows) => {
assert.equal(await post("/v1/traces", TESTCLIENT_STAGING, "application/x-protobuf"), 200);
assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["staging"]));
assert.deepEqual((rows.runtime_error_occurrences ?? []).map((row) => row.environment), ["staging", "staging"]);
for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server"))
assert.equal(attributesOf(row)["autter.test_client"], true);
await backgroundInserts(rows);
});
});

test("with the user agent kept (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) it is stored, and the environment is still `production`", async () => {
await withIngester({ keepIdentifyingAttributes: true }, async (post, rows) => {
assert.equal(await post("/v1/traces", TESTCLIENT, "application/x-protobuf"), 200);
assert.deepEqual(new Set((rows.runtime_spans ?? []).map((row) => row.environment)), new Set(["production"]));
for (const row of (rows.runtime_spans ?? []).filter((r) => r.kind === "server")) {
assert.equal(attributesOf(row)["http.user_agent"], "testclient");
// Nothing is dropped, so the filter passes the request through:
// the raw user agent is stored instead of the hint.
assert.equal(attributesOf(row)["autter.test_client"], undefined);
}
await backgroundInserts(rows);
});
});
46 changes: 38 additions & 8 deletions packages/otlp-ingester/src/identifying.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,17 +121,25 @@ export class IdentifyingAttributeFilter {
private readonly hasher: ClientIpHasher | null = null,
) {}

/** The attributes of a span or log record: the client IP hash added (from the raw list), then the drop. */
/**
* The attributes of a span or log record: what is read from the raw list
* added first (the client IP hash; on a SERVER span, the test-client
* marker, a hint only), then the drop.
*/
private recordAttributes = (obj: unknown, serverSpan: boolean): unknown => {
if (!isObject(obj)) return this.attributesOf(obj);
const added: Json[] = [];
const hasher = this.hasher;
if (!hasher || !isObject(obj) || hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) return this.attributesOf(obj);
const ip = clientIpOfList(obj.attributes, serverSpan);
if (hasher && !hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) {
const ip = clientIpOfList(obj.attributes, serverSpan);
if (ip) added.push({ key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } });
}
if (serverSpan && testClientUserAgentIn(obj.attributes) && !hasKey(obj.attributes, TEST_CLIENT_ATTRIBUTE)) {
added.push({ key: TEST_CLIENT_ATTRIBUTE, value: { boolValue: true } });
}
const filtered = this.attributesOf(obj);
if (!ip || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered;
return {
...filtered,
attributes: [...filtered.attributes, { key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }],
};
if (!added.length || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered;
return { ...filtered, attributes: [...filtered.attributes, ...added] };
};

/** An OTLP key/value list without the dropped keys; the same array when none is present. */
Expand Down Expand Up @@ -262,3 +270,25 @@ export function clientIpOfList(list: unknown, serverSpan: boolean): string | nul

/** OTLP SERVER span kind, as a number or the enum name. */
export const isServerKind = (kind: unknown) => kind === 2 || kind === "SPAN_KIND_SERVER";

// ---------------------------------------------------------------------------
// Test-client marker (a hint, never trusted)

/**
* On a SERVER span whose raw user agent is exactly `testclient` (what
* Starlette's and FastAPI's in-process TestClient sends): added before the
* user agent is dropped, so the stored span still says so. A hint only: the
* user agent is whatever the caller sent, so anyone can produce it, and
* nothing in the ingester acts on it. The environment is never changed
* because of it: test runs set `deployment.environment=test` themselves.
* Not added when nothing is dropped (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES=true
* and no client IP hash): the request passes through with its user agent.
*/
export const TEST_CLIENT_ATTRIBUTE = "autter.test_client";
const TEST_CLIENT_USER_AGENT = "testclient";
const USER_AGENT_KEYS: readonly string[] = ["user_agent.original", "http.user_agent"];

/** The raw OTLP attribute list names the test client's user agent, exactly. */
function testClientUserAgentIn(list: unknown): boolean {
return USER_AGENT_KEYS.some((key) => stringAttribute(list, key) === TEST_CLIENT_USER_AGENT);
}
52 changes: 52 additions & 0 deletions packages/otlp-ingester/src/logs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,3 +230,55 @@ test("server exceptions' autter.debug_ids locate source maps and are not kept (R
assert.equal("autter.debug_ids" in row!.attributes, false);
assert.equal("autter.debug_ids" in row!.occurrence!.attributes!, false);
});

// ── Environment from the resource only, and the running commit ──────────────────────────

function oneRecord(resource: Record<string, string>, attributes: Record<string, string>) {
return normalizeLogs({
resourceLogs: [{
resource: { attributes: Object.entries({ "service.name": "shop", ...resource }).map(([k, v]) => attr(k, v)) },
scopeLogs: [{
logRecords: [{
timeUnixNano: "1760000000000000000",
severityText: "error",
body: { stringValue: "checkout failed" },
attributes: Object.entries({
"autter.capture.mode": "log",
"exception.type": "OutOfStock",
"exception.message": "none left",
...attributes,
}).map(([k, v]) => attr(k, v)),
}],
}],
}],
})[0]!;
}

test("a record's own attributes never change its environment", () => {
for (const attributes of [{ "http.host": "testserver" }, { "server.address": "testserver" }, { "user_agent.original": "testclient" }]) {
const row = oneRecord({}, attributes);
assert.equal(row.environment, "production", JSON.stringify(attributes));
assert.equal(row.occurrence?.environment, "production");
assert.equal(oneRecord({ "deployment.environment": "test" }, attributes).environment, "test");
assert.equal(oneRecord({ "deployment.environment": "staging" }, attributes).environment, "staging");
}
const marked = normalizeLogs({ resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "x" }, attributes: [{ key: "autter.test_client", value: { boolValue: true } }] }] }] }] });
assert.equal(marked[0]!.environment, "production");
});

test("a promoted error carries the running commit: the record's beats the resource's; invalid or none, no field", () => {
const fromResource = oneRecord({ "deployment.commit": "ABCDEF1234567" }, {});
assert.equal(fromResource.occurrence?.commit, "abcdef1234567");
assert.equal(fromResource.occurrence?.attributes?.["deployment.commit"], "abcdef1234567");
// The log row's own attributes are left as they were.
assert.equal(fromResource.attributes["deployment.commit"], undefined);

assert.equal(oneRecord({ "deployment.commit": "1111111" }, { "deployment.commit": "2222222" }).occurrence?.commit, "2222222");
assert.equal(oneRecord({ "vcs.ref.head.revision": "3333333" }, {}).occurrence?.commit, "3333333");
for (const bad of ["https://github.com/acme/shop", "main", "123456"]) {
const row = oneRecord({ "deployment.commit": bad }, {});
assert.equal("commit" in row.occurrence!, false, bad);
assert.equal(row.occurrence?.attributes?.["deployment.commit"], undefined, bad);
}
assert.equal("commit" in oneRecord({}, {}).occurrence!, false);
});
18 changes: 16 additions & 2 deletions packages/otlp-ingester/src/logs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { declaredLink, liftErrorFields } from "./error-fields.js";
import { fingerprintOverrideOf } from "./fingerprint-override.js";
import { normalizeRoute } from "./fingerprint.js";
import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js";
import type { RuntimeOccurrenceInput } from "./types.js";
import { commitOf, withCommitAttribute, type RuntimeOccurrenceInput } from "./types.js";

export interface RuntimeLogRecord {
id: string;
Expand Down Expand Up @@ -249,7 +249,7 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta
statusCode: statusCode || null,
traceId: traceId || null,
sessionId: null,
attributes,
...promotedCommit(attributes, attrs, info),
occurredAt,
...declared,
...fingerprintOverrideOf(attributes["autter.fingerprint"]),
Expand Down Expand Up @@ -323,6 +323,20 @@ export function normalizeLogs(request: OtlpLogsRequest, stats?: NormalizeLogsSta
return rows;
}

/** A promoted occurrence's attributes, with the running commit (record
* attributes first, then the resource; 7–40 hex) as `commit` and, when the
* attributes carry none, as `deployment.commit`. The record's own
* attributes are left as they are. */
function promotedCommit(
attributes: Record<string, unknown>,
attrs: Record<string, unknown>,
info: Record<string, unknown>,
): { attributes: Record<string, unknown>; commit?: string } {
const commit = commitOf((key) => attrs[key], (key) => info[key]);
if (!commit) return { attributes };
return { attributes: withCommitAttribute(attributes, commit) ?? attributes, commit };
}

function hasException(attributes: Record<string, unknown>): boolean {
return ["exception.type", "exception.message", "exception.stacktrace"].some(
(key) => typeof attributes[key] === "string" && attributes[key] !== "",
Expand Down
Loading
Loading