From 07d5159996937c2a6b94248d9f3e80a227e84356 Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Sun, 15 Nov 2026 02:32:51 +0530 Subject: [PATCH 1/2] Drop identifying attributes from spans, logs and resources by default OpenTelemetry puts the client's IP address and port, the raw user agent, the machine's id and addresses, and the command line and its owner on every server span and on the resource. Redaction kept them (none looks like a secret) and nothing in Autter reads them. One list (runtime-core identifying.ts, mirrored in adapters/python redact.py and otlp-ingester identifying.ts, parity through test-vectors/redaction.json) is now dropped: - runtime-node: at export from span, event and link attributes and the span and metric resources, also with redactAttributes: false; - Python adapter: in RedactingSpanExporter (also with redact=False) and on init_tracing()'s resource; - otlp-ingester: on arrival for traces, logs (attributes and structured body keys) and metric resources, so plain OTel senders and old SDKs are covered. service.instance.id, host.name, process.pid, process.runtime.*, os.type and host.arch are kept. keepIdentifyingAttributes / keep_identifying_attributes / AUTTER_KEEP_IDENTIFYING_ATTRIBUTES keep them (true or a list; needed on both SDK and ingester), and dropAttributes / drop_attributes / AUTTER_DROP_ATTRIBUTES drop more keys such as host.name. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 + adapters/python/autter_tracing.py | 114 +++++++-- adapters/python/redact.py | 103 ++++++++ adapters/python/test_identifying.py | 226 +++++++++++++++++ docs/ARCHITECTURE.md | 13 + docs/GETTING-STARTED.md | 5 +- docs/INTEGRATIONS.md | 11 + packages/otlp-ingester/README.md | 30 +++ packages/otlp-ingester/src/config.ts | 13 + .../otlp-ingester/src/identifying.test.ts | 208 ++++++++++++++++ packages/otlp-ingester/src/identifying.ts | 151 ++++++++++++ packages/otlp-ingester/src/index.ts | 6 + packages/otlp-ingester/src/server.ts | 12 +- packages/runtime-core/src/identifying.ts | 138 +++++++++++ packages/runtime-core/src/index.ts | 10 + .../runtime-core/test/identifying.test.mjs | 95 ++++++++ packages/runtime-node/README.md | 41 ++++ packages/runtime-node/src/index.ts | 2 + packages/runtime-node/src/redact-exporter.ts | 116 ++++++--- packages/runtime-node/src/redact.ts | 1 + packages/runtime-node/src/server.ts | 46 ++++ .../test/identifying-attributes.test.mjs | 229 ++++++++++++++++++ test-vectors/redaction.json | 50 ++++ 23 files changed, 1583 insertions(+), 46 deletions(-) create mode 100644 adapters/python/test_identifying.py create mode 100644 packages/otlp-ingester/src/identifying.test.ts create mode 100644 packages/otlp-ingester/src/identifying.ts create mode 100644 packages/runtime-core/src/identifying.ts create mode 100644 packages/runtime-core/test/identifying.test.mjs create mode 100644 packages/runtime-node/test/identifying-attributes.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 13281dd..31b597d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## [Unreleased] +### Identifying attributes are dropped by default + +Packages: `runtime-node`, `otlp-ingester`, Python adapter (`adapters/python`). + +- OpenTelemetry adds attributes that identify a person or a machine to every server span and to the resource: the client's IP address and port (`net.peer.ip`, `client.address`, `http.client_ip` from `X-Forwarded-For`, `network.peer.address`, …), the raw user-agent string (`http.user_agent`, `user_agent.original`), the machine's id and addresses (`host.id`, `host.ip`, `host.mac`), and the command line and its owner (`process.command_args`, `process.command`, `process.command_line`, `process.owner`, `process.executable.path`). Redaction let them through because none looks like a secret, and nothing in Autter reads them. They are now dropped by default, from span, event and link attributes and from the resource: by `runtime-node` at export (also with `redactAttributes: false`), by the Python adapter's `RedactingSpanExporter` (also with `redact=False`), and by the ingester on arrival for traces, logs and metric resources, so plain OpenTelemetry senders and older SDKs are covered too. The full list is in [`test-vectors/redaction.json`](test-vectors/redaction.json) (`identifying.drop`). +- Kept: `service.instance.id` (the ingester needs it), `host.name` (container and pod names; on a laptop or VM it is the machine's name), `process.pid`, `process.runtime.*`, `os.type`, `host.arch`, and the coarse `user_agent.browser` / `.os` / `.device` / `.bot` fields. +- Keep them with `keepIdentifyingAttributes: true` or a list of keys (`initAutterServer`, `registerAutter`), `keep_identifying_attributes=` in Python `init_tracing()`, or `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` (`true`, or a comma-separated list) for the SDKs and the ingester. Drop more keys, such as `host.name`, with `dropAttributes` / `drop_attributes=` / `AUTTER_DROP_ATTRIBUTES`. An attribute the SDK keeps is stored only if the ingester keeps it too: set `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` on both. +- Spans already stored keep these attributes until their TTL expires. + ## [1.18.0] - 2026-11-14 Packages: `otlp-ingester` 1.16.0, `runtime-node` 1.14.0, and the local `dashboard` image. Release the ingester before `runtime-node` 1.14.0 (`llm_cache_tokens`); see the upgrade notes. diff --git a/adapters/python/autter_tracing.py b/adapters/python/autter_tracing.py index 5660193..a60a324 100644 --- a/adapters/python/autter_tracing.py +++ b/adapters/python/autter_tracing.py @@ -80,13 +80,31 @@ SamplingResult, TraceIdRatioBased, ) -from opentelemetry.trace import SpanContext, SpanKind, StatusCode, TraceFlags +from opentelemetry.trace import Link, SpanContext, SpanKind, StatusCode, TraceFlags from opentelemetry.trace.status import Status try: # imported as part of a package - from .redact import redact_attributes, redact_text + from .redact import ( + DROP_ATTRIBUTES_ENV, + KEEP_IDENTIFYING_ATTRIBUTES_ENV, + drop_attributes_from_env, + drop_identifying_attributes, + identifying_keys_to_drop, + keep_identifying_from_env, + redact_attributes, + redact_text, + ) except ImportError: # copied next to the app as a plain module - from redact import redact_attributes, redact_text + from redact import ( + DROP_ATTRIBUTES_ENV, + KEEP_IDENTIFYING_ATTRIBUTES_ENV, + drop_attributes_from_env, + drop_identifying_attributes, + identifying_keys_to_drop, + keep_identifying_from_env, + redact_attributes, + redact_text, + ) __all__ = [ "TAIL_RETAINED_ATTRIBUTE", @@ -190,10 +208,10 @@ def __init__(self, span: ReadableSpan, **overrides: Any): name=overrides.get("name", span.name), context=overrides.get("context", span.context), parent=span.parent, - resource=span.resource, + resource=overrides.get("resource", span.resource), attributes=overrides.get("attributes", span.attributes), events=overrides.get("events", span.events), - links=span.links, + links=overrides.get("links", span.links), kind=span.kind, # The deprecated property warns; the OTLP exporter reads the scope. instrumentation_info=getattr(span, "_instrumentation_info", None), @@ -500,20 +518,55 @@ def _env_flag(name: str) -> bool: return (os.environ.get(name) or "").strip().lower() in ("1", "true", "yes", "on") -def _redacted(span: ReadableSpan) -> ReadableSpan: - events = [Event(event.name, redact_attributes(event.attributes), event.timestamp) for event in span.events] +def _keys_to_drop(keep: Any, drop: Optional[Sequence[str]]) -> frozenset: + """``keep`` / ``drop`` as given, else from the environment.""" + if keep is None: + keep = keep_identifying_from_env(os.environ.get(KEEP_IDENTIFYING_ATTRIBUTES_ENV)) + if drop is None: + drop = drop_attributes_from_env(os.environ.get(DROP_ATTRIBUTES_ENV)) + return identifying_keys_to_drop(keep, drop) + + +_resource_cache: tuple = (None, None, None) + + +def _without_identifying_resource(resource: Resource, dropped: frozenset) -> Resource: + """The resource without the dropped keys; one copy per resource (every + span of a provider shares it), so the exporter's grouping stays cheap.""" + global _resource_cache + cached_resource, cached_dropped, cached = _resource_cache + if cached_resource is resource and cached_dropped is dropped: + return cached + attributes = drop_identifying_attributes(resource.attributes, dropped) + result = resource if attributes is resource.attributes else Resource(attributes, getattr(resource, "schema_url", None)) + _resource_cache = (resource, dropped, result) + return result + + +def _redacted(span: ReadableSpan, redact: bool = True, dropped: Optional[frozenset] = None) -> ReadableSpan: + def clean(attributes: Any) -> Any: + kept = drop_identifying_attributes(attributes, dropped) + return redact_attributes(kept) if redact else kept + + events = [Event(event.name, clean(event.attributes), event.timestamp) for event in span.events] + links = [Link(link.context, clean(link.attributes)) for link in span.links] status = span.status - if status is not None and status.description: + if redact and status is not None and status.description: status = Status(status.status_code, redact_text(status.description)) return _CopiedSpan( span, - name=redact_text(span.name), - attributes=redact_attributes(span.attributes), + name=redact_text(span.name) if redact else span.name, + attributes=clean(span.attributes), events=events, + links=links, status=status, + resource=_without_identifying_resource(span.resource, _DEFAULT_KEYS if dropped is None else dropped), ) +_DEFAULT_KEYS = identifying_keys_to_drop() + + class RedactingSpanExporter(SpanExporter): """Scrubs secrets and PII before spans leave the process. @@ -522,13 +575,30 @@ class RedactingSpanExporter(SpanExporter): ``str(exc)`` often carries a connection string or an API key) and status descriptions go through ``redact.py``, the patterns @autter/runtime-node uses. The ingester scrubs again on arrival. + + Identifying attributes (``redact.IDENTIFYING_ATTRIBUTE_KEYS``: the client's + IP address and port, the raw user agent, ``host.id``, the command line and + its owner) are dropped from span, event and link attributes and from the + resource, also with ``redact=False``. ``keep_identifying_attributes`` + (``True`` or a list of keys; default ``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES``) + keeps them; ``drop_attributes`` (default ``AUTTER_DROP_ATTRIBUTES``, a + comma-separated list) drops more keys, such as ``host.name``. """ - def __init__(self, exporter: SpanExporter): + def __init__( + self, + exporter: SpanExporter, + *, + redact: bool = True, + keep_identifying_attributes: Any = None, + drop_attributes: Optional[Sequence[str]] = None, + ): self._exporter = exporter + self._redact = redact + self._dropped = _keys_to_drop(keep_identifying_attributes, drop_attributes) def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - return self._exporter.export([_redacted(span) for span in spans]) + return self._exporter.export([_redacted(span, self._redact, self._dropped) for span in spans]) def shutdown(self) -> None: self._exporter.shutdown() @@ -573,6 +643,8 @@ def init_tracing( set_global: bool = True, count_requests: Optional[bool] = None, metric_exporter: Optional[MetricExporter] = None, + keep_identifying_attributes: Any = None, + drop_attributes: Optional[Sequence[str]] = None, ) -> TracerProvider: """A TracerProvider that head-samples as configured and always exports errors, registered as the global provider. @@ -584,7 +656,16 @@ def init_tracing( ``OTEL_EXPORTER_OTLP_HEADERS``. * ``resource``: default ``Resource.create()``, which reads ``OTEL_SERVICE_NAME`` and ``OTEL_RESOURCE_ATTRIBUTES``. - * ``redact``: wrap the exporter in :class:`RedactingSpanExporter`. + * ``redact``: scrub secrets and PII in :class:`RedactingSpanExporter`. + * ``keep_identifying_attributes`` (default + ``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES``: ``true`` or a comma-separated + list): identifying attributes (client IP address and port, raw user + agent, ``host.id``, command line; ``redact.IDENTIFYING_ATTRIBUTE_KEYS``) + are dropped from spans and the resource, also with ``redact=False``; + ``True`` keeps them all, a list keeps those keys. The ingester drops + them too unless it is given the same setting. + * ``drop_attributes`` (default ``AUTTER_DROP_ATTRIBUTES``): more keys to + drop the same way, such as ``["host.name"]``. * ``count_requests`` (default: ``AUTTER_COUNT_REQUESTS=true``): count every HTTP request, sampled or not (:class:`CountRequests`), in a delta histogram sent through ``metric_exporter`` (default @@ -620,6 +701,8 @@ def init_tracing( return existing if resource is None: resource = Resource.create() + dropped = _keys_to_drop(keep_identifying_attributes, drop_attributes) + resource = _without_identifying_resource(resource, dropped) meter_provider = None if count_requests if count_requests is not None else _env_flag("AUTTER_COUNT_REQUESTS"): if set_global and isinstance(metrics.get_meter_provider(), MeterProvider): @@ -638,8 +721,9 @@ def init_tracing( from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter exporter = OTLPSpanExporter() - if redact: - exporter = RedactingSpanExporter(exporter) + exporter = RedactingSpanExporter( + exporter, redact=redact, keep_identifying_attributes=keep_identifying_attributes, drop_attributes=drop_attributes + ) provider = TracerProvider( sampler=RecordUnsampled(sampler if sampler is not None else head_sampler_from_env()), resource=resource, diff --git a/adapters/python/redact.py b/adapters/python/redact.py index a97927c..14c018e 100644 --- a/adapters/python/redact.py +++ b/adapters/python/redact.py @@ -19,6 +19,12 @@ default; extend with :func:`configure` (your own value/key patterns). The Autter ingester scrubs again server-side as a second line of defence. +Identifying attributes (the client's IP address and port, the raw user +agent, the host's id and addresses, the command line and its owner: +:data:`IDENTIFYING_ATTRIBUTE_KEYS`) are dropped rather than scrubbed: +:func:`drop_identifying_attributes`. ``autter_tracing.RedactingSpanExporter`` +applies it to every span, also with redaction off. + Stdlib only. """ @@ -100,6 +106,46 @@ re.I, ) +#: Attributes that identify a person, a machine or a command line rather than +#: describe a request. Dropped by default from span, event and link +#: attributes and the resource (same list as @autter/runtime-node and the +#: ingester; parity: ``test-vectors/redaction.json``, ``identifying``). +#: Kept on purpose: ``service.instance.id``, ``host.name`` (container and pod +#: names; drop it with ``drop_attributes``), ``process.pid``, +#: ``process.runtime.*``, ``os.type``, ``host.arch``. +IDENTIFYING_ATTRIBUTE_KEYS: tuple[str, ...] = ( + # The client's address (and, on outgoing calls, the peer's), in the old, + # 1.2x and stable semantic conventions; the server's own IP address. + "net.peer.ip", + "net.peer.port", + "net.sock.peer.addr", + "net.sock.peer.port", + "net.host.ip", + "net.sock.host.addr", + "network.local.address", + "http.client_ip", + "client.address", + "client.port", + "network.peer.address", + "network.peer.port", + # The raw user-agent string. + "http.user_agent", + "user_agent.original", + # The machine (its id and addresses; host.name is kept). + "host.id", + "host.mac", + "host.ip", + # The command line and the user running it. + "process.command_args", + "process.command", + "process.command_line", + "process.owner", + "process.executable.path", +) + +KEEP_IDENTIFYING_ATTRIBUTES_ENV = "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES" +DROP_ATTRIBUTES_ENV = "AUTTER_DROP_ATTRIBUTES" + _extra_values: list[re.Pattern[str]] = [] _extra_keys: list[re.Pattern[str]] = [] _enabled = True @@ -212,3 +258,60 @@ def redact_attributes(attributes: Mapping[str, Any] | None) -> dict[str, Any]: if not _enabled: return dict(attributes) return _redact_mapping(attributes, 0) + + +def keep_identifying_from_env(raw: str | None) -> bool | list[str] | None: + """Parse ``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES``: ``true``/``1``/``yes``/ + ``on``/``all`` keep every identifying attribute, ``false``/``0``/``no``/ + ``off``/``none`` keep none, anything else is a comma-separated list of + keys. Unset or blank: ``None`` (drop them all).""" + if not isinstance(raw, str) or not raw.strip(): + return None + value = raw.strip().lower() + if value in ("1", "true", "yes", "on", "all"): + return True + if value in ("0", "false", "no", "off", "none"): + return False + return [key.strip() for key in raw.split(",") if key.strip()] + + +def drop_attributes_from_env(raw: str | None) -> list[str] | None: + """Parse ``AUTTER_DROP_ATTRIBUTES``: a comma-separated list of keys.""" + if not isinstance(raw, str): + return None + keys = [key.strip() for key in raw.split(",") if key.strip()] + return keys or None + + +def identifying_keys_to_drop( + keep: bool | Iterable[str] | None = None, extra: Iterable[str] | None = None +) -> frozenset[str]: + """The keys to drop: :data:`IDENTIFYING_ATTRIBUTE_KEYS` minus ``keep`` + (``True`` keeps them all, a list keeps those keys), plus ``extra`` (a key + both kept and in ``extra`` is dropped).""" + if keep is True: + kept: set[str] = set(IDENTIFYING_ATTRIBUTE_KEYS) + elif keep is None or keep is False or isinstance(keep, str): + kept = set() + else: + kept = {key for key in keep if isinstance(key, str)} + dropped = {key for key in IDENTIFYING_ATTRIBUTE_KEYS if key not in kept} + if extra is not None and not isinstance(extra, str): + dropped.update(key for key in extra if isinstance(key, str) and key) + return frozenset(dropped) + + +_DEFAULT_DROPPED = identifying_keys_to_drop() + + +def drop_identifying_attributes(attributes: Any, dropped: frozenset[str] | None = None) -> Any: + """``attributes`` without the identifying keys (``dropped``, default + :func:`identifying_keys_to_drop`). The same object when none is present; + otherwise a ``dict`` copy. Never mutates its input. Applies whether or + not redaction is enabled.""" + keys = _DEFAULT_DROPPED if dropped is None else dropped + if not attributes or not keys: + return attributes + if not any(key in keys for key in attributes): + return attributes + return {key: value for key, value in attributes.items() if key not in keys} diff --git a/adapters/python/test_identifying.py b/adapters/python/test_identifying.py new file mode 100644 index 0000000..83426cd --- /dev/null +++ b/adapters/python/test_identifying.py @@ -0,0 +1,226 @@ +"""Identifying attributes (client address, raw user agent, host id, command +line) in the Python adapter: ``redact.py``'s list and filter (stdlib) and +``autter_tracing.RedactingSpanExporter`` / ``init_tracing()`` (needs +opentelemetry-sdk; skipped without it). + +Run: python3 -m unittest discover -s adapters/python -v +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import redact # noqa: E402 +from redact import ( # noqa: E402 + IDENTIFYING_ATTRIBUTE_KEYS, + drop_attributes_from_env, + drop_identifying_attributes, + identifying_keys_to_drop, + keep_identifying_from_env, +) + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "test-vectors", "redaction.json") +with open(VECTORS_PATH, encoding="utf-8") as fh: + VECTORS = json.load(fh)["identifying"] + +CLEAN_ENV = {"AUTTER_KEEP_IDENTIFYING_ATTRIBUTES": "", "AUTTER_DROP_ATTRIBUTES": ""} + + +def _sample(): + return {key: f"v:{key}" for key in [*VECTORS["drop"], *VECTORS["keep"]]} + + +class IdentifyingList(unittest.TestCase): + def test_list_matches_the_shared_vectors(self): + self.assertEqual(sorted(IDENTIFYING_ATTRIBUTE_KEYS), sorted(VECTORS["drop"])) + self.assertEqual(len(set(IDENTIFYING_ATTRIBUTE_KEYS)), len(IDENTIFYING_ATTRIBUTE_KEYS)) + for key in VECTORS["keep"]: + self.assertNotIn(key, IDENTIFYING_ATTRIBUTE_KEYS) + self.assertNotIn("host.name", IDENTIFYING_ATTRIBUTE_KEYS) + + def test_default_drops_every_listed_key_and_keeps_the_rest(self): + source = _sample() + out = drop_identifying_attributes(source) + self.assertEqual(sorted(out), sorted(VECTORS["keep"])) + self.assertEqual(out["service.instance.id"], "v:service.instance.id") + self.assertEqual(len(source), len(VECTORS["drop"]) + len(VECTORS["keep"]), "input not mutated") + + def test_unchanged_input_is_returned_as_is(self): + attributes = {"http.route": "/x", "service.instance.id": "abc"} + self.assertIs(drop_identifying_attributes(attributes), attributes) + self.assertIsNone(drop_identifying_attributes(None)) + + def test_keep_true_list_and_extra_drop(self): + self.assertEqual(identifying_keys_to_drop(True), frozenset()) + self.assertEqual(identifying_keys_to_drop(False), frozenset(VECTORS["drop"])) + some = identifying_keys_to_drop(["http.user_agent", "host.id", "not.listed"]) + out = drop_identifying_attributes(_sample(), some) + self.assertEqual(out["http.user_agent"], "v:http.user_agent") + self.assertEqual(out["host.id"], "v:host.id") + self.assertNotIn("net.peer.ip", out) + self.assertNotIn("client.address", out) + self.assertEqual(out["service.instance.id"], "v:service.instance.id") + extra = identifying_keys_to_drop(None, ["host.name"]) + self.assertEqual( + drop_identifying_attributes({"host.name": "web-1", "service.instance.id": "a"}, extra), + {"service.instance.id": "a"}, + ) + # A key both kept and listed as extra is dropped. + self.assertIn("host.id", identifying_keys_to_drop(["host.id"], ["host.id"])) + + def test_drop_applies_with_redaction_disabled(self): + try: + redact.configure(enabled=False) + self.assertNotIn("net.peer.ip", drop_identifying_attributes({"net.peer.ip": "203.0.113.9"})) + finally: + redact.configure() + + def test_env_parsing(self): + self.assertIsNone(keep_identifying_from_env(None)) + self.assertIsNone(keep_identifying_from_env(" ")) + for raw in ("true", "TRUE", "1", "yes", "on", "all"): + self.assertIs(keep_identifying_from_env(raw), True, raw) + for raw in ("false", "0", "no", "off", "none"): + self.assertIs(keep_identifying_from_env(raw), False, raw) + self.assertEqual(keep_identifying_from_env(" http.user_agent, host.id ,,"), ["http.user_agent", "host.id"]) + self.assertIsNone(drop_attributes_from_env(" , ")) + self.assertEqual(drop_attributes_from_env("host.name, k8s.pod.name"), ["host.name", "k8s.pod.name"]) + + +try: + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from opentelemetry.sdk.trace.sampling import ALWAYS_ON + from opentelemetry.trace import Link, SpanContext, TraceFlags + + import autter_tracing + from autter_tracing import RedactingSpanExporter, init_tracing + + HAVE_OTEL = True +except ImportError: # pragma: no cover - depends on the environment + HAVE_OTEL = False + +RESOURCE = { + "service.name": "shop", + "service.instance.id": "inst-1", + "host.name": "web-7f9c", + "host.id": "machine-1", + "process.pid": 42, + "process.command_line": "python app.py --db=postgres://u:p@db", + "process.owner": "jane", + "process.runtime.name": "cpython", +} + +SPAN = { + "net.peer.ip": "203.0.113.9", + "client.address": "203.0.113.9", + "http.user_agent": "Mozilla/5.0", + "user_agent.original": "Mozilla/5.0", + "http.route": "/users/", + "user.email": "jane@example.com", +} + + +@unittest.skipUnless(HAVE_OTEL, "opentelemetry-sdk is not installed") +class IdentifyingExporter(unittest.TestCase): + def export(self, **options): + exporter = InMemorySpanExporter() + provider = TracerProvider(sampler=ALWAYS_ON, resource=Resource.create(RESOURCE)) + provider.add_span_processor(SimpleSpanProcessor(RedactingSpanExporter(exporter, **options))) + tracer = provider.get_tracer("test") + link = Link( + SpanContext(0x0AF7651916CD43DD8448EB211C80319C, 0xB7AD6B7169203331, is_remote=True, trace_flags=TraceFlags(1)), + {"network.peer.address": "198.51.100.8", "link.note": "kept"}, + ) + with tracer.start_as_current_span("GET /users", attributes=SPAN, links=[link]) as span: + span.add_event("peer", {"net.peer.port": 51234, "event.note": "kept"}) + provider.shutdown() + [span] = exporter.get_finished_spans() + return span + + def assert_kept_identity(self, span): + self.assertEqual(span.resource.attributes["service.instance.id"], "inst-1") + self.assertEqual(span.resource.attributes["host.name"], "web-7f9c") + self.assertEqual(span.resource.attributes["process.pid"], 42) + self.assertEqual(span.resource.attributes["process.runtime.name"], "cpython") + self.assertEqual(span.attributes["http.route"], "/users/") + self.assertEqual(span.events[0].attributes["event.note"], "kept") + self.assertEqual(span.links[0].attributes["link.note"], "kept") + + def assert_dropped(self, span, keys): + bags = [span.attributes, span.resource.attributes, span.events[0].attributes, span.links[0].attributes] + for key in keys: + for bag in bags: + self.assertNotIn(key, bag) + + def test_default_drops_from_spans_events_links_and_resource(self): + with mock.patch.dict(os.environ, CLEAN_ENV): + span = self.export() + self.assert_dropped(span, IDENTIFYING_ATTRIBUTE_KEYS) + self.assert_kept_identity(span) + self.assertEqual(span.attributes["user.email"], redact.MASK) + + def test_drop_applies_with_redact_false(self): + with mock.patch.dict(os.environ, CLEAN_ENV): + span = self.export(redact=False) + self.assert_dropped(span, IDENTIFYING_ATTRIBUTE_KEYS) + self.assert_kept_identity(span) + self.assertEqual(span.attributes["user.email"], "jane@example.com") + + def test_keep_true_and_list(self): + with mock.patch.dict(os.environ, CLEAN_ENV): + everything = self.export(keep_identifying_attributes=True) + some = self.export(keep_identifying_attributes=["http.user_agent", "process.owner"]) + self.assertEqual(everything.attributes["net.peer.ip"], "203.0.113.9") + self.assertEqual(everything.resource.attributes["host.id"], "machine-1") + self.assertEqual(everything.events[0].attributes["net.peer.port"], 51234) + self.assertEqual(some.attributes["http.user_agent"], "Mozilla/5.0") + self.assertEqual(some.resource.attributes["process.owner"], "jane") + self.assert_dropped(some, [k for k in IDENTIFYING_ATTRIBUTE_KEYS if k not in ("http.user_agent", "process.owner")]) + self.assert_kept_identity(some) + + def test_environment_and_extra_drop(self): + env = {"AUTTER_KEEP_IDENTIFYING_ATTRIBUTES": "client.address", "AUTTER_DROP_ATTRIBUTES": "host.name"} + with mock.patch.dict(os.environ, env): + span = self.export() + self.assertEqual(span.attributes["client.address"], "203.0.113.9") + self.assertNotIn("host.name", span.resource.attributes) + self.assertEqual(span.resource.attributes["service.instance.id"], "inst-1") + self.assert_dropped(span, [k for k in IDENTIFYING_ATTRIBUTE_KEYS if k != "client.address"]) + + def test_init_tracing_wires_the_options_also_with_redact_false(self): + exporter = InMemorySpanExporter() + with mock.patch.dict(os.environ, CLEAN_ENV): + provider = init_tracing( + sampler=ALWAYS_ON, + exporter=exporter, + resource=Resource.create(RESOURCE), + redact=False, + set_global=False, + keep_identifying_attributes=["http.user_agent"], + drop_attributes=["host.name"], + ) + self.assertNotIn("host.id", provider.resource.attributes) + self.assertNotIn("host.name", provider.resource.attributes) + self.assertEqual(provider.resource.attributes["service.instance.id"], "inst-1") + with provider.get_tracer("test").start_as_current_span("GET /x", attributes=SPAN): + pass + provider.force_flush() + [span] = exporter.get_finished_spans() + self.assertEqual(span.attributes["http.user_agent"], "Mozilla/5.0") + self.assertNotIn("net.peer.ip", span.attributes) + self.assertNotIn("process.command_line", span.resource.attributes) + self.assertEqual(span.attributes["user.email"], "jane@example.com") + provider.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index aa34645..e82bbaa 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -922,6 +922,19 @@ attribute bags, and LLM-call attributes (`src/redact.ts`; extend with of regex sources). Messages containing secrets therefore fingerprint on their scrubbed form. +Identifying attributes are dropped rather than scrubbed: the client's IP +address and port, the raw user-agent string, `host.id` / `host.ip` / +`host.mac`, and the command line and its owner. OpenTelemetry adds them by +default (resource detectors, HTTP server instrumentation) and nothing reads +them. The Node SDK drops them at export (also with `redactAttributes: +false`), the Python adapter in `RedactingSpanExporter`, and the ingester on +arrival for traces, logs and metric resources (`src/identifying.ts`). One +list, kept in parity by `test-vectors/redaction.json` (`identifying`). +`service.instance.id` and `host.name` are kept. `keepIdentifyingAttributes` +/ `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` keeps them and must be set on both +the SDK and the ingester; `dropAttributes` / `AUTTER_DROP_ATTRIBUTES` drops +more keys. + ## Sink webhook (v1) When `AUTTER_SINK_URL` is set, each ingest batch POSTs: diff --git a/docs/GETTING-STARTED.md b/docs/GETTING-STARTED.md index 7da3e8e..cf6d60b 100644 --- a/docs/GETTING-STARTED.md +++ b/docs/GETTING-STARTED.md @@ -60,7 +60,10 @@ bodies, headers, emails, full URLs with query strings. On the backend, custom attributes are additionally scrubbed before export: values that look like emails/tokens/credentials and attributes with sensitive keys (`password`, `api_key`, …) are masked by default (`redactAttributes`), so a -stray `captureException(err, { "user.email": … })` doesn't leak PII. +stray `captureException(err, { "user.email": … })` doesn't leak PII. The +client's IP address, the raw user-agent string, the machine's id and the +command line that OpenTelemetry adds to spans are dropped +(`keepIdentifyingAttributes` keeps them; `host.name` is kept). ## 2. Run the ingester diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index ca8b5c8..8e09db4 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -298,6 +298,17 @@ exceptions never leave the process. `init_tracing()` keeps them: exception messages and tracebacks, causes included, with `redact.py` before export. `redact.configure(value_patterns=[...], key_patterns=[...])` adds your own patterns. The ingester scrubs again. +- It also drops identifying attributes from spans, events, links and the + resource, also with `redact=False`: the client's IP address and port + (`net.peer.ip`, `client.address`, …), the raw user agent + (`http.user_agent`, `user_agent.original`), `host.id` / `host.ip` / + `host.mac`, and the command line and its owner (`process.command_line`, + …; `redact.IDENTIFYING_ATTRIBUTE_KEYS`). `service.instance.id` and + `host.name` are kept. `init_tracing(keep_identifying_attributes=True)` + (or a list of keys, or `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES`) keeps them; + the ingester drops them too unless it is given the same setting. + `drop_attributes=["host.name"]` (or `AUTTER_DROP_ATTRIBUTES`) drops more + keys. Every request is recorded as if sampled (instrumentations collect their attributes), but only sampled and failing traces are exported. To wire the diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 507c142..040aa57 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -172,6 +172,36 @@ Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILIT | `ARCHIVE_TABLES` / `ARCHIVE_BACKSTOP_DAYS` / `ARCHIVE_INTERVAL_MINUTES` / `ARCHIVE_MAX_DAYS_PER_PASS` / `ARCHIVE_SCHEDULER` | all four / `auto` (hot window + 7 days; `0` = none) / `60` / `4` / `auto` | Which tables, the disk-guard TTL, how often and how much per pass, and which process runs the job (`auto`: Redis lease holder or the only replica) | | `AUTTER_REDACT_VALUE_PATTERNS` | — | JSON array of extra regex sources masked in stored/forwarded text and attribute values (built-in secret/PII patterns always apply) | | `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale | +| `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` | unset (drop them all) | Identifying attributes (client IP and port, raw user agent, `host.id` / `host.ip` / `host.mac`, command line and owner) are dropped on arrival; `true` keeps them all, a comma-separated list keeps those keys. An SDK that keeps one needs this set too, or the attribute is not stored. See [Identifying attributes](#identifying-attributes) | +| `AUTTER_DROP_ATTRIBUTES` | unset | Comma-separated attribute keys dropped on arrival as well, such as `host.name` | + +## Identifying attributes + +OpenTelemetry SDKs put attributes that identify a person or a machine on +spans and resources by default: the client's IP address and port, the raw +user-agent string, the machine's id and addresses, and the command line and +its owner. Nothing in Autter reads them, so the ingester drops them on +arrival, before anything is stored or forwarded to the sink: from span, +span event and span link attributes, log record attributes (and the keys of +a structured log body), and the resource of traces, logs and metrics. This +covers every sender: plain OpenTelemetry SDKs in any language, the Python +adapter, and older `@autter/runtime-node` versions. The keys are listed in +[`src/identifying.ts`](src/identifying.ts) and +[`test-vectors/redaction.json`](../../test-vectors/redaction.json) +(`identifying.drop`); the SDKs drop the same list before export. + +`service.instance.id`, `host.name`, `process.pid`, `process.runtime.*`, +`os.type` and `host.arch` are kept. `host.name` is usually a container or +pod name, but on a laptop or a VM it is the machine's name: drop it with +`AUTTER_DROP_ATTRIBUTES=host.name`. + +To store identifying attributes, set `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` +(`true`, or a comma-separated list of keys) here **and** in the SDK +(`keepIdentifyingAttributes` in `@autter/runtime-node`, +`keep_identifying_attributes` in the Python adapter, or the same +environment variable). Each side drops what it was not told to keep, so a +key kept by only one of them is not stored. The ingester logs what it keeps +at startup. Rows already stored keep these attributes until their TTL. ## Rate limits and key lookups (1.12.0) diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index 134670e..fbceb07 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -1,6 +1,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { archiveConfigFromEnv, type ArchiveConfig } from "./archive.js"; +import { dropAttributesEnv, keepIdentifyingEnv } from "./identifying.js"; import { loadLlmPriceOverrides, type LlmPriceOverrides } from "./llm-pricing.js"; import type { IngestKeyScope } from "./types.js"; @@ -199,6 +200,16 @@ export interface IngesterConfig { */ redactValuePatterns?: string[]; redactKeyPatterns?: string[]; + /** + * Identifying attributes (client IP address and port, raw user agent, + * host id and addresses, command line and owner: identifying.ts) are + * dropped on arrival. `true` keeps them all, a list keeps those keys + * (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: `true` or a comma-separated + * list). An SDK that keeps one gets it stored only with this set too. + */ + keepIdentifyingAttributes?: boolean | string[]; + /** More attribute keys to drop on arrival, such as `host.name` (AUTTER_DROP_ATTRIBUTES, comma-separated). */ + dropAttributes?: string[]; /** * In-memory budget (MiB, decoded size) for parsed source maps used to * symbolicate browser frames at ingest (SOURCE_MAP_CACHE_MB, default 64). @@ -409,6 +420,8 @@ export function loadConfig(): IngesterConfig { logTtlDaysExplicit: Number.parseInt(process.env.LOG_TTL_DAYS ?? "", 10) > 0, redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"), redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"), + keepIdentifyingAttributes: keepIdentifyingEnv(process.env.AUTTER_KEEP_IDENTIFYING_ATTRIBUTES), + dropAttributes: dropAttributesEnv(process.env.AUTTER_DROP_ATTRIBUTES), sourceMapCacheMb: intEnv("SOURCE_MAP_CACHE_MB", 64), loopCollapseAfter: loopCollapseAfterEnv(process.env.LOOP_COLLAPSE_AFTER), archive: archiveConfigFromEnv(process.env), diff --git a/packages/otlp-ingester/src/identifying.test.ts b/packages/otlp-ingester/src/identifying.test.ts new file mode 100644 index 0000000..f6f686d --- /dev/null +++ b/packages/otlp-ingester/src/identifying.test.ts @@ -0,0 +1,208 @@ +// Identifying attributes are dropped on arrival, whoever sent them: plain +// OpenTelemetry SDKs in any language and older Autter SDKs. The list must +// match runtime-core's and the Python adapter's (test-vectors/redaction.json). +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { test } from "node:test"; +import type { IngesterConfig } from "./config.js"; +import { + dropAttributesEnv, + IDENTIFYING_ATTRIBUTE_KEYS, + IdentifyingAttributeFilter, + identifyingKeysToDrop, + keepIdentifyingEnv, +} from "./identifying.js"; +import { createIngesterApp } from "./server.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +).identifying as { drop: string[]; keep: string[] }; + +const kv = (attributes: Record) => + Object.entries(attributes).map(([key, value]) => ({ + key, + value: typeof value === "number" ? { intValue: String(value) } : { stringValue: value }, + })); +const keysOf = (list: unknown) => (list as Array<{ key: string }>).map((entry) => entry.key); +const everything = () => Object.fromEntries([...vectors.drop, ...vectors.keep].map((key) => [key, `v:${key}`])); + +test("the list is exactly the shared vectors' drop list; host.name is kept", () => { + assert.deepEqual([...IDENTIFYING_ATTRIBUTE_KEYS].sort(), [...vectors.drop].sort()); + for (const key of vectors.keep) assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes(key), key); + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes("host.name")); + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes("service.instance.id")); +}); + +test("traces: resource, span, event and link attributes lose the identifying keys", () => { + const filter = new IdentifyingAttributeFilter(); + const request = { + resourceSpans: [{ + resource: { attributes: kv(everything()) }, + scopeSpans: [{ + spans: [{ + name: "GET /", + attributes: kv(everything()), + events: [{ name: "e", attributes: kv({ "net.peer.ip": "1.2.3.4", note: "kept" }) }], + links: [{ traceId: "a", spanId: "b", attributes: kv({ "client.address": "1.2.3.4", note: "kept" }) }], + }], + }], + }], + }; + const before = JSON.stringify(request); + const out = filter.traces(request); + assert.equal(JSON.stringify(request), before, "input not mutated"); + const rs = out.resourceSpans[0]!; + assert.deepEqual(keysOf(rs.resource.attributes).sort(), [...vectors.keep].sort()); + const span = rs.scopeSpans[0]!.spans[0]!; + assert.deepEqual(keysOf(span.attributes).sort(), [...vectors.keep].sort()); + assert.deepEqual(keysOf(span.events[0]!.attributes), ["note"]); + assert.deepEqual(keysOf(span.links[0]!.attributes), ["note"]); + assert.equal(span.name, "GET /"); +}); + +test("logs: resource, record attributes and structured body keys; metrics: resource", () => { + const filter = new IdentifyingAttributeFilter(); + const logs = filter.logs({ + resourceLogs: [{ + resource: { attributes: kv({ "service.instance.id": "i", "host.id": "m", "process.owner": "jane" }) }, + scopeLogs: [{ + logRecords: [{ + attributes: kv({ "user_agent.original": "curl/8", "http.route": "/x" }), + body: { kvlistValue: { values: kv({ "client.address": "1.2.3.4", message: "hi" }) } }, + }], + }], + }], + }); + assert.deepEqual(keysOf(logs.resourceLogs[0]!.resource.attributes), ["service.instance.id"]); + const record = logs.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]!; + assert.deepEqual(keysOf(record.attributes), ["http.route"]); + assert.deepEqual(keysOf(record.body.kvlistValue.values), ["message"]); + + const metrics = filter.metrics({ + resourceMetrics: [{ resource: { attributes: kv({ "service.instance.id": "i", "host.ip": "10.0.0.1", "host.name": "web-1" }) }, scopeMetrics: [] }], + }); + assert.deepEqual(keysOf(metrics.resourceMetrics[0]!.resource.attributes), ["service.instance.id", "host.name"]); +}); + +test("requests without identifying keys, and malformed ones, come back untouched", () => { + const filter = new IdentifyingAttributeFilter(); + const clean = { resourceSpans: [{ resource: { attributes: kv({ "service.name": "s" }) }, scopeSpans: [{ spans: [{ attributes: kv({ "http.route": "/" }) }] }] }] }; + assert.equal(filter.traces(clean), clean); + for (const bad of [null, "x", { resourceSpans: "bad" }, { resourceSpans: [null, { scopeSpans: "bad" }] }, { resourceSpans: [{ scopeSpans: [{ spans: [{ attributes: "x" }] }] }] }]) + assert.deepEqual(filter.traces(bad), bad); + assert.deepEqual(filter.logs({ resourceLogs: "bad" }), { resourceLogs: "bad" }); + assert.deepEqual(filter.metrics({}), {}); +}); + +test("keep (true or a list) and extra drops; env parsing", () => { + assert.equal(identifyingKeysToDrop(true).size, 0); + const some = new IdentifyingAttributeFilter(identifyingKeysToDrop(["http.user_agent"], ["host.name"])); + const out = some.traces({ resourceSpans: [{ resource: { attributes: kv({ "host.name": "laptop", "service.instance.id": "i" }) }, scopeSpans: [{ spans: [{ attributes: kv({ "http.user_agent": "ua", "net.peer.ip": "1.2.3.4" }) }] }] }] }); + assert.deepEqual(keysOf(out.resourceSpans[0]!.resource.attributes), ["service.instance.id"]); + assert.deepEqual(keysOf(out.resourceSpans[0]!.scopeSpans[0]!.spans[0]!.attributes), ["http.user_agent"]); + const all = new IdentifyingAttributeFilter(identifyingKeysToDrop(true)); + const request = { resourceSpans: [{ scopeSpans: [{ spans: [{ attributes: kv({ "net.peer.ip": "1.2.3.4" }) }] }] }] }; + assert.equal(all.traces(request), request); + + assert.equal(keepIdentifyingEnv(undefined), undefined); + assert.equal(keepIdentifyingEnv("TRUE"), true); + assert.equal(keepIdentifyingEnv("off"), false); + assert.deepEqual(keepIdentifyingEnv("host.id, http.user_agent"), ["host.id", "http.user_agent"]); + assert.deepEqual(dropAttributesEnv("host.name,"), ["host.name"]); + assert.equal(dropAttributesEnv(""), undefined); +}); + +// --------------------------------------------------------------------------- +// Through the ingester: a real Python OpenTelemetry export (protobuf, Flask +// instrumentation: net.peer.ip, net.peer.port, http.user_agent) and a plain +// OTLP/JSON logs export, against a fake ClickHouse. + +const listen = async (server: Server) => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}; +const close = (server: Server) => + new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }); + +async function withIngester( + extra: Partial, + fn: (post: (path: string, body: Buffer | string, type: string) => Promise, rows: Record>>) => Promise, +) { + const rows: Record>> = {}; + const ch = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + const query = new URL(req.url!, "http://localhost").searchParams.get("query") ?? ""; + const table = /INSERT INTO\s+\S*?(runtime_[a-z0-9_]+)/.exec(query)?.[1]; + if (table) (rows[table] ??= []).push(...body.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))); + res.end(); + }); + }); + const chUrl = await listen(ch); + const config: IngesterConfig = { + port: 0, clickhouseUrl: chUrl, clickhouseUser: "default", clickhousePassword: "", clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "server", orgId: "org", repositoryId: "repo" }], + keyValidatorUrl: null, keyValidatorToken: null, sinkUrl: null, sinkToken: null, sinkMaxAttempts: 1, + sinkMaxBufferedBatches: 10, sinkMaxBufferedMb: 2, maxBodyBytes: 1024 * 1024, rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, occurrenceTtlDays: 14, spanTtlDays: 7, metricsTtlDays: 90, llmCallTtlDays: 90, + logTtlDays: 14, ...extra, + }; + const app = createIngesterApp(config).app.listen(); + const url = await new Promise((resolve) => + app.once("listening", () => resolve(`http://127.0.0.1:${(app.address() as AddressInfo).port}`)), + ); + const post = async (path: string, body: Buffer | string, type: string) => + (await fetch(`${url}${path}`, { method: "POST", headers: { "content-type": type, authorization: "Bearer server" }, body })).status; + try { + await fn(post, rows); + } finally { + await close(app); + await close(ch); + } +} + +const PYTHON_TRACES = readFileSync(new URL("../test-fixtures/python/flask-unhandled.otlp.pb", import.meta.url)); +const LOGS = JSON.stringify({ + resourceLogs: [{ + resource: { attributes: kv({ "service.name": "shop", "service.instance.id": "inst-1", "host.name": "web-1", "process.command_line": "node app.js --db=x" }) }, + scopeLogs: [{ logRecords: [{ timeUnixNano: String(Date.now() * 1e6), severityText: "info", body: { stringValue: "hello" }, + attributes: kv({ "client.address": "203.0.113.9", "user_agent.original": "Mozilla/5.0", "http.route": "/x" }) }] }], + }], +}); + +/** Every key stored in a JSON `attributes` column of `table`. */ +const storedKeys = (rows: Record>>, table: string) => + new Set((rows[table] ?? []).flatMap((row) => Object.keys(JSON.parse(String(row.attributes ?? "{}"))))); + +test("the ingester stores no identifying attribute from a plain OpenTelemetry sender", async () => { + await withIngester({}, async (post, rows) => { + assert.equal(await post("/v1/traces", PYTHON_TRACES, "application/x-protobuf"), 200); + assert.equal(await post("/v1/logs", LOGS, "application/json"), 200); + assert.ok(rows.runtime_spans?.length, "spans stored"); + assert.ok(rows.runtime_logs?.length, "logs stored"); + for (const table of ["runtime_spans", "runtime_logs", "runtime_error_occurrences"]) + for (const key of vectors.drop) assert.ok(!storedKeys(rows, table).has(key), `${key} stored in ${table}`); + assert.ok(storedKeys(rows, "runtime_spans").has("http.route"), "other attributes are stored"); + assert.ok(storedKeys(rows, "runtime_logs").has("http.route")); + }); +}); + +test("keepIdentifyingAttributes (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES) stores them; a list stores those keys", async () => { + await withIngester({ keepIdentifyingAttributes: true }, async (post, rows) => { + assert.equal(await post("/v1/traces", PYTHON_TRACES, "application/x-protobuf"), 200); + assert.equal(await post("/v1/logs", LOGS, "application/json"), 200); + for (const key of ["net.peer.ip", "net.peer.port", "http.user_agent"]) assert.ok(storedKeys(rows, "runtime_spans").has(key), key); + assert.ok(storedKeys(rows, "runtime_logs").has("client.address")); + }); + await withIngester({ keepIdentifyingAttributes: ["http.user_agent"] }, async (post, rows) => { + assert.equal(await post("/v1/traces", PYTHON_TRACES, "application/x-protobuf"), 200); + assert.ok(storedKeys(rows, "runtime_spans").has("http.user_agent")); + assert.ok(!storedKeys(rows, "runtime_spans").has("net.peer.ip")); + }); +}); diff --git a/packages/otlp-ingester/src/identifying.ts b/packages/otlp-ingester/src/identifying.ts new file mode 100644 index 0000000..5e5d24f --- /dev/null +++ b/packages/otlp-ingester/src/identifying.ts @@ -0,0 +1,151 @@ +/** + * Identifying attributes, dropped on arrival: the client's IP address and + * port, the raw user-agent string, the host's id and addresses, and the + * process's command line and owner. OpenTelemetry SDKs add them by default + * (resource detectors, HTTP server instrumentations) and nothing in Autter + * reads them. The SDKs drop them before export (@autter/runtime-node, + * adapters/python); the ingester drops them again so plain + * OpenTelemetry senders, other languages and older SDK versions are covered. + * + * Dropped from span, span event and span link attributes, log record + * attributes (and the keys of a structured log body), and the resource of + * traces, logs and metrics, before anything is stored or forwarded. + * + * Same list as runtime-core's identifying.ts and adapters/python/redact.py; + * parity is checked by test-vectors/redaction.json (`identifying`). + * `service.instance.id`, `host.name`, `process.pid`, `process.runtime.*`, + * `os.type` and `host.arch` are kept. + * + * Self-hosters opt back in with AUTTER_KEEP_IDENTIFYING_ATTRIBUTES (`true`, + * or a comma-separated list of keys) and drop more keys with + * AUTTER_DROP_ATTRIBUTES (a comma-separated list, `host.name` say). An SDK + * that keeps an attribute gets it stored only when the ingester keeps it too. + */ +export const IDENTIFYING_ATTRIBUTE_KEYS: readonly string[] = Object.freeze([ + "net.peer.ip", + "net.peer.port", + "net.sock.peer.addr", + "net.sock.peer.port", + "net.host.ip", + "net.sock.host.addr", + "network.local.address", + "http.client_ip", + "client.address", + "client.port", + "network.peer.address", + "network.peer.port", + "http.user_agent", + "user_agent.original", + "host.id", + "host.mac", + "host.ip", + "process.command_args", + "process.command", + "process.command_line", + "process.owner", + "process.executable.path", +]); + +export type KeepIdentifyingAttributes = boolean | string[]; + +/** AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: `true`/`1`/`yes`/`on`/`all`, `false`/`0`/`no`/`off`/`none`, or a comma-separated list. */ +export function keepIdentifyingEnv(raw: string | undefined): KeepIdentifyingAttributes | undefined { + if (typeof raw !== "string" || !raw.trim()) return undefined; + const value = raw.trim(); + if (/^(?:1|true|yes|on|all)$/i.test(value)) return true; + if (/^(?:0|false|no|off|none)$/i.test(value)) return false; + return value.split(",").map((key) => key.trim()).filter(Boolean); +} + +/** AUTTER_DROP_ATTRIBUTES: a comma-separated list of keys. */ +export function dropAttributesEnv(raw: string | undefined): string[] | undefined { + if (typeof raw !== "string") return undefined; + const keys = raw.split(",").map((key) => key.trim()).filter(Boolean); + return keys.length ? keys : undefined; +} + +/** The keys to drop: the list minus `keep`, plus `extra` (a key both kept and extra is dropped). */ +export function identifyingKeysToDrop( + keep?: KeepIdentifyingAttributes | null, + extra?: readonly string[] | null, +): ReadonlySet { + const kept = keep === true + ? new Set(IDENTIFYING_ATTRIBUTE_KEYS) + : new Set(Array.isArray(keep) ? keep.filter((key) => typeof key === "string") : []); + return new Set([ + ...IDENTIFYING_ATTRIBUTE_KEYS.filter((key) => !kept.has(key)), + ...(Array.isArray(extra) ? extra.filter((key) => typeof key === "string" && key !== "") : []), + ]); +} + +type Json = Record; + +const isObject = (value: unknown): value is Json => typeof value === "object" && value !== null && !Array.isArray(value); + +/** `list` with `fn` applied to each item; the same array when nothing changed. */ +function mapSame(list: T[], fn: (item: T) => T): T[] { + let out: T[] | null = null; + for (let i = 0; i < list.length; i++) { + const next = fn(list[i]!); + if (next !== list[i] && !out) out = list.slice(0, i); + if (out) out.push(next); + } + return out ?? list; +} + +/** `obj` with `field` replaced by `fn(obj[field])`; the same object when it did not change. */ +function withField(obj: unknown, field: string, fn: (value: unknown) => unknown): unknown { + if (!isObject(obj)) return obj; + const value = obj[field]; + const next = fn(value); + return next === value ? obj : { ...obj, [field]: next }; +} + +/** Each item of the array under `field` through `fn` (anything not an array is left for the normaliser to refuse). */ +function eachIn(obj: unknown, field: string, fn: (item: unknown) => unknown): unknown { + return withField(obj, field, (list) => (Array.isArray(list) ? mapSame(list as unknown[], fn) : list)); +} + +export class IdentifyingAttributeFilter { + constructor(readonly dropped: ReadonlySet = identifyingKeysToDrop()) {} + + /** An OTLP key/value list without the dropped keys; the same array when none is present. */ + list = (attributes: unknown): unknown => { + if (!Array.isArray(attributes) || !this.dropped.size) return attributes; + const dropped = (entry: unknown) => + isObject(entry) && typeof entry.key === "string" && this.dropped.has(entry.key); + if (!attributes.some(dropped)) return attributes; + return attributes.filter((entry) => !dropped(entry)); + }; + + private attributesOf = (obj: unknown): unknown => withField(obj, "attributes", this.list); + + private resourceOf = (container: unknown): unknown => withField(container, "resource", this.attributesOf); + + /** A traces request without identifying attributes (resource, spans, events, links). */ + traces(request: T): T { + if (!this.dropped.size) return request; + const span = (item: unknown) => + eachIn(eachIn(this.attributesOf(item), "events", this.attributesOf), "links", this.attributesOf); + return eachIn(request, "resourceSpans", (rs) => + eachIn(this.resourceOf(rs), "scopeSpans", (ss) => eachIn(ss, "spans", span)), + ) as T; + } + + /** A logs request without identifying attributes (resource, record attributes, the keys of a structured body). */ + logs(request: T): T { + if (!this.dropped.size) return request; + const body = (value: unknown) => + withField(value, "kvlistValue", (kv) => withField(kv, "values", this.list)); + const record = (item: unknown) => withField(this.attributesOf(item), "body", body); + return eachIn(request, "resourceLogs", (rl) => + eachIn(this.resourceOf(rl), "scopeLogs", (sl) => eachIn(sl, "logRecords", record)), + ) as T; + } + + /** A metrics request without identifying resource attributes (data points are left alone). */ + metrics(request: T): T { + if (!this.dropped.size) return request; + return eachIn(request, "resourceMetrics", this.resourceOf) as T; + } +} diff --git a/packages/otlp-ingester/src/index.ts b/packages/otlp-ingester/src/index.ts index 7bc2683..14052d5 100644 --- a/packages/otlp-ingester/src/index.ts +++ b/packages/otlp-ingester/src/index.ts @@ -10,6 +10,12 @@ if (config.llmPriceOverrides) { `LLM prices: built-in table as of ${llmPricesAsOf()}, ${config.llmPriceOverrides.models.length} model price(s) from AUTTER_LLM_PRICES_FILE (as of ${config.llmPriceOverrides.asOf})`, ); } +if (config.keepIdentifyingAttributes || config.dropAttributes?.length) { + const kept = config.keepIdentifyingAttributes === true ? "all" : (config.keepIdentifyingAttributes || []).join(", ") || "none"; + console.log( + `Identifying attributes: kept ${kept} (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES); also dropped: ${config.dropAttributes?.join(", ") || "none"} (AUTTER_DROP_ATTRIBUTES)`, + ); +} const sharingWarning = limitsWarning(config); if (sharingWarning) console.warn(`WARNING: ${sharingWarning}`); diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 33601ed..599c3c5 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -84,6 +84,7 @@ import type { SinkDeadLetterRow } from "./sink-dead-letters.js"; import { LoopCollapser } from "./loop-collapse.js"; import { applySuppression } from "./suppression.js"; import { configureRedaction, scrubOccurrence } from "./redact.js"; +import { IdentifyingAttributeFilter, identifyingKeysToDrop } from "./identifying.js"; import { configureLlmPricing } from "./llm-pricing.js"; import { buildCompatReport, @@ -219,6 +220,11 @@ const INGESTER_INFO_INTERVAL_MS = 6 * 60 * 60 * 1000; export function createIngesterApp(config: IngesterConfig, deps: IngesterDeps = {}): IngesterApp { configureRedaction(config); + // Identifying attributes (client IP, user agent, host id, command line) + // are dropped before anything below reads, stores or forwards a request. + const identifying = new IdentifyingAttributeFilter( + identifyingKeysToDrop(config.keepIdentifyingAttributes, config.dropAttributes), + ); // LLM prices: the built-in table, with a self-hosted override file // (AUTTER_LLM_PRICES_FILE, validated by loadConfig) merged over it. configureLlmPricing(config); @@ -1213,6 +1219,7 @@ export function createIngesterApp(config: IngesterConfig, deps: IngesterDeps = { } else { request = req.body as OtlpTraceRequest; } + request = identifying.traces(request); const totalSpans = spansInRequest(request); let normalized: ReturnType; let truncated: number; @@ -1295,7 +1302,9 @@ export function createIngesterApp(config: IngesterConfig, deps: IngesterDeps = { let truncated: number; const stats = { overLimit: 0 }; try { - logsRequest = req.is("application/x-protobuf") ? decodeLogsRequest(req.body as Buffer) : req.body as OtlpLogsRequest; + logsRequest = identifying.logs( + req.is("application/x-protobuf") ? decodeLogsRequest(req.body as Buffer) : req.body as OtlpLogsRequest, + ); ({ result: logs, truncated } = countTruncations(() => normalizeLogs(logsRequest, stats))); } catch { refuse(req, res, ctx, "invalid_payload", 400, { error: "invalid OTLP logs payload" }); @@ -1379,6 +1388,7 @@ export function createIngesterApp(config: IngesterConfig, deps: IngesterDeps = { } else { request = req.body as OtlpMetricsRequest; } + request = identifying.metrics(request); const memoryStats = { invalid: 0, overLimit: 0 }; let metricPoints: ReturnType; let memorySamples: ReturnType; diff --git a/packages/runtime-core/src/identifying.ts b/packages/runtime-core/src/identifying.ts new file mode 100644 index 0000000..1ac85d6 --- /dev/null +++ b/packages/runtime-core/src/identifying.ts @@ -0,0 +1,138 @@ +/** + * Attributes that identify a person, a machine or a command line rather + * than describe a request: the client's IP address and port, the raw + * user-agent string, the host's id and addresses, and the process's + * command line and owner. OpenTelemetry adds them by default (sdk-node's + * process and host resource detectors, HttpInstrumentation on every server + * span, the Python process detector, framework instrumentations), and + * redaction keeps them because none of them looks like a secret. + * + * Nothing in Autter reads them, so they are dropped by default: by + * @autter/runtime-node's exporters (span, span event and link attributes, + * and the resource), by adapters/python (`redact.py`) and by the ingester + * on arrival (spans, logs and resources, whatever sent them). The same list + * is mirrored there; parity is checked by test-vectors/redaction.json + * (`identifying`). + * + * Kept on purpose: `service.instance.id` (a random id the ingester needs to + * tell processes apart), `host.name` (container and pod names; on a laptop + * or a VM it is the machine's name, so drop it with `dropAttributes` if + * that matters to you), `process.pid`, `process.runtime.*`, + * `process.executable.name`, `os.type`, `host.arch`, the server's own + * address and port (`server.address`, `net.host.name`), and the coarse + * `user_agent.browser` / `.os` / `.device` / `.bot` fields the request + * enricher derives. + * + * Opt back in with `keepIdentifyingAttributes: true` (all of them) or a list + * of keys, or `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` (`true`, or a + * comma-separated list). The ingester drops them too unless it is given the + * same setting. Drop more keys (`host.name`, say) with `dropAttributes` / + * `AUTTER_DROP_ATTRIBUTES`. + */ +export const IDENTIFYING_ATTRIBUTE_KEYS: readonly string[] = Object.freeze([ + // The client's address (and, on outgoing calls, the peer's), in the old, + // 1.2x and stable semantic conventions; the server's own IP address. + "net.peer.ip", + "net.peer.port", + "net.sock.peer.addr", + "net.sock.peer.port", + "net.host.ip", + "net.sock.host.addr", + "network.local.address", + "http.client_ip", + "client.address", + "client.port", + "network.peer.address", + "network.peer.port", + // The raw user-agent string. + "http.user_agent", + "user_agent.original", + // The machine (its id and addresses; host.name is kept, see above). + "host.id", + "host.mac", + "host.ip", + // The command line and the user running it. + "process.command_args", + "process.command", + "process.command_line", + "process.owner", + "process.executable.path", +]); + +/** `true` keeps every identifying attribute; a list keeps those keys only. */ +export type KeepIdentifyingAttributes = boolean | readonly string[]; + +/** Environment variable read by the SDKs (and the ingester) when the option is not given. */ +export const KEEP_IDENTIFYING_ATTRIBUTES_ENV = "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES"; +/** Extra keys to drop (comma-separated), when `dropAttributes` is not given. */ +export const DROP_ATTRIBUTES_ENV = "AUTTER_DROP_ATTRIBUTES"; + +/** Parse `AUTTER_DROP_ATTRIBUTES`: a comma-separated list of keys. */ +export function dropAttributesFromEnv(raw: string | undefined | null): string[] | undefined { + if (typeof raw !== "string") return undefined; + const keys = raw.split(",").map((key) => key.trim()).filter(Boolean); + return keys.length ? keys : undefined; +} + +/** + * Parse `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES`: `true` / `1` / `yes` / `on` / + * `all` keep all of them, `false` / `0` / `no` / `off` / `none` keep none, + * anything else is a comma-separated list of keys. Unset or blank: + * `undefined` (the default, drop them all). + */ +export function keepIdentifyingFromEnv(raw: string | undefined | null): KeepIdentifyingAttributes | undefined { + if (typeof raw !== "string") return undefined; + const value = raw.trim(); + if (!value) return undefined; + if (/^(?:1|true|yes|on|all)$/i.test(value)) return true; + if (/^(?:0|false|no|off|none)$/i.test(value)) return false; + return value.split(",").map((key) => key.trim()).filter(Boolean); +} + +export interface IdentifyingAttributeFilter { + /** The keys this filter removes (the list minus what is kept, plus the extra keys). */ + readonly dropped: ReadonlySet; + /** + * `attributes` without the dropped keys: the same object when none of + * them is present (or nothing is dropped), a copy otherwise. Never + * mutates its input. + */ + apply(attributes: T): T; +} + +/** + * Build the filter once per process; `keep` as for + * `keepIdentifyingAttributes`, `extra` as for `dropAttributes` (exact keys + * dropped as well; a key both kept and listed in `extra` is dropped). + */ +export function makeIdentifyingAttributeFilter( + keep?: KeepIdentifyingAttributes | null, + extra?: readonly string[] | null, +): IdentifyingAttributeFilter { + const kept = keep === true + ? new Set(IDENTIFYING_ATTRIBUTE_KEYS) + : new Set(Array.isArray(keep) ? keep.filter((key): key is string => typeof key === "string") : []); + const dropped: ReadonlySet = new Set([ + ...IDENTIFYING_ATTRIBUTE_KEYS.filter((key) => !kept.has(key)), + ...(Array.isArray(extra) ? extra.filter((key): key is string => typeof key === "string" && key !== "") : []), + ]); + return { + dropped, + apply(attributes: T): T { + if (!dropped.size || !attributes || typeof attributes !== "object") return attributes; + let hit = false; + for (const key of Object.keys(attributes)) { + if (dropped.has(key)) { + hit = true; + break; + } + } + if (!hit) return attributes; + const out: Record = {}; + for (const key of Object.keys(attributes)) { + if (!dropped.has(key)) out[key] = (attributes as Record)[key]; + } + return out as T; + }, + }; +} diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index 014766e..743ca8a 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -12,6 +12,16 @@ export { type RedactOptions, type Redactor as AttributeRedactor, } from "./redact.js"; +export { + DROP_ATTRIBUTES_ENV, + IDENTIFYING_ATTRIBUTE_KEYS, + KEEP_IDENTIFYING_ATTRIBUTES_ENV, + dropAttributesFromEnv, + keepIdentifyingFromEnv, + makeIdentifyingAttributeFilter, + type IdentifyingAttributeFilter, + type KeepIdentifyingAttributes, +} from "./identifying.js"; export { LEVEL_SEVERITY, maxLevel, diff --git a/packages/runtime-core/test/identifying.test.mjs b/packages/runtime-core/test/identifying.test.mjs new file mode 100644 index 0000000..c52286b --- /dev/null +++ b/packages/runtime-core/test/identifying.test.mjs @@ -0,0 +1,95 @@ +// Identifying attributes (client address, raw user agent, host, command +// line): the shared list and filter the Node SDK's exporters use. The +// Python adapter and the ingester hold the same list, checked against the +// same vectors (test-vectors/redaction.json, `identifying`). +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { + DROP_ATTRIBUTES_ENV, + dropAttributesFromEnv, + IDENTIFYING_ATTRIBUTE_KEYS, + KEEP_IDENTIFYING_ATTRIBUTES_ENV, + keepIdentifyingFromEnv, + makeIdentifyingAttributeFilter, +} from "../dist/index.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +).identifying; + +const sample = () => ({ + ...Object.fromEntries(vectors.drop.map((key) => [key, `v:${key}`])), + ...Object.fromEntries(vectors.keep.map((key) => [key, `v:${key}`])), +}); + +test("the list is exactly the shared vectors' drop list", () => { + assert.deepEqual([...IDENTIFYING_ATTRIBUTE_KEYS].sort(), [...vectors.drop].sort()); + assert.equal(new Set(IDENTIFYING_ATTRIBUTE_KEYS).size, IDENTIFYING_ATTRIBUTE_KEYS.length, "no duplicates"); + for (const key of vectors.keep) assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes(key), `${key} must not be dropped`); + assert.ok(Object.isFrozen(IDENTIFYING_ATTRIBUTE_KEYS)); +}); + +test("by default every listed key is dropped and every other key survives", () => { + const filter = makeIdentifyingAttributeFilter(); + const input = sample(); + const out = filter.apply(input); + assert.deepEqual(Object.keys(out).sort(), [...vectors.keep].sort()); + for (const key of vectors.keep) assert.equal(out[key], `v:${key}`); + assert.equal(out["service.instance.id"], "v:service.instance.id"); + // Never mutates its input. + assert.equal(Object.keys(input).length, vectors.drop.length + vectors.keep.length); +}); + +test("an attribute bag without identifying keys is returned as is", () => { + const filter = makeIdentifyingAttributeFilter(); + const attributes = { "http.route": "/users/:id", "service.instance.id": "abc" }; + assert.equal(filter.apply(attributes), attributes); +}); + +test("keep: true keeps everything; a list keeps only its keys", () => { + const all = makeIdentifyingAttributeFilter(true); + const input = sample(); + assert.equal(all.apply(input), input); + assert.equal(all.dropped.size, 0); + + const some = makeIdentifyingAttributeFilter(["http.user_agent", "host.id", "not.a.listed.key"]); + const out = some.apply(sample()); + assert.equal(out["http.user_agent"], "v:http.user_agent"); + assert.equal(out["host.id"], "v:host.id"); + assert.equal(out["host.name"], "v:host.name"); + assert.equal(out["net.peer.ip"], undefined); + assert.equal(out["client.address"], undefined); + assert.equal(out["process.command_args"], undefined); + assert.equal(out["service.instance.id"], "v:service.instance.id"); + assert.equal(some.dropped.size, vectors.drop.length - 2); + + // false and an empty list are the default. + assert.equal(makeIdentifyingAttributeFilter(false).dropped.size, vectors.drop.length); + assert.equal(makeIdentifyingAttributeFilter([]).dropped.size, vectors.drop.length); +}); + +test("AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: true, false or a comma-separated list", () => { + assert.equal(KEEP_IDENTIFYING_ATTRIBUTES_ENV, "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES"); + assert.equal(keepIdentifyingFromEnv(undefined), undefined); + assert.equal(keepIdentifyingFromEnv(" "), undefined); + for (const raw of ["true", "TRUE", "1", "yes", "on", "all"]) assert.equal(keepIdentifyingFromEnv(raw), true, raw); + for (const raw of ["false", "0", "no", "off", "none"]) assert.equal(keepIdentifyingFromEnv(raw), false, raw); + assert.deepEqual(keepIdentifyingFromEnv(" http.user_agent, host.name ,,"), ["http.user_agent", "host.name"]); +}); + +test("host.name is kept by default; the extra drop list removes it (and wins over keep)", () => { + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes("host.name")); + const input = { "host.name": "web-7f9c", "service.instance.id": "abc", "host.id": "m-1" }; + assert.deepEqual(makeIdentifyingAttributeFilter().apply(input), { "host.name": "web-7f9c", "service.instance.id": "abc" }); + const extra = makeIdentifyingAttributeFilter(undefined, ["host.name"]); + assert.deepEqual(extra.apply(input), { "service.instance.id": "abc" }); + assert.equal(extra.dropped.size, vectors.drop.length + 1); + assert.deepEqual(makeIdentifyingAttributeFilter(true, ["host.name"]).apply(input), { "service.instance.id": "abc", "host.id": "m-1" }); + assert.deepEqual(makeIdentifyingAttributeFilter(["host.name"], ["host.name"]).apply({ "host.name": "x" }), {}); + + assert.equal(DROP_ATTRIBUTES_ENV, "AUTTER_DROP_ATTRIBUTES"); + assert.equal(dropAttributesFromEnv(undefined), undefined); + assert.equal(dropAttributesFromEnv(" , "), undefined); + assert.deepEqual(dropAttributesFromEnv("host.name, k8s.pod.name"), ["host.name", "k8s.pod.name"]); +}); diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index daa2dca..8d6993e 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -218,6 +218,7 @@ Defaults (cheap by construction): | Logs | not collected | | PII in custom attributes | redacted before export (`redactAttributes`) | | Query strings in span URLs | parameter names only (`keepQueryParams`) | +| Client IP, raw user agent, host id, command line | dropped (`keepIdentifyingAttributes`, `dropAttributes`) | | Forgotten `shutdown()` | exporters still flushed on exit (`autoFlush`) | **Error-linked trace retention.** Errors export at 100% while traces are @@ -614,6 +615,46 @@ This runs at export, like the scrubbing above, and also with `redactAttributes: false`. Route templates (`http.route`) and request summaries never contained a query. +**Identifying attributes.** OpenTelemetry puts attributes that identify a +person or a machine on every server span and on the resource. None of them +looks like a secret, so scrubbing keeps them, and nothing in Autter reads +them. They are dropped at export from span, event and link attributes and +from the resource (spans and metrics), whichever instrumentation set them, +also with `redactAttributes: false`: + +| What | Keys | +| --- | --- | +| The client's address and port (the peer's, on outgoing calls), and the server's own IP | `net.peer.ip`, `net.peer.port`, `net.sock.peer.addr`, `net.sock.peer.port`, `http.client_ip`, `client.address`, `client.port`, `network.peer.address`, `network.peer.port`, `net.host.ip`, `net.sock.host.addr`, `network.local.address` | +| The raw user-agent string | `http.user_agent`, `user_agent.original` | +| The machine's id and addresses | `host.id`, `host.ip`, `host.mac` | +| The command line and the user running it | `process.command_args`, `process.command`, `process.command_line`, `process.owner`, `process.executable.path` | + +Kept: `service.instance.id` (a random id; the ingester needs it to tell +processes apart), `host.name`, `process.pid`, `process.runtime.*`, +`os.type`, `host.arch`, `server.address`, and the coarse +`user_agent.browser` / `.os` / `.device` / `.bot` fields of request +summaries. `host.name` is kept because it is usually a container or pod +name; on a laptop or a VM it is the machine's name, which can be a person's. +Drop it, or any other key, with `dropAttributes`: + +```ts +initAutterServer({ ..., dropAttributes: ["host.name"] }); +// or AUTTER_DROP_ATTRIBUTES=host.name +``` + +To keep identifying attributes, list them, or pass `true` for all of them: + +```ts +initAutterServer({ ..., keepIdentifyingAttributes: ["http.user_agent"] }); +// or AUTTER_KEEP_IDENTIFYING_ATTRIBUTES=http.user_agent (true for all) +``` + +The ingester drops the same keys on arrival, so an attribute kept here is +stored only when the ingester keeps it too: set +`AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` on the ingester as well +([otlp-ingester README](../otlp-ingester/README.md#identifying-attributes)). +`IDENTIFYING_ATTRIBUTE_KEYS` exports the list. + ### One SDK per process, however many copies are loaded Bundlers can load this package more than once: Next.js bundles one copy diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index 7c78a36..db0e498 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -44,6 +44,8 @@ export { export { redactAttributes, redactText, + IDENTIFYING_ATTRIBUTE_KEYS, + type KeepIdentifyingAttributes, type RedactOptions, } from "./redact.js"; export { diff --git a/packages/runtime-node/src/redact-exporter.ts b/packages/runtime-node/src/redact-exporter.ts index 9616b9d..7ee3160 100644 --- a/packages/runtime-node/src/redact-exporter.ts +++ b/packages/runtime-node/src/redact-exporter.ts @@ -7,6 +7,7 @@ import { type ResourceMetrics, } from "@opentelemetry/sdk-metrics"; import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; +import { makeIdentifyingAttributeFilter, type IdentifyingAttributeFilter } from "@autter/runtime-core"; import type { Redactor } from "./redact.js"; import { makeUrlQueryPolicy, type UrlQueryPolicy } from "./url-query.js"; @@ -27,30 +28,85 @@ import { makeUrlQueryPolicy, type UrlQueryPolicy } from "./url-query.js"; * `process.command_args` (`--db=postgres://user:pass@…`), and * OTEL_RESOURCE_ATTRIBUTES can carry anything. * + * Identifying attributes (the client's IP address and port, the raw + * user agent, host.id and the host's addresses, the command line and its owner: see + * runtime-core's identifying.ts) are dropped from span, event and link + * attributes and from the resource, unless `keepIdentifyingAttributes` + * keeps them (and `dropAttributes` adds keys to drop). + * * The span is re-wrapped, never mutated (spans are shared with other - * processors). `redactAttributes: false` disables the scrubbing but not the - * query policy, which `keepQueryParams` tunes. + * processors). `redactAttributes: false` disables the scrubbing but neither + * the query policy, which `keepQueryParams` tunes, nor the identifying + * attribute drop, which `keepIdentifyingAttributes` tunes. */ -/** One scrubbed copy per Resource object: the OTLP serializer groups spans - * by resource identity, so a fresh copy per span would split the batch. */ -const scrubbedResources = new WeakMap(); +/** One scrubbed copy per Resource object (per redactor and filter): the + * OTLP serializer groups spans by resource identity, so a fresh copy per + * span would split the batch. */ +const scrubbedResources = new WeakMap>>(); -function redactResource(resource: R, redactor: Redactor): Resource { - const cached = scrubbedResources.get(resource); +function redactResource( + resource: R, + redactor: Redactor, + identifying: IdentifyingAttributeFilter, +): Resource { + let byFilter = scrubbedResources.get(redactor); + if (!byFilter) scrubbedResources.set(redactor, (byFilter = new WeakMap())); + let cache = byFilter.get(identifying); + if (!cache) byFilter.set(identifying, (cache = new WeakMap())); + const cached = cache.get(resource); if (cached) return cached; - const scrubbed = resourceFromAttributes(redactor.exported(resource.attributes)); - scrubbedResources.set(resource, scrubbed); + const attributes = identifying.apply(resource.attributes); + // Nothing to scrub or drop: the resource itself (keeps its schema URL). + if (!redactor.enabled && attributes === resource.attributes) { + cache.set(resource, resource); + return resource; + } + const scrubbed = resourceFromAttributes(redactor.enabled ? redactor.exported(attributes) : attributes); + cache.set(resource, scrubbed); return scrubbed; } const DEFAULT_QUERY_POLICY = makeUrlQueryPolicy(); +const DEFAULT_IDENTIFYING = makeIdentifyingAttributeFilter(); -/** Query policy only (`redactAttributes: false`): name and attributes. */ -function stripQueries(span: ReadableSpan, query: UrlQueryPolicy): ReadableSpan { +/** Without scrubbing (`redactAttributes: false`): the query policy and the + * identifying attribute drop only. */ +function stripQueries( + span: ReadableSpan, + redactor: Redactor, + query: UrlQueryPolicy, + identifying: IdentifyingAttributeFilter, +): ReadableSpan { const name = query.name(span.name); - const attributes = query.attributes(span.attributes); - if (name === span.name && attributes === span.attributes) return span; - return wrap(span, { name, attributes }); + const attributes = query.attributes(identifying.apply(span.attributes)); + let eventsChanged = false; + const events = span.events.map((event) => { + if (!event.attributes) return event; + const kept = identifying.apply(event.attributes); + if (kept === event.attributes) return event; + eventsChanged = true; + return { ...event, attributes: kept }; + }); + let linksChanged = false; + const links = span.links.map((link) => { + if (!link.attributes) return link; + const kept = identifying.apply(link.attributes); + if (kept === link.attributes) return link; + linksChanged = true; + return { ...link, attributes: kept }; + }); + const resource = redactResource(span.resource, redactor, identifying); + if ( + name === span.name && attributes === span.attributes && !eventsChanged && !linksChanged + && resource === span.resource + ) return span; + return wrap(span, { + name, + attributes, + ...(eventsChanged ? { events } : {}), + ...(linksChanged ? { links } : {}), + resource, + }); } function wrap( @@ -82,25 +138,27 @@ export function redactSpan( span: ReadableSpan, redactor: Redactor, query: UrlQueryPolicy = DEFAULT_QUERY_POLICY, + identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, ): ReadableSpan { - if (!redactor.enabled) return stripQueries(span, query); + if (!redactor.enabled) return stripQueries(span, redactor, query, identifying); const status = span.status.message ? { ...span.status, message: redactor.text(span.status.message) } : span.status; - // Scrub first (a `user:pass@` URL is recognised whole), then cut queries. + // Drop identifying keys, scrub (a `user:pass@` URL is recognised whole), + // then cut queries. return wrap(span, { name: query.name(redactor.text(span.name)), status, - attributes: query.attributes(redactor.exported(span.attributes)), + attributes: query.attributes(redactor.exported(identifying.apply(span.attributes))), links: span.links.map((link) => - link.attributes ? { ...link, attributes: redactor.exported(link.attributes) } : link, + link.attributes ? { ...link, attributes: redactor.exported(identifying.apply(link.attributes)) } : link, ), events: span.events.map((event) => ({ ...event, name: redactor.text(event.name), - ...(event.attributes ? { attributes: redactor.exported(event.attributes) } : {}), + ...(event.attributes ? { attributes: redactor.exported(identifying.apply(event.attributes)) } : {}), })), - resource: redactResource(span.resource, redactor), + resource: redactResource(span.resource, redactor, identifying), }); } @@ -109,6 +167,7 @@ export class RedactingSpanExporter implements SpanExporter { private readonly inner: SpanExporter, private readonly redactor: () => Redactor, private readonly query: UrlQueryPolicy = DEFAULT_QUERY_POLICY, + private readonly identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, ) {} export( @@ -120,7 +179,7 @@ export class RedactingSpanExporter implements SpanExporter { let failed = 0; for (const span of spans) { try { - scrubbed.push(redactSpan(span, redactor, this.query)); + scrubbed.push(redactSpan(span, redactor, this.query, this.identifying)); } catch { // Fail closed: a span we could not scrub is not exported. failed++; @@ -154,29 +213,28 @@ export class RedactingSpanExporter implements SpanExporter { /** * Metrics carry the same resource as spans (process.command_args, - * OTEL_RESOURCE_ATTRIBUTES), so their resource is scrubbed the same way. - * Data-point attributes are the SDK's own low-cardinality HTTP attributes. + * host.id, OTEL_RESOURCE_ATTRIBUTES), so their resource is scrubbed and + * loses its identifying attributes the same way (the drop also with + * `redactAttributes: false`). Data-point attributes are the SDK's own + * low-cardinality HTTP attributes. */ export class RedactingMetricExporter implements PushMetricExporter { constructor( private readonly inner: PushMetricExporter, private readonly redactor: () => Redactor, + private readonly identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, ) {} export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void { const redactor = this.redactor(); - if (!redactor.enabled) { - this.inner.export(metrics, resultCallback); - return; - } let resource: ResourceMetrics["resource"]; try { - resource = redactResource(metrics.resource, redactor); + resource = redactResource(metrics.resource, redactor, this.identifying); } catch (error) { resultCallback({ code: ExportResultCode.FAILED, error: error as Error }); return; } - this.inner.export({ ...metrics, resource }, resultCallback); + this.inner.export(resource === metrics.resource ? metrics : { ...metrics, resource }, resultCallback); } forceFlush(): Promise { diff --git a/packages/runtime-node/src/redact.ts b/packages/runtime-node/src/redact.ts index c128dee..bcfaba5 100644 --- a/packages/runtime-node/src/redact.ts +++ b/packages/runtime-node/src/redact.ts @@ -11,3 +11,4 @@ export { type RedactOptions, type AttributeRedactor as Redactor, } from "@autter/runtime-core"; +export { IDENTIFYING_ATTRIBUTE_KEYS, type KeepIdentifyingAttributes } from "@autter/runtime-core"; diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index f0d84af..9acd176 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -104,9 +104,15 @@ import { errorInternal, fingerprintEventAttributes, isError, + DROP_ATTRIBUTES_ENV, + dropAttributesFromEnv, + KEEP_IDENTIFYING_ATTRIBUTES_ENV, + keepIdentifyingFromEnv, + makeIdentifyingAttributeFilter, splitFingerprint, type CaptureLoggedErrorsOption, type IgnoreErrorPattern, + type KeepIdentifyingAttributes, } from "@autter/runtime-core"; import { setHandledCapture, @@ -377,6 +383,38 @@ export interface AutterServerOptions { * Applies even with `redactAttributes: false`. Default `[]`. */ keepQueryParams?: string[]; + /** + * Identifying attributes never leave the process by default: the + * client's IP address and port (`net.peer.ip`, `client.address`, + * `http.client_ip`, `network.peer.address`, …), the raw user-agent + * string (`http.user_agent`, `user_agent.original`), the machine's id + * and addresses (`host.id`, `host.ip`, `host.mac`) and the command line + * and its owner (`process.command_args`, `process.command`, + * `process.command_line`, `process.owner`, `process.executable.path`). + * OpenTelemetry adds them to every server span and to the resource; + * nothing in Autter reads them. They are dropped at export from span, + * event and link attributes and from the resource, whichever + * instrumentation set them, also with `redactAttributes: false`. + * `service.instance.id`, `host.name` (container and pod names; on a + * laptop or VM, the machine's name: drop it with `dropAttributes`), + * `process.pid`, `process.runtime.*`, `os.type`, `host.arch` and the + * coarse `user_agent.browser` / `.os` / `.device` fields are kept. + * + * `true` keeps all of them; a list keeps those keys: + * `keepIdentifyingAttributes: ["host.name"]`. Default: the + * AUTTER_KEEP_IDENTIFYING_ATTRIBUTES environment variable (`true`, or a + * comma-separated list), else none. The ingester drops them as well + * unless it is given the same setting. + */ + keepIdentifyingAttributes?: KeepIdentifyingAttributes; + /** + * More attribute keys to drop at export, exactly like the identifying + * ones (span, event and link attributes and the resource, also with + * `redactAttributes: false`): `dropAttributes: ["host.name"]`. A key both + * kept and listed here is dropped. Default: the AUTTER_DROP_ATTRIBUTES + * environment variable (a comma-separated list), else none. + */ + dropAttributes?: string[]; /** * Verbose lifecycle logging to stderr: "exported N spans", flushes and * auto-flush activity — handy while wiring the SDK up. Also enabled by @@ -1261,6 +1299,12 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { // Every trace exporter scrubs spans on the way out, whoever created them // (see redact-exporter.ts): secrets and PII, and URL query values. const queryPolicy = makeUrlQueryPolicy(options.keepQueryParams); + // Identifying attributes (client IP, user agent, host name, command + // line) are dropped at export unless kept (runtime-core identifying.ts). + const identifying = makeIdentifyingAttributeFilter( + options.keepIdentifyingAttributes ?? keepIdentifyingFromEnv(process.env[KEEP_IDENTIFYING_ATTRIBUTES_ENV]), + options.dropAttributes ?? dropAttributesFromEnv(process.env[DROP_ATTRIBUTES_ENV]), + ); const slowPerMinute = Number.isFinite(options.retainSlowTracesPerMinute) ? Math.max(0, options.retainSlowTracesPerMinute!) : DEFAULT_SLOW_TRACES_PER_MINUTE; // One policy for every trace exporter: LLM spans keep prompt/completion @@ -1279,6 +1323,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { new IngestSpanExporter(new OTLPTraceExporter({ url: `${endpoint}/v1/traces`, headers }), ingestTarget), () => redactor, queryPolicy, + identifying, ), llmContent, ), @@ -1376,6 +1421,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { temporalityPreference: AggregationTemporalityPreference.DELTA, }), ingestTarget), () => redactor, + identifying, ), "metrics", ), diff --git a/packages/runtime-node/test/identifying-attributes.test.mjs b/packages/runtime-node/test/identifying-attributes.test.mjs new file mode 100644 index 0000000..30fe703 --- /dev/null +++ b/packages/runtime-node/test/identifying-attributes.test.mjs @@ -0,0 +1,229 @@ +// Identifying attributes (client address, raw user agent, host id and +// addresses, command line and owner) never leave the process by default, whoever set +// them: HttpInstrumentation on server and client spans, sdk-node's process +// and host resource detectors, the app's own spans, events and links. Each +// case runs initAutterServer in a child process against a local collector, +// so the real detectors and instrumentation run. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { dirname, join } from "node:path"; +import { after, before, test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); +const vectors = JSON.parse(readFileSync(join(here, "../../../test-vectors/redaction.json"), "utf8")).identifying; +const DROP = vectors.drop; + +/** Every OTLP body received: { path, body }. */ +const received = []; +let collector; +let endpoint; + +before(async () => { + collector = createServer((req, res) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + try { + received.push({ path: req.url, body: JSON.parse(Buffer.concat(chunks).toString("utf8")) }); + } catch { + /* not JSON: not ours */ + } + res.end("{}"); + }); + }); + await new Promise((resolve) => collector.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${collector.address().port}`; +}); + +after(async () => { + await new Promise((resolve) => { + collector.close(resolve); + collector.closeAllConnections(); + }); +}); + +const until = async (check, ms = 5000) => { + const start = Date.now(); + while (!check() && Date.now() - start < ms) await new Promise((r) => setTimeout(r, 10)); + return check(); +}; + +const value = (v) => v?.stringValue ?? v?.intValue ?? v?.boolValue ?? v?.doubleValue ?? v?.arrayValue; +const attrs = (list) => Object.fromEntries((list ?? []).map((a) => [a.key, value(a.value)])); + +/** What one service exported: span, event and link attributes, and the + * resources of its spans and metrics. */ +function exportedBy(service) { + const out = { spans: [], events: [], links: [], spanResources: [], metricResources: [] }; + for (const { path, body } of received) { + for (const rs of body.resourceSpans ?? []) { + const resource = attrs(rs.resource?.attributes); + if (resource["service.name"] !== service) continue; + out.spanResources.push(resource); + for (const ss of rs.scopeSpans ?? []) + for (const span of ss.spans ?? []) { + out.spans.push({ name: span.name, kind: span.kind, attributes: attrs(span.attributes) }); + for (const event of span.events ?? []) out.events.push(attrs(event.attributes)); + for (const link of span.links ?? []) out.links.push(attrs(link.attributes)); + } + } + if (path === "/v1/metrics") + for (const rm of body.resourceMetrics ?? []) { + const resource = attrs(rm.resource?.attributes); + if (resource["service.name"] === service) out.metricResources.push(resource); + } + } + return out; +} + +/** Every attribute bag the service exported. */ +const bags = (exported) => [ + ...exported.spans.map((span) => span.attributes), + ...exported.events, + ...exported.links, + ...exported.spanResources, + ...exported.metricResources, +]; + +/** initAutterServer with `options`, one incoming request through + * HttpInstrumentation (with a user agent and X-Forwarded-For) and its + * outgoing client call, one manual span carrying identifying keys on its + * attributes, an event and a link; then shutdown (flushes spans and metrics). */ +async function run(service, options = {}, env = {}) { + const script = ` + import { createRequire } from "node:module"; + import { trace } from "@opentelemetry/api"; + import { initAutterServer } from "./dist/index.js"; + const sdk = initAutterServer({ + apiKey: "k", service: process.env.SERVICE, endpoint: process.env.ENDPOINT, traceSampleRate: 1, + captureGlobalErrors: false, autoFlush: false, memoryMetrics: false, compatCheck: false, + metricIntervalMs: 3_600_000, logging: { console: false }, + ...JSON.parse(process.env.OPTIONS), + }); + const http = createRequire(import.meta.url)("http"); + const server = http.createServer((req, res) => res.end("ok")); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + await new Promise((resolve, reject) => { + http.get({ + host: "127.0.0.1", port: server.address().port, path: "/hello", + headers: { "user-agent": "Mozilla/5.0 (Macintosh) identifying-test", "x-forwarded-for": "203.0.113.9" }, + }, (res) => { res.resume(); res.on("end", resolve); }).on("error", reject); + }); + await new Promise((resolve) => server.close(resolve)); + const span = trace.getTracer("t").startSpan("manual", { + attributes: { "client.address": "198.51.100.7", "user_agent.original": "curl/8", "host.name": "janes-laptop", + "http.route": "/x", "user.email": "kept@example.com" }, + links: [{ + context: { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "b7ad6b7169203331", traceFlags: 1 }, + attributes: { "network.peer.address": "198.51.100.8", "link.note": "kept" }, + }], + }); + span.addEvent("peer", { "net.peer.ip": "198.51.100.9", "event.note": "kept" }); + span.end(); + await sdk.shutdown(); + `; + const childEnv = { ...process.env, ENDPOINT: endpoint, SERVICE: service, OPTIONS: JSON.stringify(options), ...env }; + for (const name of ["AUTTER_KEEP_IDENTIFYING_ATTRIBUTES", "AUTTER_DROP_ATTRIBUTES"]) + if (!(name in env)) delete childEnv[name]; + const child = spawn(process.execPath, ["--input-type=module", "-e", script], { + cwd: join(here, ".."), + env: childEnv, + stdio: ["ignore", "inherit", "inherit"], + }); + const code = await new Promise((resolve) => child.on("exit", resolve)); + assert.equal(code, 0); + // shutdown() awaited both exports; poll in case a response raced the exit. + await until(() => { + const exported = exportedBy(service); + return exported.spans.some((span) => span.name === "manual") && exported.metricResources.length > 0; + }); + const exported = exportedBy(service); + assert.ok(exported.spans.some((span) => span.name === "manual"), "manual span exported"); + assert.ok(exported.spans.some((span) => span.kind === 2), "HTTP server span exported"); + assert.ok(exported.metricResources.length > 0, "metrics exported"); + return exported; +} + +const present = (exported, key) => bags(exported).some((bag) => key in bag); + +function assertKeptIdentity(exported) { + // Never dropped: the ingester needs service.instance.id; the runtime, + // pid and architecture describe the process without identifying anyone. + for (const resource of [...exported.spanResources, ...exported.metricResources]) + assert.ok(resource["service.instance.id"], "service.instance.id survives"); + assert.ok(exported.spanResources.some((resource) => resource["process.pid"] !== undefined), "process.pid survives"); + assert.ok(exported.spanResources.some((resource) => resource["process.runtime.name"] === "nodejs"), "process.runtime.* survives"); + assert.ok(exported.spanResources.some((resource) => resource["host.arch"]), "host.arch survives"); + // host.name is kept (container and pod names) unless dropAttributes lists it. + assert.ok(exported.spanResources.some((resource) => resource["host.name"]), "host.name survives"); + const manual = exported.spans.find((span) => span.name === "manual"); + assert.equal(manual.attributes["http.route"], "/x"); + assert.equal(exported.events.find((event) => "event.note" in event)?.["event.note"], "kept"); + assert.equal(exported.links.find((link) => "link.note" in link)?.["link.note"], "kept"); +} + +test("by default no identifying attribute leaves the process (spans, events, links, resources)", async () => { + const exported = await run("identifying-default"); + for (const key of DROP) assert.ok(!present(exported, key), `${key} was exported`); + assertKeptIdentity(exported); +}); + +test("the drop applies with redactAttributes: false", async () => { + const exported = await run("identifying-no-redact", { redactAttributes: false }); + for (const key of DROP) assert.ok(!present(exported, key), `${key} was exported`); + assertKeptIdentity(exported); + // Redaction itself stays off, as configured. + assert.equal(exported.spans.find((span) => span.name === "manual").attributes["user.email"], "kept@example.com"); +}); + +test("keepIdentifyingAttributes: true keeps them all", async () => { + const exported = await run("identifying-keep-all", { keepIdentifyingAttributes: true }); + // What OpenTelemetry really emits here, now let through. + for (const key of ["net.peer.ip", "net.peer.port", "net.host.ip", "http.client_ip", "http.user_agent", "client.address", + "user_agent.original", "network.peer.address"]) + assert.ok(present(exported, key), `${key} kept`); + for (const key of ["host.id", "process.command_args", "process.executable.path", "process.owner"]) + assert.ok(exported.spanResources.some((resource) => key in resource), `${key} kept on the resource`); + assert.ok(exported.metricResources.some((resource) => "host.id" in resource), "host.id kept on the metrics resource"); + const server = exported.spans.find((span) => span.kind === 2); + assert.equal(server.attributes["http.user_agent"], "Mozilla/5.0 (Macintosh) identifying-test"); + assert.equal(server.attributes["http.client_ip"], "203.0.113.9"); + assertKeptIdentity(exported); +}); + +test("keepIdentifyingAttributes: [keys] keeps those keys only", async () => { + const exported = await run("identifying-keep-some", { keepIdentifyingAttributes: ["http.user_agent", "host.id"] }); + assert.ok(present(exported, "http.user_agent"), "http.user_agent kept"); + assert.ok(exported.spanResources.some((resource) => "host.id" in resource), "host.id kept"); + for (const key of DROP.filter((key) => key !== "http.user_agent" && key !== "host.id")) + assert.ok(!present(exported, key), `${key} was exported`); + assertKeptIdentity(exported); +}); + +test("AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: a comma-separated list, or true", async () => { + const some = await run("identifying-env-some", {}, { AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: "net.peer.ip, process.owner" }); + assert.ok(present(some, "net.peer.ip"), "net.peer.ip kept"); + assert.ok(some.spanResources.some((resource) => "process.owner" in resource), "process.owner kept"); + for (const key of DROP.filter((key) => key !== "net.peer.ip" && key !== "process.owner")) + assert.ok(!present(some, key), `${key} was exported`); + + const all = await run("identifying-env-all", {}, { AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: "true" }); + assert.ok(present(all, "http.user_agent") && present(all, "host.id"), "everything kept"); + + // The option wins over the environment. + const option = await run("identifying-env-overridden", { keepIdentifyingAttributes: false }, { AUTTER_KEEP_IDENTIFYING_ATTRIBUTES: "true" }); + for (const key of DROP) assert.ok(!present(option, key), `${key} was exported`); +}); + +test("dropAttributes / AUTTER_DROP_ATTRIBUTES drop more keys (host.name)", async () => { + const option = await run("identifying-drop-option", { dropAttributes: ["host.name", "http.route"], redactAttributes: false }); + for (const key of [...DROP, "host.name", "http.route"]) assert.ok(!present(option, key), `${key} was exported`); + assert.ok(option.spanResources.some((resource) => resource["service.instance.id"]), "service.instance.id survives"); + + const env = await run("identifying-drop-env", {}, { AUTTER_DROP_ATTRIBUTES: "host.name" }); + assert.ok(!present(env, "host.name"), "host.name was exported"); + for (const key of DROP) assert.ok(!present(env, key), `${key} was exported`); +}); diff --git a/test-vectors/redaction.json b/test-vectors/redaction.json index 2801ce6..29921ba 100644 --- a/test-vectors/redaction.json +++ b/test-vectors/redaction.json @@ -194,5 +194,55 @@ "route", "message" ] + }, + "identifying": { + "$comment": "Identifying attributes (client address, raw user agent, host, command line) every implementation drops by default: runtime-core identifying.ts (runtime-node exporters), adapters/python redact.py, otlp-ingester identifying.ts. `drop` is the exact list each one must hold; `keep` are keys that must survive the drop.", + "drop": [ + "net.peer.ip", + "net.peer.port", + "net.sock.peer.addr", + "net.sock.peer.port", + "net.host.ip", + "net.sock.host.addr", + "network.local.address", + "http.client_ip", + "client.address", + "client.port", + "network.peer.address", + "network.peer.port", + "http.user_agent", + "user_agent.original", + "host.id", + "host.mac", + "host.ip", + "process.command_args", + "process.command", + "process.command_line", + "process.owner", + "process.executable.path" + ], + "keep": [ + "service.instance.id", + "service.name", + "host.name", + "process.pid", + "process.runtime.name", + "process.runtime.version", + "process.runtime.description", + "process.executable.name", + "os.type", + "host.arch", + "server.address", + "server.port", + "net.host.name", + "net.host.port", + "net.peer.name", + "user_agent.browser", + "user_agent.os", + "user_agent.device", + "user_agent.bot", + "http.route", + "http.method" + ] } } From 75cd4d03c866384f6220c7a6ab5ec7b9cfe8cbfb Mon Sep 17 00:00:00 2001 From: Sagnik Ghosh Date: Sun, 15 Nov 2026 02:40:46 +0530 Subject: [PATCH 2/2] Opt-in hashed client IP (client.address_hash) hashClientIp (runtime-node), hash_client_ip (Python adapter) and AUTTER_HASH_CLIENT_IP (SDKs and ingester), with a secret salt (AUTTER_CLIENT_IP_SALT, 16+ characters), add client.address_hash: HMAC-SHA256 of the normalised client address, 32 hex characters. The raw address is still dropped. - Source order: client.address, http.client_ip, then on server spans the socket peer (net.peer.ip, network.peer.address, net.sock.peer.addr). - Normalisation: first list entry, port, brackets and zone id removed, IPv6 compressed and lower-cased, IPv4-mapped IPv6 read as IPv4. - Off by default. On without a usable salt: nothing hashed, one warning, never an unsalted hash. The salt is never logged or exported. - The ingester hashes only spans and log records without a hash and never re-hashes one. - Shared vectors (test-vectors/redaction.json, clientIpHash) checked by runtime-core, runtime-node, the Python adapter and the ingester. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + adapters/python/autter_tracing.py | 73 ++++++- adapters/python/redact.py | 85 +++++++- adapters/python/test_client_ip_hash.py | 173 ++++++++++++++++ docs/ARCHITECTURE.md | 9 + docs/INTEGRATIONS.md | 9 + packages/otlp-ingester/README.md | 24 +++ .../otlp-ingester/src/client-ip-hash.test.ts | 148 ++++++++++++++ packages/otlp-ingester/src/config.ts | 10 + packages/otlp-ingester/src/identifying.ts | 123 +++++++++++- packages/otlp-ingester/src/index.ts | 9 + packages/otlp-ingester/src/server.ts | 3 +- packages/runtime-core/src/client-ip.ts | 110 +++++++++++ packages/runtime-core/src/index.ts | 13 ++ packages/runtime-core/test/client-ip.test.mjs | 49 +++++ packages/runtime-node/README.md | 32 +++ packages/runtime-node/src/client-ip.ts | 47 +++++ packages/runtime-node/src/index.ts | 1 + packages/runtime-node/src/redact-exporter.ts | 41 +++- packages/runtime-node/src/server.ts | 22 +++ .../runtime-node/test/client-ip-hash.test.mjs | 171 ++++++++++++++++ test-vectors/redaction.json | 187 ++++++++++++++++++ 22 files changed, 1320 insertions(+), 20 deletions(-) create mode 100644 adapters/python/test_client_ip_hash.py create mode 100644 packages/otlp-ingester/src/client-ip-hash.test.ts create mode 100644 packages/runtime-core/src/client-ip.ts create mode 100644 packages/runtime-core/test/client-ip.test.mjs create mode 100644 packages/runtime-node/src/client-ip.ts create mode 100644 packages/runtime-node/test/client-ip-hash.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 31b597d..9b22e66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Packages: `runtime-node`, `otlp-ingester`, Python adapter (`adapters/python`). - Kept: `service.instance.id` (the ingester needs it), `host.name` (container and pod names; on a laptop or VM it is the machine's name), `process.pid`, `process.runtime.*`, `os.type`, `host.arch`, and the coarse `user_agent.browser` / `.os` / `.device` / `.bot` fields. - Keep them with `keepIdentifyingAttributes: true` or a list of keys (`initAutterServer`, `registerAutter`), `keep_identifying_attributes=` in Python `init_tracing()`, or `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` (`true`, or a comma-separated list) for the SDKs and the ingester. Drop more keys, such as `host.name`, with `dropAttributes` / `drop_attributes=` / `AUTTER_DROP_ATTRIBUTES`. An attribute the SDK keeps is stored only if the ingester keeps it too: set `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` on both. - Spans already stored keep these attributes until their TTL expires. +- Opt-in hashed client IP (`runtime-node`, Python adapter, `otlp-ingester`), to count distinct clients without storing addresses: `hashClientIp: true | { salt }`, `init_tracing(hash_client_ip=True, client_ip_salt=...)`, or `AUTTER_HASH_CLIENT_IP=true` with `AUTTER_CLIENT_IP_SALT` (SDKs and ingester). The client's address (`client.address`, then `http.client_ip`, then on server spans the socket peer, which behind a proxy is the proxy) is normalised and exported as `client.address_hash`: HMAC-SHA256 under the secret salt, 32 hex characters, identical in the Node SDK, the Python adapter and the ingester for the same salt (shared vectors in `test-vectors/redaction.json`, `clientIpHash`). The raw address is still dropped. Off by default; on without a salt of at least 16 characters it hashes nothing and warns once, never an unsalted hash. The ingester keeps a hash it receives and does not hash it again. A hashed IP is still pseudonymous personal data in some readings of privacy law: treat the salt as a secret. ## [1.18.0] - 2026-11-14 diff --git a/adapters/python/autter_tracing.py b/adapters/python/autter_tracing.py index a60a324..7c6bb82 100644 --- a/adapters/python/autter_tracing.py +++ b/adapters/python/autter_tracing.py @@ -85,7 +85,14 @@ try: # imported as part of a package from .redact import ( + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, DROP_ATTRIBUTES_ENV, + HASH_CLIENT_IP_ENV, + client_ip_of, + hash_client_ip, + usable_client_ip_salt, KEEP_IDENTIFYING_ATTRIBUTES_ENV, drop_attributes_from_env, drop_identifying_attributes, @@ -96,7 +103,14 @@ ) except ImportError: # copied next to the app as a plain module from redact import ( + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, DROP_ATTRIBUTES_ENV, + HASH_CLIENT_IP_ENV, + client_ip_of, + hash_client_ip, + usable_client_ip_salt, KEEP_IDENTIFYING_ATTRIBUTES_ENV, drop_attributes_from_env, drop_identifying_attributes, @@ -543,11 +557,42 @@ def _without_identifying_resource(resource: Resource, dropped: frozenset) -> Res return result -def _redacted(span: ReadableSpan, redact: bool = True, dropped: Optional[frozenset] = None) -> ReadableSpan: +def _client_ip_hasher(enabled: Optional[bool], salt: Optional[str]) -> Optional[Any]: + """The hasher for ``hash_client_ip`` (default ``AUTTER_HASH_CLIENT_IP``) + with ``salt`` (default ``AUTTER_CLIENT_IP_SALT``), or ``None`` when off. + On without a usable salt it stays off with one warning: never an + unsalted hash. The salt is never logged.""" + if enabled is None: + enabled = _env_flag(HASH_CLIENT_IP_ENV) + if not enabled: + return None + usable = usable_client_ip_salt(salt if salt is not None else os.environ.get(CLIENT_IP_SALT_ENV)) + if usable is None: + LOGGER.warning( + "[autter-runtime] hash_client_ip is on but no salt of at least %d characters is set " + "(client_ip_salt= or %s): client IP addresses are dropped and not hashed.", + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SALT_ENV, + ) + return None + return lambda ip: hash_client_ip(usable, ip) + + +def _redacted( + span: ReadableSpan, redact: bool = True, dropped: Optional[frozenset] = None, hasher: Optional[Any] = None +) -> ReadableSpan: def clean(attributes: Any) -> Any: kept = drop_identifying_attributes(attributes, dropped) return redact_attributes(kept) if redact else kept + attributes = clean(span.attributes) + if hasher is not None and (span.attributes or {}).get(CLIENT_IP_HASH_ATTRIBUTE) is None: + # From the raw address, before it was dropped; a hash already on the + # span (set upstream) is left alone. + ip = client_ip_of(span.attributes, span.kind == SpanKind.SERVER) + if ip: + attributes = {**(attributes or {}), CLIENT_IP_HASH_ATTRIBUTE: hasher(ip)} + events = [Event(event.name, clean(event.attributes), event.timestamp) for event in span.events] links = [Link(link.context, clean(link.attributes)) for link in span.links] status = span.status @@ -556,7 +601,7 @@ def clean(attributes: Any) -> Any: return _CopiedSpan( span, name=redact_text(span.name) if redact else span.name, - attributes=clean(span.attributes), + attributes=attributes, events=events, links=links, status=status, @@ -583,6 +628,12 @@ class RedactingSpanExporter(SpanExporter): (``True`` or a list of keys; default ``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES``) keeps them; ``drop_attributes`` (default ``AUTTER_DROP_ATTRIBUTES``, a comma-separated list) drops more keys, such as ``host.name``. + + ``hash_client_ip`` (default ``AUTTER_HASH_CLIENT_IP``, off) adds + ``client.address_hash``, HMAC-SHA256 of the client's normalised address + under ``client_ip_salt`` (default ``AUTTER_CLIENT_IP_SALT``, at least 16 + characters; without one nothing is hashed). Same hash as + @autter/runtime-node and the ingester for the same salt. """ def __init__( @@ -592,13 +643,16 @@ def __init__( redact: bool = True, keep_identifying_attributes: Any = None, drop_attributes: Optional[Sequence[str]] = None, + hash_client_ip: Optional[bool] = None, + client_ip_salt: Optional[str] = None, ): self._exporter = exporter self._redact = redact self._dropped = _keys_to_drop(keep_identifying_attributes, drop_attributes) + self._hasher = _client_ip_hasher(hash_client_ip, client_ip_salt) def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - return self._exporter.export([_redacted(span, self._redact, self._dropped) for span in spans]) + return self._exporter.export([_redacted(span, self._redact, self._dropped, self._hasher) for span in spans]) def shutdown(self) -> None: self._exporter.shutdown() @@ -645,6 +699,8 @@ def init_tracing( metric_exporter: Optional[MetricExporter] = None, keep_identifying_attributes: Any = None, drop_attributes: Optional[Sequence[str]] = None, + hash_client_ip: Optional[bool] = None, + client_ip_salt: Optional[str] = None, ) -> TracerProvider: """A TracerProvider that head-samples as configured and always exports errors, registered as the global provider. @@ -666,6 +722,10 @@ def init_tracing( them too unless it is given the same setting. * ``drop_attributes`` (default ``AUTTER_DROP_ATTRIBUTES``): more keys to drop the same way, such as ``["host.name"]``. + * ``hash_client_ip`` (default ``AUTTER_HASH_CLIENT_IP``, off) and + ``client_ip_salt`` (default ``AUTTER_CLIENT_IP_SALT``): add + ``client.address_hash``, a keyed hash of the client's address (see + :class:`RedactingSpanExporter`). * ``count_requests`` (default: ``AUTTER_COUNT_REQUESTS=true``): count every HTTP request, sampled or not (:class:`CountRequests`), in a delta histogram sent through ``metric_exporter`` (default @@ -722,7 +782,12 @@ def init_tracing( exporter = OTLPSpanExporter() exporter = RedactingSpanExporter( - exporter, redact=redact, keep_identifying_attributes=keep_identifying_attributes, drop_attributes=drop_attributes + exporter, + redact=redact, + keep_identifying_attributes=keep_identifying_attributes, + drop_attributes=drop_attributes, + hash_client_ip=hash_client_ip, + client_ip_salt=client_ip_salt, ) provider = TracerProvider( sampler=RecordUnsampled(sampler if sampler is not None else head_sampler_from_env()), diff --git a/adapters/python/redact.py b/adapters/python/redact.py index 14c018e..c14d25f 100644 --- a/adapters/python/redact.py +++ b/adapters/python/redact.py @@ -23,13 +23,17 @@ agent, the host's id and addresses, the command line and its owner: :data:`IDENTIFYING_ATTRIBUTE_KEYS`) are dropped rather than scrubbed: :func:`drop_identifying_attributes`. ``autter_tracing.RedactingSpanExporter`` -applies it to every span, also with redaction off. +applies it to every span, also with redaction off. Opt in to a keyed hash of +the client's address instead (``client.address_hash``): :func:`hash_client_ip`. Stdlib only. """ from __future__ import annotations +import hashlib +import hmac +import ipaddress import re from collections.abc import Iterable, Mapping from typing import Any @@ -315,3 +319,82 @@ def drop_identifying_attributes(attributes: Any, dropped: frozenset[str] | None if not any(key in keys for key in attributes): return attributes return {key: value for key, value in attributes.items() if key not in keys} + + +#: Opt-in hashed client IP: the attribute, its length in hex characters, the +#: shortest salt accepted and the environment variables. Same values as +#: @autter/runtime-node and the ingester (``test-vectors/redaction.json``, +#: ``clientIpHash``). +CLIENT_IP_HASH_ATTRIBUTE = "client.address_hash" +CLIENT_IP_HASH_LENGTH = 32 +CLIENT_IP_SALT_MIN_LENGTH = 16 +HASH_CLIENT_IP_ENV = "AUTTER_HASH_CLIENT_IP" +CLIENT_IP_SALT_ENV = "AUTTER_CLIENT_IP_SALT" + +#: Where the client address is read from, first valid wins: ``client.address`` +#: and ``http.client_ip`` (from ``X-Forwarded-For`` behind a proxy), then the +#: socket peer, on server spans only (behind a proxy it is the proxy; on a +#: client span it is the server being called). +CLIENT_IP_SOURCE_KEYS = ("client.address", "http.client_ip", "net.peer.ip", "network.peer.address", "net.sock.peer.addr") +CLIENT_IP_CLIENT_KEYS = ("client.address", "http.client_ip") + +_IPV4_PORT = re.compile(r"^[\d.]+:\d{1,5}$") +_BRACKETED = re.compile(r"^\[([^\]]+)\](?::\d{1,5})?$") +_IPV4 = re.compile(r"^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$") +_IPV6_CHARS = re.compile(r"^[0-9a-f:.]+$") + + +def normalize_client_ip(raw: Any) -> str | None: + """The canonical text of an IP address, or ``None`` when ``raw`` is not + one: the first entry of a comma-separated list, port and brackets + removed, zone id removed, IPv6 lower-cased and compressed (RFC 5952), + IPv4-mapped IPv6 turned into IPv4.""" + if not isinstance(raw, str): + return None + value = raw.split(",")[0].strip() + if not value or len(value) > 100: + return None + bracketed = _BRACKETED.match(value) + if bracketed: + value = bracketed.group(1) + elif _IPV4_PORT.match(value): + value = value[: value.rindex(":")] + if "%" in value: + value = value[: value.index("%")] + value = value.lower() + if _IPV4.match(value): + return value + if ":" not in value or not _IPV6_CHARS.match(value): + return None + try: + address = ipaddress.IPv6Address(value) + except ValueError: + return None + if address.ipv4_mapped is not None: + return str(address.ipv4_mapped) + return address.compressed + + +def client_ip_of(attributes: Mapping[str, Any] | None, server_span: bool) -> str | None: + """The normalised client address of a span or log record: the first + :data:`CLIENT_IP_SOURCE_KEYS` entry holding a valid IP (the peer keys + only when ``server_span``).""" + if not attributes: + return None + for key in CLIENT_IP_SOURCE_KEYS if server_span else CLIENT_IP_CLIENT_KEYS: + ip = normalize_client_ip(attributes.get(key)) + if ip: + return ip + return None + + +def usable_client_ip_salt(salt: Any) -> str | None: + """``salt`` when it is a string of at least :data:`CLIENT_IP_SALT_MIN_LENGTH` characters.""" + return salt if isinstance(salt, str) and len(salt) >= CLIENT_IP_SALT_MIN_LENGTH else None + + +def hash_client_ip(salt: str, ip: str) -> str: + """HMAC-SHA256(salt, ip) as hex, first :data:`CLIENT_IP_HASH_LENGTH` + characters. ``ip`` is normalised (:func:`normalize_client_ip`). Never + call it without a secret salt: the IPv4 space can be hashed in full.""" + return hmac.new(salt.encode("utf-8"), ip.encode("utf-8"), hashlib.sha256).hexdigest()[:CLIENT_IP_HASH_LENGTH] diff --git a/adapters/python/test_client_ip_hash.py b/adapters/python/test_client_ip_hash.py new file mode 100644 index 0000000..d12c766 --- /dev/null +++ b/adapters/python/test_client_ip_hash.py @@ -0,0 +1,173 @@ +"""Opt-in hashed client IP in the Python adapter: ``redact.py``'s +normalisation, source order and HMAC (stdlib, shared vectors) and +``RedactingSpanExporter`` / ``init_tracing()`` (needs opentelemetry-sdk). + +Run: python3 -m unittest discover -s adapters/python -v +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from redact import ( # noqa: E402 + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SOURCE_KEYS, + IDENTIFYING_ATTRIBUTE_KEYS, + client_ip_of, + hash_client_ip, + normalize_client_ip, + usable_client_ip_salt, +) + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "test-vectors", "redaction.json") +with open(VECTORS_PATH, encoding="utf-8") as fh: + VECTORS = json.load(fh)["clientIpHash"] + +SALT = "project-secret-salt-7f3a9c2e41" +CLEAN_ENV = { + "AUTTER_HASH_CLIENT_IP": "", + "AUTTER_CLIENT_IP_SALT": "", + "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES": "", + "AUTTER_DROP_ATTRIBUTES": "", +} + + +class SharedVectors(unittest.TestCase): + def test_constants(self): + self.assertEqual(CLIENT_IP_HASH_ATTRIBUTE, VECTORS["attribute"]) + self.assertEqual(CLIENT_IP_HASH_LENGTH, VECTORS["length"]) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, IDENTIFYING_ATTRIBUTE_KEYS) + for key in CLIENT_IP_SOURCE_KEYS: + self.assertIn(key, IDENTIFYING_ATTRIBUTE_KEYS) + + def test_normalize(self): + for case in VECTORS["normalize"]: + with self.subTest(case["input"]): + self.assertEqual(normalize_client_ip(case["input"]), case["expect"]) + self.assertIsNone(normalize_client_ip(None)) + self.assertIsNone(normalize_client_ip(42)) + + def test_source_order(self): + for case in VECTORS["source"]: + with self.subTest(case): + self.assertEqual(client_ip_of(case["attributes"], case["server"]), case["expect"]) + + def test_hash(self): + for case in VECTORS["hash"]: + with self.subTest(case["input"]): + self.assertEqual(hash_client_ip(VECTORS["salt"], normalize_client_ip(case["input"])), case["expect"]) + + def test_short_salts_are_refused(self): + self.assertIsNone(usable_client_ip_salt("x" * (CLIENT_IP_SALT_MIN_LENGTH - 1))) + self.assertEqual(usable_client_ip_salt("x" * CLIENT_IP_SALT_MIN_LENGTH), "x" * CLIENT_IP_SALT_MIN_LENGTH) + self.assertIsNone(usable_client_ip_salt(None)) + + +try: + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from opentelemetry.sdk.trace.sampling import ALWAYS_ON + from opentelemetry.trace import SpanKind + + from autter_tracing import RedactingSpanExporter, init_tracing + + HAVE_OTEL = True +except ImportError: # pragma: no cover - depends on the environment + HAVE_OTEL = False + + +@unittest.skipUnless(HAVE_OTEL, "opentelemetry-sdk is not installed") +class HashingExporter(unittest.TestCase): + def export(self, env=None, **options): + exporter = InMemorySpanExporter() + with mock.patch.dict(os.environ, {**CLEAN_ENV, **(env or {})}): + wrapped = RedactingSpanExporter(exporter, **options) + provider = TracerProvider(sampler=ALWAYS_ON, resource=Resource.create({"service.name": "shop"})) + provider.add_span_processor(SimpleSpanProcessor(wrapped)) + tracer = provider.get_tracer("test") + # Django/Flask server span: X-Forwarded-For in http.client_ip, the proxy as peer. + with tracer.start_as_current_span( + "GET /", kind=SpanKind.SERVER, attributes={"http.client_ip": "203.0.113.9", "net.peer.ip": "10.0.0.2"} + ): + pass + # An outgoing call: the peer is the server called, never hashed. + with tracer.start_as_current_span("GET api", kind=SpanKind.CLIENT, attributes={"net.peer.ip": "10.0.0.9"}): + pass + with tracer.start_as_current_span("manual", attributes={"client.address": "::ffff:198.51.100.7"}): + pass + with tracer.start_as_current_span( + "prehashed", attributes={"client.address": "198.51.100.8", CLIENT_IP_HASH_ATTRIBUTE: "set-upstream"} + ): + pass + provider.shutdown() + return {span.name: span for span in exporter.get_finished_spans()} + + def assert_no_raw_address(self, spans): + for span in spans.values(): + for key in IDENTIFYING_ATTRIBUTE_KEYS: + self.assertNotIn(key, span.attributes) + + def test_off_by_default(self): + spans = self.export() + self.assert_no_raw_address(spans) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, spans["GET /"].attributes) + self.assertEqual(spans["prehashed"].attributes[CLIENT_IP_HASH_ATTRIBUTE], "set-upstream") + + def test_hash_with_salt_argument(self): + spans = self.export(hash_client_ip=True, client_ip_salt=SALT) + self.assert_no_raw_address(spans) + self.assertEqual(spans["GET /"].attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "203.0.113.9")) + self.assertEqual(spans["manual"].attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "198.51.100.7")) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, spans["GET api"].attributes) + self.assertEqual(spans["prehashed"].attributes[CLIENT_IP_HASH_ATTRIBUTE], "set-upstream") + for span in spans.values(): + self.assertNotIn(SALT, json.dumps(dict(span.attributes))) + + def test_environment_and_redact_false(self): + spans = self.export(env={"AUTTER_HASH_CLIENT_IP": "true", "AUTTER_CLIENT_IP_SALT": SALT}, redact=False) + self.assert_no_raw_address(spans) + self.assertEqual(spans["GET /"].attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "203.0.113.9")) + + def test_without_a_usable_salt_nothing_is_hashed_and_it_warns(self): + with self.assertLogs("autter.runtime", level="WARNING") as logs: + spans = self.export(hash_client_ip=True, client_ip_salt="short-salt") + self.assert_no_raw_address(spans) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, spans["GET /"].attributes) + self.assertEqual(len(logs.output), 1) + self.assertIn("no salt", logs.output[0]) + self.assertNotIn("short-salt", logs.output[0]) + + def test_init_tracing_wires_the_options(self): + exporter = InMemorySpanExporter() + with mock.patch.dict(os.environ, CLEAN_ENV): + provider = init_tracing( + sampler=ALWAYS_ON, + exporter=exporter, + resource=Resource.create({"service.name": "shop"}), + set_global=False, + hash_client_ip=True, + client_ip_salt=SALT, + ) + with provider.get_tracer("test").start_as_current_span( + "GET /", kind=SpanKind.SERVER, attributes={"client.address": "203.0.113.9"} + ): + pass + provider.force_flush() + [span] = exporter.get_finished_spans() + self.assertEqual(span.attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "203.0.113.9")) + self.assertNotIn("client.address", span.attributes) + provider.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e82bbaa..3623e9f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -935,6 +935,15 @@ list, kept in parity by `test-vectors/redaction.json` (`identifying`). the SDK and the ingester; `dropAttributes` / `AUTTER_DROP_ATTRIBUTES` drops more keys. +Opt-in, `hashClientIp` / `AUTTER_HASH_CLIENT_IP` with a secret salt +(`AUTTER_CLIENT_IP_SALT`, 16+ characters) adds `client.address_hash`: +HMAC-SHA256 of the normalised client address, 32 hex characters, the same in +the Node SDK, the Python adapter and the ingester (vectors: +`test-vectors/redaction.json`, `clientIpHash`). The ingester hashes only +records without a hash. No salt, no hash: there is no unsalted fallback. +A hashed IP is still pseudonymous personal data in some readings of privacy +law, which is why it is off by default. + ## Sink webhook (v1) When `AUTTER_SINK_URL` is set, each ingest batch POSTs: diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 8e09db4..6989b90 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -309,6 +309,15 @@ exceptions never leave the process. `init_tracing()` keeps them: the ingester drops them too unless it is given the same setting. `drop_attributes=["host.name"]` (or `AUTTER_DROP_ATTRIBUTES`) drops more keys. +- Opt-in: `init_tracing(hash_client_ip=True, client_ip_salt=...)` (or + `AUTTER_HASH_CLIENT_IP=true` and `AUTTER_CLIENT_IP_SALT`) adds + `client.address_hash`, an HMAC-SHA256 of the client's address under your + secret salt (16+ characters), the same value `@autter/runtime-node` and + the ingester compute from the same salt. The address is `client.address`, + then `http.client_ip` (from `X-Forwarded-For`), then on server spans + `net.peer.ip`. Without a salt nothing is hashed and a warning is logged. + A hashed IP is still pseudonymous personal data in some readings of + privacy law; keep the salt secret. Every request is recorded as if sampled (instrumentations collect their attributes), but only sampled and failing traces are exported. To wire the diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 040aa57..0c5b887 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -174,6 +174,8 @@ Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILIT | `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale | | `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` | unset (drop them all) | Identifying attributes (client IP and port, raw user agent, `host.id` / `host.ip` / `host.mac`, command line and owner) are dropped on arrival; `true` keeps them all, a comma-separated list keeps those keys. An SDK that keeps one needs this set too, or the attribute is not stored. See [Identifying attributes](#identifying-attributes) | | `AUTTER_DROP_ATTRIBUTES` | unset | Comma-separated attribute keys dropped on arrival as well, such as `host.name` | +| `AUTTER_HASH_CLIENT_IP` | off | `true` adds `client.address_hash`, a keyed hash of the client's address, to spans and log records that carry no hash yet (plain OpenTelemetry senders). Needs `AUTTER_CLIENT_IP_SALT`. See [Hashed client IP](#hashed-client-ip-opt-in) | +| `AUTTER_CLIENT_IP_SALT` | unset | Secret salt for `AUTTER_HASH_CLIENT_IP`, at least 16 characters. Never logged. Without it nothing is hashed (a warning at startup) | ## Identifying attributes @@ -203,6 +205,28 @@ environment variable). Each side drops what it was not told to keep, so a key kept by only one of them is not stored. The ingester logs what it keeps at startup. Rows already stored keep these attributes until their TTL. +### Hashed client IP (opt-in) + +`AUTTER_HASH_CLIENT_IP=true` with `AUTTER_CLIENT_IP_SALT` adds +`client.address_hash` to every span and log record that names a client +address and has no hash yet: HMAC-SHA256 of the normalised address under the +salt, 32 hex characters. The address is the first valid one of +`client.address`, `http.client_ip`, then on server spans the socket peer +(`net.peer.ip`, `network.peer.address`, `net.sock.peer.addr`). The raw +address is dropped as before. A hash sent by an SDK is stored as it is and +never hashed again. + +The SDKs (`hashClientIp` in `@autter/runtime-node`, `hash_client_ip` in the +Python adapter) compute the same hash from the same salt, so use one salt +per project and hash in the SDK where you can. The ingester has one salt for +every project it serves: on an ingester shared by several projects their +hashes are comparable to whoever can read all of them, so give each project +its own salt in its SDKs instead. Without a salt of at least 16 characters +nothing is hashed (never an unsalted hash, which could be reversed by +hashing every IPv4 address) and the ingester warns at startup. A hashed IP +is still pseudonymous personal data in some readings of privacy law: it is +off by default, and the salt should be kept and rotated like any secret. + ## Rate limits and key lookups (1.12.0) In the order a request meets them: diff --git a/packages/otlp-ingester/src/client-ip-hash.test.ts b/packages/otlp-ingester/src/client-ip-hash.test.ts new file mode 100644 index 0000000..414671c --- /dev/null +++ b/packages/otlp-ingester/src/client-ip-hash.test.ts @@ -0,0 +1,148 @@ +// Opt-in hashed client IP at the ingester (AUTTER_HASH_CLIENT_IP with +// AUTTER_CLIENT_IP_SALT): plain OpenTelemetry senders get the same +// `client.address_hash` the SDKs compute, a hash already sent is kept, and +// the raw address is dropped either way. Shared vectors: +// test-vectors/redaction.json (`clientIpHash`). +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { test } from "node:test"; +import type { IngesterConfig } from "./config.js"; +import { + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SOURCE_KEYS, + IDENTIFYING_ATTRIBUTE_KEYS, + IdentifyingAttributeFilter, + clientIpHasher, + clientIpOfList, + hashClientIp, + identifyingKeysToDrop, + normalizeClientIp, +} from "./identifying.js"; +import { createIngesterApp } from "./server.js"; + +const ALL = JSON.parse(readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8")); +const vectors = ALL.clientIpHash as { + attribute: string; salt: string; length: number; + normalize: Array<{ input: string; expect: string | null }>; + hash: Array<{ input: string; expect: string }>; + source: Array<{ attributes: Record; server: boolean; expect: string | null }>; +}; +const SALT = "project-secret-salt-7f3a9c2e41"; +const kv = (attributes: Record) => + Object.entries(attributes).map(([key, value]) => ({ key, value: { stringValue: value } })); +const asMap = (list: unknown) => + Object.fromEntries((list as Array<{ key: string; value: { stringValue?: string } }>).map((e) => [e.key, e.value.stringValue])); + +test("the shared vectors: constants, normalisation, source order and hash", () => { + assert.equal(CLIENT_IP_HASH_ATTRIBUTE, vectors.attribute); + assert.equal(CLIENT_IP_HASH_LENGTH, vectors.length); + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes(CLIENT_IP_HASH_ATTRIBUTE)); + for (const key of CLIENT_IP_SOURCE_KEYS) assert.ok(IDENTIFYING_ATTRIBUTE_KEYS.includes(key), key); + for (const { input, expect } of vectors.normalize) assert.equal(normalizeClientIp(input), expect, input); + for (const { attributes, server, expect } of vectors.source) assert.equal(clientIpOfList(kv(attributes), server), expect); + for (const { input, expect } of vectors.hash) assert.equal(hashClientIp(vectors.salt, normalizeClientIp(input)!), expect, input); +}); + +test("no hasher without a salt of 16 characters, or when off", () => { + assert.equal(clientIpHasher(true, undefined), null); + assert.equal(clientIpHasher(true, "short"), null); + assert.equal(clientIpHasher(false, SALT), null); + assert.equal(clientIpHasher(true, SALT)!("203.0.113.9"), hashClientIp(SALT, "203.0.113.9")); +}); + +test("traces and logs: hash added from the raw address, kept when already sent, peer only on server spans", () => { + const filter = new IdentifyingAttributeFilter(identifyingKeysToDrop(), clientIpHasher(true, SALT)); + const traces = filter.traces({ + resourceSpans: [{ scopeSpans: [{ spans: [ + { name: "server", kind: "SPAN_KIND_SERVER", attributes: kv({ "net.peer.ip": "10.0.0.2", "http.client_ip": "203.0.113.9", "http.route": "/x" }) }, + { name: "server-peer", kind: 2, attributes: kv({ "network.peer.address": "[2001:DB8::1]:443" }) }, + { name: "client", kind: 3, attributes: kv({ "net.peer.ip": "10.0.0.9" }) }, + { name: "hashed", kind: 2, attributes: kv({ "client.address": "198.51.100.8", [CLIENT_IP_HASH_ATTRIBUTE]: "abc123" }) }, + ] }] }], + }); + const spans = traces.resourceSpans[0]!.scopeSpans[0]!.spans.map((span) => asMap(span.attributes)); + assert.deepEqual(spans[0], { "http.route": "/x", [CLIENT_IP_HASH_ATTRIBUTE]: hashClientIp(SALT, "203.0.113.9") }); + assert.deepEqual(spans[1], { [CLIENT_IP_HASH_ATTRIBUTE]: hashClientIp(SALT, "2001:db8::1") }); + assert.deepEqual(spans[2], {}); + assert.deepEqual(spans[3], { [CLIENT_IP_HASH_ATTRIBUTE]: "abc123" }, "an existing hash is never hashed again"); + + const logs = filter.logs({ resourceLogs: [{ scopeLogs: [{ logRecords: [ + { attributes: kv({ "client.address": "::ffff:203.0.113.9", message: "m" }) }, + { attributes: kv({ "net.peer.ip": "203.0.113.9" }) }, + ] }] }] }); + const records = logs.resourceLogs[0]!.scopeLogs[0]!.logRecords.map((record) => asMap(record.attributes)); + assert.deepEqual(records[0], { message: "m", [CLIENT_IP_HASH_ATTRIBUTE]: hashClientIp(SALT, "203.0.113.9") }); + assert.deepEqual(records[1], {}); +}); + +// Through the ingester, against a fake ClickHouse: a real Python Flask +// export (protobuf; its server span's address is net.peer.ip 127.0.0.1). + +const listen = async (server: Server) => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}; +const close = (server: Server) => + new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }); + +async function storedSpans(extra: Partial): Promise>> { + const rows: Array> = []; + const ch = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + const query = new URL(req.url!, "http://localhost").searchParams.get("query") ?? ""; + if (query.includes("INSERT INTO") && query.includes("runtime_spans")) + rows.push(...body.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))); + res.end(); + }); + }); + const chUrl = await listen(ch); + const app = createIngesterApp({ + port: 0, clickhouseUrl: chUrl, clickhouseUser: "default", clickhousePassword: "", clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "server", orgId: "org", repositoryId: "repo" }], + keyValidatorUrl: null, keyValidatorToken: null, sinkUrl: null, sinkToken: null, sinkMaxAttempts: 1, + sinkMaxBufferedBatches: 10, sinkMaxBufferedMb: 2, maxBodyBytes: 1024 * 1024, rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, occurrenceTtlDays: 14, spanTtlDays: 7, metricsTtlDays: 90, llmCallTtlDays: 90, + logTtlDays: 14, ...extra, + }).app.listen(); + const url = await new Promise((resolve) => + app.once("listening", () => resolve(`http://127.0.0.1:${(app.address() as AddressInfo).port}`)), + ); + try { + const response = await fetch(`${url}/v1/traces`, { + method: "POST", + headers: { "content-type": "application/x-protobuf", authorization: "Bearer server" }, + body: readFileSync(new URL("../test-fixtures/python/flask-unhandled.otlp.pb", import.meta.url)), + }); + assert.equal(response.status, 200); + } finally { + await close(app); + await close(ch); + } + return rows.map((row) => JSON.parse(String(row.attributes))); +} + +test("a plain OpenTelemetry sender gets client.address_hash when the ingester hashes, and none otherwise", async () => { + const hashed = await storedSpans({ hashClientIp: true, clientIpSalt: SALT }); + const server = hashed.find((attributes) => attributes["http.route"]); + assert.ok(server, "server span stored"); + assert.equal(server[CLIENT_IP_HASH_ATTRIBUTE], hashClientIp(SALT, "127.0.0.1")); + assert.equal(server["net.peer.ip"], undefined); + assert.ok(!JSON.stringify(hashed).includes(SALT), "salt stored"); + + for (const config of [{}, { hashClientIp: true }, { hashClientIp: true, clientIpSalt: "short" }]) { + const plain = await storedSpans(config); + assert.ok(plain.length); + for (const attributes of plain) { + assert.equal(attributes[CLIENT_IP_HASH_ATTRIBUTE], undefined, JSON.stringify(config)); + assert.equal(attributes["net.peer.ip"], undefined); + } + } +}); diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index fbceb07..71b6078 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -210,6 +210,14 @@ export interface IngesterConfig { keepIdentifyingAttributes?: boolean | string[]; /** More attribute keys to drop on arrival, such as `host.name` (AUTTER_DROP_ATTRIBUTES, comma-separated). */ dropAttributes?: string[]; + /** + * Opt-in (AUTTER_HASH_CLIENT_IP=true): add `client.address_hash`, a keyed + * hash of the client's address, to spans and log records that carry no + * hash yet (identifying.ts). Needs clientIpSalt. + */ + hashClientIp?: boolean; + /** The secret salt (AUTTER_CLIENT_IP_SALT, at least 16 characters). Never logged. */ + clientIpSalt?: string | null; /** * In-memory budget (MiB, decoded size) for parsed source maps used to * symbolicate browser frames at ingest (SOURCE_MAP_CACHE_MB, default 64). @@ -422,6 +430,8 @@ export function loadConfig(): IngesterConfig { redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"), keepIdentifyingAttributes: keepIdentifyingEnv(process.env.AUTTER_KEEP_IDENTIFYING_ATTRIBUTES), dropAttributes: dropAttributesEnv(process.env.AUTTER_DROP_ATTRIBUTES), + hashClientIp: /^(?:1|true|yes|on)$/i.test((process.env.AUTTER_HASH_CLIENT_IP ?? "").trim()), + clientIpSalt: process.env.AUTTER_CLIENT_IP_SALT || null, sourceMapCacheMb: intEnv("SOURCE_MAP_CACHE_MB", 64), loopCollapseAfter: loopCollapseAfterEnv(process.env.LOOP_COLLAPSE_AFTER), archive: archiveConfigFromEnv(process.env), diff --git a/packages/otlp-ingester/src/identifying.ts b/packages/otlp-ingester/src/identifying.ts index 5e5d24f..f0e3d23 100644 --- a/packages/otlp-ingester/src/identifying.ts +++ b/packages/otlp-ingester/src/identifying.ts @@ -20,7 +20,16 @@ * or a comma-separated list of keys) and drop more keys with * AUTTER_DROP_ATTRIBUTES (a comma-separated list, `host.name` say). An SDK * that keeps an attribute gets it stored only when the ingester keeps it too. + * + * Opt-in hashed client IP (AUTTER_HASH_CLIENT_IP=true with + * AUTTER_CLIENT_IP_SALT): a span or log record that names a client address + * and carries no `client.address_hash` yet gets one, HMAC-SHA256 of the + * normalised address under the salt (32 hex characters), the same value the + * SDKs compute from the same salt (test-vectors/redaction.json, + * `clientIpHash`). A hash already sent is kept as it is, never hashed again. + * The raw address is dropped either way. */ +import { createHmac } from "node:crypto"; export const IDENTIFYING_ATTRIBUTE_KEYS: readonly string[] = Object.freeze([ "net.peer.ip", "net.peer.port", @@ -107,7 +116,23 @@ function eachIn(obj: unknown, field: string, fn: (item: unknown) => unknown): un } export class IdentifyingAttributeFilter { - constructor(readonly dropped: ReadonlySet = identifyingKeysToDrop()) {} + constructor( + readonly dropped: ReadonlySet = identifyingKeysToDrop(), + private readonly hasher: ClientIpHasher | null = null, + ) {} + + /** The attributes of a span or log record: the client IP hash added (from the raw list), then the drop. */ + private recordAttributes = (obj: unknown, serverSpan: boolean): unknown => { + const hasher = this.hasher; + if (!hasher || !isObject(obj) || hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) return this.attributesOf(obj); + const ip = clientIpOfList(obj.attributes, serverSpan); + const filtered = this.attributesOf(obj); + if (!ip || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered; + return { + ...filtered, + attributes: [...filtered.attributes, { key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }], + }; + }; /** An OTLP key/value list without the dropped keys; the same array when none is present. */ list = (attributes: unknown): unknown => { @@ -124,9 +149,13 @@ export class IdentifyingAttributeFilter { /** A traces request without identifying attributes (resource, spans, events, links). */ traces(request: T): T { - if (!this.dropped.size) return request; + if (!this.dropped.size && !this.hasher) return request; const span = (item: unknown) => - eachIn(eachIn(this.attributesOf(item), "events", this.attributesOf), "links", this.attributesOf); + eachIn( + eachIn(this.recordAttributes(item, isObject(item) && isServerKind(item.kind)), "events", this.attributesOf), + "links", + this.attributesOf, + ); return eachIn(request, "resourceSpans", (rs) => eachIn(this.resourceOf(rs), "scopeSpans", (ss) => eachIn(ss, "spans", span)), ) as T; @@ -134,10 +163,10 @@ export class IdentifyingAttributeFilter { /** A logs request without identifying attributes (resource, record attributes, the keys of a structured body). */ logs(request: T): T { - if (!this.dropped.size) return request; + if (!this.dropped.size && !this.hasher) return request; const body = (value: unknown) => withField(value, "kvlistValue", (kv) => withField(kv, "values", this.list)); - const record = (item: unknown) => withField(this.attributesOf(item), "body", body); + const record = (item: unknown) => withField(this.recordAttributes(item, false), "body", body); return eachIn(request, "resourceLogs", (rl) => eachIn(this.resourceOf(rl), "scopeLogs", (sl) => eachIn(sl, "logRecords", record)), ) as T; @@ -149,3 +178,87 @@ export class IdentifyingAttributeFilter { return eachIn(request, "resourceMetrics", this.resourceOf) as T; } } + +// --------------------------------------------------------------------------- +// Hashed client IP + +export const CLIENT_IP_HASH_ATTRIBUTE = "client.address_hash"; +export const CLIENT_IP_HASH_LENGTH = 32; +export const CLIENT_IP_SALT_MIN_LENGTH = 16; +/** First valid wins; the peer keys only on server spans (on a client span the peer is the server called). */ +export const CLIENT_IP_SOURCE_KEYS: readonly string[] = ["client.address", "http.client_ip", "net.peer.ip", "network.peer.address", "net.sock.peer.addr"]; +const CLIENT_IP_CLIENT_KEYS: readonly string[] = ["client.address", "http.client_ip"]; + +const IPV4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; +const MAPPED_V4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/; + +/** Canonical IP text or null (same rules as runtime-core normalizeClientIp and redact.py). */ +export function normalizeClientIp(raw: unknown): string | null { + if (typeof raw !== "string") return null; + let value = raw.split(",")[0]!.trim(); + if (!value || value.length > 100) return null; + const bracketed = /^\[([^\]]+)\](?::\d{1,5})?$/.exec(value); + if (bracketed) value = bracketed[1]!; + else if (/^[\d.]+:\d{1,5}$/.test(value)) value = value.slice(0, value.lastIndexOf(":")); + const zone = value.indexOf("%"); + if (zone >= 0) value = value.slice(0, zone); + value = value.toLowerCase(); + if (IPV4.test(value)) return value; + if (!value.includes(":") || !/^[0-9a-f:.]+$/.test(value)) return null; + let canonical: string; + try { + canonical = new URL(`http://[${value}]/`).hostname.slice(1, -1); + } catch { + return null; + } + const mapped = MAPPED_V4.exec(canonical); + if (mapped) { + const high = Number.parseInt(mapped[1]!, 16); + const low = Number.parseInt(mapped[2]!, 16); + return `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`; + } + return canonical; +} + +/** HMAC-SHA256(salt, ip), hex, first CLIENT_IP_HASH_LENGTH characters. */ +export function hashClientIp(salt: string, ip: string): string { + return createHmac("sha256", salt).update(ip).digest("hex").slice(0, CLIENT_IP_HASH_LENGTH); +} + +export type ClientIpHasher = (ip: string) => string; + +/** + * The hasher for AUTTER_HASH_CLIENT_IP / AUTTER_CLIENT_IP_SALT, or null: + * off, or on without a salt of CLIENT_IP_SALT_MIN_LENGTH characters (never + * an unsalted hash; index.ts warns at startup). The salt stays in the + * closure. + */ +export function clientIpHasher(enabled: boolean | undefined, salt: string | null | undefined): ClientIpHasher | null { + if (!enabled || typeof salt !== "string" || salt.length < CLIENT_IP_SALT_MIN_LENGTH) return null; + return (ip) => hashClientIp(salt, ip); +} + +/** The string value of `key` in an OTLP key/value list. */ +function stringAttribute(list: unknown, key: string): string | undefined { + if (!Array.isArray(list)) return undefined; + for (const entry of list) { + if (isObject(entry) && entry.key === key && isObject(entry.value) && typeof entry.value.stringValue === "string") + return entry.value.stringValue; + } + return undefined; +} + +const hasKey = (list: unknown, key: string) => + Array.isArray(list) && list.some((entry) => isObject(entry) && entry.key === key); + +/** The normalised client address in an OTLP attribute list (peer keys only on server spans). */ +export function clientIpOfList(list: unknown, serverSpan: boolean): string | null { + for (const key of serverSpan ? CLIENT_IP_SOURCE_KEYS : CLIENT_IP_CLIENT_KEYS) { + const ip = normalizeClientIp(stringAttribute(list, key)); + if (ip) return ip; + } + return null; +} + +/** OTLP SERVER span kind, as a number or the enum name. */ +export const isServerKind = (kind: unknown) => kind === 2 || kind === "SPAN_KIND_SERVER"; diff --git a/packages/otlp-ingester/src/index.ts b/packages/otlp-ingester/src/index.ts index 14052d5..881aeb1 100644 --- a/packages/otlp-ingester/src/index.ts +++ b/packages/otlp-ingester/src/index.ts @@ -1,3 +1,4 @@ +import { CLIENT_IP_SALT_MIN_LENGTH, clientIpHasher } from "./identifying.js"; import { limitsWarning, loadConfig } from "./config.js"; import { createIngesterApp } from "./server.js"; import { ingesterVersion } from "./sdk-versions.js"; @@ -16,6 +17,14 @@ if (config.keepIdentifyingAttributes || config.dropAttributes?.length) { `Identifying attributes: kept ${kept} (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES); also dropped: ${config.dropAttributes?.join(", ") || "none"} (AUTTER_DROP_ATTRIBUTES)`, ); } +if (config.hashClientIp) { + // Never the salt itself: only whether a usable one is set. + if (clientIpHasher(config.hashClientIp, config.clientIpSalt)) console.log("Client IP hashing: on (client.address_hash)"); + else + console.warn( + `WARNING: AUTTER_HASH_CLIENT_IP is on but AUTTER_CLIENT_IP_SALT is not set or shorter than ${CLIENT_IP_SALT_MIN_LENGTH} characters: client IP addresses are dropped and not hashed.`, + ); +} const sharingWarning = limitsWarning(config); if (sharingWarning) console.warn(`WARNING: ${sharingWarning}`); diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 599c3c5..e15a9be 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -84,7 +84,7 @@ import type { SinkDeadLetterRow } from "./sink-dead-letters.js"; import { LoopCollapser } from "./loop-collapse.js"; import { applySuppression } from "./suppression.js"; import { configureRedaction, scrubOccurrence } from "./redact.js"; -import { IdentifyingAttributeFilter, identifyingKeysToDrop } from "./identifying.js"; +import { IdentifyingAttributeFilter, clientIpHasher, identifyingKeysToDrop } from "./identifying.js"; import { configureLlmPricing } from "./llm-pricing.js"; import { buildCompatReport, @@ -224,6 +224,7 @@ export function createIngesterApp(config: IngesterConfig, deps: IngesterDeps = { // are dropped before anything below reads, stores or forwards a request. const identifying = new IdentifyingAttributeFilter( identifyingKeysToDrop(config.keepIdentifyingAttributes, config.dropAttributes), + clientIpHasher(config.hashClientIp, config.clientIpSalt), ); // LLM prices: the built-in table, with a self-hosted override file // (AUTTER_LLM_PRICES_FILE, validated by loadConfig) merged over it. diff --git a/packages/runtime-core/src/client-ip.ts b/packages/runtime-core/src/client-ip.ts new file mode 100644 index 0000000..b80acbf --- /dev/null +++ b/packages/runtime-core/src/client-ip.ts @@ -0,0 +1,110 @@ +/** + * Hashed client IP (opt-in): the client's address is dropped with the other + * identifying attributes (identifying.ts), and with `hashClientIp` one + * attribute, `client.address_hash`, carries a keyed hash of it so distinct + * clients can still be counted. + * + * The hash is HMAC-SHA256(salt, normalized IP), hex, first 32 characters + * (128 bits). Never a plain hash: the IPv4 space is small enough to hash in + * full, so without a secret salt any hash could be reversed. The salt is a + * secret of the project: it is never logged, exported or written to an + * attribute. Without one (or with one shorter than CLIENT_IP_SALT_MIN_LENGTH) + * nothing is hashed. The same function, normalisation and source order run + * in @autter/runtime-node, adapters/python and the ingester; parity is + * checked by test-vectors/redaction.json (`clientIpHash`). + * + * This module holds the parts that need no crypto (runtime-core runs on + * edge runtimes without node:crypto): which attribute to hash and how to + * normalise it. + */ + +/** The attribute carrying the hash. */ +export const CLIENT_IP_HASH_ATTRIBUTE = "client.address_hash"; +/** Hex characters kept from the HMAC-SHA256 digest (128 bits). */ +export const CLIENT_IP_HASH_LENGTH = 32; +/** Shorter salts are refused (nothing is hashed). */ +export const CLIENT_IP_SALT_MIN_LENGTH = 16; +/** `true` turns hashing on (the SDKs and the ingester). */ +export const HASH_CLIENT_IP_ENV = "AUTTER_HASH_CLIENT_IP"; +/** The secret salt (the SDKs and the ingester). */ +export const CLIENT_IP_SALT_ENV = "AUTTER_CLIENT_IP_SALT"; + +/** + * Where the client address is read from, first present and valid wins: + * `client.address` (stable semconv; behind a proxy the instrumentation fills + * it from `Forwarded` / `X-Forwarded-For`), `http.client_ip` (old semconv, + * from `X-Forwarded-For`), then the socket peer (`net.peer.ip`, + * `network.peer.address`, `net.sock.peer.addr`), which behind a proxy is + * the proxy. The peer keys are read on server spans only: on a client span + * the peer is the server being called. + */ +export const CLIENT_IP_SOURCE_KEYS: readonly string[] = Object.freeze([ + "client.address", + "http.client_ip", + "net.peer.ip", + "network.peer.address", + "net.sock.peer.addr", +]); +/** The source keys read on any span or log record (not only server spans). */ +export const CLIENT_IP_CLIENT_KEYS: readonly string[] = Object.freeze(["client.address", "http.client_ip"]); + +const IPV4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; +const MAPPED_V4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/; + +/** + * The canonical text of an IP address, or null when `raw` is not one: the + * first entry of a comma-separated list (`X-Forwarded-For`), port and + * brackets removed (`1.2.3.4:5678`, `[2001:db8::1]:443`), zone id removed + * (`fe80::1%eth0`), IPv6 lower-cased and compressed (RFC 5952), and + * IPv4-mapped IPv6 (`::ffff:1.2.3.4`) turned into IPv4. + */ +export function normalizeClientIp(raw: unknown): string | null { + if (typeof raw !== "string") return null; + let value = raw.split(",")[0]!.trim(); + if (!value || value.length > 100) return null; + const bracketed = /^\[([^\]]+)\](?::\d{1,5})?$/.exec(value); + if (bracketed) value = bracketed[1]!; + else if (/^[\d.]+:\d{1,5}$/.test(value)) value = value.slice(0, value.lastIndexOf(":")); + const zone = value.indexOf("%"); + if (zone >= 0) value = value.slice(0, zone); + value = value.toLowerCase(); + if (IPV4.test(value)) return value; + if (!value.includes(":") || !/^[0-9a-f:.]+$/.test(value)) return null; + let canonical: string; + try { + const host = new URL(`http://[${value}]/`).hostname; + canonical = host.slice(1, -1); + } catch { + return null; + } + const mapped = MAPPED_V4.exec(canonical); + if (mapped) { + const high = Number.parseInt(mapped[1]!, 16); + const low = Number.parseInt(mapped[2]!, 16); + return `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`; + } + return canonical; +} + +/** + * The normalised client address of a span or log record, or null: the + * first CLIENT_IP_SOURCE_KEYS entry holding a valid IP, the peer keys only + * when `serverSpan`. + */ +export function clientIpOf(attributes: Record, serverSpan: boolean): string | null { + for (const key of serverSpan ? CLIENT_IP_SOURCE_KEYS : CLIENT_IP_CLIENT_KEYS) { + const ip = normalizeClientIp(attributes[key]); + if (ip) return ip; + } + return null; +} + +/** A usable salt: a string of at least CLIENT_IP_SALT_MIN_LENGTH characters, else null. */ +export function usableClientIpSalt(salt: unknown): string | null { + return typeof salt === "string" && salt.length >= CLIENT_IP_SALT_MIN_LENGTH ? salt : null; +} + +/** `true` / `1` / `yes` / `on` (AUTTER_HASH_CLIENT_IP). */ +export function envFlag(raw: string | undefined | null): boolean { + return typeof raw === "string" && /^(?:1|true|yes|on)$/i.test(raw.trim()); +} diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index 743ca8a..36a2132 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -12,6 +12,19 @@ export { type RedactOptions, type Redactor as AttributeRedactor, } from "./redact.js"; +export { + CLIENT_IP_CLIENT_KEYS, + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SOURCE_KEYS, + HASH_CLIENT_IP_ENV, + clientIpOf, + envFlag, + normalizeClientIp, + usableClientIpSalt, +} from "./client-ip.js"; export { DROP_ATTRIBUTES_ENV, IDENTIFYING_ATTRIBUTE_KEYS, diff --git a/packages/runtime-core/test/client-ip.test.mjs b/packages/runtime-core/test/client-ip.test.mjs new file mode 100644 index 0000000..804bede --- /dev/null +++ b/packages/runtime-core/test/client-ip.test.mjs @@ -0,0 +1,49 @@ +// Hashed client IP, the crypto-free half (runtime-core also runs on edge +// runtimes): which address of a span is hashed and its canonical form. The +// Python adapter and the ingester run the same vectors. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SOURCE_KEYS, + IDENTIFYING_ATTRIBUTE_KEYS, + clientIpOf, + envFlag, + normalizeClientIp, + usableClientIpSalt, +} from "../dist/index.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +).clientIpHash; + +test("constants match the shared vectors", () => { + assert.equal(CLIENT_IP_HASH_ATTRIBUTE, vectors.attribute); + assert.equal(CLIENT_IP_HASH_LENGTH, vectors.length); + // The hash is not an identifying attribute: it survives the drop. + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes(CLIENT_IP_HASH_ATTRIBUTE)); + // Every source is itself dropped. + for (const key of CLIENT_IP_SOURCE_KEYS) assert.ok(IDENTIFYING_ATTRIBUTE_KEYS.includes(key), key); +}); + +test("normalizeClientIp: ports, brackets, zone ids, case, IPv4-mapped IPv6, lists", () => { + for (const { input, expect } of vectors.normalize) assert.equal(normalizeClientIp(input), expect, JSON.stringify(input)); + assert.equal(normalizeClientIp(undefined), null); + assert.equal(normalizeClientIp(42), null); +}); + +test("clientIpOf: client.address, then http.client_ip, then the peer on server spans only", () => { + for (const { attributes, server, expect } of vectors.source) + assert.equal(clientIpOf(attributes, server), expect, JSON.stringify({ attributes, server })); +}); + +test("salts shorter than the minimum are refused; env flag", () => { + assert.equal(usableClientIpSalt("x".repeat(CLIENT_IP_SALT_MIN_LENGTH - 1)), null); + assert.equal(usableClientIpSalt("x".repeat(CLIENT_IP_SALT_MIN_LENGTH)), "x".repeat(CLIENT_IP_SALT_MIN_LENGTH)); + assert.equal(usableClientIpSalt(undefined), null); + for (const raw of ["true", "1", "YES", " on "]) assert.equal(envFlag(raw), true, raw); + for (const raw of [undefined, "", "false", "0", "salt"]) assert.equal(envFlag(raw), false, String(raw)); +}); diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index 8d6993e..8b9d191 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -655,6 +655,38 @@ stored only when the ingester keeps it too: set ([otlp-ingester README](../otlp-ingester/README.md#identifying-attributes)). `IDENTIFYING_ATTRIBUTE_KEYS` exports the list. +**Hashed client IP (opt-in).** To count distinct clients without storing +their addresses, turn on `hashClientIp` with a secret salt. The client's +address is normalised (port, brackets and zone id removed, IPv6 lower-cased +and compressed, `::ffff:1.2.3.4` read as `1.2.3.4`) and exported only as +`client.address_hash`: HMAC-SHA256 under the salt, the first 32 hex +characters. The raw address is still dropped. + +```ts +initAutterServer({ ..., hashClientIp: { salt: process.env.AUTTER_CLIENT_IP_SALT } }); +// or AUTTER_HASH_CLIENT_IP=true and AUTTER_CLIENT_IP_SALT= +``` + +- The address is the first valid one of `client.address`, `http.client_ip` + (both filled from `X-Forwarded-For` / `Forwarded` behind a proxy), then, + on server spans only, the socket peer `net.peer.ip` / + `network.peer.address` / `net.sock.peer.addr`. Behind a proxy that does + not forward the client address, the peer is the proxy, so every request + hashes to the proxy. On client spans the peer is the server being called + and is never hashed. +- The salt is a secret of the project: at least 16 characters (32 random + ones recommended), the same in every service and instance of the + project, so the same client gives the same hash everywhere. It is never + logged, exported or written to a span. Without a usable salt nothing is + hashed and one warning is printed: there is no unsalted fallback, since + every IPv4 address can be hashed in a few minutes. +- The Python adapter and the ingester compute the same hash from the same + salt. A span that already has `client.address_hash` keeps it. +- A hashed IP is still pseudonymous personal data in some readings of + privacy law (GDPR, for one): whoever holds the salt can test a known + address against it. That is why it is opt-in; keep the salt with your + other secrets and rotate it to unlink old hashes. + ### One SDK per process, however many copies are loaded Bundlers can load this package more than once: Next.js bundles one copy diff --git a/packages/runtime-node/src/client-ip.ts b/packages/runtime-node/src/client-ip.ts new file mode 100644 index 0000000..a24f2e8 --- /dev/null +++ b/packages/runtime-node/src/client-ip.ts @@ -0,0 +1,47 @@ +import { createHmac } from "node:crypto"; +import { + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, + HASH_CLIENT_IP_ENV, + envFlag, + usableClientIpSalt, +} from "@autter/runtime-core"; + +/** `hashClientIp`: `true` (salt from AUTTER_CLIENT_IP_SALT) or `{ salt }`. */ +export type HashClientIpOption = boolean | { salt?: string }; + +/** Hashes one normalised IP address (runtime-core normalizeClientIp). */ +export type ClientIpHasher = (ip: string) => string; + +/** HMAC-SHA256(salt, ip), hex, first CLIENT_IP_HASH_LENGTH characters. */ +export function hashClientIp(salt: string, ip: string): string { + return createHmac("sha256", salt).update(ip).digest("hex").slice(0, CLIENT_IP_HASH_LENGTH); +} + +/** + * The hasher for `hashClientIp` (else AUTTER_HASH_CLIENT_IP), or null when + * hashing is off. On without a usable salt (`{ salt }`, else + * AUTTER_CLIENT_IP_SALT; at least CLIENT_IP_SALT_MIN_LENGTH characters) it is + * off too, with one warning: never an unsalted hash. The salt stays in this + * closure; it is never logged or exported. + */ +export function clientIpHasher( + option: HashClientIpOption | undefined, + env: Record = process.env, + warn: (message: string) => void = (message) => console.warn(`[autter-runtime] ${message}`), +): ClientIpHasher | null { + const enabled = option === undefined ? envFlag(env[HASH_CLIENT_IP_ENV]) : option !== false; + if (!enabled) return null; + const salt = usableClientIpSalt( + (typeof option === "object" && option !== null ? option.salt : undefined) ?? env[CLIENT_IP_SALT_ENV], + ); + if (!salt) { + warn( + `hashClientIp is on but no salt of at least ${CLIENT_IP_SALT_MIN_LENGTH} characters is set ` + + `(hashClientIp: { salt } or ${CLIENT_IP_SALT_ENV}): client IP addresses are dropped and not hashed.`, + ); + return null; + } + return (ip) => hashClientIp(salt, ip); +} diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index db0e498..608effa 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -48,6 +48,7 @@ export { type KeepIdentifyingAttributes, type RedactOptions, } from "./redact.js"; +export { hashClientIp, type HashClientIpOption } from "./client-ip.js"; export { instrumentLlmClient, type InstrumentLlmOptions, diff --git a/packages/runtime-node/src/redact-exporter.ts b/packages/runtime-node/src/redact-exporter.ts index 7ee3160..b84d08f 100644 --- a/packages/runtime-node/src/redact-exporter.ts +++ b/packages/runtime-node/src/redact-exporter.ts @@ -7,7 +7,14 @@ import { type ResourceMetrics, } from "@opentelemetry/sdk-metrics"; import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; -import { makeIdentifyingAttributeFilter, type IdentifyingAttributeFilter } from "@autter/runtime-core"; +import { SpanKind, type Attributes } from "@opentelemetry/api"; +import { + CLIENT_IP_HASH_ATTRIBUTE, + clientIpOf, + makeIdentifyingAttributeFilter, + type IdentifyingAttributeFilter, +} from "@autter/runtime-core"; +import type { ClientIpHasher } from "./client-ip.js"; import type { Redactor } from "./redact.js"; import { makeUrlQueryPolicy, type UrlQueryPolicy } from "./url-query.js"; @@ -32,7 +39,9 @@ import { makeUrlQueryPolicy, type UrlQueryPolicy } from "./url-query.js"; * user agent, host.id and the host's addresses, the command line and its owner: see * runtime-core's identifying.ts) are dropped from span, event and link * attributes and from the resource, unless `keepIdentifyingAttributes` - * keeps them (and `dropAttributes` adds keys to drop). + * keeps them (and `dropAttributes` adds keys to drop). With `hashClientIp`, + * a keyed hash of the client's address is added as `client.address_hash` + * (client-ip.ts) before the raw address is dropped. * * The span is re-wrapped, never mutated (spans are shared with other * processors). `redactAttributes: false` disables the scrubbing but neither @@ -69,16 +78,28 @@ function redactResource( const DEFAULT_QUERY_POLICY = makeUrlQueryPolicy(); const DEFAULT_IDENTIFYING = makeIdentifyingAttributeFilter(); -/** Without scrubbing (`redactAttributes: false`): the query policy and the - * identifying attribute drop only. */ +/** + * `attributes` plus `client.address_hash` from the span's raw attributes, + * when hashing is on, the span names a client address and carries no hash + * already (one set by the app or an earlier hop is left alone). + */ +function withClientIpHash(span: ReadableSpan, attributes: Attributes, hasher: ClientIpHasher | null): Attributes { + if (!hasher || span.attributes[CLIENT_IP_HASH_ATTRIBUTE] !== undefined) return attributes; + const ip = clientIpOf(span.attributes, span.kind === SpanKind.SERVER); + return ip ? { ...attributes, [CLIENT_IP_HASH_ATTRIBUTE]: hasher(ip) } : attributes; +} + +/** Without scrubbing (`redactAttributes: false`): the query policy, the + * identifying attribute drop and the client IP hash only. */ function stripQueries( span: ReadableSpan, redactor: Redactor, query: UrlQueryPolicy, identifying: IdentifyingAttributeFilter, + hasher: ClientIpHasher | null, ): ReadableSpan { const name = query.name(span.name); - const attributes = query.attributes(identifying.apply(span.attributes)); + const attributes = withClientIpHash(span, query.attributes(identifying.apply(span.attributes)), hasher); let eventsChanged = false; const events = span.events.map((event) => { if (!event.attributes) return event; @@ -139,17 +160,18 @@ export function redactSpan( redactor: Redactor, query: UrlQueryPolicy = DEFAULT_QUERY_POLICY, identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, + hasher: ClientIpHasher | null = null, ): ReadableSpan { - if (!redactor.enabled) return stripQueries(span, redactor, query, identifying); + if (!redactor.enabled) return stripQueries(span, redactor, query, identifying, hasher); const status = span.status.message ? { ...span.status, message: redactor.text(span.status.message) } : span.status; // Drop identifying keys, scrub (a `user:pass@` URL is recognised whole), - // then cut queries. + // cut queries, then add the client IP hash (from the raw address). return wrap(span, { name: query.name(redactor.text(span.name)), status, - attributes: query.attributes(redactor.exported(identifying.apply(span.attributes))), + attributes: withClientIpHash(span, query.attributes(redactor.exported(identifying.apply(span.attributes))), hasher), links: span.links.map((link) => link.attributes ? { ...link, attributes: redactor.exported(identifying.apply(link.attributes)) } : link, ), @@ -168,6 +190,7 @@ export class RedactingSpanExporter implements SpanExporter { private readonly redactor: () => Redactor, private readonly query: UrlQueryPolicy = DEFAULT_QUERY_POLICY, private readonly identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, + private readonly hasher: ClientIpHasher | null = null, ) {} export( @@ -179,7 +202,7 @@ export class RedactingSpanExporter implements SpanExporter { let failed = 0; for (const span of spans) { try { - scrubbed.push(redactSpan(span, redactor, this.query, this.identifying)); + scrubbed.push(redactSpan(span, redactor, this.query, this.identifying, this.hasher)); } catch { // Fail closed: a span we could not scrub is not exported. failed++; diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index 9acd176..1a79f4e 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -61,6 +61,7 @@ import { type Redactor, } from "./redact.js"; import { RedactingMetricExporter, RedactingSpanExporter } from "./redact-exporter.js"; +import { clientIpHasher, type HashClientIpOption } from "./client-ip.js"; import { authHeaders, resolveIngestKey, warnKeyKind, warnMissingKey } from "./ingest-auth.js"; import { IngestMetricExporter, IngestSpanExporter } from "./ingest-exporters.js"; import { makeUrlQueryPolicy } from "./url-query.js"; @@ -415,6 +416,24 @@ export interface AutterServerOptions { * environment variable (a comma-separated list), else none. */ dropAttributes?: string[]; + /** + * Count distinct clients without storing their IP addresses. Off by + * default. When on, the client's address (first of `client.address`, + * `http.client_ip`, then on server spans the socket peer `net.peer.ip` / + * `network.peer.address` / `net.sock.peer.addr`, which behind a proxy is + * the proxy) is normalised and exported only as `client.address_hash`: + * HMAC-SHA256 with your secret salt, 32 hex characters. The raw address + * is still dropped. A hashed IP is still pseudonymous personal data in + * some readings of privacy law; treat the salt as a secret and use the + * same one in every service of the project (the Python adapter and the + * ingester compute the same hash from the same salt). + * + * `true` reads the salt from AUTTER_CLIENT_IP_SALT; `{ salt }` passes it. + * A salt shorter than 16 characters, or none, turns hashing off with a + * warning (never an unsalted hash). Default: AUTTER_HASH_CLIENT_IP=true, + * else off. + */ + hashClientIp?: HashClientIpOption; /** * Verbose lifecycle logging to stderr: "exported N spans", flushes and * auto-flush activity — handy while wiring the SDK up. Also enabled by @@ -1305,6 +1324,8 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { options.keepIdentifyingAttributes ?? keepIdentifyingFromEnv(process.env[KEEP_IDENTIFYING_ATTRIBUTES_ENV]), options.dropAttributes ?? dropAttributesFromEnv(process.env[DROP_ATTRIBUTES_ENV]), ); + // Opt-in keyed hash of the client IP (client-ip.ts); null when off. + const ipHasher = clientIpHasher(options.hashClientIp); const slowPerMinute = Number.isFinite(options.retainSlowTracesPerMinute) ? Math.max(0, options.retainSlowTracesPerMinute!) : DEFAULT_SLOW_TRACES_PER_MINUTE; // One policy for every trace exporter: LLM spans keep prompt/completion @@ -1324,6 +1345,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { () => redactor, queryPolicy, identifying, + ipHasher, ), llmContent, ), diff --git a/packages/runtime-node/test/client-ip-hash.test.mjs b/packages/runtime-node/test/client-ip-hash.test.mjs new file mode 100644 index 0000000..b34ba67 --- /dev/null +++ b/packages/runtime-node/test/client-ip-hash.test.mjs @@ -0,0 +1,171 @@ +// Opt-in hashed client IP: with hashClientIp the client's address leaves +// the process only as `client.address_hash`, HMAC-SHA256 under the +// project's secret salt, the same value the Python adapter and the ingester +// compute (test-vectors/redaction.json, `clientIpHash`). Off by default; +// on without a salt it hashes nothing and warns once. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { dirname, join } from "node:path"; +import { after, before, test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { hashClientIp } from "../dist/index.js"; +import { normalizeClientIp } from "../../runtime-core/dist/index.js"; + +const here = dirname(fileURLToPath(import.meta.url)); +const VECTORS = JSON.parse(readFileSync(join(here, "../../../test-vectors/redaction.json"), "utf8")); +const vectors = VECTORS.clientIpHash; +const DROP = VECTORS.identifying.drop; +const SALT = "project-secret-salt-7f3a9c2e41"; + +test("hashClientIp matches the shared vectors (normalised first)", () => { + for (const { input, expect } of vectors.hash) assert.equal(hashClientIp(vectors.salt, normalizeClientIp(input)), expect, input); + assert.equal(hashClientIp(vectors.salt, "203.0.113.9").length, vectors.length); + assert.notEqual(hashClientIp(`${vectors.salt}x`, "203.0.113.9"), hashClientIp(vectors.salt, "203.0.113.9")); +}); + +const received = []; +let collector; +let endpoint; + +before(async () => { + collector = createServer((req, res) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + try { + received.push({ path: req.url, text, body: JSON.parse(text) }); + } catch { + /* not JSON */ + } + res.end("{}"); + }); + }); + await new Promise((resolve) => collector.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${collector.address().port}`; +}); + +after(async () => { + await new Promise((resolve) => { + collector.close(resolve); + collector.closeAllConnections(); + }); +}); + +const until = async (check, ms = 5000) => { + const start = Date.now(); + while (!check() && Date.now() - start < ms) await new Promise((r) => setTimeout(r, 10)); + return check(); +}; +const value = (v) => v?.stringValue ?? v?.intValue ?? v?.boolValue ?? v?.doubleValue; +const attrs = (list) => Object.fromEntries((list ?? []).map((a) => [a.key, value(a.value)])); + +function spansOf(service) { + const out = []; + for (const { body } of received) + for (const rs of body.resourceSpans ?? []) { + if (attrs(rs.resource?.attributes)["service.name"] !== service) continue; + for (const ss of rs.scopeSpans ?? []) + for (const span of ss.spans ?? []) out.push({ name: span.name, kind: span.kind, attributes: attrs(span.attributes) }); + } + return out; +} + +/** One request through HttpInstrumentation (X-Forwarded-For: 203.0.113.9, + * socket peer 127.0.0.1) and its client span; a manual span with + * client.address; a manual span already carrying a hash. */ +async function run(service, options = {}, env = {}) { + const script = ` + import { createRequire } from "node:module"; + import { trace } from "@opentelemetry/api"; + import { initAutterServer } from "./dist/index.js"; + const sdk = initAutterServer({ + apiKey: "k", service: process.env.SERVICE, endpoint: process.env.ENDPOINT, traceSampleRate: 1, + captureGlobalErrors: false, autoFlush: false, memoryMetrics: false, compatCheck: false, + metricIntervalMs: 3_600_000, logging: { console: false }, + ...JSON.parse(process.env.OPTIONS), + }); + const http = createRequire(import.meta.url)("http"); + const server = http.createServer((req, res) => res.end("ok")); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + await new Promise((resolve, reject) => { + http.get({ host: "127.0.0.1", port: server.address().port, path: "/hello", + headers: { "x-forwarded-for": "203.0.113.9, 10.0.0.1" } }, (res) => { res.resume(); res.on("end", resolve); }).on("error", reject); + }); + await new Promise((resolve) => server.close(resolve)); + const tracer = trace.getTracer("t"); + tracer.startSpan("manual", { attributes: { "client.address": "::ffff:198.51.100.7" } }).end(); + tracer.startSpan("prehashed", { attributes: { "client.address": "198.51.100.8", "client.address_hash": "set-upstream" } }).end(); + await sdk.shutdown(); + `; + const childEnv = { ...process.env, ENDPOINT: endpoint, SERVICE: service, OPTIONS: JSON.stringify(options), ...env }; + for (const name of ["AUTTER_HASH_CLIENT_IP", "AUTTER_CLIENT_IP_SALT", "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES", "AUTTER_DROP_ATTRIBUTES"]) + if (!(name in env)) delete childEnv[name]; + const child = spawn(process.execPath, ["--input-type=module", "-e", script], { + cwd: join(here, ".."), + env: childEnv, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => (stderr += chunk)); + const code = await new Promise((resolve) => child.on("exit", resolve)); + assert.equal(code, 0, stderr); + await until(() => spansOf(service).some((span) => span.name === "prehashed")); + const spans = spansOf(service); + const server = spans.find((span) => span.kind === 2); + assert.ok(server, "HTTP server span exported"); + return { + spans, + server, + client: spans.find((span) => span.kind === 3), + manual: spans.find((span) => span.name === "manual"), + prehashed: spans.find((span) => span.name === "prehashed"), + stderr, + }; +} + +function assertNoRawAddress(out) { + for (const span of out.spans) for (const key of DROP) assert.ok(!(key in span.attributes), `${key} on ${span.name}`); +} + +test("off by default: the address is dropped and no hash is added", async () => { + const out = await run("ip-hash-off"); + assertNoRawAddress(out); + assert.equal(out.server.attributes["client.address_hash"], undefined); + assert.equal(out.manual.attributes["client.address_hash"], undefined); + assert.equal(out.prehashed.attributes["client.address_hash"], "set-upstream"); +}); + +test("hashClientIp: { salt } adds client.address_hash from the forwarded client address", async () => { + const out = await run("ip-hash-option", { hashClientIp: { salt: SALT } }); + assertNoRawAddress(out); + // X-Forwarded-For's first entry (http.client_ip) wins over the socket peer. + assert.equal(out.server.attributes["client.address_hash"], hashClientIp(SALT, "203.0.113.9")); + // IPv4-mapped IPv6 is hashed as the IPv4 address. + assert.equal(out.manual.attributes["client.address_hash"], hashClientIp(SALT, "198.51.100.7")); + // An existing hash is left alone, never hashed again. + assert.equal(out.prehashed.attributes["client.address_hash"], "set-upstream"); + // On a client span the peer is the server being called: no hash. + assert.ok(out.client, "HTTP client span exported"); + assert.equal(out.client.attributes["client.address_hash"], undefined); + // The salt is never exported or printed. + for (const { text } of received) assert.ok(!text.includes(SALT), "salt on the wire"); + assert.ok(!out.stderr.includes(SALT), "salt printed"); +}); + +test("AUTTER_HASH_CLIENT_IP + AUTTER_CLIENT_IP_SALT, also with redactAttributes: false", async () => { + const out = await run("ip-hash-env", { redactAttributes: false }, { AUTTER_HASH_CLIENT_IP: "true", AUTTER_CLIENT_IP_SALT: SALT }); + assertNoRawAddress(out); + assert.equal(out.server.attributes["client.address_hash"], hashClientIp(SALT, "203.0.113.9")); + assert.equal(out.manual.attributes["client.address_hash"], hashClientIp(SALT, "198.51.100.7")); +}); + +test("on without a usable salt: nothing is hashed (never unsalted) and one warning is printed", async () => { + const out = await run("ip-hash-no-salt", { hashClientIp: true }, { AUTTER_CLIENT_IP_SALT: "short" }); + assertNoRawAddress(out); + for (const span of out.spans) if (span.name !== "prehashed") assert.equal(span.attributes["client.address_hash"], undefined, span.name); + assert.equal(out.stderr.match(/hashClientIp is on but no salt/g)?.length, 1, out.stderr); + assert.ok(!out.stderr.includes("short"), "salt printed"); +}); diff --git a/test-vectors/redaction.json b/test-vectors/redaction.json index 29921ba..1c1ce87 100644 --- a/test-vectors/redaction.json +++ b/test-vectors/redaction.json @@ -244,5 +244,192 @@ "http.route", "http.method" ] + }, + "clientIpHash": { + "$comment": "Opt-in hashed client IP (client.address_hash): HMAC-SHA256(salt, normalized IP) as hex, first `length` characters. `normalize` maps raw values to the canonical IP (null: not an IP, nothing hashed); `hash` maps raw values to the hash under `salt`; `source` says which address of a span or log record is hashed (`server`: an HTTP server span, where the socket peer may be read). Runtime-core/runtime-node, adapters/python and the otlp-ingester must agree on all three.", + "attribute": "client.address_hash", + "salt": "autter-test-salt-0123456789", + "length": 32, + "normalize": [ + { + "input": "203.0.113.9", + "expect": "203.0.113.9" + }, + { + "input": " 203.0.113.9 ", + "expect": "203.0.113.9" + }, + { + "input": "203.0.113.9:51234", + "expect": "203.0.113.9" + }, + { + "input": "203.0.113.9, 10.0.0.1", + "expect": "203.0.113.9" + }, + { + "input": "::ffff:203.0.113.9", + "expect": "203.0.113.9" + }, + { + "input": "::FFFF:CB00:7109", + "expect": "203.0.113.9" + }, + { + "input": "[::ffff:203.0.113.9]:443", + "expect": "203.0.113.9" + }, + { + "input": "2001:DB8:0:0::1", + "expect": "2001:db8::1" + }, + { + "input": "[2001:db8::1]:8080", + "expect": "2001:db8::1" + }, + { + "input": "fe80::1%eth0", + "expect": "fe80::1" + }, + { + "input": "2001:0db8:0000:0000:0000:0000:0000:0001", + "expect": "2001:db8::1" + }, + { + "input": "1:0:0:2:0:0:0:3", + "expect": "1:0:0:2::3" + }, + { + "input": "unknown", + "expect": null + }, + { + "input": "", + "expect": null + }, + { + "input": "999.1.1.1", + "expect": null + }, + { + "input": "01.2.3.4", + "expect": null + }, + { + "input": "example.com", + "expect": null + }, + { + "input": "1.2.3", + "expect": null + }, + { + "input": "::ffff:zz", + "expect": null + } + ], + "hash": [ + { + "input": "203.0.113.9", + "expect": "ca02c4212f9f108bfc1f1c799ddac123" + }, + { + "input": "::ffff:203.0.113.9", + "expect": "ca02c4212f9f108bfc1f1c799ddac123" + }, + { + "input": "203.0.113.9:443", + "expect": "ca02c4212f9f108bfc1f1c799ddac123" + }, + { + "input": "2001:db8::1", + "expect": "7f8f52c731e08ed237dc17921591cbe0" + }, + { + "input": "[2001:DB8::1]:443", + "expect": "7f8f52c731e08ed237dc17921591cbe0" + }, + { + "input": "10.0.0.2", + "expect": "8397b1a98831ffa12e2b88a4aa7e25bc" + } + ], + "source": [ + { + "attributes": { + "client.address": "203.0.113.9", + "net.peer.ip": "10.0.0.2" + }, + "server": true, + "expect": "203.0.113.9" + }, + { + "attributes": { + "http.client_ip": "198.51.100.1", + "net.peer.ip": "10.0.0.2" + }, + "server": true, + "expect": "198.51.100.1" + }, + { + "attributes": { + "client.address": "203.0.113.9", + "http.client_ip": "198.51.100.1" + }, + "server": true, + "expect": "203.0.113.9" + }, + { + "attributes": { + "net.peer.ip": "10.0.0.2", + "network.peer.address": "10.0.0.3" + }, + "server": true, + "expect": "10.0.0.2" + }, + { + "attributes": { + "network.peer.address": "::ffff:10.0.0.3" + }, + "server": true, + "expect": "10.0.0.3" + }, + { + "attributes": { + "net.sock.peer.addr": "10.0.0.5" + }, + "server": true, + "expect": "10.0.0.5" + }, + { + "attributes": { + "net.peer.ip": "10.0.0.2" + }, + "server": false, + "expect": null + }, + { + "attributes": { + "client.address": "unknown", + "network.peer.address": "10.0.0.3" + }, + "server": true, + "expect": "10.0.0.3" + }, + { + "attributes": { + "client.address": "10.0.0.4" + }, + "server": false, + "expect": "10.0.0.4" + }, + { + "attributes": { + "http.route": "/x" + }, + "server": true, + "expect": null + } + ] } }