diff --git a/CHANGELOG.md b/CHANGELOG.md index d9ca493..bdb0f31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Packages: `runtime-node`, `otlp-ingester`, Python adapter (`adapters/python`). - Kept: `service.instance.id` (the ingester needs it), `host.name` (container and pod names; on a laptop or VM it is the machine's name), `process.pid`, `process.runtime.*`, `os.type`, `host.arch`, and the coarse `user_agent.browser` / `.os` / `.device` / `.bot` fields. - Keep them with `keepIdentifyingAttributes: true` or a list of keys (`initAutterServer`, `registerAutter`), `keep_identifying_attributes=` in Python `init_tracing()`, or `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` (`true`, or a comma-separated list) for the SDKs and the ingester. Drop more keys, such as `host.name`, with `dropAttributes` / `drop_attributes=` / `AUTTER_DROP_ATTRIBUTES`. An attribute the SDK keeps is stored only if the ingester keeps it too: set `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` on both. - Spans already stored keep these attributes until their TTL expires. +- Opt-in hashed client IP (`runtime-node`, Python adapter, `otlp-ingester`), to count distinct clients without storing addresses: `hashClientIp: true | { salt }`, `init_tracing(hash_client_ip=True, client_ip_salt=...)`, or `AUTTER_HASH_CLIENT_IP=true` with `AUTTER_CLIENT_IP_SALT` (SDKs and ingester). The client's address (`client.address`, then `http.client_ip`, then on server spans the socket peer, which behind a proxy is the proxy) is normalised and exported as `client.address_hash`: HMAC-SHA256 under the secret salt, 32 hex characters, identical in the Node SDK, the Python adapter and the ingester for the same salt (shared vectors in `test-vectors/redaction.json`, `clientIpHash`). The raw address is still dropped. Off by default; on without a salt of at least 16 characters it hashes nothing and warns once, never an unsalted hash. The ingester keeps a hash it receives and does not hash it again. A hashed IP is still pseudonymous personal data in some readings of privacy law: treat the salt as a secret. ## [1.18.0] - 2026-11-14 diff --git a/adapters/python/autter_tracing.py b/adapters/python/autter_tracing.py index 62c0639..406bdc0 100644 --- a/adapters/python/autter_tracing.py +++ b/adapters/python/autter_tracing.py @@ -89,7 +89,14 @@ try: # imported as part of a package from .redact import ( + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, DROP_ATTRIBUTES_ENV, + HASH_CLIENT_IP_ENV, + client_ip_of, + hash_client_ip, + usable_client_ip_salt, KEEP_IDENTIFYING_ATTRIBUTES_ENV, drop_attributes_from_env, drop_identifying_attributes, @@ -100,7 +107,14 @@ ) except ImportError: # copied next to the app as a plain module from redact import ( + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, DROP_ATTRIBUTES_ENV, + HASH_CLIENT_IP_ENV, + client_ip_of, + hash_client_ip, + usable_client_ip_salt, KEEP_IDENTIFYING_ATTRIBUTES_ENV, drop_attributes_from_env, drop_identifying_attributes, @@ -561,11 +575,42 @@ def _without_identifying_resource(resource: Resource, dropped: frozenset) -> Res return result -def _redacted(span: ReadableSpan, redact: bool = True, dropped: Optional[frozenset] = None) -> ReadableSpan: +def _client_ip_hasher(enabled: Optional[bool], salt: Optional[str]) -> Optional[Any]: + """The hasher for ``hash_client_ip`` (default ``AUTTER_HASH_CLIENT_IP``) + with ``salt`` (default ``AUTTER_CLIENT_IP_SALT``), or ``None`` when off. + On without a usable salt it stays off with one warning: never an + unsalted hash. The salt is never logged.""" + if enabled is None: + enabled = _env_flag(HASH_CLIENT_IP_ENV) + if not enabled: + return None + usable = usable_client_ip_salt(salt if salt is not None else os.environ.get(CLIENT_IP_SALT_ENV)) + if usable is None: + LOGGER.warning( + "[autter-runtime] hash_client_ip is on but no salt of at least %d characters is set " + "(client_ip_salt= or %s): client IP addresses are dropped and not hashed.", + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SALT_ENV, + ) + return None + return lambda ip: hash_client_ip(usable, ip) + + +def _redacted( + span: ReadableSpan, redact: bool = True, dropped: Optional[frozenset] = None, hasher: Optional[Any] = None +) -> ReadableSpan: def clean(attributes: Any) -> Any: kept = drop_identifying_attributes(attributes, dropped) return redact_attributes(kept) if redact else kept + attributes = clean(span.attributes) + if hasher is not None and (span.attributes or {}).get(CLIENT_IP_HASH_ATTRIBUTE) is None: + # From the raw address, before it was dropped; a hash already on the + # span (set upstream) is left alone. + ip = client_ip_of(span.attributes, span.kind == SpanKind.SERVER) + if ip: + attributes = {**(attributes or {}), CLIENT_IP_HASH_ATTRIBUTE: hasher(ip)} + events = [Event(event.name, clean(event.attributes), event.timestamp) for event in span.events] links = [Link(link.context, clean(link.attributes)) for link in span.links] status = span.status @@ -574,7 +619,7 @@ def clean(attributes: Any) -> Any: return _CopiedSpan( span, name=redact_text(span.name) if redact else span.name, - attributes=clean(span.attributes), + attributes=attributes, events=events, links=links, status=status, @@ -601,6 +646,12 @@ class RedactingSpanExporter(SpanExporter): (``True`` or a list of keys; default ``AUTTER_KEEP_IDENTIFYING_ATTRIBUTES``) keeps them; ``drop_attributes`` (default ``AUTTER_DROP_ATTRIBUTES``, a comma-separated list) drops more keys, such as ``host.name``. + + ``hash_client_ip`` (default ``AUTTER_HASH_CLIENT_IP``, off) adds + ``client.address_hash``, HMAC-SHA256 of the client's normalised address + under ``client_ip_salt`` (default ``AUTTER_CLIENT_IP_SALT``, at least 16 + characters; without one nothing is hashed). Same hash as + @autter/runtime-node and the ingester for the same salt. """ def __init__( @@ -610,13 +661,16 @@ def __init__( redact: bool = True, keep_identifying_attributes: Any = None, drop_attributes: Optional[Sequence[str]] = None, + hash_client_ip: Optional[bool] = None, + client_ip_salt: Optional[str] = None, ): self._exporter = exporter self._redact = redact self._dropped = _keys_to_drop(keep_identifying_attributes, drop_attributes) + self._hasher = _client_ip_hasher(hash_client_ip, client_ip_salt) def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - return self._exporter.export([_redacted(span, self._redact, self._dropped) for span in spans]) + return self._exporter.export([_redacted(span, self._redact, self._dropped, self._hasher) for span in spans]) def shutdown(self) -> None: self._exporter.shutdown() @@ -663,6 +717,8 @@ def init_tracing( metric_exporter: Optional[MetricExporter] = None, keep_identifying_attributes: Any = None, drop_attributes: Optional[Sequence[str]] = None, + hash_client_ip: Optional[bool] = None, + client_ip_salt: Optional[str] = None, ) -> TracerProvider: """A TracerProvider that head-samples as configured and always exports errors, registered as the global provider. @@ -684,6 +740,10 @@ def init_tracing( them too unless it is given the same setting. * ``drop_attributes`` (default ``AUTTER_DROP_ATTRIBUTES``): more keys to drop the same way, such as ``["host.name"]``. + * ``hash_client_ip`` (default ``AUTTER_HASH_CLIENT_IP``, off) and + ``client_ip_salt`` (default ``AUTTER_CLIENT_IP_SALT``): add + ``client.address_hash``, a keyed hash of the client's address (see + :class:`RedactingSpanExporter`). * ``count_requests`` (default: on, unless ``AUTTER_COUNT_REQUESTS=false``; ``False`` turns it off): count every HTTP request, sampled or not (:class:`CountRequests`), in a delta histogram sent through @@ -747,7 +807,12 @@ def init_tracing( exporter = OTLPSpanExporter() exporter = RedactingSpanExporter( - exporter, redact=redact, keep_identifying_attributes=keep_identifying_attributes, drop_attributes=drop_attributes + exporter, + redact=redact, + keep_identifying_attributes=keep_identifying_attributes, + drop_attributes=drop_attributes, + hash_client_ip=hash_client_ip, + client_ip_salt=client_ip_salt, ) provider = TracerProvider( sampler=RecordUnsampled(sampler if sampler is not None else head_sampler_from_env()), diff --git a/adapters/python/redact.py b/adapters/python/redact.py index 14c018e..c14d25f 100644 --- a/adapters/python/redact.py +++ b/adapters/python/redact.py @@ -23,13 +23,17 @@ agent, the host's id and addresses, the command line and its owner: :data:`IDENTIFYING_ATTRIBUTE_KEYS`) are dropped rather than scrubbed: :func:`drop_identifying_attributes`. ``autter_tracing.RedactingSpanExporter`` -applies it to every span, also with redaction off. +applies it to every span, also with redaction off. Opt in to a keyed hash of +the client's address instead (``client.address_hash``): :func:`hash_client_ip`. Stdlib only. """ from __future__ import annotations +import hashlib +import hmac +import ipaddress import re from collections.abc import Iterable, Mapping from typing import Any @@ -315,3 +319,82 @@ def drop_identifying_attributes(attributes: Any, dropped: frozenset[str] | None if not any(key in keys for key in attributes): return attributes return {key: value for key, value in attributes.items() if key not in keys} + + +#: Opt-in hashed client IP: the attribute, its length in hex characters, the +#: shortest salt accepted and the environment variables. Same values as +#: @autter/runtime-node and the ingester (``test-vectors/redaction.json``, +#: ``clientIpHash``). +CLIENT_IP_HASH_ATTRIBUTE = "client.address_hash" +CLIENT_IP_HASH_LENGTH = 32 +CLIENT_IP_SALT_MIN_LENGTH = 16 +HASH_CLIENT_IP_ENV = "AUTTER_HASH_CLIENT_IP" +CLIENT_IP_SALT_ENV = "AUTTER_CLIENT_IP_SALT" + +#: Where the client address is read from, first valid wins: ``client.address`` +#: and ``http.client_ip`` (from ``X-Forwarded-For`` behind a proxy), then the +#: socket peer, on server spans only (behind a proxy it is the proxy; on a +#: client span it is the server being called). +CLIENT_IP_SOURCE_KEYS = ("client.address", "http.client_ip", "net.peer.ip", "network.peer.address", "net.sock.peer.addr") +CLIENT_IP_CLIENT_KEYS = ("client.address", "http.client_ip") + +_IPV4_PORT = re.compile(r"^[\d.]+:\d{1,5}$") +_BRACKETED = re.compile(r"^\[([^\]]+)\](?::\d{1,5})?$") +_IPV4 = re.compile(r"^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$") +_IPV6_CHARS = re.compile(r"^[0-9a-f:.]+$") + + +def normalize_client_ip(raw: Any) -> str | None: + """The canonical text of an IP address, or ``None`` when ``raw`` is not + one: the first entry of a comma-separated list, port and brackets + removed, zone id removed, IPv6 lower-cased and compressed (RFC 5952), + IPv4-mapped IPv6 turned into IPv4.""" + if not isinstance(raw, str): + return None + value = raw.split(",")[0].strip() + if not value or len(value) > 100: + return None + bracketed = _BRACKETED.match(value) + if bracketed: + value = bracketed.group(1) + elif _IPV4_PORT.match(value): + value = value[: value.rindex(":")] + if "%" in value: + value = value[: value.index("%")] + value = value.lower() + if _IPV4.match(value): + return value + if ":" not in value or not _IPV6_CHARS.match(value): + return None + try: + address = ipaddress.IPv6Address(value) + except ValueError: + return None + if address.ipv4_mapped is not None: + return str(address.ipv4_mapped) + return address.compressed + + +def client_ip_of(attributes: Mapping[str, Any] | None, server_span: bool) -> str | None: + """The normalised client address of a span or log record: the first + :data:`CLIENT_IP_SOURCE_KEYS` entry holding a valid IP (the peer keys + only when ``server_span``).""" + if not attributes: + return None + for key in CLIENT_IP_SOURCE_KEYS if server_span else CLIENT_IP_CLIENT_KEYS: + ip = normalize_client_ip(attributes.get(key)) + if ip: + return ip + return None + + +def usable_client_ip_salt(salt: Any) -> str | None: + """``salt`` when it is a string of at least :data:`CLIENT_IP_SALT_MIN_LENGTH` characters.""" + return salt if isinstance(salt, str) and len(salt) >= CLIENT_IP_SALT_MIN_LENGTH else None + + +def hash_client_ip(salt: str, ip: str) -> str: + """HMAC-SHA256(salt, ip) as hex, first :data:`CLIENT_IP_HASH_LENGTH` + characters. ``ip`` is normalised (:func:`normalize_client_ip`). Never + call it without a secret salt: the IPv4 space can be hashed in full.""" + return hmac.new(salt.encode("utf-8"), ip.encode("utf-8"), hashlib.sha256).hexdigest()[:CLIENT_IP_HASH_LENGTH] diff --git a/adapters/python/test_client_ip_hash.py b/adapters/python/test_client_ip_hash.py new file mode 100644 index 0000000..d12c766 --- /dev/null +++ b/adapters/python/test_client_ip_hash.py @@ -0,0 +1,173 @@ +"""Opt-in hashed client IP in the Python adapter: ``redact.py``'s +normalisation, source order and HMAC (stdlib, shared vectors) and +``RedactingSpanExporter`` / ``init_tracing()`` (needs opentelemetry-sdk). + +Run: python3 -m unittest discover -s adapters/python -v +""" + +from __future__ import annotations + +import json +import os +import sys +import unittest +from unittest import mock + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from redact import ( # noqa: E402 + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SOURCE_KEYS, + IDENTIFYING_ATTRIBUTE_KEYS, + client_ip_of, + hash_client_ip, + normalize_client_ip, + usable_client_ip_salt, +) + +VECTORS_PATH = os.path.join(os.path.dirname(__file__), "..", "..", "test-vectors", "redaction.json") +with open(VECTORS_PATH, encoding="utf-8") as fh: + VECTORS = json.load(fh)["clientIpHash"] + +SALT = "project-secret-salt-7f3a9c2e41" +CLEAN_ENV = { + "AUTTER_HASH_CLIENT_IP": "", + "AUTTER_CLIENT_IP_SALT": "", + "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES": "", + "AUTTER_DROP_ATTRIBUTES": "", +} + + +class SharedVectors(unittest.TestCase): + def test_constants(self): + self.assertEqual(CLIENT_IP_HASH_ATTRIBUTE, VECTORS["attribute"]) + self.assertEqual(CLIENT_IP_HASH_LENGTH, VECTORS["length"]) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, IDENTIFYING_ATTRIBUTE_KEYS) + for key in CLIENT_IP_SOURCE_KEYS: + self.assertIn(key, IDENTIFYING_ATTRIBUTE_KEYS) + + def test_normalize(self): + for case in VECTORS["normalize"]: + with self.subTest(case["input"]): + self.assertEqual(normalize_client_ip(case["input"]), case["expect"]) + self.assertIsNone(normalize_client_ip(None)) + self.assertIsNone(normalize_client_ip(42)) + + def test_source_order(self): + for case in VECTORS["source"]: + with self.subTest(case): + self.assertEqual(client_ip_of(case["attributes"], case["server"]), case["expect"]) + + def test_hash(self): + for case in VECTORS["hash"]: + with self.subTest(case["input"]): + self.assertEqual(hash_client_ip(VECTORS["salt"], normalize_client_ip(case["input"])), case["expect"]) + + def test_short_salts_are_refused(self): + self.assertIsNone(usable_client_ip_salt("x" * (CLIENT_IP_SALT_MIN_LENGTH - 1))) + self.assertEqual(usable_client_ip_salt("x" * CLIENT_IP_SALT_MIN_LENGTH), "x" * CLIENT_IP_SALT_MIN_LENGTH) + self.assertIsNone(usable_client_ip_salt(None)) + + +try: + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from opentelemetry.sdk.trace.sampling import ALWAYS_ON + from opentelemetry.trace import SpanKind + + from autter_tracing import RedactingSpanExporter, init_tracing + + HAVE_OTEL = True +except ImportError: # pragma: no cover - depends on the environment + HAVE_OTEL = False + + +@unittest.skipUnless(HAVE_OTEL, "opentelemetry-sdk is not installed") +class HashingExporter(unittest.TestCase): + def export(self, env=None, **options): + exporter = InMemorySpanExporter() + with mock.patch.dict(os.environ, {**CLEAN_ENV, **(env or {})}): + wrapped = RedactingSpanExporter(exporter, **options) + provider = TracerProvider(sampler=ALWAYS_ON, resource=Resource.create({"service.name": "shop"})) + provider.add_span_processor(SimpleSpanProcessor(wrapped)) + tracer = provider.get_tracer("test") + # Django/Flask server span: X-Forwarded-For in http.client_ip, the proxy as peer. + with tracer.start_as_current_span( + "GET /", kind=SpanKind.SERVER, attributes={"http.client_ip": "203.0.113.9", "net.peer.ip": "10.0.0.2"} + ): + pass + # An outgoing call: the peer is the server called, never hashed. + with tracer.start_as_current_span("GET api", kind=SpanKind.CLIENT, attributes={"net.peer.ip": "10.0.0.9"}): + pass + with tracer.start_as_current_span("manual", attributes={"client.address": "::ffff:198.51.100.7"}): + pass + with tracer.start_as_current_span( + "prehashed", attributes={"client.address": "198.51.100.8", CLIENT_IP_HASH_ATTRIBUTE: "set-upstream"} + ): + pass + provider.shutdown() + return {span.name: span for span in exporter.get_finished_spans()} + + def assert_no_raw_address(self, spans): + for span in spans.values(): + for key in IDENTIFYING_ATTRIBUTE_KEYS: + self.assertNotIn(key, span.attributes) + + def test_off_by_default(self): + spans = self.export() + self.assert_no_raw_address(spans) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, spans["GET /"].attributes) + self.assertEqual(spans["prehashed"].attributes[CLIENT_IP_HASH_ATTRIBUTE], "set-upstream") + + def test_hash_with_salt_argument(self): + spans = self.export(hash_client_ip=True, client_ip_salt=SALT) + self.assert_no_raw_address(spans) + self.assertEqual(spans["GET /"].attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "203.0.113.9")) + self.assertEqual(spans["manual"].attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "198.51.100.7")) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, spans["GET api"].attributes) + self.assertEqual(spans["prehashed"].attributes[CLIENT_IP_HASH_ATTRIBUTE], "set-upstream") + for span in spans.values(): + self.assertNotIn(SALT, json.dumps(dict(span.attributes))) + + def test_environment_and_redact_false(self): + spans = self.export(env={"AUTTER_HASH_CLIENT_IP": "true", "AUTTER_CLIENT_IP_SALT": SALT}, redact=False) + self.assert_no_raw_address(spans) + self.assertEqual(spans["GET /"].attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "203.0.113.9")) + + def test_without_a_usable_salt_nothing_is_hashed_and_it_warns(self): + with self.assertLogs("autter.runtime", level="WARNING") as logs: + spans = self.export(hash_client_ip=True, client_ip_salt="short-salt") + self.assert_no_raw_address(spans) + self.assertNotIn(CLIENT_IP_HASH_ATTRIBUTE, spans["GET /"].attributes) + self.assertEqual(len(logs.output), 1) + self.assertIn("no salt", logs.output[0]) + self.assertNotIn("short-salt", logs.output[0]) + + def test_init_tracing_wires_the_options(self): + exporter = InMemorySpanExporter() + with mock.patch.dict(os.environ, CLEAN_ENV): + provider = init_tracing( + sampler=ALWAYS_ON, + exporter=exporter, + resource=Resource.create({"service.name": "shop"}), + set_global=False, + hash_client_ip=True, + client_ip_salt=SALT, + ) + with provider.get_tracer("test").start_as_current_span( + "GET /", kind=SpanKind.SERVER, attributes={"client.address": "203.0.113.9"} + ): + pass + provider.force_flush() + [span] = exporter.get_finished_spans() + self.assertEqual(span.attributes[CLIENT_IP_HASH_ATTRIBUTE], hash_client_ip(SALT, "203.0.113.9")) + self.assertNotIn("client.address", span.attributes) + provider.shutdown() + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 7171c80..50421d8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -936,6 +936,15 @@ list, kept in parity by `test-vectors/redaction.json` (`identifying`). the SDK and the ingester; `dropAttributes` / `AUTTER_DROP_ATTRIBUTES` drops more keys. +Opt-in, `hashClientIp` / `AUTTER_HASH_CLIENT_IP` with a secret salt +(`AUTTER_CLIENT_IP_SALT`, 16+ characters) adds `client.address_hash`: +HMAC-SHA256 of the normalised client address, 32 hex characters, the same in +the Node SDK, the Python adapter and the ingester (vectors: +`test-vectors/redaction.json`, `clientIpHash`). The ingester hashes only +records without a hash. No salt, no hash: there is no unsalted fallback. +A hashed IP is still pseudonymous personal data in some readings of privacy +law, which is why it is off by default. + ## Sink webhook (v1) When `AUTTER_SINK_URL` is set, each ingest batch POSTs: diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 302d475..6930136 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -310,6 +310,15 @@ exceptions never leave the process. `init_tracing()` keeps them: the ingester drops them too unless it is given the same setting. `drop_attributes=["host.name"]` (or `AUTTER_DROP_ATTRIBUTES`) drops more keys. +- Opt-in: `init_tracing(hash_client_ip=True, client_ip_salt=...)` (or + `AUTTER_HASH_CLIENT_IP=true` and `AUTTER_CLIENT_IP_SALT`) adds + `client.address_hash`, an HMAC-SHA256 of the client's address under your + secret salt (16+ characters), the same value `@autter/runtime-node` and + the ingester compute from the same salt. The address is `client.address`, + then `http.client_ip` (from `X-Forwarded-For`), then on server spans + `net.peer.ip`. Without a salt nothing is hashed and a warning is logged. + A hashed IP is still pseudonymous personal data in some readings of + privacy law; keep the salt secret. Every request is recorded as if sampled (instrumentations collect their attributes), but only sampled and failing traces are exported. To wire the diff --git a/packages/otlp-ingester/README.md b/packages/otlp-ingester/README.md index 040aa57..0c5b887 100644 --- a/packages/otlp-ingester/README.md +++ b/packages/otlp-ingester/README.md @@ -174,6 +174,8 @@ Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILIT | `AUTTER_REDACT_KEY_PATTERNS` | — | JSON array of extra regex sources for attribute keys whose values are masked wholesale | | `AUTTER_KEEP_IDENTIFYING_ATTRIBUTES` | unset (drop them all) | Identifying attributes (client IP and port, raw user agent, `host.id` / `host.ip` / `host.mac`, command line and owner) are dropped on arrival; `true` keeps them all, a comma-separated list keeps those keys. An SDK that keeps one needs this set too, or the attribute is not stored. See [Identifying attributes](#identifying-attributes) | | `AUTTER_DROP_ATTRIBUTES` | unset | Comma-separated attribute keys dropped on arrival as well, such as `host.name` | +| `AUTTER_HASH_CLIENT_IP` | off | `true` adds `client.address_hash`, a keyed hash of the client's address, to spans and log records that carry no hash yet (plain OpenTelemetry senders). Needs `AUTTER_CLIENT_IP_SALT`. See [Hashed client IP](#hashed-client-ip-opt-in) | +| `AUTTER_CLIENT_IP_SALT` | unset | Secret salt for `AUTTER_HASH_CLIENT_IP`, at least 16 characters. Never logged. Without it nothing is hashed (a warning at startup) | ## Identifying attributes @@ -203,6 +205,28 @@ environment variable). Each side drops what it was not told to keep, so a key kept by only one of them is not stored. The ingester logs what it keeps at startup. Rows already stored keep these attributes until their TTL. +### Hashed client IP (opt-in) + +`AUTTER_HASH_CLIENT_IP=true` with `AUTTER_CLIENT_IP_SALT` adds +`client.address_hash` to every span and log record that names a client +address and has no hash yet: HMAC-SHA256 of the normalised address under the +salt, 32 hex characters. The address is the first valid one of +`client.address`, `http.client_ip`, then on server spans the socket peer +(`net.peer.ip`, `network.peer.address`, `net.sock.peer.addr`). The raw +address is dropped as before. A hash sent by an SDK is stored as it is and +never hashed again. + +The SDKs (`hashClientIp` in `@autter/runtime-node`, `hash_client_ip` in the +Python adapter) compute the same hash from the same salt, so use one salt +per project and hash in the SDK where you can. The ingester has one salt for +every project it serves: on an ingester shared by several projects their +hashes are comparable to whoever can read all of them, so give each project +its own salt in its SDKs instead. Without a salt of at least 16 characters +nothing is hashed (never an unsalted hash, which could be reversed by +hashing every IPv4 address) and the ingester warns at startup. A hashed IP +is still pseudonymous personal data in some readings of privacy law: it is +off by default, and the salt should be kept and rotated like any secret. + ## Rate limits and key lookups (1.12.0) In the order a request meets them: diff --git a/packages/otlp-ingester/src/client-ip-hash.test.ts b/packages/otlp-ingester/src/client-ip-hash.test.ts new file mode 100644 index 0000000..414671c --- /dev/null +++ b/packages/otlp-ingester/src/client-ip-hash.test.ts @@ -0,0 +1,148 @@ +// Opt-in hashed client IP at the ingester (AUTTER_HASH_CLIENT_IP with +// AUTTER_CLIENT_IP_SALT): plain OpenTelemetry senders get the same +// `client.address_hash` the SDKs compute, a hash already sent is kept, and +// the raw address is dropped either way. Shared vectors: +// test-vectors/redaction.json (`clientIpHash`). +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { test } from "node:test"; +import type { IngesterConfig } from "./config.js"; +import { + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SOURCE_KEYS, + IDENTIFYING_ATTRIBUTE_KEYS, + IdentifyingAttributeFilter, + clientIpHasher, + clientIpOfList, + hashClientIp, + identifyingKeysToDrop, + normalizeClientIp, +} from "./identifying.js"; +import { createIngesterApp } from "./server.js"; + +const ALL = JSON.parse(readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8")); +const vectors = ALL.clientIpHash as { + attribute: string; salt: string; length: number; + normalize: Array<{ input: string; expect: string | null }>; + hash: Array<{ input: string; expect: string }>; + source: Array<{ attributes: Record; server: boolean; expect: string | null }>; +}; +const SALT = "project-secret-salt-7f3a9c2e41"; +const kv = (attributes: Record) => + Object.entries(attributes).map(([key, value]) => ({ key, value: { stringValue: value } })); +const asMap = (list: unknown) => + Object.fromEntries((list as Array<{ key: string; value: { stringValue?: string } }>).map((e) => [e.key, e.value.stringValue])); + +test("the shared vectors: constants, normalisation, source order and hash", () => { + assert.equal(CLIENT_IP_HASH_ATTRIBUTE, vectors.attribute); + assert.equal(CLIENT_IP_HASH_LENGTH, vectors.length); + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes(CLIENT_IP_HASH_ATTRIBUTE)); + for (const key of CLIENT_IP_SOURCE_KEYS) assert.ok(IDENTIFYING_ATTRIBUTE_KEYS.includes(key), key); + for (const { input, expect } of vectors.normalize) assert.equal(normalizeClientIp(input), expect, input); + for (const { attributes, server, expect } of vectors.source) assert.equal(clientIpOfList(kv(attributes), server), expect); + for (const { input, expect } of vectors.hash) assert.equal(hashClientIp(vectors.salt, normalizeClientIp(input)!), expect, input); +}); + +test("no hasher without a salt of 16 characters, or when off", () => { + assert.equal(clientIpHasher(true, undefined), null); + assert.equal(clientIpHasher(true, "short"), null); + assert.equal(clientIpHasher(false, SALT), null); + assert.equal(clientIpHasher(true, SALT)!("203.0.113.9"), hashClientIp(SALT, "203.0.113.9")); +}); + +test("traces and logs: hash added from the raw address, kept when already sent, peer only on server spans", () => { + const filter = new IdentifyingAttributeFilter(identifyingKeysToDrop(), clientIpHasher(true, SALT)); + const traces = filter.traces({ + resourceSpans: [{ scopeSpans: [{ spans: [ + { name: "server", kind: "SPAN_KIND_SERVER", attributes: kv({ "net.peer.ip": "10.0.0.2", "http.client_ip": "203.0.113.9", "http.route": "/x" }) }, + { name: "server-peer", kind: 2, attributes: kv({ "network.peer.address": "[2001:DB8::1]:443" }) }, + { name: "client", kind: 3, attributes: kv({ "net.peer.ip": "10.0.0.9" }) }, + { name: "hashed", kind: 2, attributes: kv({ "client.address": "198.51.100.8", [CLIENT_IP_HASH_ATTRIBUTE]: "abc123" }) }, + ] }] }], + }); + const spans = traces.resourceSpans[0]!.scopeSpans[0]!.spans.map((span) => asMap(span.attributes)); + assert.deepEqual(spans[0], { "http.route": "/x", [CLIENT_IP_HASH_ATTRIBUTE]: hashClientIp(SALT, "203.0.113.9") }); + assert.deepEqual(spans[1], { [CLIENT_IP_HASH_ATTRIBUTE]: hashClientIp(SALT, "2001:db8::1") }); + assert.deepEqual(spans[2], {}); + assert.deepEqual(spans[3], { [CLIENT_IP_HASH_ATTRIBUTE]: "abc123" }, "an existing hash is never hashed again"); + + const logs = filter.logs({ resourceLogs: [{ scopeLogs: [{ logRecords: [ + { attributes: kv({ "client.address": "::ffff:203.0.113.9", message: "m" }) }, + { attributes: kv({ "net.peer.ip": "203.0.113.9" }) }, + ] }] }] }); + const records = logs.resourceLogs[0]!.scopeLogs[0]!.logRecords.map((record) => asMap(record.attributes)); + assert.deepEqual(records[0], { message: "m", [CLIENT_IP_HASH_ATTRIBUTE]: hashClientIp(SALT, "203.0.113.9") }); + assert.deepEqual(records[1], {}); +}); + +// Through the ingester, against a fake ClickHouse: a real Python Flask +// export (protobuf; its server span's address is net.peer.ip 127.0.0.1). + +const listen = async (server: Server) => { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +}; +const close = (server: Server) => + new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }); + +async function storedSpans(extra: Partial): Promise>> { + const rows: Array> = []; + const ch = createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + const query = new URL(req.url!, "http://localhost").searchParams.get("query") ?? ""; + if (query.includes("INSERT INTO") && query.includes("runtime_spans")) + rows.push(...body.trim().split("\n").filter(Boolean).map((line) => JSON.parse(line))); + res.end(); + }); + }); + const chUrl = await listen(ch); + const app = createIngesterApp({ + port: 0, clickhouseUrl: chUrl, clickhouseUser: "default", clickhousePassword: "", clickhouseDatabase: "autter_runtime", + ingestKeys: [{ key: "server", orgId: "org", repositoryId: "repo" }], + keyValidatorUrl: null, keyValidatorToken: null, sinkUrl: null, sinkToken: null, sinkMaxAttempts: 1, + sinkMaxBufferedBatches: 10, sinkMaxBufferedMb: 2, maxBodyBytes: 1024 * 1024, rateLimitPerMinute: 300, + clientRateLimitPerMinute: 120, occurrenceTtlDays: 14, spanTtlDays: 7, metricsTtlDays: 90, llmCallTtlDays: 90, + logTtlDays: 14, ...extra, + }).app.listen(); + const url = await new Promise((resolve) => + app.once("listening", () => resolve(`http://127.0.0.1:${(app.address() as AddressInfo).port}`)), + ); + try { + const response = await fetch(`${url}/v1/traces`, { + method: "POST", + headers: { "content-type": "application/x-protobuf", authorization: "Bearer server" }, + body: readFileSync(new URL("../test-fixtures/python/flask-unhandled.otlp.pb", import.meta.url)), + }); + assert.equal(response.status, 200); + } finally { + await close(app); + await close(ch); + } + return rows.map((row) => JSON.parse(String(row.attributes))); +} + +test("a plain OpenTelemetry sender gets client.address_hash when the ingester hashes, and none otherwise", async () => { + const hashed = await storedSpans({ hashClientIp: true, clientIpSalt: SALT }); + const server = hashed.find((attributes) => attributes["http.route"]); + assert.ok(server, "server span stored"); + assert.equal(server[CLIENT_IP_HASH_ATTRIBUTE], hashClientIp(SALT, "127.0.0.1")); + assert.equal(server["net.peer.ip"], undefined); + assert.ok(!JSON.stringify(hashed).includes(SALT), "salt stored"); + + for (const config of [{}, { hashClientIp: true }, { hashClientIp: true, clientIpSalt: "short" }]) { + const plain = await storedSpans(config); + assert.ok(plain.length); + for (const attributes of plain) { + assert.equal(attributes[CLIENT_IP_HASH_ATTRIBUTE], undefined, JSON.stringify(config)); + assert.equal(attributes["net.peer.ip"], undefined); + } + } +}); diff --git a/packages/otlp-ingester/src/config.ts b/packages/otlp-ingester/src/config.ts index fbceb07..71b6078 100644 --- a/packages/otlp-ingester/src/config.ts +++ b/packages/otlp-ingester/src/config.ts @@ -210,6 +210,14 @@ export interface IngesterConfig { keepIdentifyingAttributes?: boolean | string[]; /** More attribute keys to drop on arrival, such as `host.name` (AUTTER_DROP_ATTRIBUTES, comma-separated). */ dropAttributes?: string[]; + /** + * Opt-in (AUTTER_HASH_CLIENT_IP=true): add `client.address_hash`, a keyed + * hash of the client's address, to spans and log records that carry no + * hash yet (identifying.ts). Needs clientIpSalt. + */ + hashClientIp?: boolean; + /** The secret salt (AUTTER_CLIENT_IP_SALT, at least 16 characters). Never logged. */ + clientIpSalt?: string | null; /** * In-memory budget (MiB, decoded size) for parsed source maps used to * symbolicate browser frames at ingest (SOURCE_MAP_CACHE_MB, default 64). @@ -422,6 +430,8 @@ export function loadConfig(): IngesterConfig { redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"), keepIdentifyingAttributes: keepIdentifyingEnv(process.env.AUTTER_KEEP_IDENTIFYING_ATTRIBUTES), dropAttributes: dropAttributesEnv(process.env.AUTTER_DROP_ATTRIBUTES), + hashClientIp: /^(?:1|true|yes|on)$/i.test((process.env.AUTTER_HASH_CLIENT_IP ?? "").trim()), + clientIpSalt: process.env.AUTTER_CLIENT_IP_SALT || null, sourceMapCacheMb: intEnv("SOURCE_MAP_CACHE_MB", 64), loopCollapseAfter: loopCollapseAfterEnv(process.env.LOOP_COLLAPSE_AFTER), archive: archiveConfigFromEnv(process.env), diff --git a/packages/otlp-ingester/src/identifying.ts b/packages/otlp-ingester/src/identifying.ts index 5e5d24f..f0e3d23 100644 --- a/packages/otlp-ingester/src/identifying.ts +++ b/packages/otlp-ingester/src/identifying.ts @@ -20,7 +20,16 @@ * or a comma-separated list of keys) and drop more keys with * AUTTER_DROP_ATTRIBUTES (a comma-separated list, `host.name` say). An SDK * that keeps an attribute gets it stored only when the ingester keeps it too. + * + * Opt-in hashed client IP (AUTTER_HASH_CLIENT_IP=true with + * AUTTER_CLIENT_IP_SALT): a span or log record that names a client address + * and carries no `client.address_hash` yet gets one, HMAC-SHA256 of the + * normalised address under the salt (32 hex characters), the same value the + * SDKs compute from the same salt (test-vectors/redaction.json, + * `clientIpHash`). A hash already sent is kept as it is, never hashed again. + * The raw address is dropped either way. */ +import { createHmac } from "node:crypto"; export const IDENTIFYING_ATTRIBUTE_KEYS: readonly string[] = Object.freeze([ "net.peer.ip", "net.peer.port", @@ -107,7 +116,23 @@ function eachIn(obj: unknown, field: string, fn: (item: unknown) => unknown): un } export class IdentifyingAttributeFilter { - constructor(readonly dropped: ReadonlySet = identifyingKeysToDrop()) {} + constructor( + readonly dropped: ReadonlySet = identifyingKeysToDrop(), + private readonly hasher: ClientIpHasher | null = null, + ) {} + + /** The attributes of a span or log record: the client IP hash added (from the raw list), then the drop. */ + private recordAttributes = (obj: unknown, serverSpan: boolean): unknown => { + const hasher = this.hasher; + if (!hasher || !isObject(obj) || hasKey(obj.attributes, CLIENT_IP_HASH_ATTRIBUTE)) return this.attributesOf(obj); + const ip = clientIpOfList(obj.attributes, serverSpan); + const filtered = this.attributesOf(obj); + if (!ip || !isObject(filtered) || !Array.isArray(filtered.attributes)) return filtered; + return { + ...filtered, + attributes: [...filtered.attributes, { key: CLIENT_IP_HASH_ATTRIBUTE, value: { stringValue: hasher(ip) } }], + }; + }; /** An OTLP key/value list without the dropped keys; the same array when none is present. */ list = (attributes: unknown): unknown => { @@ -124,9 +149,13 @@ export class IdentifyingAttributeFilter { /** A traces request without identifying attributes (resource, spans, events, links). */ traces(request: T): T { - if (!this.dropped.size) return request; + if (!this.dropped.size && !this.hasher) return request; const span = (item: unknown) => - eachIn(eachIn(this.attributesOf(item), "events", this.attributesOf), "links", this.attributesOf); + eachIn( + eachIn(this.recordAttributes(item, isObject(item) && isServerKind(item.kind)), "events", this.attributesOf), + "links", + this.attributesOf, + ); return eachIn(request, "resourceSpans", (rs) => eachIn(this.resourceOf(rs), "scopeSpans", (ss) => eachIn(ss, "spans", span)), ) as T; @@ -134,10 +163,10 @@ export class IdentifyingAttributeFilter { /** A logs request without identifying attributes (resource, record attributes, the keys of a structured body). */ logs(request: T): T { - if (!this.dropped.size) return request; + if (!this.dropped.size && !this.hasher) return request; const body = (value: unknown) => withField(value, "kvlistValue", (kv) => withField(kv, "values", this.list)); - const record = (item: unknown) => withField(this.attributesOf(item), "body", body); + const record = (item: unknown) => withField(this.recordAttributes(item, false), "body", body); return eachIn(request, "resourceLogs", (rl) => eachIn(this.resourceOf(rl), "scopeLogs", (sl) => eachIn(sl, "logRecords", record)), ) as T; @@ -149,3 +178,87 @@ export class IdentifyingAttributeFilter { return eachIn(request, "resourceMetrics", this.resourceOf) as T; } } + +// --------------------------------------------------------------------------- +// Hashed client IP + +export const CLIENT_IP_HASH_ATTRIBUTE = "client.address_hash"; +export const CLIENT_IP_HASH_LENGTH = 32; +export const CLIENT_IP_SALT_MIN_LENGTH = 16; +/** First valid wins; the peer keys only on server spans (on a client span the peer is the server called). */ +export const CLIENT_IP_SOURCE_KEYS: readonly string[] = ["client.address", "http.client_ip", "net.peer.ip", "network.peer.address", "net.sock.peer.addr"]; +const CLIENT_IP_CLIENT_KEYS: readonly string[] = ["client.address", "http.client_ip"]; + +const IPV4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; +const MAPPED_V4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/; + +/** Canonical IP text or null (same rules as runtime-core normalizeClientIp and redact.py). */ +export function normalizeClientIp(raw: unknown): string | null { + if (typeof raw !== "string") return null; + let value = raw.split(",")[0]!.trim(); + if (!value || value.length > 100) return null; + const bracketed = /^\[([^\]]+)\](?::\d{1,5})?$/.exec(value); + if (bracketed) value = bracketed[1]!; + else if (/^[\d.]+:\d{1,5}$/.test(value)) value = value.slice(0, value.lastIndexOf(":")); + const zone = value.indexOf("%"); + if (zone >= 0) value = value.slice(0, zone); + value = value.toLowerCase(); + if (IPV4.test(value)) return value; + if (!value.includes(":") || !/^[0-9a-f:.]+$/.test(value)) return null; + let canonical: string; + try { + canonical = new URL(`http://[${value}]/`).hostname.slice(1, -1); + } catch { + return null; + } + const mapped = MAPPED_V4.exec(canonical); + if (mapped) { + const high = Number.parseInt(mapped[1]!, 16); + const low = Number.parseInt(mapped[2]!, 16); + return `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`; + } + return canonical; +} + +/** HMAC-SHA256(salt, ip), hex, first CLIENT_IP_HASH_LENGTH characters. */ +export function hashClientIp(salt: string, ip: string): string { + return createHmac("sha256", salt).update(ip).digest("hex").slice(0, CLIENT_IP_HASH_LENGTH); +} + +export type ClientIpHasher = (ip: string) => string; + +/** + * The hasher for AUTTER_HASH_CLIENT_IP / AUTTER_CLIENT_IP_SALT, or null: + * off, or on without a salt of CLIENT_IP_SALT_MIN_LENGTH characters (never + * an unsalted hash; index.ts warns at startup). The salt stays in the + * closure. + */ +export function clientIpHasher(enabled: boolean | undefined, salt: string | null | undefined): ClientIpHasher | null { + if (!enabled || typeof salt !== "string" || salt.length < CLIENT_IP_SALT_MIN_LENGTH) return null; + return (ip) => hashClientIp(salt, ip); +} + +/** The string value of `key` in an OTLP key/value list. */ +function stringAttribute(list: unknown, key: string): string | undefined { + if (!Array.isArray(list)) return undefined; + for (const entry of list) { + if (isObject(entry) && entry.key === key && isObject(entry.value) && typeof entry.value.stringValue === "string") + return entry.value.stringValue; + } + return undefined; +} + +const hasKey = (list: unknown, key: string) => + Array.isArray(list) && list.some((entry) => isObject(entry) && entry.key === key); + +/** The normalised client address in an OTLP attribute list (peer keys only on server spans). */ +export function clientIpOfList(list: unknown, serverSpan: boolean): string | null { + for (const key of serverSpan ? CLIENT_IP_SOURCE_KEYS : CLIENT_IP_CLIENT_KEYS) { + const ip = normalizeClientIp(stringAttribute(list, key)); + if (ip) return ip; + } + return null; +} + +/** OTLP SERVER span kind, as a number or the enum name. */ +export const isServerKind = (kind: unknown) => kind === 2 || kind === "SPAN_KIND_SERVER"; diff --git a/packages/otlp-ingester/src/index.ts b/packages/otlp-ingester/src/index.ts index 14052d5..881aeb1 100644 --- a/packages/otlp-ingester/src/index.ts +++ b/packages/otlp-ingester/src/index.ts @@ -1,3 +1,4 @@ +import { CLIENT_IP_SALT_MIN_LENGTH, clientIpHasher } from "./identifying.js"; import { limitsWarning, loadConfig } from "./config.js"; import { createIngesterApp } from "./server.js"; import { ingesterVersion } from "./sdk-versions.js"; @@ -16,6 +17,14 @@ if (config.keepIdentifyingAttributes || config.dropAttributes?.length) { `Identifying attributes: kept ${kept} (AUTTER_KEEP_IDENTIFYING_ATTRIBUTES); also dropped: ${config.dropAttributes?.join(", ") || "none"} (AUTTER_DROP_ATTRIBUTES)`, ); } +if (config.hashClientIp) { + // Never the salt itself: only whether a usable one is set. + if (clientIpHasher(config.hashClientIp, config.clientIpSalt)) console.log("Client IP hashing: on (client.address_hash)"); + else + console.warn( + `WARNING: AUTTER_HASH_CLIENT_IP is on but AUTTER_CLIENT_IP_SALT is not set or shorter than ${CLIENT_IP_SALT_MIN_LENGTH} characters: client IP addresses are dropped and not hashed.`, + ); +} const sharingWarning = limitsWarning(config); if (sharingWarning) console.warn(`WARNING: ${sharingWarning}`); diff --git a/packages/otlp-ingester/src/server.ts b/packages/otlp-ingester/src/server.ts index 599c3c5..e15a9be 100644 --- a/packages/otlp-ingester/src/server.ts +++ b/packages/otlp-ingester/src/server.ts @@ -84,7 +84,7 @@ import type { SinkDeadLetterRow } from "./sink-dead-letters.js"; import { LoopCollapser } from "./loop-collapse.js"; import { applySuppression } from "./suppression.js"; import { configureRedaction, scrubOccurrence } from "./redact.js"; -import { IdentifyingAttributeFilter, identifyingKeysToDrop } from "./identifying.js"; +import { IdentifyingAttributeFilter, clientIpHasher, identifyingKeysToDrop } from "./identifying.js"; import { configureLlmPricing } from "./llm-pricing.js"; import { buildCompatReport, @@ -224,6 +224,7 @@ export function createIngesterApp(config: IngesterConfig, deps: IngesterDeps = { // are dropped before anything below reads, stores or forwards a request. const identifying = new IdentifyingAttributeFilter( identifyingKeysToDrop(config.keepIdentifyingAttributes, config.dropAttributes), + clientIpHasher(config.hashClientIp, config.clientIpSalt), ); // LLM prices: the built-in table, with a self-hosted override file // (AUTTER_LLM_PRICES_FILE, validated by loadConfig) merged over it. diff --git a/packages/runtime-core/src/client-ip.ts b/packages/runtime-core/src/client-ip.ts new file mode 100644 index 0000000..b80acbf --- /dev/null +++ b/packages/runtime-core/src/client-ip.ts @@ -0,0 +1,110 @@ +/** + * Hashed client IP (opt-in): the client's address is dropped with the other + * identifying attributes (identifying.ts), and with `hashClientIp` one + * attribute, `client.address_hash`, carries a keyed hash of it so distinct + * clients can still be counted. + * + * The hash is HMAC-SHA256(salt, normalized IP), hex, first 32 characters + * (128 bits). Never a plain hash: the IPv4 space is small enough to hash in + * full, so without a secret salt any hash could be reversed. The salt is a + * secret of the project: it is never logged, exported or written to an + * attribute. Without one (or with one shorter than CLIENT_IP_SALT_MIN_LENGTH) + * nothing is hashed. The same function, normalisation and source order run + * in @autter/runtime-node, adapters/python and the ingester; parity is + * checked by test-vectors/redaction.json (`clientIpHash`). + * + * This module holds the parts that need no crypto (runtime-core runs on + * edge runtimes without node:crypto): which attribute to hash and how to + * normalise it. + */ + +/** The attribute carrying the hash. */ +export const CLIENT_IP_HASH_ATTRIBUTE = "client.address_hash"; +/** Hex characters kept from the HMAC-SHA256 digest (128 bits). */ +export const CLIENT_IP_HASH_LENGTH = 32; +/** Shorter salts are refused (nothing is hashed). */ +export const CLIENT_IP_SALT_MIN_LENGTH = 16; +/** `true` turns hashing on (the SDKs and the ingester). */ +export const HASH_CLIENT_IP_ENV = "AUTTER_HASH_CLIENT_IP"; +/** The secret salt (the SDKs and the ingester). */ +export const CLIENT_IP_SALT_ENV = "AUTTER_CLIENT_IP_SALT"; + +/** + * Where the client address is read from, first present and valid wins: + * `client.address` (stable semconv; behind a proxy the instrumentation fills + * it from `Forwarded` / `X-Forwarded-For`), `http.client_ip` (old semconv, + * from `X-Forwarded-For`), then the socket peer (`net.peer.ip`, + * `network.peer.address`, `net.sock.peer.addr`), which behind a proxy is + * the proxy. The peer keys are read on server spans only: on a client span + * the peer is the server being called. + */ +export const CLIENT_IP_SOURCE_KEYS: readonly string[] = Object.freeze([ + "client.address", + "http.client_ip", + "net.peer.ip", + "network.peer.address", + "net.sock.peer.addr", +]); +/** The source keys read on any span or log record (not only server spans). */ +export const CLIENT_IP_CLIENT_KEYS: readonly string[] = Object.freeze(["client.address", "http.client_ip"]); + +const IPV4 = /^(?:(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)\.){3}(?:25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)$/; +const MAPPED_V4 = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/; + +/** + * The canonical text of an IP address, or null when `raw` is not one: the + * first entry of a comma-separated list (`X-Forwarded-For`), port and + * brackets removed (`1.2.3.4:5678`, `[2001:db8::1]:443`), zone id removed + * (`fe80::1%eth0`), IPv6 lower-cased and compressed (RFC 5952), and + * IPv4-mapped IPv6 (`::ffff:1.2.3.4`) turned into IPv4. + */ +export function normalizeClientIp(raw: unknown): string | null { + if (typeof raw !== "string") return null; + let value = raw.split(",")[0]!.trim(); + if (!value || value.length > 100) return null; + const bracketed = /^\[([^\]]+)\](?::\d{1,5})?$/.exec(value); + if (bracketed) value = bracketed[1]!; + else if (/^[\d.]+:\d{1,5}$/.test(value)) value = value.slice(0, value.lastIndexOf(":")); + const zone = value.indexOf("%"); + if (zone >= 0) value = value.slice(0, zone); + value = value.toLowerCase(); + if (IPV4.test(value)) return value; + if (!value.includes(":") || !/^[0-9a-f:.]+$/.test(value)) return null; + let canonical: string; + try { + const host = new URL(`http://[${value}]/`).hostname; + canonical = host.slice(1, -1); + } catch { + return null; + } + const mapped = MAPPED_V4.exec(canonical); + if (mapped) { + const high = Number.parseInt(mapped[1]!, 16); + const low = Number.parseInt(mapped[2]!, 16); + return `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`; + } + return canonical; +} + +/** + * The normalised client address of a span or log record, or null: the + * first CLIENT_IP_SOURCE_KEYS entry holding a valid IP, the peer keys only + * when `serverSpan`. + */ +export function clientIpOf(attributes: Record, serverSpan: boolean): string | null { + for (const key of serverSpan ? CLIENT_IP_SOURCE_KEYS : CLIENT_IP_CLIENT_KEYS) { + const ip = normalizeClientIp(attributes[key]); + if (ip) return ip; + } + return null; +} + +/** A usable salt: a string of at least CLIENT_IP_SALT_MIN_LENGTH characters, else null. */ +export function usableClientIpSalt(salt: unknown): string | null { + return typeof salt === "string" && salt.length >= CLIENT_IP_SALT_MIN_LENGTH ? salt : null; +} + +/** `true` / `1` / `yes` / `on` (AUTTER_HASH_CLIENT_IP). */ +export function envFlag(raw: string | undefined | null): boolean { + return typeof raw === "string" && /^(?:1|true|yes|on)$/i.test(raw.trim()); +} diff --git a/packages/runtime-core/src/index.ts b/packages/runtime-core/src/index.ts index 743ca8a..36a2132 100644 --- a/packages/runtime-core/src/index.ts +++ b/packages/runtime-core/src/index.ts @@ -12,6 +12,19 @@ export { type RedactOptions, type Redactor as AttributeRedactor, } from "./redact.js"; +export { + CLIENT_IP_CLIENT_KEYS, + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SOURCE_KEYS, + HASH_CLIENT_IP_ENV, + clientIpOf, + envFlag, + normalizeClientIp, + usableClientIpSalt, +} from "./client-ip.js"; export { DROP_ATTRIBUTES_ENV, IDENTIFYING_ATTRIBUTE_KEYS, diff --git a/packages/runtime-core/test/client-ip.test.mjs b/packages/runtime-core/test/client-ip.test.mjs new file mode 100644 index 0000000..804bede --- /dev/null +++ b/packages/runtime-core/test/client-ip.test.mjs @@ -0,0 +1,49 @@ +// Hashed client IP, the crypto-free half (runtime-core also runs on edge +// runtimes): which address of a span is hashed and its canonical form. The +// Python adapter and the ingester run the same vectors. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; +import { + CLIENT_IP_HASH_ATTRIBUTE, + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_MIN_LENGTH, + CLIENT_IP_SOURCE_KEYS, + IDENTIFYING_ATTRIBUTE_KEYS, + clientIpOf, + envFlag, + normalizeClientIp, + usableClientIpSalt, +} from "../dist/index.js"; + +const vectors = JSON.parse( + readFileSync(new URL("../../../test-vectors/redaction.json", import.meta.url), "utf8"), +).clientIpHash; + +test("constants match the shared vectors", () => { + assert.equal(CLIENT_IP_HASH_ATTRIBUTE, vectors.attribute); + assert.equal(CLIENT_IP_HASH_LENGTH, vectors.length); + // The hash is not an identifying attribute: it survives the drop. + assert.ok(!IDENTIFYING_ATTRIBUTE_KEYS.includes(CLIENT_IP_HASH_ATTRIBUTE)); + // Every source is itself dropped. + for (const key of CLIENT_IP_SOURCE_KEYS) assert.ok(IDENTIFYING_ATTRIBUTE_KEYS.includes(key), key); +}); + +test("normalizeClientIp: ports, brackets, zone ids, case, IPv4-mapped IPv6, lists", () => { + for (const { input, expect } of vectors.normalize) assert.equal(normalizeClientIp(input), expect, JSON.stringify(input)); + assert.equal(normalizeClientIp(undefined), null); + assert.equal(normalizeClientIp(42), null); +}); + +test("clientIpOf: client.address, then http.client_ip, then the peer on server spans only", () => { + for (const { attributes, server, expect } of vectors.source) + assert.equal(clientIpOf(attributes, server), expect, JSON.stringify({ attributes, server })); +}); + +test("salts shorter than the minimum are refused; env flag", () => { + assert.equal(usableClientIpSalt("x".repeat(CLIENT_IP_SALT_MIN_LENGTH - 1)), null); + assert.equal(usableClientIpSalt("x".repeat(CLIENT_IP_SALT_MIN_LENGTH)), "x".repeat(CLIENT_IP_SALT_MIN_LENGTH)); + assert.equal(usableClientIpSalt(undefined), null); + for (const raw of ["true", "1", "YES", " on "]) assert.equal(envFlag(raw), true, raw); + for (const raw of [undefined, "", "false", "0", "salt"]) assert.equal(envFlag(raw), false, String(raw)); +}); diff --git a/packages/runtime-node/README.md b/packages/runtime-node/README.md index 8d6993e..8b9d191 100644 --- a/packages/runtime-node/README.md +++ b/packages/runtime-node/README.md @@ -655,6 +655,38 @@ stored only when the ingester keeps it too: set ([otlp-ingester README](../otlp-ingester/README.md#identifying-attributes)). `IDENTIFYING_ATTRIBUTE_KEYS` exports the list. +**Hashed client IP (opt-in).** To count distinct clients without storing +their addresses, turn on `hashClientIp` with a secret salt. The client's +address is normalised (port, brackets and zone id removed, IPv6 lower-cased +and compressed, `::ffff:1.2.3.4` read as `1.2.3.4`) and exported only as +`client.address_hash`: HMAC-SHA256 under the salt, the first 32 hex +characters. The raw address is still dropped. + +```ts +initAutterServer({ ..., hashClientIp: { salt: process.env.AUTTER_CLIENT_IP_SALT } }); +// or AUTTER_HASH_CLIENT_IP=true and AUTTER_CLIENT_IP_SALT= +``` + +- The address is the first valid one of `client.address`, `http.client_ip` + (both filled from `X-Forwarded-For` / `Forwarded` behind a proxy), then, + on server spans only, the socket peer `net.peer.ip` / + `network.peer.address` / `net.sock.peer.addr`. Behind a proxy that does + not forward the client address, the peer is the proxy, so every request + hashes to the proxy. On client spans the peer is the server being called + and is never hashed. +- The salt is a secret of the project: at least 16 characters (32 random + ones recommended), the same in every service and instance of the + project, so the same client gives the same hash everywhere. It is never + logged, exported or written to a span. Without a usable salt nothing is + hashed and one warning is printed: there is no unsalted fallback, since + every IPv4 address can be hashed in a few minutes. +- The Python adapter and the ingester compute the same hash from the same + salt. A span that already has `client.address_hash` keeps it. +- A hashed IP is still pseudonymous personal data in some readings of + privacy law (GDPR, for one): whoever holds the salt can test a known + address against it. That is why it is opt-in; keep the salt with your + other secrets and rotate it to unlink old hashes. + ### One SDK per process, however many copies are loaded Bundlers can load this package more than once: Next.js bundles one copy diff --git a/packages/runtime-node/src/client-ip.ts b/packages/runtime-node/src/client-ip.ts new file mode 100644 index 0000000..a24f2e8 --- /dev/null +++ b/packages/runtime-node/src/client-ip.ts @@ -0,0 +1,47 @@ +import { createHmac } from "node:crypto"; +import { + CLIENT_IP_HASH_LENGTH, + CLIENT_IP_SALT_ENV, + CLIENT_IP_SALT_MIN_LENGTH, + HASH_CLIENT_IP_ENV, + envFlag, + usableClientIpSalt, +} from "@autter/runtime-core"; + +/** `hashClientIp`: `true` (salt from AUTTER_CLIENT_IP_SALT) or `{ salt }`. */ +export type HashClientIpOption = boolean | { salt?: string }; + +/** Hashes one normalised IP address (runtime-core normalizeClientIp). */ +export type ClientIpHasher = (ip: string) => string; + +/** HMAC-SHA256(salt, ip), hex, first CLIENT_IP_HASH_LENGTH characters. */ +export function hashClientIp(salt: string, ip: string): string { + return createHmac("sha256", salt).update(ip).digest("hex").slice(0, CLIENT_IP_HASH_LENGTH); +} + +/** + * The hasher for `hashClientIp` (else AUTTER_HASH_CLIENT_IP), or null when + * hashing is off. On without a usable salt (`{ salt }`, else + * AUTTER_CLIENT_IP_SALT; at least CLIENT_IP_SALT_MIN_LENGTH characters) it is + * off too, with one warning: never an unsalted hash. The salt stays in this + * closure; it is never logged or exported. + */ +export function clientIpHasher( + option: HashClientIpOption | undefined, + env: Record = process.env, + warn: (message: string) => void = (message) => console.warn(`[autter-runtime] ${message}`), +): ClientIpHasher | null { + const enabled = option === undefined ? envFlag(env[HASH_CLIENT_IP_ENV]) : option !== false; + if (!enabled) return null; + const salt = usableClientIpSalt( + (typeof option === "object" && option !== null ? option.salt : undefined) ?? env[CLIENT_IP_SALT_ENV], + ); + if (!salt) { + warn( + `hashClientIp is on but no salt of at least ${CLIENT_IP_SALT_MIN_LENGTH} characters is set ` + + `(hashClientIp: { salt } or ${CLIENT_IP_SALT_ENV}): client IP addresses are dropped and not hashed.`, + ); + return null; + } + return (ip) => hashClientIp(salt, ip); +} diff --git a/packages/runtime-node/src/index.ts b/packages/runtime-node/src/index.ts index db0e498..608effa 100644 --- a/packages/runtime-node/src/index.ts +++ b/packages/runtime-node/src/index.ts @@ -48,6 +48,7 @@ export { type KeepIdentifyingAttributes, type RedactOptions, } from "./redact.js"; +export { hashClientIp, type HashClientIpOption } from "./client-ip.js"; export { instrumentLlmClient, type InstrumentLlmOptions, diff --git a/packages/runtime-node/src/redact-exporter.ts b/packages/runtime-node/src/redact-exporter.ts index 7ee3160..b84d08f 100644 --- a/packages/runtime-node/src/redact-exporter.ts +++ b/packages/runtime-node/src/redact-exporter.ts @@ -7,7 +7,14 @@ import { type ResourceMetrics, } from "@opentelemetry/sdk-metrics"; import type { ReadableSpan, SpanExporter } from "@opentelemetry/sdk-trace-base"; -import { makeIdentifyingAttributeFilter, type IdentifyingAttributeFilter } from "@autter/runtime-core"; +import { SpanKind, type Attributes } from "@opentelemetry/api"; +import { + CLIENT_IP_HASH_ATTRIBUTE, + clientIpOf, + makeIdentifyingAttributeFilter, + type IdentifyingAttributeFilter, +} from "@autter/runtime-core"; +import type { ClientIpHasher } from "./client-ip.js"; import type { Redactor } from "./redact.js"; import { makeUrlQueryPolicy, type UrlQueryPolicy } from "./url-query.js"; @@ -32,7 +39,9 @@ import { makeUrlQueryPolicy, type UrlQueryPolicy } from "./url-query.js"; * user agent, host.id and the host's addresses, the command line and its owner: see * runtime-core's identifying.ts) are dropped from span, event and link * attributes and from the resource, unless `keepIdentifyingAttributes` - * keeps them (and `dropAttributes` adds keys to drop). + * keeps them (and `dropAttributes` adds keys to drop). With `hashClientIp`, + * a keyed hash of the client's address is added as `client.address_hash` + * (client-ip.ts) before the raw address is dropped. * * The span is re-wrapped, never mutated (spans are shared with other * processors). `redactAttributes: false` disables the scrubbing but neither @@ -69,16 +78,28 @@ function redactResource( const DEFAULT_QUERY_POLICY = makeUrlQueryPolicy(); const DEFAULT_IDENTIFYING = makeIdentifyingAttributeFilter(); -/** Without scrubbing (`redactAttributes: false`): the query policy and the - * identifying attribute drop only. */ +/** + * `attributes` plus `client.address_hash` from the span's raw attributes, + * when hashing is on, the span names a client address and carries no hash + * already (one set by the app or an earlier hop is left alone). + */ +function withClientIpHash(span: ReadableSpan, attributes: Attributes, hasher: ClientIpHasher | null): Attributes { + if (!hasher || span.attributes[CLIENT_IP_HASH_ATTRIBUTE] !== undefined) return attributes; + const ip = clientIpOf(span.attributes, span.kind === SpanKind.SERVER); + return ip ? { ...attributes, [CLIENT_IP_HASH_ATTRIBUTE]: hasher(ip) } : attributes; +} + +/** Without scrubbing (`redactAttributes: false`): the query policy, the + * identifying attribute drop and the client IP hash only. */ function stripQueries( span: ReadableSpan, redactor: Redactor, query: UrlQueryPolicy, identifying: IdentifyingAttributeFilter, + hasher: ClientIpHasher | null, ): ReadableSpan { const name = query.name(span.name); - const attributes = query.attributes(identifying.apply(span.attributes)); + const attributes = withClientIpHash(span, query.attributes(identifying.apply(span.attributes)), hasher); let eventsChanged = false; const events = span.events.map((event) => { if (!event.attributes) return event; @@ -139,17 +160,18 @@ export function redactSpan( redactor: Redactor, query: UrlQueryPolicy = DEFAULT_QUERY_POLICY, identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, + hasher: ClientIpHasher | null = null, ): ReadableSpan { - if (!redactor.enabled) return stripQueries(span, redactor, query, identifying); + if (!redactor.enabled) return stripQueries(span, redactor, query, identifying, hasher); const status = span.status.message ? { ...span.status, message: redactor.text(span.status.message) } : span.status; // Drop identifying keys, scrub (a `user:pass@` URL is recognised whole), - // then cut queries. + // cut queries, then add the client IP hash (from the raw address). return wrap(span, { name: query.name(redactor.text(span.name)), status, - attributes: query.attributes(redactor.exported(identifying.apply(span.attributes))), + attributes: withClientIpHash(span, query.attributes(redactor.exported(identifying.apply(span.attributes))), hasher), links: span.links.map((link) => link.attributes ? { ...link, attributes: redactor.exported(identifying.apply(link.attributes)) } : link, ), @@ -168,6 +190,7 @@ export class RedactingSpanExporter implements SpanExporter { private readonly redactor: () => Redactor, private readonly query: UrlQueryPolicy = DEFAULT_QUERY_POLICY, private readonly identifying: IdentifyingAttributeFilter = DEFAULT_IDENTIFYING, + private readonly hasher: ClientIpHasher | null = null, ) {} export( @@ -179,7 +202,7 @@ export class RedactingSpanExporter implements SpanExporter { let failed = 0; for (const span of spans) { try { - scrubbed.push(redactSpan(span, redactor, this.query, this.identifying)); + scrubbed.push(redactSpan(span, redactor, this.query, this.identifying, this.hasher)); } catch { // Fail closed: a span we could not scrub is not exported. failed++; diff --git a/packages/runtime-node/src/server.ts b/packages/runtime-node/src/server.ts index 9acd176..1a79f4e 100644 --- a/packages/runtime-node/src/server.ts +++ b/packages/runtime-node/src/server.ts @@ -61,6 +61,7 @@ import { type Redactor, } from "./redact.js"; import { RedactingMetricExporter, RedactingSpanExporter } from "./redact-exporter.js"; +import { clientIpHasher, type HashClientIpOption } from "./client-ip.js"; import { authHeaders, resolveIngestKey, warnKeyKind, warnMissingKey } from "./ingest-auth.js"; import { IngestMetricExporter, IngestSpanExporter } from "./ingest-exporters.js"; import { makeUrlQueryPolicy } from "./url-query.js"; @@ -415,6 +416,24 @@ export interface AutterServerOptions { * environment variable (a comma-separated list), else none. */ dropAttributes?: string[]; + /** + * Count distinct clients without storing their IP addresses. Off by + * default. When on, the client's address (first of `client.address`, + * `http.client_ip`, then on server spans the socket peer `net.peer.ip` / + * `network.peer.address` / `net.sock.peer.addr`, which behind a proxy is + * the proxy) is normalised and exported only as `client.address_hash`: + * HMAC-SHA256 with your secret salt, 32 hex characters. The raw address + * is still dropped. A hashed IP is still pseudonymous personal data in + * some readings of privacy law; treat the salt as a secret and use the + * same one in every service of the project (the Python adapter and the + * ingester compute the same hash from the same salt). + * + * `true` reads the salt from AUTTER_CLIENT_IP_SALT; `{ salt }` passes it. + * A salt shorter than 16 characters, or none, turns hashing off with a + * warning (never an unsalted hash). Default: AUTTER_HASH_CLIENT_IP=true, + * else off. + */ + hashClientIp?: HashClientIpOption; /** * Verbose lifecycle logging to stderr: "exported N spans", flushes and * auto-flush activity — handy while wiring the SDK up. Also enabled by @@ -1305,6 +1324,8 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { options.keepIdentifyingAttributes ?? keepIdentifyingFromEnv(process.env[KEEP_IDENTIFYING_ATTRIBUTES_ENV]), options.dropAttributes ?? dropAttributesFromEnv(process.env[DROP_ATTRIBUTES_ENV]), ); + // Opt-in keyed hash of the client IP (client-ip.ts); null when off. + const ipHasher = clientIpHasher(options.hashClientIp); const slowPerMinute = Number.isFinite(options.retainSlowTracesPerMinute) ? Math.max(0, options.retainSlowTracesPerMinute!) : DEFAULT_SLOW_TRACES_PER_MINUTE; // One policy for every trace exporter: LLM spans keep prompt/completion @@ -1324,6 +1345,7 @@ export function initAutterServer(options: AutterServerOptions): AutterServer { () => redactor, queryPolicy, identifying, + ipHasher, ), llmContent, ), diff --git a/packages/runtime-node/test/client-ip-hash.test.mjs b/packages/runtime-node/test/client-ip-hash.test.mjs new file mode 100644 index 0000000..b34ba67 --- /dev/null +++ b/packages/runtime-node/test/client-ip-hash.test.mjs @@ -0,0 +1,171 @@ +// Opt-in hashed client IP: with hashClientIp the client's address leaves +// the process only as `client.address_hash`, HMAC-SHA256 under the +// project's secret salt, the same value the Python adapter and the ingester +// compute (test-vectors/redaction.json, `clientIpHash`). Off by default; +// on without a salt it hashes nothing and warns once. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { dirname, join } from "node:path"; +import { after, before, test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { hashClientIp } from "../dist/index.js"; +import { normalizeClientIp } from "../../runtime-core/dist/index.js"; + +const here = dirname(fileURLToPath(import.meta.url)); +const VECTORS = JSON.parse(readFileSync(join(here, "../../../test-vectors/redaction.json"), "utf8")); +const vectors = VECTORS.clientIpHash; +const DROP = VECTORS.identifying.drop; +const SALT = "project-secret-salt-7f3a9c2e41"; + +test("hashClientIp matches the shared vectors (normalised first)", () => { + for (const { input, expect } of vectors.hash) assert.equal(hashClientIp(vectors.salt, normalizeClientIp(input)), expect, input); + assert.equal(hashClientIp(vectors.salt, "203.0.113.9").length, vectors.length); + assert.notEqual(hashClientIp(`${vectors.salt}x`, "203.0.113.9"), hashClientIp(vectors.salt, "203.0.113.9")); +}); + +const received = []; +let collector; +let endpoint; + +before(async () => { + collector = createServer((req, res) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + const text = Buffer.concat(chunks).toString("utf8"); + try { + received.push({ path: req.url, text, body: JSON.parse(text) }); + } catch { + /* not JSON */ + } + res.end("{}"); + }); + }); + await new Promise((resolve) => collector.listen(0, "127.0.0.1", resolve)); + endpoint = `http://127.0.0.1:${collector.address().port}`; +}); + +after(async () => { + await new Promise((resolve) => { + collector.close(resolve); + collector.closeAllConnections(); + }); +}); + +const until = async (check, ms = 5000) => { + const start = Date.now(); + while (!check() && Date.now() - start < ms) await new Promise((r) => setTimeout(r, 10)); + return check(); +}; +const value = (v) => v?.stringValue ?? v?.intValue ?? v?.boolValue ?? v?.doubleValue; +const attrs = (list) => Object.fromEntries((list ?? []).map((a) => [a.key, value(a.value)])); + +function spansOf(service) { + const out = []; + for (const { body } of received) + for (const rs of body.resourceSpans ?? []) { + if (attrs(rs.resource?.attributes)["service.name"] !== service) continue; + for (const ss of rs.scopeSpans ?? []) + for (const span of ss.spans ?? []) out.push({ name: span.name, kind: span.kind, attributes: attrs(span.attributes) }); + } + return out; +} + +/** One request through HttpInstrumentation (X-Forwarded-For: 203.0.113.9, + * socket peer 127.0.0.1) and its client span; a manual span with + * client.address; a manual span already carrying a hash. */ +async function run(service, options = {}, env = {}) { + const script = ` + import { createRequire } from "node:module"; + import { trace } from "@opentelemetry/api"; + import { initAutterServer } from "./dist/index.js"; + const sdk = initAutterServer({ + apiKey: "k", service: process.env.SERVICE, endpoint: process.env.ENDPOINT, traceSampleRate: 1, + captureGlobalErrors: false, autoFlush: false, memoryMetrics: false, compatCheck: false, + metricIntervalMs: 3_600_000, logging: { console: false }, + ...JSON.parse(process.env.OPTIONS), + }); + const http = createRequire(import.meta.url)("http"); + const server = http.createServer((req, res) => res.end("ok")); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + await new Promise((resolve, reject) => { + http.get({ host: "127.0.0.1", port: server.address().port, path: "/hello", + headers: { "x-forwarded-for": "203.0.113.9, 10.0.0.1" } }, (res) => { res.resume(); res.on("end", resolve); }).on("error", reject); + }); + await new Promise((resolve) => server.close(resolve)); + const tracer = trace.getTracer("t"); + tracer.startSpan("manual", { attributes: { "client.address": "::ffff:198.51.100.7" } }).end(); + tracer.startSpan("prehashed", { attributes: { "client.address": "198.51.100.8", "client.address_hash": "set-upstream" } }).end(); + await sdk.shutdown(); + `; + const childEnv = { ...process.env, ENDPOINT: endpoint, SERVICE: service, OPTIONS: JSON.stringify(options), ...env }; + for (const name of ["AUTTER_HASH_CLIENT_IP", "AUTTER_CLIENT_IP_SALT", "AUTTER_KEEP_IDENTIFYING_ATTRIBUTES", "AUTTER_DROP_ATTRIBUTES"]) + if (!(name in env)) delete childEnv[name]; + const child = spawn(process.execPath, ["--input-type=module", "-e", script], { + cwd: join(here, ".."), + env: childEnv, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr.on("data", (chunk) => (stderr += chunk)); + const code = await new Promise((resolve) => child.on("exit", resolve)); + assert.equal(code, 0, stderr); + await until(() => spansOf(service).some((span) => span.name === "prehashed")); + const spans = spansOf(service); + const server = spans.find((span) => span.kind === 2); + assert.ok(server, "HTTP server span exported"); + return { + spans, + server, + client: spans.find((span) => span.kind === 3), + manual: spans.find((span) => span.name === "manual"), + prehashed: spans.find((span) => span.name === "prehashed"), + stderr, + }; +} + +function assertNoRawAddress(out) { + for (const span of out.spans) for (const key of DROP) assert.ok(!(key in span.attributes), `${key} on ${span.name}`); +} + +test("off by default: the address is dropped and no hash is added", async () => { + const out = await run("ip-hash-off"); + assertNoRawAddress(out); + assert.equal(out.server.attributes["client.address_hash"], undefined); + assert.equal(out.manual.attributes["client.address_hash"], undefined); + assert.equal(out.prehashed.attributes["client.address_hash"], "set-upstream"); +}); + +test("hashClientIp: { salt } adds client.address_hash from the forwarded client address", async () => { + const out = await run("ip-hash-option", { hashClientIp: { salt: SALT } }); + assertNoRawAddress(out); + // X-Forwarded-For's first entry (http.client_ip) wins over the socket peer. + assert.equal(out.server.attributes["client.address_hash"], hashClientIp(SALT, "203.0.113.9")); + // IPv4-mapped IPv6 is hashed as the IPv4 address. + assert.equal(out.manual.attributes["client.address_hash"], hashClientIp(SALT, "198.51.100.7")); + // An existing hash is left alone, never hashed again. + assert.equal(out.prehashed.attributes["client.address_hash"], "set-upstream"); + // On a client span the peer is the server being called: no hash. + assert.ok(out.client, "HTTP client span exported"); + assert.equal(out.client.attributes["client.address_hash"], undefined); + // The salt is never exported or printed. + for (const { text } of received) assert.ok(!text.includes(SALT), "salt on the wire"); + assert.ok(!out.stderr.includes(SALT), "salt printed"); +}); + +test("AUTTER_HASH_CLIENT_IP + AUTTER_CLIENT_IP_SALT, also with redactAttributes: false", async () => { + const out = await run("ip-hash-env", { redactAttributes: false }, { AUTTER_HASH_CLIENT_IP: "true", AUTTER_CLIENT_IP_SALT: SALT }); + assertNoRawAddress(out); + assert.equal(out.server.attributes["client.address_hash"], hashClientIp(SALT, "203.0.113.9")); + assert.equal(out.manual.attributes["client.address_hash"], hashClientIp(SALT, "198.51.100.7")); +}); + +test("on without a usable salt: nothing is hashed (never unsalted) and one warning is printed", async () => { + const out = await run("ip-hash-no-salt", { hashClientIp: true }, { AUTTER_CLIENT_IP_SALT: "short" }); + assertNoRawAddress(out); + for (const span of out.spans) if (span.name !== "prehashed") assert.equal(span.attributes["client.address_hash"], undefined, span.name); + assert.equal(out.stderr.match(/hashClientIp is on but no salt/g)?.length, 1, out.stderr); + assert.ok(!out.stderr.includes("short"), "salt printed"); +}); diff --git a/test-vectors/redaction.json b/test-vectors/redaction.json index 29921ba..1c1ce87 100644 --- a/test-vectors/redaction.json +++ b/test-vectors/redaction.json @@ -244,5 +244,192 @@ "http.route", "http.method" ] + }, + "clientIpHash": { + "$comment": "Opt-in hashed client IP (client.address_hash): HMAC-SHA256(salt, normalized IP) as hex, first `length` characters. `normalize` maps raw values to the canonical IP (null: not an IP, nothing hashed); `hash` maps raw values to the hash under `salt`; `source` says which address of a span or log record is hashed (`server`: an HTTP server span, where the socket peer may be read). Runtime-core/runtime-node, adapters/python and the otlp-ingester must agree on all three.", + "attribute": "client.address_hash", + "salt": "autter-test-salt-0123456789", + "length": 32, + "normalize": [ + { + "input": "203.0.113.9", + "expect": "203.0.113.9" + }, + { + "input": " 203.0.113.9 ", + "expect": "203.0.113.9" + }, + { + "input": "203.0.113.9:51234", + "expect": "203.0.113.9" + }, + { + "input": "203.0.113.9, 10.0.0.1", + "expect": "203.0.113.9" + }, + { + "input": "::ffff:203.0.113.9", + "expect": "203.0.113.9" + }, + { + "input": "::FFFF:CB00:7109", + "expect": "203.0.113.9" + }, + { + "input": "[::ffff:203.0.113.9]:443", + "expect": "203.0.113.9" + }, + { + "input": "2001:DB8:0:0::1", + "expect": "2001:db8::1" + }, + { + "input": "[2001:db8::1]:8080", + "expect": "2001:db8::1" + }, + { + "input": "fe80::1%eth0", + "expect": "fe80::1" + }, + { + "input": "2001:0db8:0000:0000:0000:0000:0000:0001", + "expect": "2001:db8::1" + }, + { + "input": "1:0:0:2:0:0:0:3", + "expect": "1:0:0:2::3" + }, + { + "input": "unknown", + "expect": null + }, + { + "input": "", + "expect": null + }, + { + "input": "999.1.1.1", + "expect": null + }, + { + "input": "01.2.3.4", + "expect": null + }, + { + "input": "example.com", + "expect": null + }, + { + "input": "1.2.3", + "expect": null + }, + { + "input": "::ffff:zz", + "expect": null + } + ], + "hash": [ + { + "input": "203.0.113.9", + "expect": "ca02c4212f9f108bfc1f1c799ddac123" + }, + { + "input": "::ffff:203.0.113.9", + "expect": "ca02c4212f9f108bfc1f1c799ddac123" + }, + { + "input": "203.0.113.9:443", + "expect": "ca02c4212f9f108bfc1f1c799ddac123" + }, + { + "input": "2001:db8::1", + "expect": "7f8f52c731e08ed237dc17921591cbe0" + }, + { + "input": "[2001:DB8::1]:443", + "expect": "7f8f52c731e08ed237dc17921591cbe0" + }, + { + "input": "10.0.0.2", + "expect": "8397b1a98831ffa12e2b88a4aa7e25bc" + } + ], + "source": [ + { + "attributes": { + "client.address": "203.0.113.9", + "net.peer.ip": "10.0.0.2" + }, + "server": true, + "expect": "203.0.113.9" + }, + { + "attributes": { + "http.client_ip": "198.51.100.1", + "net.peer.ip": "10.0.0.2" + }, + "server": true, + "expect": "198.51.100.1" + }, + { + "attributes": { + "client.address": "203.0.113.9", + "http.client_ip": "198.51.100.1" + }, + "server": true, + "expect": "203.0.113.9" + }, + { + "attributes": { + "net.peer.ip": "10.0.0.2", + "network.peer.address": "10.0.0.3" + }, + "server": true, + "expect": "10.0.0.2" + }, + { + "attributes": { + "network.peer.address": "::ffff:10.0.0.3" + }, + "server": true, + "expect": "10.0.0.3" + }, + { + "attributes": { + "net.sock.peer.addr": "10.0.0.5" + }, + "server": true, + "expect": "10.0.0.5" + }, + { + "attributes": { + "net.peer.ip": "10.0.0.2" + }, + "server": false, + "expect": null + }, + { + "attributes": { + "client.address": "unknown", + "network.peer.address": "10.0.0.3" + }, + "server": true, + "expect": "10.0.0.3" + }, + { + "attributes": { + "client.address": "10.0.0.4" + }, + "server": false, + "expect": "10.0.0.4" + }, + { + "attributes": { + "http.route": "/x" + }, + "server": true, + "expect": null + } + ] } }