diff --git a/authbridge/authlib/go.mod b/authbridge/authlib/go.mod index 191d9a8f..ae443262 100644 --- a/authbridge/authlib/go.mod +++ b/authbridge/authlib/go.mod @@ -14,6 +14,10 @@ require ( github.com/spiffe/go-spiffe/v2 v2.8.1 github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 + go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 @@ -47,6 +51,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect @@ -58,6 +63,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/go-clone v1.7.3 // indirect github.com/huandu/go-sqlbuilder v1.42.1 // indirect @@ -109,10 +115,9 @@ require ( github.com/yashtewari/glob-intersection v0.2.0 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -120,6 +125,7 @@ require ( golang.org/x/crypto v0.55.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/ini.v1 v1.67.3 // indirect oras.land/oras-go/v2 v2.6.2 // indirect diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go new file mode 100644 index 00000000..8b32992e --- /dev/null +++ b/authbridge/authlib/plugins/lineage/config.go @@ -0,0 +1,308 @@ +package lineage + +import ( + "bytes" + "encoding/json" + "fmt" + "net/url" + "path" + "strings" +) + +// defaultOTelEndpoint is the OTLP gRPC target used when otel_endpoint is unset: +// an in-pod collector reached over plaintext loopback. +const defaultOTelEndpoint = "localhost:4317" + +// defaultMaxPayloadBytes bounds a captured input.value / output.value as a +// deliberate producer-side cap. It is NOT a mirror of any SDK limit: the OTel +// SDK's default attribute-value length limit is unlimited (-1) and Init sets no +// SpanLimits, so an oversized value is not dropped or truncated downstream. This +// bound is our own guard against unbounded spans (and against any backend value +// limit); anything longer is cut here with an explicit marker so the loss is +// visible in the span. 4096 is a conservative default, not a hard requirement. +const defaultMaxPayloadBytes = 4096 + +// unboundedPayload is the sole opt-out from the cap. Any other negative is +// refused at decode, so a typo cannot quietly attach whole payloads. +const unboundedPayload = -1 + +// defaultMaxAttrBytes bounds every variable-content string attribute and the +// span name. Several of those values are caller-controlled (url.path, Host, +// mcp.tool from the request body, a2a.session_id) and the SDK never truncates +// on its own — without this cap one request can put a 100 KB span name into +// the backend. 256 comfortably fits real paths, hosts and tool names. +const defaultMaxAttrBytes = 256 + +// Config holds the per-plugin configuration decoded from the pipeline YAML. +type Config struct { + // OTelEndpoint is the OTLP gRPC endpoint (host:port, http://host:port, or + // https://host:port). An https:// scheme implies OTelTLS=true. Any other + // URL scheme is rejected at decode (see decodeConfig). + // Default: "localhost:4317" + OTelEndpoint string `json:"otel_endpoint" description:"OTLP gRPC target: host:port, http://host:port or https://host:port; any other scheme is refused." default:"localhost:4317"` + + // OTelTLS selects the OTLP transport. False (the default) dials plaintext, + // which is correct for the in-pod loopback collector but sends spans — + // including lineage.principal.* on every inbound request, and full payloads + // under CaptureIO — in cleartext. Set true for any collector off-pod: it + // dials with TLS and verifies the collector against the system root CAs, + // or against OTelCAFile when set. An https:// otel_endpoint turns this on + // automatically; a bare host:port with otel_tls:true is honoured (TLS to + // that host:port). Two combinations are refused at decode as + // contradictions rather than silently resolved: an https:// endpoint with + // an explicit otel_tls:false, and an http:// endpoint with otel_tls:true + // or OTelCAFile — the scheme states a transport intent, and the knobs + // must agree with it. + OTelTLS bool `json:"otel_tls" description:"Dial the collector with TLS, verified against the system roots or otel_ca_file; an https:// endpoint implies it." default:"false"` + + // OTelCAFile is a PEM bundle of CA certificates to verify the collector's + // serving certificate against, for a collector whose certificate is not + // signed by a system root — an in-cluster collector with a cert-manager + // issued certificate, typically (mount its TLS Secret's ca.crt). Setting it + // implies OTelTLS=true; an explicit otel_tls:false alongside it, or an + // http:// endpoint, is a refused contradiction. Read at Init into the cert + // pool the dial verifies against: an unreadable file, or one with no + // certificate in it, refuses to start rather than falling back to the + // system roots. Empty (the default) verifies against the system roots. + OTelCAFile string `json:"otel_ca_file" description:"PEM bundle to verify the collector certificate against (a private CA); implies otel_tls."` + + // CaptureIO when true attaches parsed request/response content as + // input.value (request span) and output.value (response span) + // attributes, enabling Phoenix to display message content inline. + // + // For A2A (inbound agent calls): input = user message parts, output = artifact. + // For MCP tools/call: input = tool params JSON, output = tool result JSON. + // For Inference (LLM): input = messages array JSON, output = completion text. + // + // Off by default — enable only if traces do not contain PII or the + // OTel backend enforces appropriate access controls. + CaptureIO bool `json:"capture_io" description:"Attach parsed request/response content as input.value / output.value." default:"false"` + + // MaxPayloadBytes caps the size of the input.value / output.value + // attributes attached under CaptureIO. A payload longer than this is cut on + // a UTF-8 boundary and suffixed with a truncation marker, so the loss is + // explicit in the span. This is a deliberate producer-side bound; the OTel + // SDK does not itself drop or truncate an oversized value (its default + // attribute-value limit is unlimited and Init sets no SpanLimits), so + // without this cap the whole payload would be emitted. Zero (or unset) uses + // defaultMaxPayloadBytes; unboundedPayload (-1) attaches the whole value, + // and any other negative is refused at decode. + // Ignored when CaptureIO is false. + // Default: 4096 + MaxPayloadBytes int `json:"max_payload_bytes" description:"Byte cap on input.value / output.value; 0 or unset takes the default, -1 attaches whole values." default:"4096"` + + // MaxAttrBytes caps every variable-content string attribute (url.path, + // lineage.peer.host, mcp.tool, a2a.session_id, …) and the span name, cut + // on a UTF-8 boundary with the same truncation marker as payloads. + // input.value / output.value keep their own MaxPayloadBytes cap, and + // fixed-vocabulary facts (lineage.role, lineage.outcome, …) are bounded by + // construction. Zero (or unset) uses defaultMaxAttrBytes; -1 removes the + // cap, and any other negative is refused at decode. + // Default: 256 + MaxAttrBytes int `json:"max_attr_bytes" description:"Byte cap on every variable-content string attribute and the span name; 0 or unset takes the default, -1 removes the cap." default:"256"` + + // MintTraceparent — both directions — forwards a W3C traceparent naming + // this exchange's request span when the request arrived with no + // valid traceparent. Without one the next element has nothing to + // extract: an app's propagate-only shim roots a fresh trace of its own, + // and the tracestate stamp (which W3C reads only alongside a valid + // traceparent) never leaves this pod — so the entry exchange lands alone + // in its own trace and every call it caused derives as a parentless root. + // Absent, empty and malformed traceparents are all restarted, which is + // W3C's processing model for an unparseable one; a valid traceparent is + // never modified. This is the one place the plugin writes a traceparent. + // Set false for a pure observer that must not add a header the + // application would see (the exchange then fragments, visibly). + // Default: true + MintTraceparent bool `json:"mint_traceparent" description:"Forward a traceparent naming this request span when no valid one arrived; false = a pure observer that writes no traceparent." default:"true"` + + // BypassPaths lists URL path globs that should not generate lineage + // hops. Useful for suppressing infrastructure polling (agent-card + // discovery, health checks) that would otherwise flood the lineage graph. + // Matched by the shared bypass package (path.Match, query stripped, path + // normalized) — the same package and semantics jwt-validation and sparc + // use for this key, so a pattern copied between plugins means the same + // thing. Note path.Match's "*" does not cross "/": "/.well-known/*" + // matches "/.well-known/agent.json" but not "/.well-known/a/b". An + // earlier prefix match here silently bypassed real traffic under the + // "/health" default ("/health-records/..."). + // + // Setting the key REPLACES this list rather than extending it, the same + // convention ibac, sparc and cpex use for their bypass keys: an operator + // who adds one glob must restate the defaults they want to keep. + // Entries are trimmed of surrounding whitespace; bypass.NewMatcher + // refuses invalid path.Match syntax and match-everything patterns + // (empty, "*", "/*") at boot. + // Default: ["/.well-known/*", "/healthz", "/readyz", "/health"] + BypassPaths []string `json:"bypass_paths" description:"URL path globs (path.Match) that produce no spans; setting the key replaces the default list." default:"/.well-known/*, /healthz, /readyz, /health"` + + // BypassHosts lists host globs whose exchanges should not generate lineage + // hops. Useful for suppressing infrastructure outbound calls such as OTel + // trace exports. Matched with path.Match against the request Host with the + // port stripped and case folded — see matchesAnyHost — so "otel-collector" + // matches only that exact name and "otel-collector.*" matches + // otel-collector.rossoctl-system.svc. This is the glob convention ibac, + // sparc and cpex already use for the key of the same name; the defaults + // carry both forms because in-cluster short-name calls are ordinary. + // + // Honoured on the outbound phase only: an inbound Host is the caller's own + // header, and a bypass driven by it would be an opt-out from being graphed. + // + // Setting the key REPLACES this list rather than extending it, as with + // BypassPaths. Entries are trimmed of surrounding whitespace; one that is + // empty, "*", or not valid path.Match syntax is refused at decode. + // Default: ["otel-collector", "otel-collector.*", "jaeger", "jaeger.*", + // "zipkin", "zipkin.*", "prometheus", "prometheus.*"] + BypassHosts []string `json:"bypass_hosts" description:"Outbound host globs (path.Match, port stripped, case folded) that produce no spans; ignored inbound; replaces the default list." default:"otel-collector, otel-collector.*, jaeger, jaeger.*, zipkin, zipkin.*, prometheus, prometheus.*"` + + // SelfID is the agent's own stable identifier, emitted as the + // lineage.self.id fact on every span. Typically the Keycloak client ID + // of this workload. If empty, SelfIDFile is consulted instead. A value + // containing "/" (a SPIFFE ID) is reduced to its last non-empty path + // segment before emission — see serviceLabel — so two identities that + // differ only above that segment emit the same lineage.self.id. + SelfID string `json:"self_id" description:"This workload identity, emitted as lineage.self.id; a SPIFFE ID is reduced to its last path segment."` + + // SelfIDFile is the path to a file containing the agent's own client ID. + // Defaults to /shared/client-id.txt (the operator-mounted credential). + // Ignored when SelfID is set. + SelfIDFile string `json:"self_id_file" description:"Read when self_id is empty; the plugin refuses to start if neither yields an identity." default:"/shared/client-id.txt"` +} + +func defaultConfig() Config { + return Config{ + OTelEndpoint: defaultOTelEndpoint, + MaxPayloadBytes: defaultMaxPayloadBytes, + MaxAttrBytes: defaultMaxAttrBytes, + MintTraceparent: true, + BypassPaths: []string{"/.well-known/*", "/healthz", "/readyz", "/health"}, + BypassHosts: []string{ + "otel-collector", "otel-collector.*", + "jaeger", "jaeger.*", + "zipkin", "zipkin.*", + "prometheus", "prometheus.*", + }, + SelfIDFile: "/shared/client-id.txt", + } +} + +func decodeConfig(raw json.RawMessage) (Config, error) { + cfg := defaultConfig() + if len(raw) == 0 { + return cfg, nil + } + // Unknown keys are a boot error: a typo'd knob (capture-io, selfid_file) + // must not silently run with defaults. + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("lineage-telemetry config: %w", err) + } + if cfg.OTelEndpoint == "" { + cfg.OTelEndpoint = defaultOTelEndpoint + } + // Zero means "unset" → the safe default; a negative value is the explicit + // opt-out (no cap). This keeps an omitted key and an explicit 0 identical. + if cfg.MaxPayloadBytes == 0 { + cfg.MaxPayloadBytes = defaultMaxPayloadBytes + } + if cfg.MaxPayloadBytes < unboundedPayload { + return Config{}, fmt.Errorf("lineage-telemetry config: max_payload_bytes %d is invalid; use -1 to attach whole values or a positive byte cap", cfg.MaxPayloadBytes) + } + // Same convention as max_payload_bytes: 0 = the default, -1 = no cap. + if cfg.MaxAttrBytes == 0 { + cfg.MaxAttrBytes = defaultMaxAttrBytes + } + if cfg.MaxAttrBytes < unboundedPayload { + return Config{}, fmt.Errorf("lineage-telemetry config: max_attr_bytes %d is invalid; use -1 to remove the cap or a positive byte cap", cfg.MaxAttrBytes) + } + // gRPC NewClient expects host:port only, so reduce a URL form (e.g. + // http://collector:4317/v1/traces) to its host — TrimPrefix left any path + // behind and produced an invalid dial target. A URL scheme also carries an + // intent about transport: https:// asks for TLS. Honour it (or fail on a + // contradiction) rather than silently dropping to cleartext. + if strings.Contains(cfg.OTelEndpoint, "://") { + u, err := url.Parse(cfg.OTelEndpoint) + if err != nil || u.Host == "" { + return Config{}, fmt.Errorf("lineage-telemetry config: invalid otel_endpoint %q", cfg.OTelEndpoint) + } + // Only http/https carry a meaningful OTLP transport intent. Reject any + // other scheme (ftp://, ftps://, …) rather than strip it and dial the + // bare host:port insecurely — that would silently send principal facts + // and payloads in cleartext. Fail closed, matching this package's + // DisallowUnknownFields / https+otel_tls:false posture. + if u.Scheme != "http" && u.Scheme != "https" { + return Config{}, fmt.Errorf("lineage-telemetry config: unsupported otel_endpoint scheme %q (want http or https)", u.Scheme) + } + if u.Scheme == "https" { + // An explicit otel_tls:false alongside an https:// endpoint is a + // contradiction: one asks for encryption, the other for cleartext. + // Fail closed rather than pick one, consistent with the + // DisallowUnknownFields fail-on-ambiguity choice this package makes. + if tlsExplicitlyFalse(raw) { + return Config{}, fmt.Errorf("lineage-telemetry config: otel_endpoint %q is https but otel_tls is false", cfg.OTelEndpoint) + } + cfg.OTelTLS = true + } + // The mirror image: an http:// endpoint asks for cleartext, so a TLS + // knob beside it is the same contradiction the other way round. + if u.Scheme == "http" && (cfg.OTelTLS || cfg.OTelCAFile != "") { + return Config{}, fmt.Errorf("lineage-telemetry config: otel_endpoint %q is http but otel_tls or otel_ca_file asks for TLS", cfg.OTelEndpoint) + } + cfg.OTelEndpoint = u.Host + } + // A CA file is only meaningful for a TLS dial: it implies otel_tls, and + // pairing it with an explicit otel_tls:false is the same contradiction as + // https:// + otel_tls:false above. + if cfg.OTelCAFile != "" { + if tlsExplicitlyFalse(raw) { + return Config{}, fmt.Errorf("lineage-telemetry config: otel_ca_file is set but otel_tls is false") + } + cfg.OTelTLS = true + } + if err := validateBypass(&cfg); err != nil { + return Config{}, fmt.Errorf("lineage-telemetry config: %w", err) + } + return cfg, nil +} + +// validateBypass trims and checks the bypass lists in place. An entry that +// matches everything disables the plugin silently — every exchange takes the +// skip, no span is ever emitted, and Ready() still reports true — so it is a +// boot error rather than a runtime surprise. Paths are only trimmed here: +// their validation (invalid path.Match syntax, match-everything patterns) +// lives in bypass.NewMatcher, called at Configure. Hosts are checked here, +// mirroring ibac / sparc / cpex, including the advice to remove the plugin +// from the pipeline if disabling it is what was meant. +func validateBypass(cfg *Config) error { + for i, entry := range cfg.BypassPaths { + cfg.BypassPaths[i] = strings.TrimSpace(entry) + } + for i, entry := range cfg.BypassHosts { + entry = strings.TrimSpace(entry) + if _, err := path.Match(entry, ""); err != nil { + return fmt.Errorf("invalid bypass_hosts glob %q: %w", cfg.BypassHosts[i], err) + } + if entry == "" || entry == "*" { + return fmt.Errorf("bypass_hosts entry %q matches every host; "+ + "to disable lineage-telemetry, remove it from the pipeline instead", cfg.BypassHosts[i]) + } + cfg.BypassHosts[i] = entry + } + return nil +} + +// tlsExplicitlyFalse reports whether the raw config carries otel_tls set to a +// literal false, as opposed to being absent (whose decoded value is also false +// but carries no intent). Used only to reject the https:// + otel_tls:false +// contradiction; a decode failure here is treated as "not explicitly false" +// since the DisallowUnknownFields pass above already validated the shape. +func tlsExplicitlyFalse(raw json.RawMessage) bool { + var probe struct { + OTelTLS *bool `json:"otel_tls"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return false + } + return probe.OTelTLS != nil && !*probe.OTelTLS +} diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go new file mode 100644 index 00000000..4e40903a --- /dev/null +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -0,0 +1,1094 @@ +// Package lineage provides the lineage-telemetry authbridge plugin. +// +// Two-span model (see authbridge/docs/lineage-wire-contract.md — the wire +// contract this file implements, kept byte-identical with the consumer's copy +// in the lab-data-governance repo). Each HTTP exchange through the sidecar +// produces TWO OTLP spans: +// +// - a request span, emitted as soon as the request has been seen and +// forwarded, carrying caller-side facts + input.value; and +// - a response span, emitted at stream end (even when no response was +// produced), carrying status/outcome facts + output.value. +// +// Both spans are ended immediately at emission — no span is held open across +// the wait. lineage.exchange.id (= the request span's own span id) is echoed +// on both so the consumer pairs them. The plugin emits FACTS ONLY (direction, +// protocol, endpoints, parsed payloads); all vocabulary — hop kinds, entity +// kinds, caller/callee — lives in the consumer's classify(). See the "removed +// vs today" migration map in the contract for the attrs this no longer emits. +// +// The plugin implements Finisher so the response span is emitted at stream +// end whatever the outcome — including denials that happen AFTER the request +// span was recorded (a response-phase deny, or a request-phase deny by a +// plugin ordered after this one); pctx.Outcome() is available at that point +// and maps to lineage.outcome=denied + lineage.denied_by. LIMITATION: the +// pipeline YAML places this plugin after the gate plugins (ordering is not +// soft-declared under this capabilities model — position in the list is the +// contract), and the pipeline short-circuits on a request-phase Reject, +// so an exchange denied by a gate BEFORE OnRequest ran emits NO spans at all — +// it is invisible to lineage. Moving lineage ahead of the gates (spans for +// denied traffic too) is a named follow-up, not current behavior. +// +// What this producer writes onto forwarded requests — two things, both +// directions. Always: one tracestate member (tracestateStampKey) so the next +// lineage element can parent its exchange on this one. Only when the request +// carried no VALID traceparent — absent, empty or malformed, as the W3C +// propagator judges it: a traceparent naming this request span +// (mintTraceparent, config mint_traceparent, default on), because without one +// the next element has nothing to extract and the stamp has nothing to ride +// on. A valid traceparent is never modified; an invalid one is replaced, which +// is W3C's processing model for it (restart the trace, drop tracestate). +// +// Read-only variant ("Option 4"). A deployment that wants a pure observer — +// no header written, parenting on the wire context alone — sets +// mint_traceparent: false and deletes exactly the selectParent and +// restampTracestate calls in OnRequest (and the lineage.parent.source fact); +// the span emit itself stays. The trade-off: without the stamp, two sidecarred +// pods can only be joined through the app's own propagation, so cross-pod +// parenting degrades to whatever the wire parent carries. The call site is +// marked so the choice stays locatable; the variant is not built here. +package lineage + +import ( + "context" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "os" + "path" + "strings" + "sync/atomic" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/rossoctl/cortex/authbridge/authlib/bypass" + "github.com/rossoctl/cortex/authbridge/authlib/pipeline" + "github.com/rossoctl/cortex/authbridge/authlib/plugins" +) + +const pluginName = "lineage-telemetry" + +// tracestateStampKey is the W3C tracestate member that carries the sidecar +// parent chain — the single channel every lineage element reads its parent +// from and writes its own request span id into (wire contract v1.5). Inbound +// stamps the request it forwards to its own app; the app's propagate-only shim +// carries tracestate through its per-request causal chain (contextvars), so +// the member surfaces on exactly the outbound calls that inbound caused. +// Outbound re-stamps the request it forwards to the peer, whose inbound +// sidecar reads it as its parent. Parent precedence is stamp > wire parent > +// none (nothing valid on the wire: the span roots a trace) in BOTH +// directions, and the chosen source is recorded as the +// lineage.parent.source fact. The forwarded traceparent is never modified: +// the sidecar chain lives entirely in this member, so an app that emits its +// own spans keeps an intact traceparent chain toward its own backend while +// the sidecar chain stays self-consistent in ours. (Until v1.4 the outbound +// instead rewrote the forwarded traceparent — the splice; v1.5 removed it.) +// The one exception: a request that arrived with no valid traceparent is +// forwarded with one naming this request span (mintTraceparent), so this +// member has a header to ride on — W3C reads tracestate only alongside a +// valid traceparent. +// +// The key is producer-owned and names the lineage domain (W3C convention: the +// key identifies the owner of the entry): the member carries the sidecar +// chain's own parent link and names no consumer. It was `kglin` until +// 2026-08-04 and `dg-parent` until 2026-09-03; the name never lands in stored +// data, so renaming is wire-only — but every sidecar on a hop must run the +// same key, so it changes in one release. +// +// A trace-keyed map (one entry per trace, "the last inbound seen") used to sit +// between the two. It was removed: its answer is correct only while exactly one +// inbound of that trace is in flight — a precondition it never checked and could +// not verify — and when it was wrong it produced a real, exported, walkable +// parent that was simply untrue. Un-stamped traffic falls to the wire +// parent, which is an app-internal span this pipeline never exported: the +// interaction still derives in full, but as a trace entry rather than a child. +// A visibly missing edge is recoverable; a silently wrong one is not. +const tracestateStampKey = "lineage-parent" + +// truncatedSuffix marks a captured payload that MaxPayloadBytes cut short. +const truncatedSuffix = "…[truncated]" + +func init() { + plugins.RegisterPlugin(pluginName, func() pipeline.Plugin { return NewLineageTelemetry() }) +} + +// exchangeState carries what OnFinish needs to emit the response span as the +// twin of the request span emitted in OnRequest. +type exchangeState struct { + // reqCtx is the (already-ended) request span's context — the parent of + // the response span. An ended span's SpanContext is a valid parent. + reqCtx trace.SpanContext + // common holds the attributes shared by both spans (lineage.direction, + // self.id, peer.*, protocol, exchange.id) — NOT lineage.role, which + // differs per span. Computed once so both spans agree byte-for-byte. + common []attribute.KeyValue + spanKind trace.SpanKind + // spanName is the request span's name; the response span appends " response". + spanName string + // protocol is the request span's lineage.protocol fact; the response + // span's output.value must be read through the SAME protocol's parser + // (parsers are precedence-ordered, not mutually exclusive — mcp-parser + // also matches any JSON-RPC body, including every a2a exchange). + protocol string +} + +// LineageTelemetry emits OTel spans for each request hop observed by authbridge. +type LineageTelemetry struct { + cfg Config + bypassPaths *bypass.Matcher // built in Configure from cfg.BypassPaths + tp *sdktrace.TracerProvider + tracer trace.Tracer + conn *grpc.ClientConn // OTLP gRPC connection; owned by us, closed on Shutdown + ready atomic.Bool + propagator propagation.TextMapPropagator + selfID string // agent's own client ID for the lineage.self.id fact + // exportFailures counts batches the collector refused or never received. + // Export is asynchronous and the dial is lazy, so this — with the WARN + // exportObserver logs — is how an unreachable collector or a TLS chain + // that does not verify becomes visible; see Ready for why readiness + // deliberately does not follow it. + exportFailures atomic.Uint64 +} + +// NewLineageTelemetry constructs an unconfigured plugin. Configure + Init must +// run before it serves traffic (guarded by Ready()). +func NewLineageTelemetry() *LineageTelemetry { + return &LineageTelemetry{ + propagator: propagation.TraceContext{}, + } +} + +func (p *LineageTelemetry) Name() string { return pluginName } + +// ConfigSchema exposes the Config fields for schema-aware tooling +// (/v1/plugins, /v1/pipeline, abctl edit templates), the same one-line +// delegation every configurable sibling plugin uses. +func (p *LineageTelemetry) ConfigSchema() []pipeline.FieldSchema { + return pipeline.SchemaOf(Config{}) +} + +func (p *LineageTelemetry) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{ + // At least one protocol parser must be present and earlier in the + // chain: the protocol fact and both payload reductions read the + // parsers' Extensions. A chain that misorders lineage before its + // parsers (or has none) fails at startup instead of silently + // emitting lineage.protocol="http" on everything. jwt-validation + // ordering (for the principal facts) cannot be soft-declared under + // this capabilities model — list it before lineage in the YAML. + RequiresAny: []string{"a2a-parser", "mcp-parser", "inference-parser"}, + // The contract is cited major.minor only, deliberately: patch + // revisions (v1.5.x) clarify prose and never change span semantics, + // so a patch bump must not imply a producer change. + Description: "Emits two facts-only lineage spans per HTTP exchange (wire contract v1.6).", + } +} + +func (p *LineageTelemetry) Configure(raw json.RawMessage) error { + cfg, err := decodeConfig(raw) + if err != nil { + return err + } + // The shared bypass matcher (path.Match globs, path normalization) — the + // same package and wiring jwt-validation and sparc use for this key. It + // validates at boot: invalid syntax and match-everything patterns refuse + // to start. + matcher, err := bypass.NewMatcher(cfg.BypassPaths) + if err != nil { + return fmt.Errorf("lineage-telemetry bypass_paths: %w", err) + } + p.cfg = cfg + p.bypassPaths = matcher + return nil +} + +func (p *LineageTelemetry) Init(ctx context.Context) error { + // Resolve self identity for the lineage.self.id fact FIRST, before any + // exporter or tracer resource is allocated. Every span this plugin emits is + // a claim of the form "X did Y"; with no X there is no claim to make, so an + // unresolvable identity refuses to start rather than serving traffic under a + // plausible-but-wrong label ("no mechanism may guess", contract v1.3). Note + // the asymmetry with this file's other unknowns: a missing status, payload + // or parent anchor is a missing PART of a fact and degrades honestly + // (abandoned / NULL / parent.source=wire or none). Identity is the fact's subject — + // it has no degraded form, and a shared placeholder would collapse every + // unidentified pod onto one entity row (entity id = uuid5("{kind}:{self.id}"), + // and entities is upsert-only). Resolving it up front also means a refused + // identity leaks nothing: the gRPC client and batch-span-processor goroutine + // below are never created on that path. + if p.cfg.SelfID != "" { + p.selfID = p.cfg.SelfID + } else if p.cfg.SelfIDFile != "" { + raw, err := os.ReadFile(p.cfg.SelfIDFile) + if err != nil { + return fmt.Errorf("lineage-telemetry: no inline self_id and self_id_file unreadable: %w", err) + } + p.selfID = strings.TrimSpace(string(raw)) + } + if p.selfID == "" { + return fmt.Errorf("lineage-telemetry: self identity unresolved (empty self_id and self_id_file %q)", p.cfg.SelfIDFile) + } + + endpoint := p.cfg.OTelEndpoint + // Transport credentials: plaintext by default (the loopback in-pod + // collector), TLS when otel_tls is set — an https:// endpoint or an + // otel_ca_file sets it in decodeConfig. Spans carry principal facts on + // every inbound request and, under capture_io, user messages and model + // output, so a remote collector must not receive them in cleartext. + creds, plaintext := insecure.NewCredentials(), true + switch { + case p.cfg.OTelCAFile != "": + // A private CA (a cert-manager issued in-cluster collector, typically): + // the dial verifies against this bundle only. Keyed on the file alone, + // not on OTelTLS, so a Config built without the decoder still cannot + // dial cleartext with a CA configured. Fails closed — an unreadable + // file or one with no certificate refuses to start rather than falling + // back to the system roots. Empty serverName = derive from endpoint. + pemBytes, err := os.ReadFile(p.cfg.OTelCAFile) + if err != nil { + return fmt.Errorf("lineage-telemetry: otel_ca_file %q: %w", p.cfg.OTelCAFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pemBytes) { + return fmt.Errorf("lineage-telemetry: otel_ca_file %q: no CA certificate found in PEM", p.cfg.OTelCAFile) + } + creds, plaintext = credentials.NewClientTLSFromCert(pool, ""), false + case p.cfg.OTelTLS: + // nil cert pool = system roots; empty serverName = derive from endpoint. + creds, plaintext = credentials.NewClientTLSFromCert(nil, ""), false + } + // A plaintext dial to anything but loopback puts principal facts, and under + // capture_io whole prompts and tool output, on the network in the clear. + // Whether that hop leaves the cluster is not knowable from a host:port, and + // the platform's own collector is plaintext gRPC, so this is the operator's + // call to make rather than ours to refuse — but they get told at the one + // moment they can act on it. + if plaintext && !isLoopback(endpoint) { + slog.Warn("lineage-telemetry: exporting spans in cleartext to a non-loopback collector; "+ + "set otel_tls (or otel_ca_file for a private CA) to encrypt them", + "endpoint", endpoint) + } + + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(creds), + ) + if err != nil { + return fmt.Errorf("lineage-telemetry: gRPC dial %s: %w", endpoint, err) + } + + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithGRPCConn(conn), + ) + if err != nil { + // The dial succeeded but the exporter did not adopt the conn, so close + // it here rather than leaking the gRPC client on this error path. + _ = conn.Close() + return fmt.Errorf("lineage-telemetry: OTLP exporter: %w", err) + } + // WithGRPCConn leaves connection ownership with the caller: the exporter's + // Shutdown will not close conn, so we retain it and close it ourselves in + // Shutdown. Store it only now, past the error path above. + p.conn = conn + + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String("authbridge"), + attribute.String("authbridge.component", pluginName), + ), + ) + if err != nil { + slog.Warn("lineage-telemetry: resource detection failed, using default", "error", err) + res = resource.Default() + } + + p.tp = newTracerProvider(&exportObserver{SpanExporter: exporter, failures: &p.exportFailures}, res) + p.tracer = p.tp.Tracer("authbridge/" + pluginName) + + p.ready.Store(true) + slog.Info("lineage-telemetry: initialized", "endpoint", endpoint, "self_id", p.selfID) + return nil +} + +// newTracerProvider builds the provider Init installs. AlwaysSample is +// explicit and deliberate: lineage is an audit record, and under the SDK +// default ParentBased sampler a caller sending a valid traceparent with the +// sampled-out flag (…-00) suppressed both spans — the same caller-controlled +// opt-out from being graphed that the inbound bypass_hosts decision refuses. +// The forwarded traceparent keeps the caller's flags (a valid one is never +// modified); only what this producer exports ignores them. An explicit +// sampler also means OTEL_TRACES_SAMPLER no longer overrides it. +func newTracerProvider(exporter sdktrace.SpanExporter, res *resource.Resource) *sdktrace.TracerProvider { + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) +} + +// isLoopback reports whether endpoint (host:port, already reduced from any URL +// form by decodeConfig) names this pod. Loopback traffic never leaves the +// network namespace, so cleartext there carries no exposure and earns no WARN. +func isLoopback(endpoint string) bool { + host := endpoint + if h, _, err := net.SplitHostPort(endpoint); err == nil { + host = h + } + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// Shutdown flushes and stops the tracer provider and then closes the OTLP gRPC +// connection. The exporter created with WithGRPCConn does not own conn, so +// closing it here is what actually releases the socket; both errors are joined +// so neither is lost. Safe to call after a failed Init (tp/conn may be nil). +// +// Readiness is cleared first, unconditionally: after Shutdown the plugin is no +// longer ready even if tp/conn are nil (post-failed-Init) or their shutdown +// errors, so a pipeline orchestrator checking Ready() before routing sees the +// lifecycle transition. This mirrors the p.ready.Store(true) in Init. +func (p *LineageTelemetry) Shutdown(ctx context.Context) error { + p.ready.Store(false) + var tpErr error + if p.tp != nil { + tpErr = p.tp.Shutdown(ctx) + } + var connErr error + if p.conn != nil { + connErr = p.conn.Close() + } + return errors.Join(tpErr, connErr) +} + +// Ready reports that the plugin is configured, has an identity and can +// record spans — not that the collector is reachable. grpc.NewClient dials +// lazily and the batch processor exports asynchronously, so no point in Init +// can prove the collector or its TLS chain; and readiness must not follow the +// collector anyway: an unready plugin skips OnRequest, which is where the +// tracestate stamp and the minted traceparent are written, so a collector +// outage would fragment every trace on the wire instead of merely delaying +// its export. A collector that cannot be reached surfaces through +// exportObserver instead: a plugin-namespaced WARN and the exportFailures +// counter. +func (p *LineageTelemetry) Ready() bool { return p.ready.Load() } + +// Metrics exposes the export-failure counter through pipeline.MetricsProvider, +// so an operator can see a collector outage on /v1/pipeline rather than only in +// the logs. The counter is a running total since Init, not a rate. Name and +// Note carry no request or response content, as that endpoint is unauthenticated. +func (p *LineageTelemetry) Metrics() []pipeline.Metric { + return []pipeline.Metric{{ + Name: "lineage.export_failures", + Value: float64(p.exportFailures.Load()), + Unit: "count", + Note: "OTLP batches the collector refused or never received, since start", + }} +} + +// exportObserver wraps the OTLP exporter so a failed export is visible from +// this plugin — a plugin-namespaced WARN and a counter — instead of only +// through the OTel SDK's default error handler on stderr. The error is +// returned unchanged; the batch processor's retry/drop behaviour is untouched. +// The WARN is throttled to the 1st, 2nd, 4th, 8th… failure: a dead collector +// fails a batch every export interval, and one line per failure would bury +// the logs exactly when they matter. The running total is in every line. +type exportObserver struct { + sdktrace.SpanExporter + failures *atomic.Uint64 +} + +func (e *exportObserver) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error { + err := e.SpanExporter.ExportSpans(ctx, spans) + if err != nil { + if n := e.failures.Add(1); logExportFailure(n) { + slog.Warn("lineage-telemetry: span export failed; the collector is unreachable or refused the batch", + "spans", len(spans), "failures", n, "error", err) + } + } + return err +} + +// logExportFailure reports whether the n-th consecutive-count failure is one +// of the logged ones: powers of two, so the volume grows with the log of the +// outage length rather than with the outage length. +func logExportFailure(n uint64) bool { return n&(n-1) == 0 } + +func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if !p.ready.Load() { + pctx.Skip("not_ready") + return pipeline.Action{Type: pipeline.Continue} + } + + // Skip infrastructure paths (health checks, agent-card discovery, etc.). + if p.bypassPaths.Match(pctx.Path) { + pctx.Skip("bypass_path") + return pipeline.Action{Type: pipeline.Continue} + } + + // Skip infrastructure outbound targets (OTel exporters, metrics scrapers, etc.). + // Outbound only: on the inbound phase Host is the caller's own header, so + // honouring the list there would let any caller suppress its own exchange + // by sending "Host: otel-collector". cpex reached the same conclusion for + // the same reason (its bypass_hosts is outbound-only for an attacker- + // controlled inbound Host). + if pctx.Direction == pipeline.Outbound && matchesAnyHost(p.cfg.BypassHosts, pctx.Host) { + pctx.Skip("bypass_host") + return pipeline.Action{Type: pipeline.Continue} + } + + // Extract remote trace context from the incoming W3C traceparent header. + // HeaderCarrier wraps http.Header and uses case-insensitive Get/Keys so + // canonical-form keys ("Traceparent") match the propagator's lowercase + // lookups. + remoteCtx := p.propagator.Extract(ctx, propagation.HeaderCarrier(pctx.Headers)) + + protocol := protocolOf(pctx) + self := serviceLabel(p.selfID) + spanKind := spanKindFor(pctx.Direction) + spanName := truncate(requestSpanName(self, protocol, spanOp(pctx, protocol)), p.cfg.MaxAttrBytes) + + // Facts shared by both spans (exchange.id is appended once the request + // span exists, since it IS the request span id). + base := p.baseAttrs(pctx, self, protocol) + + // Request-span attributes: role + shared facts + request-only facts. + reqAttrs := make([]attribute.KeyValue, 0, len(base)+8) + reqAttrs = append(reqAttrs, attribute.String("lineage.role", "request")) + reqAttrs = append(reqAttrs, base...) + reqAttrs = p.appendRequestFacts(reqAttrs, pctx, protocol) + + // (3) parent · (4) emit · (4b) mint · (5) re-stamp — wire contract v1.6. + // The emit is unconditional; the calls around it are the header + // machinery, and (4b) only acts when no valid traceparent arrived. The + // read-only "Option 4" variant deletes selectParent and restampTracestate + // (and the parent.source fact) and sets mint_traceparent: false — see the + // package doc for the trade-off. + parent, parentSource := selectParent(ctx, remoteCtx) + reqAttrs = append(reqAttrs, attribute.String("lineage.parent.source", parentSource)) + reqCtx := p.emitRequestSpan(parent, spanName, spanKind, reqAttrs) + exchangeID := reqCtx.SpanID().String() + remoteCtx = p.mintTraceparent(ctx, pctx, remoteCtx, reqCtx) + stamped := restampTracestate(pctx, remoteCtx, exchangeID) + + common := make([]attribute.KeyValue, 0, len(base)+1) + common = append(common, base...) + common = append(common, attribute.String("lineage.exchange.id", exchangeID)) + + pipeline.SetState(pctx, pluginName, &exchangeState{ + reqCtx: reqCtx, + common: common, + spanKind: spanKind, + spanName: spanName, + protocol: protocol, + }) + // modify vs observe follows what actually happened to the message: the + // stamp (and any minted traceparent under it) is a header rewrite; when + // nothing was written — pure-observer config, or a refused Insert — the + // exchange was only recorded. + if stamped { + pctx.Modify("stamped_tracestate") + } else { + pctx.Observe("recorded_request") + } + return pipeline.Action{Type: pipeline.Continue} +} + +// selectParent is step (3) of the single-channel parenting mechanism (wire +// contract v1.6): the parent is the tracestate stamp — the previous sidecar +// element in the chain (the caller's outbound for an inbound, this pod's +// inbound for an outbound) — else the wire parent; and when nothing valid is +// on the wire at all, no parent: the request span roots a trace and the fact +// says so ("none") rather than claiming a wire parent that was never there. +// Same precedence in both directions. Guessing an attribution is deliberately +// not among the options: it is worse than declining to give one. Returns the +// parent context and the source label the caller emits as the +// lineage.parent.source fact. +func selectParent(ctx, remoteCtx context.Context) (context.Context, string) { + rsc := trace.SpanContextFromContext(remoteCtx) + if !rsc.IsValid() { + return remoteCtx, "none" + } + if psc, ok := stampedParent(rsc); ok { + return trace.ContextWithRemoteSpanContext(ctx, psc), "tracestate" + } + return remoteCtx, "wire" +} + +// emitRequestSpan is step (4): emit the request span under parent and end it +// immediately — no span is held open across the exchange. lineage.exchange.id +// is the span's OWN id, so it can only be set after Start. Returns the span's +// context; an ended span's SpanContext remains a valid parent for the response +// span, and its span id is the exchange id. +func (p *LineageTelemetry) emitRequestSpan( + parent context.Context, + spanName string, + spanKind trace.SpanKind, + reqAttrs []attribute.KeyValue, +) trace.SpanContext { + _, span := p.tracer.Start(parent, spanName, + trace.WithSpanKind(spanKind), + trace.WithAttributes(reqAttrs...), + ) + sc := span.SpanContext() + span.SetAttributes(attribute.String("lineage.exchange.id", sc.SpanID().String())) + span.End() + return sc +} + +// mintTraceparent is step (4b), both directions: when the request arrived +// with NO VALID traceparent, forward one naming this request span, and return +// that context for the re-stamp to build on. Without it the next element has +// nothing to extract — an app's propagate-only shim roots a fresh trace of +// its own and the tracestate stamp never leaves this pod, because W3C reads +// tracestate only alongside a valid traceparent — so the entry exchange lands +// alone in its own trace and every call it caused derives as a parentless +// root (measured live 2026-09-02: one traceparent-less turn, 32 spans, 9 +// derived roots instead of 1). Validity is the propagator's verdict, the same +// one selectParent used to record parent.source=none: absent, empty and +// malformed all extract as no context, and all three are restarted here — +// exactly W3C's processing model for an unparseable traceparent (new +// traceparent, tracestate dropped; the re-stamp then writes ours alone). A +// valid traceparent is never modified. Disabled by mint_traceparent: false, +// for a pure observer. +func (p *LineageTelemetry) mintTraceparent(ctx context.Context, pctx *pipeline.Context, remoteCtx context.Context, reqCtx trace.SpanContext) context.Context { + if !p.cfg.MintTraceparent || trace.SpanContextFromContext(remoteCtx).IsValid() { + return remoteCtx + } + minted := trace.ContextWithRemoteSpanContext(ctx, reqCtx) + // TraceContext.Inject writes traceparent only; tracestate follows in + // restampTracestate (reqCtx carries an empty TraceState). + p.propagator.Inject(minted, propagation.HeaderCarrier(pctx.Headers)) + return minted +} + +// restampTracestate is step (5): rewrite the forwarded request's tracestate +// member with this exchange id — both directions. Inbound: the app's +// propagate-only shim couriers it to exactly the outbound calls this inbound +// caused. Outbound: the peer sidecar's inbound reads it as its parent. The +// forwarded traceparent is never modified (see tracestateStampKey). A valid +// traceparent to ride on is required — the wire's, or the one mintTraceparent +// just forwarded; with neither (mint_traceparent off) the shim would root a +// fresh trace and drop the tracestate anyway, so there is nothing to stamp. On +// a minted context the base TraceState is empty, so the caller's tracestate, +// if any rode in with an invalid traceparent, is dropped — W3C's restart +// semantics, not an accident. The listener is +// responsible for propagating these header mutations (ext_proc emits a +// SetHeaders diff). +// +// Reports whether it wrote — which is exactly whether the plugin mutated the +// forwarded message: whenever mintTraceparent wrote, the restamp that follows +// succeeds too (a minted context's TraceState is empty, so Insert cannot +// fail), so a false here means no header of either kind was written. +func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchangeID string) bool { + rsc := trace.SpanContextFromContext(remoteCtx) + if !rsc.IsValid() { + return false + } + ts, err := rsc.TraceState().Insert(tracestateStampKey, exchangeID) + if err != nil { + // Stamp attempted and refused (tracestate full or a member malformed, + // W3C caps at 32 members / 512 bytes). Without this line the outcome + // is indistinguishable from "app has no shim". + slog.Warn("lineage-telemetry: tracestate stamp rejected; the next element will attribute as wire", + "exchange_id", exchangeID, "error", err) + return false + } + pctx.Headers.Set("tracestate", ts.String()) + return true +} + +// matchesAnyHost reports whether host matches any configured bypass_hosts +// glob. Semantics follow the convention ibac, sparc and cpex already share +// for this key: path.Match against the host with its port stripped, so +// "otel-collector.*" matches otel-collector.rossoctl-system.svc:4317. +// +// The anchoring is the point. The earlier strings.Contains match was +// unanchored against bare-word defaults, so a legitimate workload at +// prometheus-metrics-agent.team1.svc silently left the lineage graph, and a +// tenant could opt out of being graphed at all by naming a service to contain +// one of the default words. A glob has to match from the first character. +// +// Two deliberate differences from the siblings, both strict improvements: +// the port is split with net.SplitHostPort so an IPv6 literal ([::1]:4317) +// survives, and matching is case-folded because an authority is +// case-insensitive (RFC 3986) — a target spelled OTel-Collector would +// otherwise be recorded while otel-collector was skipped. +func matchesAnyHost(patterns []string, host string) bool { + if host == "" { + return false + } + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.ToLower(host) + for _, pattern := range patterns { + if matched, _ := path.Match(strings.ToLower(pattern), host); matched { + return true + } + } + return false +} + +// stampedParent resolves the tracestate stamp on an outbound wire context: +// the inbound exchange id this pod's sidecar wrote into tracestate on the +// forwarded request, carried back by the app's shim. Returns ok=false when +// the member is absent or malformed (caller falls back to the wire parent). +func stampedParent(rsc trace.SpanContext) (trace.SpanContext, bool) { + raw := rsc.TraceState().Get(tracestateStampKey) + if raw == "" { + return trace.SpanContext{}, false + } + sid, err := trace.SpanIDFromHex(raw) + if err != nil { + return trace.SpanContext{}, false + } + psc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: rsc.TraceID(), + SpanID: sid, + TraceFlags: rsc.TraceFlags(), + Remote: true, + }) + return psc, psc.IsValid() +} + +// OnResponse is a no-op. The response span is emitted in OnFinish (which fires +// on every finished exchange, including denials and abandonments), not here. +// The method exists only to satisfy the base pipeline.Plugin interface, which +// mandates OnResponse; it carries no logic in the two-span model. +func (p *LineageTelemetry) OnResponse(_ context.Context, _ *pipeline.Context) pipeline.Action { + return pipeline.Action{Type: pipeline.Continue} +} + +// OnFinish emits the response span — the twin of the request span, parented +// under it and echoing the same exchange.id — carrying outcome/status/output. +// Always fires at stream end, so a bodyless or failed exchange still completes +// as a first-class pair. No recover here: the Finisher contract states that +// OnFinish runs best-effort and that panics are recovered and logged, and +// dispatchFinish scopes that recover to one plugin, so a second net would only +// hide the same panic under a different logger. +func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) { + state := pipeline.GetState[exchangeState](pctx, pluginName) + if state == nil || !state.reqCtx.IsValid() { + return + } + + outcome, status, hasStatus, deniedBy := lineageOutcome(pctx.Outcome()) + + attrs := make([]attribute.KeyValue, 0, len(state.common)+5) + attrs = append(attrs, attribute.String("lineage.role", "response")) + attrs = append(attrs, state.common...) + attrs = append(attrs, attribute.String("lineage.outcome", outcome)) + if hasStatus { + // "http.status_code" (like "http.method" on the request span) is the + // pre-v1.21 OTel semconv key, kept deliberately: this producer's + // contract vocabulary is lineage.* + these two well-known HTTP keys, + // pinned to the wire contract, not the stable OTel names + // (http.response.status_code / http.request.method). Interop with + // generic OTel tooling is a non-goal here. + attrs = append(attrs, attribute.Int("http.status_code", status)) + } + if deniedBy != "" { + attrs = append(attrs, p.capped("lineage.denied_by", deniedBy)) + } + if p.cfg.CaptureIO { + if v := ioOutputValue(pctx, state.protocol); v != "" { + attrs = append(attrs, attribute.String("output.value", truncate(v, p.cfg.MaxPayloadBytes))) + } + } + + parent := trace.ContextWithRemoteSpanContext(ctx, state.reqCtx) + _, span := p.tracer.Start(parent, state.spanName+" response", + trace.WithSpanKind(state.spanKind), + trace.WithAttributes(attrs...), + ) + span.End() +} + +// lineageOutcome maps the pipeline's 3-value Outcome (allow/deny/error, nil +// outside OnFinish) onto the contract's lineage.outcome vocabulary +// (ok|denied|error|abandoned) plus the http.status_code fact. "ok" and "denied" +// are the pipeline's own verdicts and carry a status only if one was written — +// an allow that produced none is still an allow. "abandoned" is a nil Outcome, +// or an error that never wrote a status (upstream reset, client disconnect, +// listener death) — the row completes as in-flight-turned-failed rather than +// dangling. hasStatus is false when no status code was produced. +func lineageOutcome(o *pipeline.Outcome) (outcome string, status int, hasStatus bool, deniedBy string) { + if o == nil { + return "abandoned", 0, false, "" + } + switch o.FinalAction { + case pipeline.OutcomeAllow: + return "ok", o.StatusCode, o.StatusCode > 0, "" + case pipeline.OutcomeDeny: + return "denied", o.StatusCode, o.StatusCode > 0, o.DenyingPlugin + case pipeline.OutcomeError: + if o.StatusCode > 0 { + return "error", o.StatusCode, true, "" + } + return "abandoned", 0, false, "" + default: + return "error", o.StatusCode, o.StatusCode > 0, "" + } +} + +// protocolOf reports which parser populated Extensions — the lineage.protocol +// fact. "http" means no parser matched. +func protocolOf(pctx *pipeline.Context) string { + switch { + case pctx.Extensions.A2A != nil: + return "a2a" + case pctx.Extensions.MCP != nil: + return "mcp" + case pctx.Extensions.Inference != nil: + return "inference" + default: + return "http" + } +} + +// spanKindFor maps direction to OTel SpanKind: inbound is SERVER, outbound is +// CLIENT. The response span reuses its request span's kind. +func spanKindFor(dir pipeline.Direction) trace.SpanKind { + if dir == pipeline.Inbound { + return trace.SpanKindServer + } + return trace.SpanKindClient +} + +// baseAttrs returns the facts carried on BOTH spans except exchange.id (added +// once the request span id is known) and role (differs per span). +func (p *LineageTelemetry) baseAttrs(pctx *pipeline.Context, self, protocol string) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("lineage.direction", pctx.Direction.String()), + p.capped("lineage.self.id", self), + attribute.String("lineage.protocol", protocol), + } + if pctx.Host != "" { + attrs = append(attrs, p.capped("lineage.peer.host", pctx.Host)) + } + return attrs +} + +// capped returns a string span attribute whose value is bounded by +// max_attr_bytes. Every variable-content string attribute goes through here — +// several are caller-controlled (url.path, Host, mcp.tool from the request +// body, a2a.session_id) and the SDK never truncates on its own. The +// fixed-vocabulary facts (lineage.role, lineage.direction, lineage.protocol, +// lineage.parent.source, lineage.outcome) and the 16-hex exchange id are +// bounded by construction and skip it; input.value / output.value carry their +// own max_payload_bytes cap. +func (p *LineageTelemetry) capped(key, val string) attribute.KeyValue { + return attribute.String(key, truncate(val, p.cfg.MaxAttrBytes)) +} + +// appendRequestFacts adds the request-only facts: HTTP method/path/scheme, the +// protocol-specific parsed facts, validated-JWT principal (inbound only), and +// input.value when capture_io is on. protocolOf guarantees the matching +// extension pointer is non-nil. +func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx *pipeline.Context, protocol string) []attribute.KeyValue { + if pctx.Method != "" { + // "http.method" is the pre-v1.21 OTel semconv key, kept deliberately — + // see the http.status_code note on the response span. + attrs = append(attrs, p.capped("http.method", pctx.Method)) + } + if path := urlPath(pctx); path != "" { + attrs = append(attrs, p.capped("url.path", path)) + } + if pctx.Scheme != "" { + attrs = append(attrs, p.capped("url.scheme", pctx.Scheme)) + } + switch protocol { + case "a2a": + a := pctx.Extensions.A2A + if a.Method != "" { + attrs = append(attrs, p.capped("a2a.method", a.Method)) + } + if a.SessionID != "" { + attrs = append(attrs, p.capped("a2a.session_id", a.SessionID)) + } + case "mcp": + m := pctx.Extensions.MCP + if m.Method != "" { + attrs = append(attrs, p.capped("mcp.method", m.Method)) + } + if t := mcpTool(pctx); t != "" { + attrs = append(attrs, p.capped("mcp.tool", t)) + } + case "inference": + if model := pctx.Extensions.Inference.Model; model != "" { + attrs = append(attrs, p.capped("inference.model", model)) + } + } + // Principal facts: request span, inbound only, and only from a validated + // JWT (pctx.Identity non-nil). + if pctx.Direction == pipeline.Inbound && pctx.Identity != nil { + if s := pctx.Identity.Subject(); s != "" { + attrs = append(attrs, p.capped("lineage.principal.sub", s)) + } + if c := pctx.Identity.ClientID(); c != "" { + attrs = append(attrs, p.capped("lineage.principal.client", c)) + } + } + if p.cfg.CaptureIO { + if v := ioInputValue(pctx, protocol); v != "" { + attrs = append(attrs, attribute.String("input.value", truncate(v, p.cfg.MaxPayloadBytes))) + } + } + return attrs +} + +// truncate bounds a captured payload to max bytes, cutting on a UTF-8 +// rune boundary and appending truncatedSuffix so the loss is explicit in the +// span rather than a silent drop at the OTLP exporter's attribute-length limit. +// A non-positive max disables the cap; decode narrows that to exactly -1. The +// returned string, suffix included, never exceeds max bytes. +func truncate(s string, max int) string { + if max <= 0 || len(s) <= max { + return s + } + // Reserve room for the marker; if the marker alone would not fit, drop it + // and return the prefix. Still back up to a rune boundary so the fallback + // never emits invalid UTF-8, and never exceed max. + budget := max - len(truncatedSuffix) + if budget <= 0 { + budget = max + for budget > 0 && !utf8.RuneStart(s[budget]) { + budget-- + } + return s[:budget] + } + // Back up to a rune boundary so we never split a multi-byte character. + for budget > 0 && !utf8.RuneStart(s[budget]) { + budget-- + } + return s[:budget] + truncatedSuffix +} + +// requestSpanName builds "{self} {protocol} {op}", dropping the trailing op +// when it is empty. The response span appends " response". +func requestSpanName(self, protocol, op string) string { + if op == "" { + return self + " " + protocol + } + return self + " " + protocol + " " + op +} + +// spanOp picks the operation label for the span name per protocol: +// mcp.tool / a2a.method / inference.model, falling back to url.path. +func spanOp(pctx *pipeline.Context, protocol string) string { + var op string + switch protocol { + case "a2a": + if pctx.Extensions.A2A != nil { + op = pctx.Extensions.A2A.Method + } + case "mcp": + op = mcpTool(pctx) + if op == "" && pctx.Extensions.MCP != nil { + op = pctx.Extensions.MCP.Method + } + case "inference": + if pctx.Extensions.Inference != nil { + op = pctx.Extensions.Inference.Model + } + } + if op == "" { + op = urlPath(pctx) + } + return op +} + +// urlPath returns pctx.Path without any query string. The extproc listener +// populates Path from the raw :path pseudo-header, query included; the proxy +// listeners use the parsed r.URL.Path, which excludes it. Stripping here keeps +// url.path and the span-name fallback query-free under every listener — a +// query can carry secrets that must not reach the trace store even with +// capture_io off. +func urlPath(pctx *pipeline.Context) string { + path, _, _ := strings.Cut(pctx.Path, "?") + return path +} + +// mcpTool returns the tool name for an MCP tools/call, or "" otherwise. +func mcpTool(pctx *pipeline.Context) string { + m := pctx.Extensions.MCP + if m == nil || m.Method != "tools/call" || m.Params == nil { + return "" + } + if name, ok := m.Params["name"].(string); ok { + return name + } + return "" +} + +// serviceLabel reduces a SPIFFE ID to its last path segment, or returns +// selfID as-is if it is not a SPIFFE URI. Used for the lineage.self.id fact +// and span names. +// +// "spiffe://trust-domain/ns/team1/sa/weather-service" → "weather-service" +// "weather-service" → "weather-service" +// +// selfID is never empty at the only call site: Init refuses to start without +// a resolved identity (v1.3). There is deliberately no empty-string fallback — +// inventing a label is the guess that rule exists to forbid. +func serviceLabel(selfID string) string { + parts := strings.Split(selfID, "/") + for i := len(parts) - 1; i >= 0; i-- { + if parts[i] != "" { + return parts[i] + } + } + return selfID +} + +// ioInputValue returns the input.value for a request span: the parsed request +// content for *protocol* — the hop's lineage.protocol fact — or "" if that +// parser produced nothing meaningful. Only that protocol's extension is read: +// parsers are precedence-ordered, not mutually exclusive (mcp-parser matches +// any JSON-RPC body, including every a2a exchange), so falling through to +// another parser's output would attach a mislabeled protocol envelope. A hop +// whose own parser yields nothing keeps a NULL payload — the contract's +// "interactions are independent of payloads". +func ioInputValue(pctx *pipeline.Context, protocol string) string { + ext := pctx.Extensions + switch { + case protocol == "a2a" && ext.A2A != nil && len(ext.A2A.Parts) > 0: + // Collect all text parts; fall back to JSON if non-text parts present. + var texts []string + for _, p := range ext.A2A.Parts { + if p.Content != "" { + texts = append(texts, p.Content) + } + } + if len(texts) > 0 { + return strings.Join(texts, "\n") + } + if b, err := json.Marshal(ext.A2A.Parts); err == nil { + return string(b) + } + case protocol == "inference" && ext.Inference != nil && len(ext.Inference.Messages) > 0: + if b, err := json.Marshal(ext.Inference.Messages); err == nil { + return string(b) + } + case protocol == "mcp" && ext.MCP != nil && ext.MCP.Params != nil: + // For tools/call, surface just the arguments (the semantically + // meaningful part) rather than the full {"name":…,"arguments":…} wrapper. + if ext.MCP.Method == "tools/call" { + if args, ok := ext.MCP.Params["arguments"]; ok { + if b, err := json.Marshal(args); err == nil { + return string(b) + } + } + } + if b, err := json.Marshal(ext.MCP.Params); err == nil { + return string(b) + } + } + return "" +} + +// isA2AProtocolEvent returns true when s is a JSON object carrying an A2A +// transport-level "kind" field (status-update, task-status-update, etc.) +// rather than actual content. Used to avoid surfacing protocol metadata +// as output.value when the a2a-parser captures a protocol event as the +// artifact instead of the real agent response text. +func isA2AProtocolEvent(s string) bool { + var obj map[string]json.RawMessage + if json.Unmarshal([]byte(s), &obj) != nil { + return false + } + var kind string + if raw, ok := obj["kind"]; ok { + _ = json.Unmarshal(raw, &kind) + } + // A2A protocol event kinds are enumerated and stable, so match exactly: + // a substring test would suppress a legitimate agent-defined artifact whose + // kind merely contains one of these words (e.g. "final-status-report"). + switch kind { + case "status-update", "task-status-update", "artifact-update", "working", "canceled": + return true + default: + return false + } +} + +// ioOutputValue returns the output.value for a response span: the parsed +// response content for *protocol* — the REQUEST span's lineage.protocol fact — +// or "" if that parser produced nothing. Only that protocol's extension is +// read, for the same reason as ioInputValue: mcp-parser also parses every a2a +// response (any JSON-RPC body), and falling through to it would emit the raw +// JSON-RPC envelope — including the protocol events isA2AProtocolEvent exists +// to suppress — as an a2a hop's payload. +func ioOutputValue(pctx *pipeline.Context, protocol string) string { + ext := pctx.Extensions + switch { + case protocol == "a2a" && ext.A2A != nil && ext.A2A.Artifact != "" && !isA2AProtocolEvent(ext.A2A.Artifact): + return ext.A2A.Artifact + case protocol == "a2a" && ext.A2A != nil && ext.A2A.ErrorMessage != "": + return ext.A2A.ErrorMessage + case protocol == "inference" && ext.Inference != nil && ext.Inference.Completion != "": + return ext.Inference.Completion + case protocol == "inference" && ext.Inference != nil && len(ext.Inference.ToolCalls) > 0: + if b, err := json.Marshal(ext.Inference.ToolCalls); err == nil { + return string(b) + } + case protocol == "mcp" && ext.MCP != nil && ext.MCP.Result != nil: + // For tools/call results, extract the text content from the MCP + // content array rather than returning the full {"content":[…],"_meta":…} + // envelope, so the output matches what Phoenix shows for the tool span. + if ext.MCP.Method == "tools/call" { + if content, ok := ext.MCP.Result["content"]; ok { + if items, ok := content.([]any); ok { + var texts []string + for _, item := range items { + if m, ok := item.(map[string]any); ok { + if m["type"] == "text" { + if t, ok := m["text"].(string); ok && t != "" { + texts = append(texts, t) + } + } + } + } + if len(texts) > 0 { + return strings.Join(texts, "\n") + } + } + } + } + if b, err := json.Marshal(ext.MCP.Result); err == nil { + return string(b) + } + case protocol == "mcp" && ext.MCP != nil && ext.MCP.Err != nil: + if b, err := json.Marshal(ext.MCP.Err); err == nil { + return string(b) + } + } + return "" +} + +// Compile-time interface assertions. +var ( + _ pipeline.Plugin = (*LineageTelemetry)(nil) + _ pipeline.Configurable = (*LineageTelemetry)(nil) + _ pipeline.Initializer = (*LineageTelemetry)(nil) + _ pipeline.Shutdowner = (*LineageTelemetry)(nil) + _ pipeline.Finisher = (*LineageTelemetry)(nil) + _ pipeline.Readier = (*LineageTelemetry)(nil) + _ pipeline.SchemaProvider = (*LineageTelemetry)(nil) + _ pipeline.MetricsProvider = (*LineageTelemetry)(nil) +) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go new file mode 100644 index 00000000..f7efd33d --- /dev/null +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -0,0 +1,1797 @@ +package lineage + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "maps" + "math/big" + "net/http" + "os" + "reflect" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + "unicode/utf8" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + + "github.com/rossoctl/cortex/authbridge/authlib/bypass" + "github.com/rossoctl/cortex/authbridge/authlib/pipeline" +) + +// newTestPlugin creates a LineageTelemetry wired to an in-memory span exporter +// (synchronous, so a span appears the instant it is ended) and marks it ready +// so Init is not needed. +func newTestPlugin(t *testing.T) (*LineageTelemetry, *tracetest.InMemoryExporter) { + t.Helper() + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + p := NewLineageTelemetry() + p.cfg = defaultConfig() // the shipped defaults, so a test sees what a deployment sees + p.bypassPaths, _ = bypass.NewMatcher(p.cfg.BypassPaths) + p.tp = tp + p.tracer = tp.Tracer("test") + p.selfID = "weather-service" + p.ready.Store(true) + return p, exp +} + +// run drives a full exchange (request pass + finish) through a single-plugin +// pipeline. Spans are read from the caller's exporter. +func run(t *testing.T, p *LineageTelemetry, pctx *pipeline.Context, outcome pipeline.Outcome) { + t.Helper() + pl, err := pipeline.New([]pipeline.Plugin{p}) + if err != nil { + t.Fatalf("pipeline.New: %v", err) + } + pl.Run(context.Background(), pctx) + pl.RunFinish(context.Background(), pctx, outcome) +} + +// allow is the ordinary success outcome. +func allow(status int) pipeline.Outcome { + return pipeline.Outcome{FinalAction: pipeline.OutcomeAllow, StatusCode: status} +} + +// fakeContext mirrors what the real listeners supply. Method is populated +// because every listener now supplies it (reverseproxy/forwardproxy from +// r.Method, ext_proc from the :method pseudo-header) — if a listener stops, +// the fixture must change with it rather than keep asserting a fiction. +func fakeContext(dir pipeline.Direction, headers http.Header) *pipeline.Context { + return &pipeline.Context{ + Direction: dir, + Method: "POST", + Host: "test-service:8000", + Path: "/test", + Headers: headers, + } +} + +// traceparent builds a header carrier naming traceID/spanID as the wire parent. +func traceparent(traceID, spanID string) http.Header { + h := http.Header{} + h.Set("traceparent", "00-"+traceID+"-"+spanID+"-01") + return h +} + +// extractParent decodes the span context named by the headers' traceparent. +func extractParent(h http.Header) trace.SpanContext { + ctx := propagation.TraceContext{}.Extract(context.Background(), propagation.HeaderCarrier(h)) + return trace.SpanContextFromContext(ctx) +} + +// roleSplit returns the request and response spans from an exported set, +// asserting exactly one of each. +func roleSplit(t *testing.T, spans tracetest.SpanStubs) (req, resp tracetest.SpanStub) { + t.Helper() + var gotReq, gotResp bool + for _, s := range spans { + switch attrStr(s, "lineage.role") { + case "request": + if gotReq { + t.Fatal("more than one request span") + } + req, gotReq = s, true + case "response": + if gotResp { + t.Fatal("more than one response span") + } + resp, gotResp = s, true + default: + t.Fatalf("span %q has no lineage.role", s.Name) + } + } + if !gotReq || !gotResp { + t.Fatalf("want one request + one response span, got %d spans (req=%v resp=%v)", len(spans), gotReq, gotResp) + } + return req, resp +} + +// ---- identifiers, pairing, parenting ---- + +func TestExchange_TwoSpansPairedAndParented(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + + pl, err := pipeline.New([]pipeline.Plugin{p}) + if err != nil { + t.Fatalf("pipeline.New: %v", err) + } + + // Emit on sight: the request span exists after the request pass, before finish. + pl.Run(context.Background(), pctx) + if got := len(exp.GetSpans()); got != 1 { + t.Fatalf("after request pass: want 1 span (request), got %d", got) + } + + pl.RunFinish(context.Background(), pctx, allow(200)) + spans := exp.GetSpans() + if len(spans) != 2 { + t.Fatalf("after finish: want 2 spans, got %d", len(spans)) + } + req, resp := roleSplit(t, spans) + + // exchange.id == request span id, echoed on both. + wantID := req.SpanContext.SpanID().String() + if got := attrStr(req, "lineage.exchange.id"); got != wantID { + t.Errorf("request exchange.id = %q, want %q", got, wantID) + } + if got := attrStr(resp, "lineage.exchange.id"); got != wantID { + t.Errorf("response exchange.id = %q, want %q", got, wantID) + } + + // Response span's parent is the request span (same trace). + if resp.Parent.SpanID() != req.SpanContext.SpanID() { + t.Errorf("response parent span = %s, want request span %s", resp.Parent.SpanID(), req.SpanContext.SpanID()) + } + if resp.SpanContext.TraceID() != req.SpanContext.TraceID() { + t.Errorf("response trace = %s, want request trace %s", resp.SpanContext.TraceID(), req.SpanContext.TraceID()) + } + + // Both spans share the same SpanKind (SERVER for inbound). + if req.SpanKind != trace.SpanKindServer || resp.SpanKind != trace.SpanKindServer { + t.Errorf("span kinds = %v/%v, want server/server", req.SpanKind, resp.SpanKind) + } +} + +// ---- the stamp (single-channel parenting, wire contract v1.5) ---- + +// TestStamp_OutboundRewritesStampNotTraceparent: the outbound reads its +// parent from the inbound's stamp, then re-stamps the forwarded tracestate +// with its OWN request span id for the peer sidecar's inbound to read. The +// forwarded traceparent is NOT modified (v1.5 removed the splice) — an app +// chain riding traceparent toward its own backend stays intact. +func TestStamp_OutboundRewritesStampNotTraceparent(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID, wireParent = "4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7" + const inboundID = "1111111111111111" + h := traceparent(traceID, wireParent) + h.Set("tracestate", tracestateStampKey+"="+inboundID) + pctx := fakeContext(pipeline.Outbound, h) + pctx.Extensions.MCP = &pipeline.MCPExtension{Method: "tools/call", Params: map[string]any{"name": "get_weather"}} + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + // Parent comes from the inbound's stamp. + if got := req.Parent.SpanID().String(); got != inboundID { + t.Errorf("parent = %s, want stamped inbound %s", got, inboundID) + } + // The forwarded traceparent is untouched — still the wire parent. + forwarded := extractParent(pctx.Headers) + if got := forwarded.SpanID().String(); got != wireParent { + t.Errorf("forwarded traceparent parent = %s, want untouched wire parent %s", got, wireParent) + } + if got := forwarded.TraceID().String(); got != traceID { + t.Errorf("forwarded trace = %s, want %s", got, traceID) + } + // The forwarded tracestate now stamps THIS outbound request span, + // replacing the inbound's stamp it consumed. + want := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != want { + t.Errorf("tracestate = %q, want re-stamp %q", got, want) + } +} + +// TestStamp_InboundParentsOnPeerStamp is the cross-pod link: the caller +// sidecar's outbound stamped tracestate with its request span id, and this +// inbound must parent on that stamp — not on the wire traceparent, whose +// span id may belong to an app chain this pipeline never exports. +func TestStamp_InboundParentsOnPeerStamp(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID, wireParent = "4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7" + const peerOutbound = "2222222222222222" + h := traceparent(traceID, wireParent) + h.Set("tracestate", tracestateStampKey+"="+peerOutbound) + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + if got := req.Parent.SpanID().String(); got != peerOutbound { + t.Errorf("parent = %s, want peer outbound stamp %s", got, peerOutbound) + } + if got := attrStr(req, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } + // The forwarded stamp now names THIS inbound request span — the app's + // shim couriers it to exactly the outbound calls this inbound causes. + want := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != want { + t.Errorf("tracestate = %q, want re-stamp %q", got, want) + } +} + +func TestStamp_InboundHeadersUntouchedExceptStamp(t *testing.T) { + p, exp := newTestPlugin(t) + h := traceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7") + before := http.Header{} + maps.Copy(before, h) + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + // The ONLY inbound mutation is the tracestate stamp; traceparent and + // everything else are forwarded as they arrived. + req, _ := roleSplit(t, exp.GetSpans()) + want := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != want { + t.Errorf("tracestate = %q, want stamp %q", got, want) + } + after := http.Header{} + maps.Copy(after, pctx.Headers) + after.Del("tracestate") + if !headersEqual(before, after) { + t.Errorf("inbound headers beyond tracestate mutated: before=%v after=%v", before, after) + } + // No stamp arrived, so the parent is the wire traceparent — recorded as such. + if got := attrStr(req, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +func TestStamp_PreservesForeignTracestateMembers(t *testing.T) { + p, exp := newTestPlugin(t) + h := traceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7") + h.Set("tracestate", "vendor=abc") + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + got := pctx.Headers.Get("tracestate") + wantStamp := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if !strings.Contains(got, wantStamp) || !strings.Contains(got, "vendor=abc") { + t.Errorf("tracestate = %q, want both %q and vendor=abc", got, wantStamp) + } +} + +// TestMint_NoTraceparentForwardsOwn is the traceparent-less entry, both +// directions: the request span roots a fresh trace, and the forwarded request +// now carries a traceparent naming that span PLUS the tracestate stamp — so +// the next element (the app's shim inbound, the peer's sidecar outbound) has +// a context to extract and the stamp has a header to ride on. +func TestMint_NoTraceparentForwardsOwn(t *testing.T) { + for _, dir := range []pipeline.Direction{pipeline.Inbound, pipeline.Outbound} { + t.Run(dir.String(), func(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(dir, http.Header{}) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + if req.Parent.IsValid() { + t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) + } + if got := attrStr(req, "lineage.parent.source"); got != "none" { + t.Errorf("lineage.parent.source = %q, want none", got) + } + want := "00-" + req.SpanContext.TraceID().String() + "-" + req.SpanContext.SpanID().String() + "-01" + if got := pctx.Headers.Get("traceparent"); got != want { + t.Errorf("forwarded traceparent = %q, want minted %q", got, want) + } + wantStamp := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != wantStamp { + t.Errorf("tracestate = %q, want stamp %q", got, wantStamp) + } + }) + } +} + +// TestMint_Disabled is the pure-observer posture: with mint_traceparent off a +// traceparent-less request is forwarded exactly as it arrived — no +// traceparent, and therefore no stamp either. +func TestMint_Disabled(t *testing.T) { + p, _ := newTestPlugin(t) + p.cfg.MintTraceparent = false + pctx := fakeContext(pipeline.Inbound, http.Header{}) + + run(t, p, pctx, allow(200)) + + if got := pctx.Headers.Get("traceparent"); got != "" { + t.Errorf("traceparent minted with mint_traceparent off: %q", got) + } + if got := pctx.Headers.Get("tracestate"); got != "" { + t.Errorf("tracestate stamped without a wire traceparent: %q", got) + } +} + +// TestMint_RestartsInvalidTraceparent: W3C's processing model for an +// unparseable traceparent is to restart the trace — a new traceparent, the +// tracestate dropped. The propagator's verdict decides: a malformed value, an +// empty one, and a version-ff one all extract as no context, so all three are +// restarted exactly like an absent header, and a foreign tracestate that rode +// in with them does not survive. (A VALID traceparent is never touched: +// TestStamp_InboundHeadersUntouchedExceptStamp.) +func TestMint_RestartsInvalidTraceparent(t *testing.T) { + for _, sent := range []string{"not-a-traceparent", "", "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} { + t.Run(fmt.Sprintf("traceparent=%q", sent), func(t *testing.T) { + p, exp := newTestPlugin(t) + h := http.Header{} + h.Set("traceparent", sent) + h.Set("tracestate", "vendor=abc") + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + if req.Parent.IsValid() { + t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) + } + if got := attrStr(req, "lineage.parent.source"); got != "none" { + t.Errorf("lineage.parent.source = %q, want none", got) + } + want := "00-" + req.SpanContext.TraceID().String() + "-" + req.SpanContext.SpanID().String() + "-01" + if got := pctx.Headers.Values("traceparent"); len(got) != 1 || got[0] != want { + t.Errorf("forwarded traceparent = %v, want restarted %q", got, want) + } + wantStamp := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != wantStamp { + t.Errorf("tracestate = %q, want the stamp alone (caller's dropped on restart) %q", got, wantStamp) + } + }) + } +} + +// TestMint_OutboundChainsIntoPeerInbound is the cross-pod twin of +// TestMint_ChainsThroughEntry: an app with no context of its own calls out +// bare, this pod's outbound mints and stamps, and the peer's inbound — +// receiving exactly the forwarded headers — parents on that outbound via the +// stamp, in the outbound's trace. Same code path, both directions, proven +// rather than assumed. +func TestMint_OutboundChainsIntoPeerInbound(t *testing.T) { + p, exp := newTestPlugin(t) + out := fakeContext(pipeline.Outbound, http.Header{}) + run(t, p, out, allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + + peer, peerExp := newTestPlugin(t) + peer.selfID = "weather-tool" + in := fakeContext(pipeline.Inbound, out.Headers.Clone()) + run(t, peer, in, allow(200)) + inReq, _ := roleSplit(t, peerExp.GetSpans()) + + if inReq.SpanContext.TraceID() != outReq.SpanContext.TraceID() { + t.Errorf("peer inbound trace %s, want the outbound's %s", inReq.SpanContext.TraceID(), outReq.SpanContext.TraceID()) + } + if inReq.Parent.SpanID() != outReq.SpanContext.SpanID() { + t.Errorf("peer inbound parent %s, want the outbound request span %s", inReq.Parent.SpanID(), outReq.SpanContext.SpanID()) + } + if got := attrStr(inReq, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } +} + +// TestMint_ChainsThroughEntry is the whole entry mechanism end to end in +// miniature: a traceparent-less inbound (the entry), then an outbound that +// arrives carrying exactly the headers the inbound forwarded — as a +// propagate-only shim would courier them — parents on the entry's request +// span via the stamp, in the entry's own trace. One tree, one root. +func TestMint_ChainsThroughEntry(t *testing.T) { + p, exp := newTestPlugin(t) + entry := fakeContext(pipeline.Inbound, http.Header{}) + run(t, p, entry, allow(200)) + entryReq, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + + out := fakeContext(pipeline.Outbound, entry.Headers.Clone()) + run(t, p, out, allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if outReq.SpanContext.TraceID() != entryReq.SpanContext.TraceID() { + t.Errorf("outbound trace %s, want the entry's %s", outReq.SpanContext.TraceID(), entryReq.SpanContext.TraceID()) + } + if outReq.Parent.SpanID() != entryReq.SpanContext.SpanID() { + t.Errorf("outbound parent %s, want the entry request span %s", outReq.Parent.SpanID(), entryReq.SpanContext.SpanID()) + } + if got := attrStr(outReq, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } +} + +func TestConfig_MintTraceparent(t *testing.T) { + cases := []struct { + name string + raw string + want bool + }{ + {"default on", `{"self_id":"x"}`, true}, + {"explicit true", `{"mint_traceparent":true,"self_id":"x"}`, true}, + {"explicit false", `{"mint_traceparent":false,"self_id":"x"}`, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if err != nil { + t.Fatalf("decodeConfig: %v", err) + } + if cfg.MintTraceparent != tc.want { + t.Errorf("MintTraceparent = %v, want %v", cfg.MintTraceparent, tc.want) + } + }) + } +} + +// TestStamp_OutboundPrefersStampOverMap is the same-trace fan-in case in +// miniature: two concurrent inbound exchanges on ONE trace (the trace-keyed +// map can only hold the later one), then an outbound whose tracestate stamp +// names the EARLIER inbound. Without the stamp this outbound would collapse +// onto the map entry — the 1/N misattribution the fanin-test.sh e2e proves. +func TestStamp_OutboundUsesTheStampedInbound(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + // Two concurrent inbounds on the SAME trace — the case no trace-keyed + // structure can disambiguate, and the reason the stamp exists. + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "1111111111111111")), allow(200)) + in1, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "2222222222222222")), allow(200)) + in2, _ := roleSplit(t, exp.GetSpans()) + + // Outbound couriered in1's stamp back through the app. It must parent + // under in1 specifically — not in2, not the wire parent. + exp.Reset() + h := traceparent(traceID, "3333333333333333") + h.Set("tracestate", tracestateStampKey+"="+in1.SpanContext.SpanID().String()) + run(t, p, fakeContext(pipeline.Outbound, h), allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if outReq.Parent.SpanID() != in1.SpanContext.SpanID() { + t.Errorf("parent = %s, want stamped inbound %s (the other in-flight inbound was %s)", + outReq.Parent.SpanID(), in1.SpanContext.SpanID(), in2.SpanContext.SpanID()) + } + if got := attrStr(outReq, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } +} + +func TestStamp_MalformedFallsBackToWire(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const wireParent = "3333333333333333" + + // An inbound on this trace exists — and must NOT be used, because a + // malformed stamp means "unknown", not "guess for me". + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "1111111111111111")), allow(200)) + in1, _ := roleSplit(t, exp.GetSpans()) + + exp.Reset() + h := traceparent(traceID, wireParent) + h.Set("tracestate", tracestateStampKey+"=nothex") + run(t, p, fakeContext(pipeline.Outbound, h), allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if got := outReq.Parent.SpanID().String(); got != wireParent { + t.Errorf("parent = %s, want wire parent %s", got, wireParent) + } + if outReq.Parent.SpanID() == in1.SpanContext.SpanID() { + t.Error("malformed stamp silently inherited this pod's inbound span") + } + if got := attrStr(outReq, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +func TestStamp_ParentSourceWireWhenUnstamped(t *testing.T) { + p, exp := newTestPlugin(t) + out := fakeContext(pipeline.Outbound, traceparent("cccccccccccccccccccccccccccccccc", "1111111111111111")) + run(t, p, out, allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + if got := attrStr(outReq, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +// TestStamp_UnstampedOutboundNeverInheritsInbound is the regression guard for +// the removal of the trace-keyed map. An outbound with no stamp must fall to the +// wire parent EVEN WHEN this pod has an inbound span for the same trace. The old +// map answered such cases from "the last inbound seen", which is correct only +// while exactly one inbound is in flight — a precondition it never checked. A +// missing edge is recoverable; a confidently wrong one is not. +func TestStamp_UnstampedOutboundNeverInheritsInbound(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const wireParent = "3333333333333333" + + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "2222222222222222")), allow(200)) + inReq, _ := roleSplit(t, exp.GetSpans()) + + exp.Reset() + run(t, p, fakeContext(pipeline.Outbound, traceparent(traceID, wireParent)), allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if outReq.Parent.SpanID() == inReq.SpanContext.SpanID() { + t.Fatal("un-stamped outbound inherited this pod's inbound span — the map is back") + } + if got := outReq.Parent.SpanID().String(); got != wireParent { + t.Errorf("parent = %s, want wire parent %s", got, wireParent) + } + if got := attrStr(outReq, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +func TestStamp_ConcurrentTracesNeverCross(t *testing.T) { + p, exp := newTestPlugin(t) + const traceA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const traceB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + // One inbound on each trace. + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceA, "1111111111111111")), allow(200)) + inA, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceB, "2222222222222222")), allow(200)) + inB, _ := roleSplit(t, exp.GetSpans()) + + // Each outbound couriers its own trace's stamp back. + exp.Reset() + hA := traceparent(traceA, "3333333333333333") + hA.Set("tracestate", tracestateStampKey+"="+inA.SpanContext.SpanID().String()) + run(t, p, fakeContext(pipeline.Outbound, hA), allow(200)) + outA, _ := roleSplit(t, exp.GetSpans()) + if outA.Parent.SpanID() != inA.SpanContext.SpanID() { + t.Errorf("outbound A parent = %s, want inbound A %s", outA.Parent.SpanID(), inA.SpanContext.SpanID()) + } + if outA.Parent.SpanID() == inB.SpanContext.SpanID() { + t.Error("outbound A crossed into inbound B's span") + } + if outA.SpanContext.TraceID().String() != traceA { + t.Errorf("outbound A trace = %s, want %s", outA.SpanContext.TraceID(), traceA) + } + + exp.Reset() + hB := traceparent(traceB, "4444444444444444") + hB.Set("tracestate", tracestateStampKey+"="+inB.SpanContext.SpanID().String()) + run(t, p, fakeContext(pipeline.Outbound, hB), allow(200)) + outB, _ := roleSplit(t, exp.GetSpans()) + if outB.Parent.SpanID() != inB.SpanContext.SpanID() { + t.Errorf("outbound B parent = %s, want inbound B %s", outB.Parent.SpanID(), inB.SpanContext.SpanID()) + } +} + +// ---- bodyless / unparsed completeness ---- + +func TestBodyless_UnparsedNoCaptureStillEmitsBothSpans(t *testing.T) { + p, exp := newTestPlugin(t) + // capture_io defaults false; no parser extensions → protocol http. + pctx := fakeContext(pipeline.Outbound, http.Header{}) + + run(t, p, pctx, allow(200)) + + req, resp := roleSplit(t, exp.GetSpans()) + if got := attrStr(req, "lineage.protocol"); got != "http" { + t.Errorf("protocol = %q, want http", got) + } + // Complete: both carry the shared facts and the exchange is paired. + if attrStr(req, "lineage.exchange.id") == "" || attrStr(resp, "lineage.exchange.id") == "" { + t.Error("exchange.id missing on a bodyless span") + } + if got := attrStr(resp, "lineage.outcome"); got != "ok" { + t.Errorf("outcome = %q, want ok", got) + } + // Deliberately no capture_io assertion here: protocol http has no payload + // extractor (ioInputValue/ioOutputValue dispatch on a2a/mcp/inference + // only), so this fixture yields "" whatever the flag says. The default is + // proven in TestCaptureIO_OffByDefaultEmitsNoPayload, on a fixture where + // the flag is the only thing that can suppress the payload. +} + +// TestCaptureIO_OffByDefaultEmitsNoPayload proves the privacy default: with +// capture_io unset, no parsed content leaves the pod. It runs on an MCP +// tools/call fixture whose arguments and result both yield a non-empty value, +// so the absence of input.value/output.value can only be the flag — the same +// assertion on an http fixture cannot fail. The second half flips the flag on +// and asserts both appear, which is what makes the first half non-vacuous. +// +// The two halves need separate fixtures: a pipeline.Context carries its own +// finished state, so a second RunFinish on the same one is dropped. +func TestCaptureIO_OffByDefaultEmitsNoPayload(t *testing.T) { + mcpContext := func() *pipeline.Context { + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Host = "weather-tool-mcp.team1.svc:8000" + pctx.Path = "/mcp" + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", + Params: map[string]any{"name": "get_weather", "arguments": map[string]any{"city": "Tokyo"}}, + Result: map[string]any{"content": []any{map[string]any{"type": "text", "text": "sunny"}}}, + } + return pctx + } + + off, offExp := newTestPlugin(t) + if off.cfg.CaptureIO { + t.Fatal("capture_io must default to false") + } + run(t, off, mcpContext(), allow(200)) + req, resp := roleSplit(t, offExp.GetSpans()) + if v, ok := findAttr(req, "input.value"); ok { + t.Errorf("input.value = %v present with capture_io off", v) + } + if v, ok := findAttr(resp, "output.value"); ok { + t.Errorf("output.value = %v present with capture_io off", v) + } + + // Same fixture, flag on: both values appear, so the assertions above are + // measuring the flag rather than an absent extractor. + on, onExp := newTestPlugin(t) + on.cfg.CaptureIO = true + run(t, on, mcpContext(), allow(200)) + req, resp = roleSplit(t, onExp.GetSpans()) + checkAttr(t, req, "input.value", `{"city":"Tokyo"}`) + checkAttr(t, resp, "output.value", "sunny") +} + +// ---- outcomes ---- + +func TestOutcome_Denied(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + + run(t, p, pctx, pipeline.Outcome{ + FinalAction: pipeline.OutcomeDeny, + StatusCode: 401, + DenyingPlugin: "jwt-validation", + }) + + _, resp := roleSplit(t, exp.GetSpans()) + if got := attrStr(resp, "lineage.outcome"); got != "denied" { + t.Errorf("outcome = %q, want denied", got) + } + if got := attrStr(resp, "lineage.denied_by"); got != "jwt-validation" { + t.Errorf("denied_by = %q, want jwt-validation", got) + } + if got, ok := intAttr(resp, "http.status_code"); !ok || got != 401 { + t.Errorf("http.status_code = %d (ok=%v), want 401", got, ok) + } +} + +func TestOutcome_AbandonedHasNoStatus(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + + // Terminal error with no response written (upstream reset / disconnect). + run(t, p, pctx, pipeline.Outcome{FinalAction: pipeline.OutcomeError, StatusCode: 0}) + + _, resp := roleSplit(t, exp.GetSpans()) + if got := attrStr(resp, "lineage.outcome"); got != "abandoned" { + t.Errorf("outcome = %q, want abandoned", got) + } + if _, ok := findAttr(resp, "http.status_code"); ok { + t.Error("http.status_code present on an abandoned exchange (none was produced)") + } +} + +// ---- request facts + capture_io + span names ---- + +func TestRequestFacts_MCPWithCapture(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Host = "weather-tool-mcp.team1.svc:8000" + pctx.Path = "/mcp" + pctx.Scheme = "http" + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", + Params: map[string]any{"name": "get_weather", "arguments": map[string]any{"city": "Tokyo"}}, + Result: map[string]any{"content": []any{map[string]any{"type": "text", "text": "sunny"}}}, + } + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "lineage.protocol", "mcp") + checkAttr(t, req, "mcp.method", "tools/call") + checkAttr(t, req, "mcp.tool", "get_weather") + checkAttr(t, req, "http.method", "POST") + checkAttr(t, req, "url.path", "/mcp") + checkAttr(t, req, "url.scheme", "http") + checkAttr(t, req, "lineage.self.id", "weather-service") + checkAttr(t, req, "lineage.peer.host", "weather-tool-mcp.team1.svc:8000") + checkAttr(t, req, "lineage.direction", "outbound") + checkAttr(t, req, "input.value", `{"city":"Tokyo"}`) + checkAttr(t, resp, "output.value", "sunny") + + if req.Name != "weather-service mcp get_weather" { + t.Errorf("request span name = %q", req.Name) + } + if resp.Name != "weather-service mcp get_weather response" { + t.Errorf("response span name = %q", resp.Name) + } + if req.SpanKind != trace.SpanKindClient { + t.Errorf("outbound request kind = %v, want client", req.SpanKind) + } +} + +func TestRequestFacts_A2AAndInference(t *testing.T) { + p, exp := newTestPlugin(t) + // A2A. + a := fakeContext(pipeline.Outbound, http.Header{}) + a.Extensions.A2A = &pipeline.A2AExtension{Method: "message/send", SessionID: "sess-123"} + run(t, p, a, allow(200)) + areq, _ := roleSplit(t, exp.GetSpans()) + checkAttr(t, areq, "lineage.protocol", "a2a") + checkAttr(t, areq, "a2a.method", "message/send") + checkAttr(t, areq, "a2a.session_id", "sess-123") + if _, ok := findAttr(areq, "url.scheme"); ok { + t.Error("url.scheme present although the context carried no scheme") + } + if areq.Name != "weather-service a2a message/send" { + t.Errorf("a2a span name = %q", areq.Name) + } + + // Inference. + exp.Reset() + i := fakeContext(pipeline.Outbound, http.Header{}) + i.Extensions.Inference = &pipeline.InferenceExtension{Model: "qwen2.5:7b"} + run(t, p, i, allow(200)) + ireq, _ := roleSplit(t, exp.GetSpans()) + checkAttr(t, ireq, "lineage.protocol", "inference") + checkAttr(t, ireq, "inference.model", "qwen2.5:7b") + if ireq.Name != "weather-service inference qwen2.5:7b" { + t.Errorf("inference span name = %q", ireq.Name) + } +} + +// mcp-parser attaches to ANY JSON-RPC body — including every a2a exchange — +// so on an a2a hop both extensions are populated. The payload read is keyed by +// the protocol fact: when the a2a parser yields nothing (no text parts, a +// protocol-event artifact), the payload stays ABSENT rather than falling +// through to the co-populated MCP parse of the same bytes (which would emit +// the raw JSON-RPC envelope on an lineage.protocol=a2a span). +func TestCaptureIO_A2ANeverFallsThroughToCoPopulatedMCP(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Extensions.A2A = &pipeline.A2AExtension{ + Method: "message/send", + // A status-update captured as the artifact — a protocol event, filtered. + Artifact: `{"kind":"status-update","taskId":"t-1"}`, + } + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "message/send", + Params: map[string]any{"message": map[string]any{"role": "user"}}, + Result: map[string]any{"artifacts": []any{map[string]any{"artifactId": "a-1"}}}, + } + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "lineage.protocol", "a2a") + if v, ok := findAttr(req, "input.value"); ok { + t.Errorf("input.value = %q on an a2a hop with no a2a parts — leaked from the co-populated MCP parse", v.String()) + } + if v, ok := findAttr(resp, "output.value"); ok { + t.Errorf("output.value = %q on an a2a hop whose artifact is a protocol event — leaked from the co-populated MCP parse", v.String()) + } + // mcp.* facts belong to mcp hops only; the a2a label must keep them off. + if v, ok := findAttr(req, "mcp.method"); ok { + t.Errorf("mcp.method = %q emitted on an a2a hop", v.String()) + } +} + +func TestPrincipalFacts_InboundRequestOnly(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + pctx.Identity = fakeIdentity{sub: "alice", client: "weather-ui"} + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "lineage.principal.sub", "alice") + checkAttr(t, req, "lineage.principal.client", "weather-ui") + // Principal facts are request-only. + if _, ok := findAttr(resp, "lineage.principal.sub"); ok { + t.Error("lineage.principal.sub leaked onto the response span") + } +} + +func TestPrincipalFacts_OutboundNeverEmitsPrincipal(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Identity = fakeIdentity{sub: "alice", client: "weather-ui"} + + run(t, p, pctx, allow(200)) + req, _ := roleSplit(t, exp.GetSpans()) + if _, ok := findAttr(req, "lineage.principal.sub"); ok { + t.Error("outbound span carried a principal fact (inbound-only)") + } +} + +// The extproc listener populates pctx.Path from the raw :path pseudo-header, +// query string included; url.path and the span-name fallback must never +// carry it. +func TestQueryStringNeverEmitted(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Path = "/api/search?token=sekret" + + run(t, p, pctx, allow(200)) + req, _ := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "url.path", "/api/search") + if req.Name != "weather-service http /api/search" { + t.Errorf("request span name = %q, want query-free", req.Name) + } +} + +// Variable-content attributes and the span name are caller-controlled (a +// long path, a huge Host header, a tool name from the request body) and the +// SDK never truncates on its own; max_attr_bytes must bound them all. +func TestAttrBytesCapsCallerControlledValues(t *testing.T) { + long := strings.Repeat("x", 100_000) + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Path = "/" + long + pctx.Host = "evil-" + long + ".example:8000" + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", + Params: map[string]any{"name": "tool_" + long}, + } + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + max := defaultMaxAttrBytes + for _, key := range []string{"url.path", "lineage.peer.host", "mcp.tool"} { + v := attrStr(req, key) + if len(v) > max { + t.Errorf("attr %q is %d bytes, cap is %d", key, len(v), max) + } + if !strings.HasSuffix(v, truncatedSuffix) { + t.Errorf("attr %q lost bytes without the truncation marker", key) + } + } + if len(req.Name) > max { + t.Errorf("request span name is %d bytes, cap is %d", len(req.Name), max) + } + // The response name is the capped request name + " response". + if want := req.Name + " response"; resp.Name != want { + t.Errorf("response span name = %q, want %q", resp.Name, want) + } +} + +func TestConfig_MaxAttrBytes(t *testing.T) { + cases := []struct { + raw string + want int + wantErr bool + }{ + {raw: `{}`, want: defaultMaxAttrBytes}, + {raw: `{"max_attr_bytes": 0}`, want: defaultMaxAttrBytes}, + {raw: `{"max_attr_bytes": 64}`, want: 64}, + {raw: `{"max_attr_bytes": -1}`, want: unboundedPayload}, + {raw: `{"max_attr_bytes": -2}`, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.raw, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if tc.wantErr { + if err == nil { + t.Fatal("accepted; it removes the cap silently") + } + return + } + if err != nil { + t.Fatalf("decode: %v", err) + } + if cfg.MaxAttrBytes != tc.want { + t.Fatalf("max_attr_bytes = %d, want %d", cfg.MaxAttrBytes, tc.want) + } + }) + } +} + +// A valid traceparent with the sampled-out flag (…-00) must not suppress the +// exchange: lineage is an audit record, and a caller-chosen flag is not an +// opt-out from being graphed. Built on the provider Init installs — the +// other tests' newTestPlugin provider has no sampler and would pass or fail +// on the SDK default instead. +func TestSampledOutParentStillExports(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := newTracerProvider(exp, resource.Empty()) + p := NewLineageTelemetry() + p.cfg = defaultConfig() + p.bypassPaths, _ = bypass.NewMatcher(p.cfg.BypassPaths) + p.tp = tp + p.tracer = tp.Tracer("test") + p.selfID = "weather-service" + p.ready.Store(true) + + h := http.Header{} + h.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") + run(t, p, fakeContext(pipeline.Inbound, h), allow(200)) + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("ForceFlush: %v", err) + } + if got := len(exp.GetSpans()); got != 2 { + t.Fatalf("exported %d spans for a sampled-out parent, want 2", got) + } +} + +// The schema must track Config exactly — every operator key present, each +// with a description — so /v1/plugins and abctl never render a blank field, +// and a twelfth key added without a description goes red here. +func TestConfigSchema_TracksConfig(t *testing.T) { + schema := NewLineageTelemetry().ConfigSchema() + byName := map[string]pipeline.FieldSchema{} + for _, f := range schema { + byName[f.Name] = f + } + cfgType := reflect.TypeOf(Config{}) + keys := 0 + for i := 0; i < cfgType.NumField(); i++ { + key, _, _ := strings.Cut(cfgType.Field(i).Tag.Get("json"), ",") + if key == "" || key == "-" { + continue + } + keys++ + f, ok := byName[key] + if !ok { + t.Errorf("schema missing config key %q", key) + continue + } + if f.Description == "" { + t.Errorf("config key %q has no description tag", key) + } + } + if len(schema) != keys { + t.Errorf("schema has %d fields, Config has %d json keys", len(schema), keys) + } +} + +// The invocation action follows what happened to the message: the tracestate +// stamp is a header rewrite (modify); a pure observer that wrote nothing +// records observe. The repo's action vocabulary is operator-facing. +func TestInvocationAction_ModifyOnlyWhenStamped(t *testing.T) { + // Default config, valid inbound context: the stamp is written. + p, _ := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, traceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7")) + run(t, p, pctx, allow(200)) + inv := pctx.Extensions.Invocations.Inbound[0] + if string(inv.Action) != "modify" { + t.Errorf("stamped exchange action = %q, want modify", inv.Action) + } + + // Pure observer: mint off, nothing valid on the wire — nothing written. + p2, _ := newTestPlugin(t) + p2.cfg.MintTraceparent = false + pctx2 := fakeContext(pipeline.Inbound, http.Header{}) + run(t, p2, pctx2, allow(200)) + inv2 := pctx2.Extensions.Invocations.Inbound[0] + if string(inv2.Action) != "observe" { + t.Errorf("unstamped exchange action = %q, want observe", inv2.Action) + } +} + +// The metric name is a promise to operators (and was promised in review): +// the export-failure total is visible on /v1/pipeline as lineage.export_failures. +func TestMetrics_ExportFailures(t *testing.T) { + p := NewLineageTelemetry() + p.exportFailures.Store(7) + m := p.Metrics() + if len(m) != 1 || m[0].Name != "lineage.export_failures" || m[0].Value != 7 { + t.Fatalf("Metrics() = %+v, want one lineage.export_failures = 7", m) + } +} + +// ---- the forbidden-keys guard ---- + +// TestForbiddenKeysNeverEmitted scans every attribute of every span emitted +// across a spread of exchange shapes and asserts none carries a key from a +// removed vocabulary. The contract deleted these; this test is the tripwire +// that keeps them deleted. +func TestForbiddenKeysNeverEmitted(t *testing.T) { + forbidden := []string{"trust.", "lineage.hop.kind", "enduser.id", "openinference.", "source", "authbridge.proxy"} + + shapes := []func() *pipeline.Context{ + func() *pipeline.Context { + c := fakeContext(pipeline.Inbound, http.Header{}) + c.Identity = fakeIdentity{sub: "alice", client: "weather-ui"} + return c + }, + func() *pipeline.Context { + c := fakeContext(pipeline.Outbound, http.Header{}) + c.Extensions.MCP = &pipeline.MCPExtension{Method: "tools/call", Params: map[string]any{"name": "get_weather"}} + return c + }, + func() *pipeline.Context { + c := fakeContext(pipeline.Outbound, http.Header{}) + c.Extensions.A2A = &pipeline.A2AExtension{Method: "message/send"} + return c + }, + func() *pipeline.Context { + c := fakeContext(pipeline.Outbound, http.Header{}) + c.Extensions.Inference = &pipeline.InferenceExtension{Model: "qwen2.5:7b"} + return c + }, + } + + for _, mk := range shapes { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + run(t, p, mk(), allow(200)) + // roleSplit fatals unless the exchange produced exactly one request + // and one response span, so the scan below can never run on an empty + // set and report green on a plugin that emitted nothing. + req, resp := roleSplit(t, exp.GetSpans()) + for _, s := range []tracetest.SpanStub{req, resp} { + for _, kv := range s.Attributes { + key := string(kv.Key) + for _, bad := range forbidden { + if key == bad || strings.HasPrefix(key, bad) { + t.Errorf("span %q emitted forbidden attribute %q", s.Name, key) + } + } + } + } + } +} + +// TestConfig_MaxPayloadBytes pins the decode boundary: the cap was only ever +// set straight onto the struct, so the remap that makes an explicit 0 mean +// "unset" - the only thing between max_payload_bytes: 0 and uncapped payloads, +// since truncate treats every non-positive max as unbounded - went untested. +func TestConfig_MaxPayloadBytes(t *testing.T) { + cases := []struct { + raw string + want int + wantErr bool + }{ + {raw: `{}`, want: defaultMaxPayloadBytes}, + {raw: `{"max_payload_bytes": 0}`, want: defaultMaxPayloadBytes}, + {raw: `{"max_payload_bytes": 128}`, want: 128}, + {raw: `{"max_payload_bytes": -1}`, want: unboundedPayload}, + {raw: `{"max_payload_bytes": -2}`, wantErr: true}, + {raw: `{"max_payload_bytes": -4096}`, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.raw, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if tc.wantErr { + if err == nil { + t.Fatal("accepted; it removes the cap silently") + } + return + } + if err != nil { + t.Fatalf("decode: %v", err) + } + if cfg.MaxPayloadBytes != tc.want { + t.Fatalf("max_payload_bytes = %d, want %d", cfg.MaxPayloadBytes, tc.want) + } + }) + } +} + +// TestIsLoopback pins which endpoints earn the cleartext WARN. Only traffic +// that never leaves the pod's network namespace is exempt; an in-cluster +// service name is indistinguishable from a collector across the internet, so +// both are warned about. +func TestIsLoopback(t *testing.T) { + cases := []struct { + endpoint string + want bool + }{ + {"localhost:4317", true}, + {"LocalHost:4317", true}, + {"127.0.0.1:4317", true}, + {"127.9.9.9:4317", true}, + {"[::1]:4317", true}, + {"localhost", true}, + {"otel-collector.rossoctl-system.svc.cluster.local:4317", false}, + {"collector.vendor.example.com:4317", false}, + {"10.0.0.7:4317", false}, + {"", false}, + } + for _, tc := range cases { + if got := isLoopback(tc.endpoint); got != tc.want { + t.Errorf("isLoopback(%q) = %v, want %v", tc.endpoint, got, tc.want) + } + } +} + +// ---- robustness ---- + +func TestOnFinish_NoStateDoesNotPanicOrEmit(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + // OnFinish without OnRequest having run — no exchangeState stored. + p.OnFinish(context.Background(), pctx) + if got := len(exp.GetSpans()); got != 0 { + t.Errorf("OnFinish with no state emitted %d spans, want 0", got) + } +} + +func TestNotReady_SkipsSpan(t *testing.T) { + p := NewLineageTelemetry() + // Do NOT set ready — Init never called. + pctx := fakeContext(pipeline.Inbound, http.Header{}) + action := p.OnRequest(context.Background(), pctx) + if action.Type != pipeline.Continue { + t.Fatalf("expected Continue, got %v", action.Type) + } + if pipeline.GetState[exchangeState](pctx, pluginName) != nil { + t.Error("exchangeState should not be set when plugin is not ready") + } +} + +// ---- config ---- + +func TestConfigure_Defaults(t *testing.T) { + p := NewLineageTelemetry() + if err := p.Configure(nil); err != nil { + t.Fatalf("Configure(nil): %v", err) + } + if p.cfg.OTelEndpoint != "localhost:4317" { + t.Errorf("default endpoint = %q, want localhost:4317", p.cfg.OTelEndpoint) + } + if p.cfg.SelfIDFile != "/shared/client-id.txt" { + t.Errorf("default self_id_file = %q", p.cfg.SelfIDFile) + } +} + +func TestConfigure_DecodesKeptKeys(t *testing.T) { + p := NewLineageTelemetry() + raw := json.RawMessage(`{"otel_endpoint":"http://collector:4317","capture_io":true,"self_id":"weather-service"}`) + if err := p.Configure(raw); err != nil { + t.Fatalf("Configure: %v", err) + } + if p.cfg.OTelEndpoint != "collector:4317" { + t.Errorf("endpoint = %q, want collector:4317 (scheme stripped)", p.cfg.OTelEndpoint) + } + if !p.cfg.CaptureIO { + t.Error("capture_io should be true") + } + if p.cfg.SelfID != "weather-service" { + t.Errorf("self_id = %q", p.cfg.SelfID) + } +} + +// ---- helpers ---- + +type fakeIdentity struct { + sub, client string + scopes []string +} + +func (f fakeIdentity) Subject() string { return f.sub } +func (f fakeIdentity) ClientID() string { return f.client } +func (f fakeIdentity) Scopes() []string { return f.scopes } + +func findAttr(span tracetest.SpanStub, key string) (attribute.Value, bool) { + for _, kv := range span.Attributes { + if string(kv.Key) == key { + return kv.Value, true + } + } + return attribute.Value{}, false +} + +func attrStr(span tracetest.SpanStub, key string) string { + if v, ok := findAttr(span, key); ok { + return v.AsString() + } + return "" +} + +func intAttr(span tracetest.SpanStub, key string) (int64, bool) { + if v, ok := findAttr(span, key); ok { + return v.AsInt64(), true + } + return 0, false +} + +// checkAttr asserts a span contains attribute key with the given string value. +func checkAttr(t *testing.T, span tracetest.SpanStub, key, want string) { + t.Helper() + got, ok := findAttr(span, key) + if !ok { + t.Errorf("attribute %q not found in span %q", key, span.Name) + return + } + if got.AsString() != want { + t.Errorf("attr %q = %q, want %q", key, got.AsString(), want) + } +} + +func headersEqual(a, b http.Header) bool { + return maps.EqualFunc(a, b, slices.Equal[[]string]) +} + +// TestInit_RefusesToStartWithoutIdentity locks the v1.3 rule at the identity +// boundary: a pod whose self identity cannot be resolved must fail at boot, +// never serve traffic under a plausible-but-wrong label (the old behavior +// emitted lineage.self.id="agent" from the empty-string serviceLabel). +func TestInit_RefusesToStartWithoutIdentity(t *testing.T) { + cases := []struct { + name string + cfg Config + wantErr bool + }{ + {"inline self_id starts", Config{OTelEndpoint: "localhost:4317", SelfID: "weather-service"}, false}, + {"missing self_id_file refuses", Config{OTelEndpoint: "localhost:4317", SelfIDFile: t.TempDir() + "/absent.txt"}, true}, + {"no identity source refuses", Config{OTelEndpoint: "localhost:4317"}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = tc.cfg + err := p.Init(context.Background()) + if p.tp != nil { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + _ = p.tp.Shutdown(ctx) + cancel() + } + if tc.wantErr && err == nil { + t.Fatal("Init succeeded without a resolvable identity") + } + if !tc.wantErr && err != nil { + t.Fatalf("Init failed with a valid inline self_id: %v", err) + } + if tc.wantErr && p.Ready() { + t.Error("plugin reports Ready after a refused Init") + } + }) + } +} + +// TestInit_ReadsSelfIDFile covers the operator-injected path (file, not inline). +func TestInit_ReadsSelfIDFile(t *testing.T) { + dir := t.TempDir() + path := dir + "/client-id.txt" + if err := os.WriteFile(path, []byte("weather-service\n"), 0o600); err != nil { + t.Fatal(err) + } + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "localhost:4317", SelfIDFile: path} + if err := p.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + _ = p.tp.Shutdown(ctx) + cancel() + if p.selfID != "weather-service" { + t.Errorf("selfID = %q, want trimmed file content", p.selfID) + } +} + +// TestConfig_UnknownKeysRefused: a typo'd knob must be a boot error, not a +// silent run-with-defaults. +func TestConfig_UnknownKeysRefused(t *testing.T) { + if _, err := decodeConfig([]byte(`{"capture-io": true}`)); err == nil { + t.Fatal("unknown config key accepted silently") + } + if _, err := decodeConfig([]byte(`{"capture_io": true, "self_id": "x"}`)); err != nil { + t.Fatalf("valid config rejected: %v", err) + } +} + +// ---- bypass config ---- +// The one failure mode of bypass_paths / bypass_hosts produces NO signal +// anywhere: a matched hop is simply absent from the graph. So both directions +// are pinned — a match emits nothing, a near-miss emits the full pair. + +func TestBypassPaths_GlobMatchEmitsNothing(t *testing.T) { + cases := []struct { + name string + path string + spans int // spans expected from the exchange + }{ + {"exact match skipped", "/health", 0}, + {"glob matches one level", "/.well-known/agent.json", 0}, + {"glob does not cross /", "/.well-known/a/b", 2}, + {"over-match fixed: not a prefix rule", "/health-records/patient/42", 2}, + {"non-canonical form normalized", "//health", 0}, + {"non-matching path emits", "/api/health-report", 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, exp := newTestPlugin(t) + // Through Configure, so the test exercises the same NewMatcher + // wiring a deployment gets (jwt-validation and sparc shape). + if err := p.Configure(json.RawMessage(`{"bypass_paths": ["/.well-known/*", "/health"]}`)); err != nil { + t.Fatalf("Configure: %v", err) + } + pctx := fakeContext(pipeline.Inbound, http.Header{}) + pctx.Path = tc.path + run(t, p, pctx, allow(200)) + if got := len(exp.GetSpans()); got != tc.spans { + t.Fatalf("path %q: got %d spans, want %d", tc.path, got, tc.spans) + } + }) + } +} + +// TestBypassHosts_GlobMatchEmitsNothing pins the anchored matcher. The +// unanchored strings.Contains it replaced skipped every host in the third and +// fourth rows: a real workload leaving the graph, and a tenant opting out of +// being graphed by choosing its own name. +func TestBypassHosts_GlobMatchEmitsNothing(t *testing.T) { + cases := []struct { + name string + host string + spans int + }{ + {"bare name skipped", "otel-collector:4317", 0}, + {"fqdn skipped by the .* form", "otel-collector.rossoctl-system.svc:4317", 0}, + {"prefixed workload is NOT skipped", "prometheus-metrics-agent.team1.svc:9090", 2}, + {"suffixed workload is NOT skipped", "my-otel-collector:4317", 2}, + {"unrelated host emits", "weather-tool:8000", 2}, + {"case is folded", "OTel-Collector:4317", 0}, + {"port is optional", "otel-collector", 0}, + {"ipv6 literal keeps its host", "[::1]:4317", 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg = defaultConfig() + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Host = tc.host + run(t, p, pctx, allow(200)) + if got := len(exp.GetSpans()); got != tc.spans { + t.Fatalf("host %q: got %d spans, want %d", tc.host, got, tc.spans) + } + }) + } +} + +// TestBypassHosts_InboundIgnoresHost: an inbound Host is the caller's own +// header. Honouring bypass_hosts there would let any caller suppress the +// record of its own request by naming the target it claims to be calling. +func TestBypassHosts_InboundIgnoresHost(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.BypassHosts = []string{"otel-collector"} + pctx := fakeContext(pipeline.Inbound, http.Header{}) + pctx.Host = "otel-collector:4317" + run(t, p, pctx, allow(200)) + if got := len(exp.GetSpans()); got != 2 { + t.Fatalf("inbound Host bypass honoured: got %d spans, want 2", got) + } +} + +// TestConfig_BypassEntriesValidated: an entry that matches everything turns +// the plugin off with no signal at all — Ready() stays true and no span is +// ever emitted — so it has to fail at boot. +func TestConfig_BypassEntriesValidated(t *testing.T) { + refused := []struct { + name string + raw string + }{ + {"empty host", `{"bypass_hosts": ["jaeger", ""]}`}, + {"whitespace-only host", `{"bypass_hosts": [" "]}`}, + {"star host", `{"bypass_hosts": ["*"]}`}, + {"invalid host glob", `{"bypass_hosts": ["[unclosed"]}`}, + } + for _, tc := range refused { + t.Run(tc.name, func(t *testing.T) { + if _, err := decodeConfig([]byte(tc.raw)); err == nil { + t.Fatalf("%s accepted; it disables the plugin silently", tc.raw) + } + }) + } + // Path entries are validated by bypass.NewMatcher at Configure — the + // shared package jwt-validation and sparc validate with. + refusedPaths := []struct { + name string + raw string + }{ + {"empty path", `{"bypass_paths": ["/healthz", ""]}`}, + {"whitespace-only path", `{"bypass_paths": [" "]}`}, + {"star path", `{"bypass_paths": ["*"]}`}, + {"slash-star path", `{"bypass_paths": ["/*"]}`}, + {"invalid path glob", `{"bypass_paths": ["[unclosed"]}`}, + } + for _, tc := range refusedPaths { + t.Run(tc.name, func(t *testing.T) { + if err := NewLineageTelemetry().Configure(json.RawMessage(tc.raw)); err == nil { + t.Fatalf("%s accepted; it disables the plugin silently", tc.raw) + } + }) + } + + cfg, err := decodeConfig([]byte(`{"bypass_hosts": [" jaeger.* "], "bypass_paths": [" /healthz "]}`)) + if err != nil { + t.Fatalf("valid bypass config rejected: %v", err) + } + // Surrounding whitespace is trimmed rather than silently never matching. + if got := cfg.BypassHosts; len(got) != 1 || got[0] != "jaeger.*" { + t.Fatalf("bypass_hosts = %q, want [\"jaeger.*\"]", got) + } + if got := cfg.BypassPaths; len(got) != 1 || got[0] != "/healthz" { + t.Fatalf("bypass_paths = %q, want [\"/healthz\"]", got) + } +} + +// TestConfig_BypassReplacesDefaults: setting either key replaces the default +// list rather than extending it — the convention ibac, sparc and cpex share. +// Undocumented until now, and the reason it is now stated on both keys. +func TestConfig_BypassReplacesDefaults(t *testing.T) { + cfg, err := decodeConfig([]byte(`{"bypass_hosts": ["my-metrics-thing"]}`)) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(cfg.BypassHosts) != 1 || cfg.BypassHosts[0] != "my-metrics-thing" { + t.Fatalf("bypass_hosts = %q, want the operator list alone", cfg.BypassHosts) + } + // The untouched key keeps its defaults. + if len(cfg.BypassPaths) != len(defaultConfig().BypassPaths) { + t.Fatalf("bypass_paths = %q, want the defaults", cfg.BypassPaths) + } +} + +// ---- lifecycle: gRPC connection ownership ---- +// WithGRPCConn leaves the conn for the caller to close; the exporter's Shutdown +// does not. A real Init dials a (never-answered) localhost target, stores the +// conn, and Shutdown must both stop the provider and close that conn without +// error. We assert the observable contract — conn stored after Init, Shutdown +// returns nil — since the closed socket itself is not introspectable here. +func TestShutdown_ClosesConn(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "localhost:4317", SelfID: "weather-service"} + if err := p.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + if p.conn == nil { + t.Fatal("Init did not store the gRPC conn for Shutdown to close") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := p.Shutdown(ctx); err != nil { + t.Fatalf("Shutdown returned an error: %v", err) + } + // A second Shutdown after conn is already closed must not panic; it may + // return an error from re-closing, which the caller can ignore. + _ = p.Shutdown(ctx) +} + +// TestShutdown_NoInitIsSafe: Shutdown on a plugin that never Init'd (tp and +// conn both nil) is a no-op, not a nil-deref — the host may call it after a +// failed Init. +func TestShutdown_NoInitIsSafe(t *testing.T) { + p := NewLineageTelemetry() + if err := p.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown on an uninitialized plugin: %v", err) + } +} + +// TestShutdown_ClearsReady: a successful Init makes the plugin ready; Shutdown +// clears readiness, so a pipeline orchestrator polling Ready() before routing +// sees the plugin fall out of rotation rather than keep receiving traffic into +// a torn-down tracer provider. +func TestShutdown_ClearsReady(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "localhost:4317", SelfID: "weather-service"} + if err := p.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + if !p.Ready() { + t.Fatal("Ready() is false after a successful Init") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := p.Shutdown(ctx); err != nil { + t.Fatalf("Shutdown returned an error: %v", err) + } + if p.Ready() { + t.Fatal("Ready() is still true after Shutdown") + } +} + +// TestShutdown_ClearsReadyAfterFailedInit: readiness is cleared unconditionally, +// even when Init never set it — Shutdown flips it false regardless of whether +// tp/conn were ever built, keeping the false→false transition idempotent. +func TestShutdown_ClearsReadyAfterFailedInit(t *testing.T) { + p := NewLineageTelemetry() + if p.Ready() { + t.Fatal("a freshly constructed plugin should not be ready") + } + if err := p.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown on an uninitialized plugin: %v", err) + } + if p.Ready() { + t.Fatal("Ready() is true after Shutdown on an uninitialized plugin") + } +} + +// ---- OTLP transport selection ---- +// The export defaults to plaintext (in-pod loopback) but must honour a request +// for TLS rather than silently downgrading it: an https:// endpoint turns TLS +// on, and the one contradiction (https:// with an explicit otel_tls:false) +// fails closed rather than sending principal facts / payloads in the clear. A +// non-http(s) scheme (ftp://, ftps://) is rejected outright, not stripped and +// dialed insecure. +func TestConfig_TLSFromScheme(t *testing.T) { + cases := []struct { + name string + raw string + wantErr bool + wantTLS bool + wantHost string + }{ + {"bare host:port stays insecure", `{"otel_endpoint":"collector:4317","self_id":"x"}`, false, false, "collector:4317"}, + {"http:// stays insecure, host reduced", `{"otel_endpoint":"http://collector:4317/v1/traces","self_id":"x"}`, false, false, "collector:4317"}, + {"https:// turns TLS on, host reduced", `{"otel_endpoint":"https://collector.example.com:4317","self_id":"x"}`, false, true, "collector.example.com:4317"}, + {"explicit otel_tls on a plaintext host is honoured", `{"otel_endpoint":"collector:4317","otel_tls":true,"self_id":"x"}`, false, true, "collector:4317"}, + {"https:// with otel_tls:false is a rejected contradiction", `{"otel_endpoint":"https://collector:4317","otel_tls":false,"self_id":"x"}`, true, false, ""}, + {"https:// with otel_tls:true is consistent", `{"otel_endpoint":"https://collector:4317","otel_tls":true,"self_id":"x"}`, false, true, "collector:4317"}, + {"ftp:// scheme is rejected", `{"otel_endpoint":"ftp://collector:4317","self_id":"x"}`, true, false, ""}, + {"ftps:// scheme is rejected", `{"otel_endpoint":"ftps://collector:4317","self_id":"x"}`, true, false, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if tc.wantErr { + if err == nil { + t.Fatal("expected a config error, got nil") + } + return + } + if err != nil { + t.Fatalf("decodeConfig: %v", err) + } + if cfg.OTelTLS != tc.wantTLS { + t.Errorf("OTelTLS = %v, want %v", cfg.OTelTLS, tc.wantTLS) + } + if cfg.OTelEndpoint != tc.wantHost { + t.Errorf("OTelEndpoint = %q, want %q", cfg.OTelEndpoint, tc.wantHost) + } + }) + } +} + +// ---- payload truncation ---- +// With capture_io on, a payload larger than max_payload_bytes must be cut at +// the producer with an explicit marker, not left whole to be silently dropped +// by the OTLP exporter's attribute-length limit. The cut is byte-bounded and +// UTF-8-safe. +func TestCaptureIO_TruncatesOversizePayload(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + p.cfg.MaxPayloadBytes = 64 + big := strings.Repeat("x", 500) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Extensions.A2A = &pipeline.A2AExtension{ + Method: "message/send", + Parts: []pipeline.A2APart{{Kind: "text", Content: big}}, + } + run(t, p, pctx, allow(200)) + req, _ := roleSplit(t, exp.GetSpans()) + v, ok := findAttr(req, "input.value") + if !ok { + t.Fatal("input.value absent on a captured a2a hop with text parts") + } + got := v.AsString() + if len(got) > p.cfg.MaxPayloadBytes { + t.Errorf("input.value is %d bytes, exceeds cap %d", len(got), p.cfg.MaxPayloadBytes) + } + if !strings.HasSuffix(got, truncatedSuffix) { + t.Errorf("truncated input.value %q missing %q suffix", got, truncatedSuffix) + } +} + +// TestTruncate covers the helper's boundaries directly: within-cap passthrough, +// the opt-out, and a UTF-8-safe cut that never splits a multi-byte rune. +func TestTruncate(t *testing.T) { + if got := truncate("short", 64); got != "short" { + t.Errorf("within cap mutated: %q", got) + } + if got := truncate("anything", -1); got != "anything" { + t.Errorf("negative cap should disable truncation, got %q", got) + } + // A run of 3-byte runes ("世") cut to a byte budget that lands mid-rune: + // the result must be valid UTF-8 and within the cap. + s := strings.Repeat("世", 100) // 300 bytes + got := truncate(s, 40) + if len(got) > 40 { + t.Errorf("truncated to %d bytes, exceeds cap 40", len(got)) + } + if !utf8.ValidString(got) { + t.Errorf("truncation split a multi-byte rune: %q is not valid UTF-8", got) + } + if !strings.HasSuffix(got, truncatedSuffix) { + t.Errorf("missing truncation marker: %q", got) + } + // The suffix-can't-fit fallback: max below len(truncatedSuffix) with a + // multi-byte payload must still return valid UTF-8 within the cap (the + // marker is dropped, but a mid-rune byte cut is not). Guards the budget<=0 + // branch, which previously did a hard s[:max] byte cut. + tiny := truncate(strings.Repeat("世", 100), 4) // 4 < len(truncatedSuffix) + if len(tiny) > 4 { + t.Errorf("suffix-can't-fit cut to %d bytes, exceeds cap 4", len(tiny)) + } + if !utf8.ValidString(tiny) { + t.Errorf("suffix-can't-fit cut split a rune: %q is not valid UTF-8", tiny) + } +} + +// ---- otel_ca_file: a private CA for the collector ---- + +// TestConfig_CAFileImpliesTLS: a CA bundle is only meaningful for a TLS dial, +// so otel_ca_file turns otel_tls on. The contradictions are refused at decode +// like https:// + otel_tls:false: an explicit otel_tls:false beside the file, +// and an http:// endpoint beside either TLS knob. +func TestConfig_CAFileImpliesTLS(t *testing.T) { + cfg, err := decodeConfig([]byte(`{"self_id":"x","otel_endpoint":"collector.ns:4317","otel_ca_file":"/etc/lineage/ca.pem"}`)) + if err != nil { + t.Fatalf("decodeConfig: %v", err) + } + if !cfg.OTelTLS { + t.Error("otel_ca_file did not imply otel_tls") + } + if cfg.OTelCAFile != "/etc/lineage/ca.pem" { + t.Errorf("OTelCAFile = %q", cfg.OTelCAFile) + } + for name, raw := range map[string]string{ + "otel_ca_file with an explicit otel_tls:false": `{"self_id":"x","otel_ca_file":"/etc/lineage/ca.pem","otel_tls":false}`, + "http:// endpoint with otel_ca_file": `{"self_id":"x","otel_endpoint":"http://collector:4317","otel_ca_file":"/etc/lineage/ca.pem"}`, + "http:// endpoint with otel_tls:true": `{"self_id":"x","otel_endpoint":"http://collector:4317","otel_tls":true}`, + } { + if _, err := decodeConfig([]byte(raw)); err == nil { + t.Errorf("%s: accepted, want a refused contradiction", name) + } + } +} + +// selfSignedCAPEM returns a PEM-encoded self-signed CA certificate, enough +// for Init to build a cert pool from. +func selfSignedCAPEM(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "lineage-test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +// TestInit_CAFile: a private-CA bundle is loaded at Init and a bad one refuses +// to start (fail closed — never a silent fallback to the system roots). The +// dial is lazy, so a valid bundle lets Init succeed without a collector. The +// Config is built directly, with OTelTLS left false, to pin that Init keys +// on the file alone. +func TestInit_CAFile(t *testing.T) { + dir := t.TempDir() + good := dir + "/ca.pem" + if err := os.WriteFile(good, selfSignedCAPEM(t), 0o600); err != nil { + t.Fatal(err) + } + garbage := dir + "/garbage.pem" + if err := os.WriteFile(garbage, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + caFile string + wantErr bool + }{ + {"valid bundle starts", good, false}, + {"missing file refuses", dir + "/absent.pem", true}, + {"file with no certificate refuses", garbage, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "collector.ns:4317", OTelCAFile: tc.caFile, SelfID: "x"} + err := p.Init(context.Background()) + if tc.wantErr { + if err == nil { + _ = p.Shutdown(context.Background()) + t.Fatal("Init succeeded with an unusable otel_ca_file") + } + if p.Ready() { + t.Error("plugin ready after a refused Init") + } + return + } + if err != nil { + t.Fatalf("Init: %v", err) + } + if !p.Ready() { + t.Error("plugin not ready after a successful Init") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = p.Shutdown(ctx) + }) + } +} + +// ---- export failure visibility ---- + +type failingExporter struct{ calls atomic.Int32 } + +func (f *failingExporter) ExportSpans(context.Context, []sdktrace.ReadOnlySpan) error { + f.calls.Add(1) + return errors.New("collector unreachable") +} +func (f *failingExporter) Shutdown(context.Context) error { return nil } + +// TestExportObserver_CountsFailures: a refused batch is counted (and logged +// under the plugin's name) and the error still reaches the SDK unchanged, so +// the batch processor's own handling is not altered. +func TestExportObserver_CountsFailures(t *testing.T) { + fe := &failingExporter{} + p := NewLineageTelemetry() + obs := &exportObserver{SpanExporter: fe, failures: &p.exportFailures} + if err := obs.ExportSpans(context.Background(), nil); err == nil { + t.Fatal("observer swallowed the export error") + } + if got := p.exportFailures.Load(); got != 1 { + t.Fatalf("exportFailures = %d after one refused batch, want 1", got) + } + // Reachable through the SDK: a span ended on a provider that exports + // through the observer increments the counter again. + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(obs)) + _, span := tp.Tracer("t").Start(context.Background(), "s") + span.End() + _ = tp.Shutdown(context.Background()) + if got := p.exportFailures.Load(); got != 2 { + t.Errorf("exportFailures = %d after a span through the SDK, want 2", got) + } + if fe.calls.Load() != 2 { + t.Errorf("underlying exporter called %d times, want 2", fe.calls.Load()) + } +} + +// TestLogExportFailure pins the throttle: powers of two are logged, nothing +// else is, so a long outage costs log lines in proportion to log2(length). +func TestLogExportFailure(t *testing.T) { + var logged []uint64 + for n := uint64(1); n <= 40; n++ { + if logExportFailure(n) { + logged = append(logged, n) + } + } + if !slices.Equal(logged, []uint64{1, 2, 4, 8, 16, 32}) { + t.Errorf("logged failures = %v, want the powers of two up to 32", logged) + } +} diff --git a/authbridge/cmd/authbridge-envoy/go.mod b/authbridge/cmd/authbridge-envoy/go.mod index 494878bf..07d5b807 100644 --- a/authbridge/cmd/authbridge-envoy/go.mod +++ b/authbridge/cmd/authbridge-envoy/go.mod @@ -39,6 +39,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect @@ -52,6 +53,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/go-clone v1.7.3 // indirect github.com/huandu/go-sqlbuilder v1.42.1 // indirect @@ -110,9 +112,12 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -124,6 +129,7 @@ require ( golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/ini.v1 v1.67.3 // indirect diff --git a/authbridge/cmd/authbridge-envoy/plugins_lineage.go b/authbridge/cmd/authbridge-envoy/plugins_lineage.go new file mode 100644 index 00000000..76ea1e64 --- /dev/null +++ b/authbridge/cmd/authbridge-envoy/plugins_lineage.go @@ -0,0 +1,5 @@ +//go:build !exclude_plugin_lineage + +package main + +import _ "github.com/rossoctl/cortex/authbridge/authlib/plugins/lineage" diff --git a/authbridge/cmd/authbridge-proxy/go.mod b/authbridge/cmd/authbridge-proxy/go.mod index 2de1aedf..a3ffb0d5 100644 --- a/authbridge/cmd/authbridge-proxy/go.mod +++ b/authbridge/cmd/authbridge-proxy/go.mod @@ -32,6 +32,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect @@ -43,6 +44,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/go-clone v1.7.3 // indirect github.com/huandu/go-sqlbuilder v1.42.1 // indirect @@ -100,9 +102,12 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -114,6 +119,7 @@ require ( golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect diff --git a/authbridge/cmd/authbridge-proxy/plugins_lineage.go b/authbridge/cmd/authbridge-proxy/plugins_lineage.go new file mode 100644 index 00000000..76ea1e64 --- /dev/null +++ b/authbridge/cmd/authbridge-proxy/plugins_lineage.go @@ -0,0 +1,5 @@ +//go:build !exclude_plugin_lineage + +package main + +import _ "github.com/rossoctl/cortex/authbridge/authlib/plugins/lineage" diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md new file mode 100644 index 00000000..88eb7270 --- /dev/null +++ b/authbridge/docs/lineage-wire-contract.md @@ -0,0 +1,346 @@ +# Lineage wire contract — two-span sidecar lineage (v1.6.2) + +What the AuthBridge `lineage-telemetry` plugin emits, what it writes onto the wire, and what the +data-governance `sidecar` interactions algorithm (ADR-0030) commits to when consuming it. + +- Producer: `cortex/authbridge/authlib/plugins/lineage/` (repo `rossoctl/cortex`). +- Consumer: `data_governance/processors/interactions/sidecar.py`; vocabulary in + `data_governance/sidecar_facts.py` (repo `rossoctl/lab-data-governance`). + +This document is kept **byte-identical in both repositories** — `cortex/authbridge/docs/lineage-wire-contract.md` +and `lab-data-governance/docs/sidecar-wire-contract.md`. The version in the title is the pin: a +minor bump means producer behaviour or vocabulary changed; a patch bump means prose only. A change +is a pull request to both repositories. + +## 1. Principles + +- **Facts, not meaning.** The producer emits what it observed on the wire plus parsed protocol + facts. All vocabulary — hop kinds, entity kinds, caller/callee — lives in the consumer's + `classify()`. +- **Emit on sight.** Two spans per exchange, each emitted and ended as soon as its half has been + seen. No span is held open across the wait and no body is buffered for the exchange's lifetime. +- **One channel for the sidecar chain.** The sidecar parent chain lives entirely in one W3C + `tracestate` member, `lineage-parent`. A valid forwarded `traceparent` is never modified. The + sidecar's spans and an application's own spans land in different backends, so a chain that + pointed across the two would always dangle somewhere; the member keeps the sidecar chain + self-consistent in this store while an instrumented application keeps its own `traceparent` chain + intact toward its own backend. +- **No mechanism may guess.** A mechanism whose correctness depends on a precondition it cannot + verify at runtime does not belong in the producer. When attribution is unknown the producer says + so — `lineage.parent.source` is `wire` or `none` — and the edge is visibly absent. A missing edge + is recoverable downstream; a confidently wrong one is not, because it is indistinguishable from a + true one. +- **Parsers reduce payloads; interactions do not depend on them.** `input.value` and + `output.value` are semantic content produced by the protocol parsers, not raw bytes, and they are + enrichment only. Every exchange the sidecar saw becomes a complete interaction — kind, endpoints, + timing, status — whether or not a body could be read. + +## 2. Span model + +One HTTP exchange through the sidecar produces two OTLP spans. + +| | request span | response span | +|---|---|---| +| emitted | when the request (headers and body) has been seen and forwarded | when the response has been fully relayed, or the stream ends or errors | +| SpanKind | SERVER for inbound, CLIENT for outbound | same as its request span | +| parent | see §3.2 | its request span | +| carries | caller-side facts and `input.value` | outcome and status facts and `output.value` | + +- `lineage.exchange.id` is the request span's own span id, echoed on both spans. No new identifier + is minted; the response span names its request twin. Exchange duration is computed downstream as + response end minus request start. +- The response span is emitted at stream end **even when no response was produced** — client + disconnect, upstream reset, plugin denial. It then carries `lineage.outcome` and whatever status + exists, so the row completes as failed instead of dangling. +- The producer samples unconditionally: a valid caller `traceparent` with the sampled-out flag + (`…-00`) does not suppress the spans — lineage is an audit record, and a caller-chosen flag is + not an opt-out from being graphed. The forwarded `traceparent` keeps the caller's flags (§3.3: a + valid one is never modified); only what this producer exports ignores them. +- A lone request span means one of four things: the sidecar died mid-exchange; a panic while + emitting the response span was recovered by the pipeline (a WARN is logged); the response span + was emitted but lost — the two halves enter a batching exporter an exchange apart, so a response + can be lost after its request has flushed; or the exchange outlived a config hot-reload — old + pipelines stop a drain window (default 30 s) after the swap, and a response span emitted into + the old, already-shut-down provider is dropped, which is routine for SSE or LLM exchanges + longer than the window. The consumer renders it as in-flight, never as a wrong + pairing. A response span whose `lineage.outcome` is absent derives with `error` NULL (honest + unknown), never `false`. +- **Scope of `denied`.** The lineage plugin runs after the gate plugins and the pipeline + short-circuits on a request-phase denial, so an exchange a gate rejects **before** the request + span exists emits **no spans at all** and is invisible to lineage. `denied` appears only for + denials after the request span exists: response-phase denials, or gates ordered after lineage. + Spans for gate-denied traffic are a named follow-up, not current behaviour. +- **Bypass.** Requests whose path matches a `bypass_paths` glob, and outbound requests whose + host matches a `bypass_hosts` glob (both `path.Match`; hosts port-stripped and case folded, + paths query-stripped and normalized), produce no spans + (defaults in §6). `bypass_hosts` is outbound-only: an inbound `Host` is the caller's own header, + so honouring it there would let a caller suppress the record of its own request. + +## 3. Trace context on the wire + +### 3.1 The stamp + +Each lineage element — inbound and outbound alike — re-stamps one W3C `tracestate` member on the +request it forwards: + +``` +tracestate: lineage-parent= +``` + +Inbound stamps toward its own application: an application that propagates W3C context carries +`tracestate` through its per-request causal chain, so the member surfaces on exactly the outbound +calls that inbound caused. Outbound stamps toward the peer, whose inbound sidecar reads it as its +parent. Foreign `tracestate` members are preserved. The key is producer-owned and names the lineage +domain; it never lands in stored data, so it is wire-only, and every sidecar on a hop must run the +same key. + +The stamp needs a valid `traceparent` to ride on: a W3C reader takes `tracestate` only alongside a +valid `traceparent`. That is why §3.3 exists. + +### 3.2 Parent selection + +The request span's parent is chosen by the first rule that applies, in both directions: + +| precedence | parent | `lineage.parent.source` | +|---|---|---| +| 1 | the `lineage-parent` stamp, if the wire context is valid and the member parses as a span id in that trace | `tracestate` | +| 2 | the wire `traceparent`'s parent span, if the wire context is valid | `wire` | +| 3 | none — the request span roots a new trace | `none` | + +There is no fourth option. A malformed stamp falls through to the wire parent silently. Under +precedence 1 the parent is the previous lineage element: the caller sidecar's outbound request span +for an inbound, this pod's inbound request span for an outbound. Under precedence 2 the parent is +usually a span this pipeline never exported (an application-internal span, or an un-sidecared +caller's); the exchange still derives as a complete interaction, but as a trace entry rather than a +child. + +### 3.3 The `traceparent` rule + +- **Valid → never modified.** A `traceparent` the W3C propagator accepts is forwarded byte for + byte. +- **Invalid → restarted.** When the request carries no valid `traceparent` — absent, empty or + malformed, as the propagator judges it, the same verdict that yields `parent.source=none` — the + producer forwards a new one naming its own request span, and the caller's `tracestate` is + dropped; the stamp is then written alone. This is W3C Trace Context's processing model for an + unparseable `traceparent`. Without it the next element has nothing to extract: a propagating + application roots a trace of its own and discards `tracestate`, the stamp never leaves the pod, + and every call the application makes derives as a separate root. +- Controlled by `mint_traceparent` (default on). Off, the producer writes no `traceparent` at all + and an entry without a valid one fragments as described. + +Consequences for the stored trace: a trace entered through a sidecar with no valid `traceparent` +has a real, exported root (the entry request span, `parent.source=none`). A trace entered with a +foreign valid `traceparent` — an un-sidecared driver or UI that propagates — keeps one dangling +parent at the trace edge, by design. + +### 3.4 What the producer writes onto a forwarded request + +| header | when | value | +|---|---|---| +| `tracestate` | both directions, whenever a valid context exists after §3.3 — except a bypassed exchange (§6), a not-yet-ready producer, or an inbound `tracestate` that refuses the insert (malformed member; skipped with a WARN) | the caller's members with `lineage-parent` set to this request span id | +| `traceparent` | only when the request carried no valid one and `mint_traceparent` is on | `00---` | + +Nothing else is written. The listener is responsible for carrying these header mutations to the +wire. + +### 3.5 Un-stamped traffic + +An application with no context propagation, one that strips `tracestate`, or a caller with no +sidecar yields precedence 2 or 3 at the next element. The trace fragments at that pod, visibly, +instead of being welded by a guess. The consequences per case: + +| case | inbound entry | that pod's outbound calls | +|---|---|---| +| caller propagates, application propagates | stamp or wire | stamp | +| caller propagates, application does not | wire | wire: each call a trace of its own, restarted by its outbound sidecar | +| caller sends no valid context, application propagates | none (restarted) | stamp: one tree under the entry | +| caller sends no valid context, application does not | none (restarted) | wire: each call a trace of its own | + +A bypassed exchange (§6) is a fifth case: no spans at that element and the stamp passes through +unchanged, so the next element parents on the last element that did stamp — the bypassed hop is +simply absent from the chain. + +## 4. Attributes + +Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-telemetry`. Both are +constant on every pod, deliberately: the resource says what produced the span, and the span says +which workload it was beside (`lineage.self.id`). A backend that groups by `service.name` — +Phoenix and Jaeger both do — therefore shows one merged service. Operators wanting per-workload +grouping map `lineage.self.id` onto `service.name` in a collector transform, the same way §8 +handles `openinference.span.kind`. + +| key | on | example | notes | +|---|---|---|---| +| `lineage.exchange.id` | both | `00f067aa0ba902b7` | the request span id, hex | +| `lineage.role` | both | `request` \| `response` | which half this span is | +| `lineage.direction` | both | `inbound` \| `outbound` | | +| `lineage.self.id` | both | `weather-service` | this workload's identity, from `self_id` or `self_id_file`, **reduced to its last non-empty `/`-segment**: a SPIFFE ID `spiffe://td/ns/team1/sa/agent` emits `agent`, and two identities that differ only above that segment emit the same value — the consumer keys entity identity on it (§7). The producer refuses to start without an identity | +| `lineage.peer.host` | both, when present | `weather-tool-mcp.team1.svc:8000` | the Host/authority header. Outbound: the service being called. Inbound: the address this workload was reached on | +| `lineage.protocol` | both | `a2a` \| `mcp` \| `inference` \| `http` | which parser matched, at fixed precedence `a2a` > `mcp` > `inference`; `http` = none. The precedence is load-bearing: the parsers are not mutually exclusive — `mcp-parser` attaches to any JSON-RPC body, including every a2a exchange — so an a2a hop is labeled `a2a`, never `mcp`. The payload reduction (§5) is keyed by this label, reading the same protocol's parser | +| `lineage.parent.source` | request | `tracestate` \| `wire` \| `none` | which precedence in §3.2 chose the parent. An audit fact; the consumer derives nothing from it | +| `http.method` | request, when the listener supplies it | `POST` | all listeners do | +| `url.path` | request, when present | `/mcp` | query-free: anything from `?` on is stripped before emission (per OTel semconv; the query can carry secrets and is never captured) | +| `url.scheme` | request, when present | `http` | the listener's observed scheme; `tcp` marks a proxy-sidecar CONNECT tunnel (§5 Limits). Optional: the consumer composes `scheme://peer.host + url.path` only when all three exist | +| `a2a.method`, `a2a.session_id` | request, a2a | `message/send` | parsed facts | +| `mcp.method`, `mcp.tool` | request, mcp | `tools/call`, `get_weather` | `mcp.tool` only for `tools/call` | +| `inference.model` | request, inference | `qwen2.5:7b` | from the parsed request body | +| `lineage.principal.sub`, `lineage.principal.client` | request, inbound, only when a gate plugin validated a JWT | `alice` | raw identity facts, never inferred from a network address | +| `input.value` | request, with `capture_io` | `{"city":"Tokyo"}` | see §5 | +| `output.value` | response, with `capture_io` | `{...}` | see §5; absent when unparsed or streamed | +| `http.status_code` | response, when a status was produced | `200` | | +| `lineage.outcome` | response | `ok` \| `denied` \| `error` \| `abandoned` | how the exchange ended as the proxy saw it; `ok` and `denied` are the pipeline's verdicts, with or without a status; `abandoned` is a nil outcome, or an error that never produced a status | +| `lineage.denied_by` | response, denials | `jwt-validation` | the plugin that denied | + +`http.method` and `http.status_code` are the pre-1.21 OpenTelemetry semantic-convention keys, kept +deliberately: this producer's vocabulary is `lineage.*` plus these two well-known keys, and +interoperability with generic OpenTelemetry tooling is not a goal. + +Span names: request = `{self.id} {protocol} {op}`, where op is `mcp.tool` (else `mcp.method`), +`a2a.method`, or `inference.model`, falling back to `url.path`, and is omitted when empty; +response = the request name + ` response`. + +Every variable-content string attribute above, and the request span name, is capped at +`max_attr_bytes` (default 256), cut on a UTF-8 boundary and suffixed `…[truncated]` — several of +these values are caller-controlled (`url.path`, `lineage.peer.host`, `mcp.tool`, +`a2a.session_id`) and the SDK never truncates on its own. `input.value` / `output.value` carry +their own `max_payload_bytes` cap (§5); the fixed-vocabulary facts and the hex ids are bounded by +construction. + +## 5. Payloads + +- `input.value` and `output.value` are the parsers' semantic reduction of the request and + response: for a2a the message or artifact text, for mcp the tool arguments and the text content + of a result, for inference the messages and the completion or tool calls. +- Two heuristics live in that reduction, affecting payloads only, never interactions: the a2a + parser falls back to the status-message text when a result carries no artifact; and the lineage + plugin suppresses an A2A protocol event from `output.value` when the reduced value is a JSON + object whose `kind` is exactly one of `status-update`, `task-status-update`, `artifact-update`, + `working`, `canceled`. Either can mislabel an unusual payload; since payload absence is legal, the + failure mode is a missing or imprecise `output.value`, never a wrong interaction. +- A value longer than `max_payload_bytes` is cut on a UTF-8 boundary and suffixed `…[truncated]`, + so the loss is visible in the span. A truncated value no longer parses as JSON; the consumer then + stores it as a string. Deployments that want whole prompts set `max_payload_bytes: -1` or raise it + (LLM chat prompts on the reference fleet reach 14 KB; a third exceed the 4096 default). +- In envoy-sidecar mode, TLS-passthrough connections bypass Envoy's HTTP filter chain entirely and + produce no exchange — a capture gap. In proxy-sidecar mode an HTTPS destination is a CONNECT + tunnel, which IS an exchange: an ordinary span pair with `http.method=CONNECT`, + `url.scheme=tcp`, `lineage.peer.host` naming the dial target, no path and no payload — the bytes + inside the tunnel are opaque. The producer does not filter tunnel exchanges; what they mean is + the consumer's call, like every other fact. + +## 6. Producer configuration + +| key | default | meaning | +|---|---|---| +| `otel_endpoint` | `localhost:4317` | OTLP gRPC target; `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused | +| `otel_tls` | `false` | TLS to the collector, verified against the system roots or `otel_ca_file`; an `https://` endpoint implies it. Refused contradictions: `https://` with `otel_tls: false`, `http://` with `otel_tls: true` or `otel_ca_file`. Plaintext to a non-loopback collector is allowed and logged as a WARN at start — spans carry principal facts, and payloads under `capture_io` | +| `otel_ca_file` | — | PEM bundle to verify the collector's certificate against, for a private CA; implies `otel_tls`, and `otel_ca_file` with `otel_tls: false` is refused. An unreadable file, or one with no certificate, refuses to start | +| `capture_io` | `false` | attach `input.value` / `output.value` | +| `max_payload_bytes` | `4096` | producer-side cap on those two values; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start | +| `max_attr_bytes` | `256` | cap on every variable-content string attribute and the span name (§4); same `0` / `-1` / negative semantics as `max_payload_bytes` | +| `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | +| `bypass_paths` | `/.well-known/*`, `/healthz`, `/readyz`, `/health` | path globs (`path.Match`; `*` does not cross `/`) that produce no spans, matched by the shared bypass package (query stripped, path normalized) — the same key and semantics as `jwt-validation` and `sparc` | +| `bypass_hosts` | `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*` | outbound host globs that produce no spans | +| `self_id` | — | this workload's identity (§4: reduced to its last `/`-segment) | +| `self_id_file` | `/shared/client-id.txt` | read when `self_id` is empty; the producer refuses to start if neither yields an identity | + +Setting `bypass_paths` or `bypass_hosts` **replaces** the default list rather than extending it — +the convention the `ibac`, `sparc` and `cpex` plugins use for their keys of the same name. An +operator who adds one entry must restate the defaults they want kept. An entry that would match +everything by its literal shape — empty, whitespace-only, `*` for a host, `*` or `/*` for a +path — is refused at start, as is an entry of either kind that is not valid `path.Match` syntax. +(The check is by shape, not by semantics: an exotic glob that happens to match every value, such +as `?*`, is the operator's own deliberate choice and is accepted — the same behaviour as the +sibling plugins' keys.) + +Unknown keys are a boot error. + +## 7. Consumer commitments + +- Interaction id = `uuid5(NS_INTERACTION, f"{trace_id}/{exchange.id}")`. The request half fills + caller, callee, request payload hash and started-at; the response half fills response payload + hash, ended-at and error. +- **Whole-trace reconcile, not per-half upsert.** Every arriving span re-derives its entire trace + from all stored spans of that trace: idempotent, order-independent, authoritative. Rows no longer + justified by the current span set are deleted, trace-scoped; `entities` is global and never + deleted. A half arriving alone still produces its row, so in-flight stays visible. The wanted set + can shrink — an inbound request is a real interaction until its outbound ancestor arrives, then it + demotes to the callee-side echo and its row is removed — which a per-half upsert cannot express. + See ADR-0030. +- Anchors: `role=request` and either `direction=outbound`, or `direction=inbound` with no stored + anchor ancestor (the trace entry). Entry detection tolerates both a NULL parent and a dangling + wire parent, since an un-sidecared caller's root span is never exported. +- Response spans are never anchors; they attach by `exchange.id`. +- Kinds and entity identity come from the facts only. `classify()` never requires `input.value` or + `output.value`; bodyless exchanges produce complete, first-class interaction rows with NULL payload + hashes, and the UI renders them like any other row. +- The consumer never welds a fragmented trace: an anchor whose parent is not a stored anchor derives + as a root. +- `content_kind` vocabulary stays ADR-0014-compatible; the classification processor consumes + `interaction_payloads` as a stream. +- Drain loop = the shared `data_governance/processors/_driver.py` StreamSpec. + +## 8. Retired names + +The producer must not emit these, and the consumer reads nothing from them. + +| retired | replaced by | +|---|---| +| `lineage.hop.kind`, `trust.hop_kind` | consumer `classify()` over (direction, protocol, mcp.method) | +| `lineage.source.id`, `lineage.target.id`, `trust.source_id`, `trust.target_id` | `lineage.self.id` + `lineage.peer.host` + `lineage.direction`; caller and callee computed downstream | +| `enduser.id`, `trust.principal_id` | `lineage.principal.*` | +| `source=sidecar` | the resource `service.name=authbridge` | +| `openinference.span.kind` | not a producer attribute; a display backend derives it from `lineage.protocol` in a collector transform | +| `lineage.peer.addr` | nothing; it was never producible in ext_proc mode. Anonymous inbound callers derive as `client:(unknown)` | +| config `is_principal`, `emit_body_hash` | nothing | +| tracestate keys `kglin`, `dg-parent` | `lineage-parent` | + +## 9. History + +Version ladder, newest first. Each line is what changed on the wire or in the vocabulary; the +mechanisms named as removed are not to be reintroduced. + +- **v1.6.2** — `url.path` and the span-name fallback derived from it are query-free: the producer + strips anything from `?` on before emission. Until now the envoy-sidecar listener's raw `:path` + pseudo-header put the query string on the wire regardless of `capture_io`; the proxy listeners + never delivered it. And every variable-content string attribute plus the span name is capped at + `max_attr_bytes` (default 256) — until now only the two payload values were bounded, so one + request could put a 100 KB span name into the backend. And the producer samples unconditionally + (§2): under the SDK-default ParentBased sampler a caller's sampled-out `traceparent` (`…-00`) + exported zero spans for the whole chain. And `bypass_paths` becomes a `path.Match` glob list via + the shared bypass matcher (it was a prefix match): the `/health` default no longer swallows + `/health-records/...`, and a pattern copied from a sibling plugin means the same thing here. + Prose: the `lineage.protocol` precedence (`a2a` > `mcp` > `inference`), always the producer's + behaviour, is stated in §4, and §2's lone-request-span causes gain config hot-reload. +- **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented + as reduced to its last `/`-segment before emission, which the producer has always done; + `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only + `path.Match` glob list (it was an unanchored substring match on both directions) and both bypass + lists are validated at start; `-1` is stated as the only `max_payload_bytes` opt-out, with any + other negative refused rather than silently unbounded. +- **v1.6** — an invalid or absent `traceparent` is restarted per W3C (`mint_traceparent`); + `lineage.parent.source` gains `none`; the stamp key becomes `lineage-parent`; `otel_tls` and + `max_payload_bytes` added; the document is vendored into the producer repository. Motivation: + one traceparent-less turn through a four-pod fleet derived nine roots instead of one — the + application's shim minted the trace, but with no `traceparent` to ride on the entry's stamp never + reached the application's outbound calls. +- **v1.5** — the single channel: both directions parent stamp-first; the outbound `traceparent` + rewrite ("the splice", v1.2–v1.4) removed; `lineage.peer.host` read on both directions; + `url.scheme` added. +- **v1.4** — `lineage.peer.addr` removed; the listener header diff made live on all handler + paths (until then the stamp never reached the wire in ext_proc mode, the mechanical cause of the + phantom-rooted per-pod trees stored before it). +- **v1.3** — the trace-keyed inbound map removed: it answered "the last inbound seen for this + trace", correct only while exactly one inbound of that trace is in flight, a precondition it could + not verify, and under same-trace concurrency it produced a real, exported, untrue parent with no + signal. A census before removal found zero spans attributed by it. `lineage.parent.source` added. +- **v1.2** — the `tracestate` stamp introduced (then keyed `kglin`), proven under same-trace + fan-in: six concurrent same-trace turns through a mid-chain agent paired 1/6 by the map and 6/6 + by the stamp. + +Naming decisions, not to be re-litigated: span attributes stay `lineage.*` — they are this +producer's own telemetry and name the domain, not a product. The `tracestate` key is a shared +channel every intermediary must preserve, so it carries a producer-owned, consumer-neutral name. + +Open item: the response-span name suffix (` response`) is cosmetic, for trace-viewer legibility +only. diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index bcfa16db..a8864a02 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -28,6 +28,7 @@ AuthBridge pipeline YAML, not whether it is compiled into the binary | [`ibac`](#ibac) | LLM-judge intent-based access control for outbound tool calls. | Alpha | Outbound | No | | [`inference-parser`](#inference-parser) | Parses LLM completions into `pctx.Extensions.Inference`. | Alpha | Outbound | No | | [`jwt-validation`](#jwt-validation) | Inbound JWT validation (signature, issuer, audience) against JWKS. | Ready | Inbound | YES | +| [`lineage-telemetry`](#lineage-telemetry) | Emits two facts-only OTel lineage spans per HTTP exchange, parented across pods through one `tracestate` member. | Alpha | Both | No | | [`litellm-budget-track`](#litellm-budget-track) | Tracks `x-litellm-response-cost` (with `-original` fallback) and enforces a daily budget limit. Place on whichever chain carries LLM traffic — inbound when fronting the LLM endpoint, outbound when hosting an agent via `authbridge exec`. | Alpha | Both | No | | [`mcp-parser`](#mcp-parser) | Parses MCP tool calls/results into `pctx.Extensions.MCP`. | Beta | Outbound | No | | [`opa`](#opa) | [OPA](https://www.openpolicyagent.org/docs) policy enforcement for inbound and outbound requests. | Alpha | Both | No | @@ -128,6 +129,32 @@ Validates inbound JWTs: signature via JWKS, issuer, and audience. - `placeholder_mode` (bool) — replace the validated inbound token with an opaque placeholder before forwarding, for later outbound resolution. Default `false`. - `placeholder_ttl` (string) — how long the real token is retained. Default `1h`. +## `lineage-telemetry` + +Emits two facts-only OpenTelemetry spans per HTTP exchange — a request +span on sight and a response span at stream end, paired by +`lineage.exchange.id` — carrying direction, protocol, endpoints, outcome +and, optionally, the parsed payload. Cross-pod parenting rides one +`tracestate` member, `lineage-parent`; a request that arrives with no +valid `traceparent` is forwarded with one naming the request span, and a +valid one is never modified. The wire format is +[lineage-wire-contract.md](./lineage-wire-contract.md). Place it after +the protocol parsers (declared in `RequiresAny`) and after +`jwt-validation` when the principal facts are wanted; a request-phase +denial by a plugin ordered before it emits no spans. + +- `otel_endpoint` (string) — OTLP gRPC target: `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused. Default `localhost:4317`. +- `otel_tls` (bool) — dial the collector with TLS, verified against the system roots or `otel_ca_file`. An `https://` endpoint implies it; `https://` with `otel_tls: false`, and `http://` with `otel_tls: true` or `otel_ca_file`, are refused. A plaintext dial to a non-loopback collector is allowed and logged as a WARN at start. Default `false`. +- `otel_ca_file` (string) — PEM bundle to verify the collector's certificate against (a private CA, e.g. cert-manager issued). Implies `otel_tls`; with an explicit `otel_tls: false` it is refused; an unreadable file or one with no certificate refuses to start. Default: system roots. +- `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. +- `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start. Default `4096`. +- `max_attr_bytes` (int) — cap on every variable-content string attribute (`url.path`, `lineage.peer.host`, `mcp.tool`, …) and the span name; same `0` / `-1` / negative semantics as `max_payload_bytes`. Default `256`. +- `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. +- `bypass_paths` (`[]string`) — path globs (`path.Match`, query stripped, path normalized — the shared bypass matcher `jwt-validation` and `sparc` use) that produce no spans. Default `/.well-known/*`, `/healthz`, `/readyz`, `/health`. Setting either bypass key replaces its default list rather than extending it, as in `ibac` / `sparc` / `cpex`; an entry matching everything is refused at start. +- `bypass_hosts` (`[]string`) — outbound host globs (`path.Match`, port stripped, case folded) that produce no spans; ignored inbound, where `Host` is caller-controlled. Default `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*`. +- `self_id` (string) — this workload's identity, emitted as `lineage.self.id`. +- `self_id_file` (string) — read when `self_id` is empty; the plugin refuses to start if neither yields an identity. Default `/shared/client-id.txt`. + ## `litellm-budget-track` Tracks the `x-litellm-response-cost` response header and enforces a diff --git a/authbridge/scripts/lite-tags/main_test.go b/authbridge/scripts/lite-tags/main_test.go index b68c258c..81ecb0af 100644 --- a/authbridge/scripts/lite-tags/main_test.go +++ b/authbridge/scripts/lite-tags/main_test.go @@ -16,7 +16,7 @@ func TestDiscover_ExpectedTags(t *testing.T) { t.Fatalf("discover: %v", err) } got := strings.Join(tags, ",") - want := "exclude_plugin_a2aparser,exclude_plugin_ibac,exclude_plugin_inferenceparser,exclude_plugin_mcpparser,exclude_plugin_opa,exclude_plugin_sparc,exclude_plugin_tokenbroker,exclude_plugin_toolprune" + want := "exclude_plugin_a2aparser,exclude_plugin_ibac,exclude_plugin_inferenceparser,exclude_plugin_lineage,exclude_plugin_mcpparser,exclude_plugin_opa,exclude_plugin_sparc,exclude_plugin_tokenbroker,exclude_plugin_toolprune" if got != want { t.Errorf("output changed\n got: %s\nwant: %s", got, want) }