From a2214f53711ef4c0142a4506ce4261ed34fb575b Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 14 Aug 2026 19:49:24 +0300 Subject: [PATCH 01/37] Feat: Add the lineage-telemetry plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emits two facts-only OTel spans per HTTP exchange crossing the sidecar: a request span when the request is seen, a response span at stream end, joined by lineage.exchange.id (the request span's own id). Span names are "{self_id} {protocol} {operation}", with the response span appending " response". The facts are lineage.role / direction / self.id / peer.host / protocol / principal.{sub,client} / outcome / denied_by / parent.source, plus url.scheme and url.path. With capture_io the parsed message content rides along as input.value and output.value, so a trace viewer shows the actual A2A message, MCP tool arguments or LLM prompt inline. capture_io is off by default — payloads may carry user messages and model output. The producer records facts, not meaning: no hop classification, no trust vocabulary, no identity guessing. Interpretation belongs to whatever consumes the spans, which is what keeps this package small and lets the vocabulary change without touching Go. Cross-pod parenting rides a single tracestate member: parent from dg-parent when present, else the wire parent, then re-stamp that member with this span's id. The forwarded traceparent is never modified, so an app with its own tracing keeps its chain intact toward its own backend. Nothing guesses a parent — missing data degrades to an explicit unknown. Config decodes with DisallowUnknownFields so a typo'd knob is a boot error rather than a silent default. self_id falls back to self_id_file, defaulting to the operator-mounted /shared/client-id.txt. bypass_paths and bypass_hosts keep agent-card discovery, health probes and telemetry backends out of the graph. Known limit, documented at plugin.go:22: this plugin orders itself after the gate plugins and the pipeline short-circuits on a request-phase reject, so an exchange denied by a gate before OnRequest ran emits no spans at all. Denials after that point are captured as outcome=denied with denied_by. Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 79 ++ authbridge/authlib/plugins/lineage/plugin.go | 746 ++++++++++++++ .../authlib/plugins/lineage/plugin_test.go | 912 ++++++++++++++++++ 3 files changed, 1737 insertions(+) create mode 100644 authbridge/authlib/plugins/lineage/config.go create mode 100644 authbridge/authlib/plugins/lineage/plugin.go create mode 100644 authbridge/authlib/plugins/lineage/plugin_test.go diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go new file mode 100644 index 000000000..c030c7e05 --- /dev/null +++ b/authbridge/authlib/plugins/lineage/config.go @@ -0,0 +1,79 @@ +package lineage + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" +) + +// Config holds the per-plugin configuration decoded from the pipeline YAML. +type Config struct { + // OTelEndpoint is the OTLP gRPC endpoint (host:port or http://host:port). + // Default: "localhost:4317" + OTelEndpoint string `json:"otel_endpoint"` + + // CaptureIO when true attaches parsed request/response content as + // input.value (request span) and output.value (response span) + // attributes, enabling Phoenix to display message content inline. + // + // For A2A (inbound agent calls): input = user message parts, output = artifact. + // For MCP tools/call: input = tool params JSON, output = tool result JSON. + // For Inference (LLM): input = messages array JSON, output = completion text. + // + // Off by default — enable only if traces do not contain PII or the + // OTel backend enforces appropriate access controls. + CaptureIO bool `json:"capture_io"` + + // BypassPaths lists URL path prefixes that should not generate lineage + // hops. Useful for suppressing infrastructure polling (agent-card + // discovery, health checks) that would otherwise flood the lineage graph. + // Default: ["/.well-known/", "/healthz", "/readyz", "/health"] + BypassPaths []string `json:"bypass_paths"` + + // BypassHosts lists target host substrings (matched against pctx.Host) + // that should not generate lineage hops. Useful for suppressing + // infrastructure outbound calls such as OTel trace exports. + // Default: ["otel-collector", "jaeger", "zipkin", "prometheus"] + BypassHosts []string `json:"bypass_hosts"` + + // SelfID is the agent's own stable identifier, emitted as the + // lineage.self.id fact on every span. Typically the Keycloak client ID + // of this workload. If empty, SelfIDFile is consulted instead. + SelfID string `json:"self_id"` + + // SelfIDFile is the path to a file containing the agent's own client ID. + // Defaults to /shared/client-id.txt (the operator-mounted credential). + // Ignored when SelfID is set. + SelfIDFile string `json:"self_id_file"` +} + +func defaultConfig() Config { + return Config{ + OTelEndpoint: "localhost:4317", + BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, + BypassHosts: []string{"otel-collector", "jaeger", "zipkin", "prometheus"}, + SelfIDFile: "/shared/client-id.txt", + } +} + +func decodeConfig(raw json.RawMessage) (Config, error) { + cfg := defaultConfig() + if len(raw) == 0 { + return cfg, nil + } + // Unknown keys are a boot error: a typo'd knob (capture-io, selfid_file) + // must not silently run with defaults. + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("lineage-telemetry config: %w", err) + } + if cfg.OTelEndpoint == "" { + cfg.OTelEndpoint = "localhost:4317" + } + // Strip http:// or https:// prefix — gRPC NewClient expects host:port only. + cfg.OTelEndpoint = strings.TrimPrefix(cfg.OTelEndpoint, "https://") + cfg.OTelEndpoint = strings.TrimPrefix(cfg.OTelEndpoint, "http://") + return cfg, nil +} diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go new file mode 100644 index 000000000..d31169f54 --- /dev/null +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -0,0 +1,746 @@ +// Package lineage provides the lineage-telemetry authbridge plugin. +// +// Two-span model (see docs/sidecar-wire-contract.md in the lab-data-governance +// repo, the consumer side — the law this file implements). Each HTTP exchange through the sidecar produces TWO OTLP spans: +// +// - a request span, emitted as soon as the request has been seen and +// forwarded, carrying caller-side facts + input.value; and +// - a response span, emitted at stream end (even when no response was +// produced), carrying status/outcome facts + output.value. +// +// Both spans are ended immediately at emission — no span is held open across +// the wait. lineage.exchange.id (= the request span's own span id) is echoed +// on both so the consumer pairs them. The plugin emits FACTS ONLY (direction, +// protocol, endpoints, parsed payloads); all vocabulary — hop kinds, entity +// kinds, caller/callee — lives in the consumer's classify(). See the "removed +// vs today" migration map in the contract for the attrs this no longer emits. +// +// The plugin implements Finisher so the response span is emitted at stream +// end whatever the outcome — including denials that happen AFTER the request +// span was recorded (a response-phase deny, or a request-phase deny by a +// plugin ordered after this one); pctx.Outcome() is available at that point +// and maps to lineage.outcome=denied + lineage.denied_by. LIMITATION: the +// pipeline YAML places this plugin after the gate plugins (ordering is not +// soft-declared under this capabilities model — position in the list is the +// contract), and the pipeline short-circuits on a request-phase Reject, +// so an exchange denied by a gate BEFORE OnRequest ran emits NO spans at all — +// it is invisible to lineage. Moving lineage ahead of the gates (spans for +// denied traffic too) is a named follow-up, not current behavior. +package lineage + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "strings" + "sync/atomic" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" + "go.opentelemetry.io/otel/trace" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/rossoctl/cortex/authbridge/authlib/pipeline" + "github.com/rossoctl/cortex/authbridge/authlib/plugins" +) + +const pluginName = "lineage-telemetry" + +// tracestateStampKey is the W3C tracestate member that carries the sidecar +// parent chain — the single channel every lineage element reads its parent +// from and writes its own request span id into (wire contract v1.5). Inbound +// stamps the request it forwards to its own app; the app's propagate-only shim +// carries tracestate through its per-request causal chain (contextvars), so +// the member surfaces on exactly the outbound calls that inbound caused. +// Outbound re-stamps the request it forwards to the peer, whose inbound +// sidecar reads it as its parent. Parent precedence is stamp > wire parent in +// BOTH directions, and the chosen source is recorded as the +// lineage.parent.source fact. The forwarded traceparent is never modified: +// the sidecar chain lives entirely in this member, so an app that emits its +// own spans keeps an intact traceparent chain toward its own backend while +// the sidecar chain stays self-consistent in ours. (Until v1.4 the outbound +// instead rewrote the forwarded traceparent — the splice; v1.5 removed it.) +// +// The key names the consuming data-governance system (W3C convention: the key +// identifies the owner of the entry) and is deliberately platform-neutral — +// it was `kglin` until 2026-08-04; the name never lands in stored data, so +// renaming is wire-only. +// +// A trace-keyed map (one entry per trace, "the last inbound seen") used to sit +// between the two. It was removed: its answer is correct only while exactly one +// inbound of that trace is in flight — a precondition it never checked and could +// not verify — and when it was wrong it produced a real, exported, walkable +// parent that was simply untrue. Un-stamped traffic falls to the wire +// parent, which is an app-internal span this pipeline never exported: the +// interaction still derives in full, but as a trace entry rather than a child. +// A visibly missing edge is recoverable; a silently wrong one is not. +const tracestateStampKey = "dg-parent" + +func init() { + plugins.RegisterPlugin(pluginName, func() pipeline.Plugin { return NewLineageTelemetry() }) +} + +// exchangeState carries what OnFinish needs to emit the response span as the +// twin of the request span emitted in OnRequest. +type exchangeState struct { + // reqCtx is the (already-ended) request span's context — the parent of + // the response span. An ended span's SpanContext is a valid parent. + reqCtx trace.SpanContext + // common holds the attributes shared by both spans (lineage.direction, + // self.id, peer.*, protocol, exchange.id) — NOT lineage.role, which + // differs per span. Computed once so both spans agree byte-for-byte. + common []attribute.KeyValue + spanKind trace.SpanKind + // spanName is the request span's name; the response span appends " response". + spanName string + // protocol is the request span's lineage.protocol fact; the response + // span's output.value must be read through the SAME protocol's parser + // (parsers are precedence-ordered, not mutually exclusive — mcp-parser + // also matches any JSON-RPC body, including every a2a exchange). + protocol string +} + +// LineageTelemetry emits OTel spans for each request hop observed by authbridge. +type LineageTelemetry struct { + cfg Config + tp *sdktrace.TracerProvider + tracer trace.Tracer + ready atomic.Bool + propagator propagation.TextMapPropagator + selfID string // agent's own client ID for the lineage.self.id fact +} + +// NewLineageTelemetry constructs an unconfigured plugin. Configure + Init must +// run before it serves traffic (guarded by Ready()). +func NewLineageTelemetry() *LineageTelemetry { + return &LineageTelemetry{ + propagator: propagation.TraceContext{}, + } +} + +func (p *LineageTelemetry) Name() string { return pluginName } + +func (p *LineageTelemetry) Capabilities() pipeline.PluginCapabilities { + return pipeline.PluginCapabilities{ + // At least one protocol parser must be present and earlier in the + // chain: the protocol fact and both payload reductions read the + // parsers' Extensions. A chain that misorders lineage before its + // parsers (or has none) fails at startup instead of silently + // emitting lineage.protocol="http" on everything. jwt-validation + // ordering (for the principal facts) cannot be soft-declared under + // this capabilities model — list it before lineage in the YAML. + RequiresAny: []string{"a2a-parser", "mcp-parser", "inference-parser"}, + // The contract is cited major.minor only, deliberately: patch + // revisions (v1.5.x) clarify prose and never change span semantics, + // so a patch bump must not imply a producer change. + Description: "Emits two facts-only lineage spans per HTTP exchange (wire contract v1.5).", + } +} + +func (p *LineageTelemetry) Configure(raw json.RawMessage) error { + cfg, err := decodeConfig(raw) + if err != nil { + return err + } + p.cfg = cfg + return nil +} + +func (p *LineageTelemetry) Init(ctx context.Context) error { + endpoint := p.cfg.OTelEndpoint + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + return fmt.Errorf("lineage-telemetry: gRPC dial %s: %w", endpoint, err) + } + + exporter, err := otlptracegrpc.New(ctx, + otlptracegrpc.WithGRPCConn(conn), + ) + if err != nil { + return fmt.Errorf("lineage-telemetry: OTLP exporter: %w", err) + } + + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceNameKey.String("authbridge"), + attribute.String("authbridge.component", pluginName), + ), + ) + if err != nil { + slog.Warn("lineage-telemetry: resource detection failed, using default", "error", err) + res = resource.Default() + } + + p.tp = sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + ) + p.tracer = p.tp.Tracer("authbridge/" + pluginName) + + // Resolve self identity for the lineage.self.id fact. Every span this + // plugin emits is a claim of the form "X did Y"; with no X there is no + // claim to make, so an unresolvable identity refuses to start rather + // than serving traffic under a plausible-but-wrong label ("no mechanism + // may guess", contract v1.3). Note the asymmetry with this file's other + // unknowns: a missing status, payload or parent anchor is a missing PART + // of a fact and degrades honestly (abandoned / NULL / parent.source=wire). + // Identity is the fact's subject — it has no degraded form, and a shared + // placeholder would collapse every unidentified pod onto one entity row + // (entity id = uuid5("{kind}:{self.id}"), and entities is upsert-only). + if p.cfg.SelfID != "" { + p.selfID = p.cfg.SelfID + } else if p.cfg.SelfIDFile != "" { + raw, err := os.ReadFile(p.cfg.SelfIDFile) + if err != nil { + return fmt.Errorf("lineage-telemetry: no inline self_id and self_id_file unreadable: %w", err) + } + p.selfID = strings.TrimSpace(string(raw)) + } + if p.selfID == "" { + return fmt.Errorf("lineage-telemetry: self identity unresolved (empty self_id and self_id_file %q)", p.cfg.SelfIDFile) + } + + p.ready.Store(true) + slog.Info("lineage-telemetry: initialized", "endpoint", endpoint, "self_id", p.selfID) + return nil +} + +func (p *LineageTelemetry) Shutdown(ctx context.Context) error { + if p.tp == nil { + return nil + } + return p.tp.Shutdown(ctx) +} + +func (p *LineageTelemetry) Ready() bool { return p.ready.Load() } + +func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context) pipeline.Action { + if !p.ready.Load() { + pctx.Skip("not_ready") + return pipeline.Action{Type: pipeline.Continue} + } + + // Skip infrastructure paths (health checks, agent-card discovery, etc.) + for _, prefix := range p.cfg.BypassPaths { + if strings.HasPrefix(pctx.Path, prefix) { + pctx.Skip("bypass_path") + return pipeline.Action{Type: pipeline.Continue} + } + } + + // Skip infrastructure outbound targets (OTel exporters, metrics scrapers, etc.) + for _, substr := range p.cfg.BypassHosts { + if strings.Contains(pctx.Host, substr) { + pctx.Skip("bypass_host") + return pipeline.Action{Type: pipeline.Continue} + } + } + + // Extract remote trace context from the incoming W3C traceparent header. + // HeaderCarrier wraps http.Header and uses case-insensitive Get/Keys so + // canonical-form keys ("Traceparent") match the propagator's lowercase + // lookups. + remoteCtx := p.propagator.Extract(ctx, propagation.HeaderCarrier(pctx.Headers)) + + protocol := protocolOf(pctx) + self := serviceLabel(p.selfID) + spanKind := spanKindFor(pctx.Direction) + spanName := requestSpanName(self, protocol, spanOp(pctx, protocol)) + + // Facts shared by both spans (exchange.id is appended once the request + // span exists, since it IS the request span id). + base := baseAttrs(pctx, self, protocol) + + // Request-span attributes: role + shared facts + request-only facts. + reqAttrs := make([]attribute.KeyValue, 0, len(base)+8) + reqAttrs = append(reqAttrs, attribute.String("lineage.role", "request")) + reqAttrs = append(reqAttrs, base...) + reqAttrs = p.appendRequestFacts(reqAttrs, pctx, protocol) + + // (3) parent · (4) emit · (5) re-stamp — wire contract v1.5. The emit is + // unconditional; the two calls around it are the stamp machinery. + // + // >>> OPTION-4 DELETION POINT <<< + // A pure read-only sidecar deletes exactly the selectParent and + // restampTracestate calls below (and the parent.source fact), parenting on + // remoteCtx alone — wire-parent-only propagation. The emit stays. The + // trade-off to weigh first: without the stamp, two sidecarred pods can + // only be joined through the app's own propagation, so cross-pod + // parenting degrades to whatever the wire parent happens to carry. + parent, parentSource := selectParent(ctx, remoteCtx) + reqAttrs = append(reqAttrs, attribute.String("lineage.parent.source", parentSource)) + reqCtx := p.emitRequestSpan(parent, spanName, spanKind, reqAttrs) + exchangeID := reqCtx.SpanID().String() + restampTracestate(pctx, remoteCtx, exchangeID) + + common := make([]attribute.KeyValue, 0, len(base)+1) + common = append(common, base...) + common = append(common, attribute.String("lineage.exchange.id", exchangeID)) + + pipeline.SetState(pctx, pluginName, &exchangeState{ + reqCtx: reqCtx, + common: common, + spanKind: spanKind, + spanName: spanName, + protocol: protocol, + }) + pctx.Observe("recorded_request") + return pipeline.Action{Type: pipeline.Continue} +} + +// selectParent is step (3) of the single-channel parenting mechanism (wire +// contract v1.5): the parent is the tracestate stamp — the previous sidecar +// element in the chain (the caller's outbound for an inbound, this pod's +// inbound for an outbound) — else the wire parent. Same precedence in both +// directions. There is deliberately no third option: guessing an attribution +// is worse than declining to give one. Returns the parent context and the +// source label the caller emits as the lineage.parent.source fact. +func selectParent(ctx, remoteCtx context.Context) (context.Context, string) { + rsc := trace.SpanContextFromContext(remoteCtx) + if rsc.IsValid() { + if psc, ok := stampedParent(rsc); ok { + return trace.ContextWithRemoteSpanContext(ctx, psc), "tracestate" + } + } + return remoteCtx, "wire" +} + +// emitRequestSpan is step (4): emit the request span under parent and end it +// immediately — no span is held open across the exchange. lineage.exchange.id +// is the span's OWN id, so it can only be set after Start. Returns the span's +// context; an ended span's SpanContext remains a valid parent for the response +// span, and its span id is the exchange id. +func (p *LineageTelemetry) emitRequestSpan( + parent context.Context, + spanName string, + spanKind trace.SpanKind, + reqAttrs []attribute.KeyValue, +) trace.SpanContext { + _, span := p.tracer.Start(parent, spanName, + trace.WithSpanKind(spanKind), + trace.WithAttributes(reqAttrs...), + ) + sc := span.SpanContext() + span.SetAttributes(attribute.String("lineage.exchange.id", sc.SpanID().String())) + span.End() + return sc +} + +// restampTracestate is step (5): rewrite the forwarded request's tracestate +// member with this exchange id — both directions. Inbound: the app's +// propagate-only shim couriers it to exactly the outbound calls this inbound +// caused. Outbound: the peer sidecar's inbound reads it as its parent. The +// forwarded traceparent is never modified (see tracestateStampKey). A valid +// wire traceparent is required — without one the app's shim starts a fresh +// root trace and drops the tracestate anyway, so there is nothing to stamp. +// The listener is responsible for propagating this header mutation (ext_proc +// emits a SetHeaders diff). +func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchangeID string) { + rsc := trace.SpanContextFromContext(remoteCtx) + if !rsc.IsValid() { + return + } + ts, err := rsc.TraceState().Insert(tracestateStampKey, exchangeID) + if err != nil { + // Stamp attempted and refused (tracestate full or a member malformed, + // W3C caps at 32 members / 512 bytes). Without this line the outcome + // is indistinguishable from "app has no shim". + slog.Warn("lineage-telemetry: tracestate stamp rejected; the next element will attribute as wire", + "exchange_id", exchangeID, "error", err) + return + } + pctx.Headers.Set("tracestate", ts.String()) +} + +// stampedParent resolves the tracestate stamp on an outbound wire context: +// the inbound exchange id this pod's sidecar wrote into tracestate on the +// forwarded request, carried back by the app's shim. Returns ok=false when +// the member is absent or malformed (caller falls back to the wire parent). +func stampedParent(rsc trace.SpanContext) (trace.SpanContext, bool) { + raw := rsc.TraceState().Get(tracestateStampKey) + if raw == "" { + return trace.SpanContext{}, false + } + sid, err := trace.SpanIDFromHex(raw) + if err != nil { + return trace.SpanContext{}, false + } + psc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: rsc.TraceID(), + SpanID: sid, + TraceFlags: rsc.TraceFlags(), + Remote: true, + }) + return psc, psc.IsValid() +} + +// OnResponse is a no-op. The response span is emitted in OnFinish (which fires +// on every finished exchange, including denials and abandonments), not here. +// The method exists only to satisfy the base pipeline.Plugin interface, which +// mandates OnResponse; it carries no logic in the two-span model. +func (p *LineageTelemetry) OnResponse(_ context.Context, _ *pipeline.Context) pipeline.Action { + return pipeline.Action{Type: pipeline.Continue} +} + +// OnFinish emits the response span — the twin of the request span, parented +// under it and echoing the same exchange.id — carrying outcome/status/output. +// Always fires at stream end, so a bodyless or failed exchange still completes +// as a first-class pair. Runs under a recover so an unexpected state never +// crashes the pipeline. +func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) { + defer func() { + if r := recover(); r != nil { + slog.Warn("lineage-telemetry: OnFinish panic recovered", "recover", r) + } + }() + + state := pipeline.GetState[exchangeState](pctx, pluginName) + if state == nil || !state.reqCtx.IsValid() { + return + } + + outcome, status, hasStatus, deniedBy := lineageOutcome(pctx.Outcome()) + + attrs := make([]attribute.KeyValue, 0, len(state.common)+5) + attrs = append(attrs, attribute.String("lineage.role", "response")) + attrs = append(attrs, state.common...) + attrs = append(attrs, attribute.String("lineage.outcome", outcome)) + if hasStatus { + attrs = append(attrs, attribute.Int("http.status_code", status)) + } + if deniedBy != "" { + attrs = append(attrs, attribute.String("lineage.denied_by", deniedBy)) + } + if p.cfg.CaptureIO { + if v := ioOutputValue(pctx, state.protocol); v != "" { + attrs = append(attrs, attribute.String("output.value", v)) + } + } + + parent := trace.ContextWithRemoteSpanContext(ctx, state.reqCtx) + _, span := p.tracer.Start(parent, state.spanName+" response", + trace.WithSpanKind(state.spanKind), + trace.WithAttributes(attrs...), + ) + span.End() +} + +// lineageOutcome maps the pipeline's 3-value Outcome (allow/deny/error, nil +// outside OnFinish) onto the contract's lineage.outcome vocabulary +// (ok|denied|error|abandoned) plus the http.status_code fact. A terminal state +// with no status written (upstream reset, client disconnect, listener death) +// is "abandoned" — the row completes as in-flight-turned-failed rather than +// dangling. hasStatus is false when no status code was produced. +func lineageOutcome(o *pipeline.Outcome) (outcome string, status int, hasStatus bool, deniedBy string) { + if o == nil { + return "abandoned", 0, false, "" + } + switch o.FinalAction { + case pipeline.OutcomeAllow: + return "ok", o.StatusCode, o.StatusCode > 0, "" + case pipeline.OutcomeDeny: + return "denied", o.StatusCode, o.StatusCode > 0, o.DenyingPlugin + case pipeline.OutcomeError: + if o.StatusCode > 0 { + return "error", o.StatusCode, true, "" + } + return "abandoned", 0, false, "" + default: + return "error", o.StatusCode, o.StatusCode > 0, "" + } +} + +// protocolOf reports which parser populated Extensions — the lineage.protocol +// fact. "http" means no parser matched. +func protocolOf(pctx *pipeline.Context) string { + switch { + case pctx.Extensions.A2A != nil: + return "a2a" + case pctx.Extensions.MCP != nil: + return "mcp" + case pctx.Extensions.Inference != nil: + return "inference" + default: + return "http" + } +} + +// spanKindFor maps direction to OTel SpanKind: inbound is SERVER, outbound is +// CLIENT. The response span reuses its request span's kind. +func spanKindFor(dir pipeline.Direction) trace.SpanKind { + if dir == pipeline.Inbound { + return trace.SpanKindServer + } + return trace.SpanKindClient +} + +// baseAttrs returns the facts carried on BOTH spans except exchange.id (added +// once the request span id is known) and role (differs per span). +func baseAttrs(pctx *pipeline.Context, self, protocol string) []attribute.KeyValue { + attrs := []attribute.KeyValue{ + attribute.String("lineage.direction", pctx.Direction.String()), + attribute.String("lineage.self.id", self), + attribute.String("lineage.protocol", protocol), + } + if pctx.Host != "" { + attrs = append(attrs, attribute.String("lineage.peer.host", pctx.Host)) + } + return attrs +} + +// appendRequestFacts adds the request-only facts: HTTP method/path/scheme, the +// protocol-specific parsed facts, validated-JWT principal (inbound only), and +// input.value when capture_io is on. protocolOf guarantees the matching +// extension pointer is non-nil. +func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx *pipeline.Context, protocol string) []attribute.KeyValue { + if pctx.Method != "" { + attrs = append(attrs, attribute.String("http.method", pctx.Method)) + } + if pctx.Path != "" { + attrs = append(attrs, attribute.String("url.path", pctx.Path)) + } + if pctx.Scheme != "" { + attrs = append(attrs, attribute.String("url.scheme", pctx.Scheme)) + } + switch protocol { + case "a2a": + a := pctx.Extensions.A2A + if a.Method != "" { + attrs = append(attrs, attribute.String("a2a.method", a.Method)) + } + if a.SessionID != "" { + attrs = append(attrs, attribute.String("a2a.session_id", a.SessionID)) + } + case "mcp": + m := pctx.Extensions.MCP + if m.Method != "" { + attrs = append(attrs, attribute.String("mcp.method", m.Method)) + } + if t := mcpTool(pctx); t != "" { + attrs = append(attrs, attribute.String("mcp.tool", t)) + } + case "inference": + if model := pctx.Extensions.Inference.Model; model != "" { + attrs = append(attrs, attribute.String("inference.model", model)) + } + } + // Principal facts: request span, inbound only, and only from a validated + // JWT (pctx.Identity non-nil). + if pctx.Direction == pipeline.Inbound && pctx.Identity != nil { + if s := pctx.Identity.Subject(); s != "" { + attrs = append(attrs, attribute.String("lineage.principal.sub", s)) + } + if c := pctx.Identity.ClientID(); c != "" { + attrs = append(attrs, attribute.String("lineage.principal.client", c)) + } + } + if p.cfg.CaptureIO { + if v := ioInputValue(pctx, protocol); v != "" { + attrs = append(attrs, attribute.String("input.value", v)) + } + } + return attrs +} + +// requestSpanName builds "{self} {protocol} {op}", dropping the trailing op +// when it is empty. The response span appends " response". +func requestSpanName(self, protocol, op string) string { + if op == "" { + return self + " " + protocol + } + return self + " " + protocol + " " + op +} + +// spanOp picks the operation label for the span name per protocol: +// mcp.tool / a2a.method / inference.model, falling back to url.path. +func spanOp(pctx *pipeline.Context, protocol string) string { + var op string + switch protocol { + case "a2a": + if pctx.Extensions.A2A != nil { + op = pctx.Extensions.A2A.Method + } + case "mcp": + op = mcpTool(pctx) + if op == "" && pctx.Extensions.MCP != nil { + op = pctx.Extensions.MCP.Method + } + case "inference": + if pctx.Extensions.Inference != nil { + op = pctx.Extensions.Inference.Model + } + } + if op == "" { + op = pctx.Path + } + return op +} + +// mcpTool returns the tool name for an MCP tools/call, or "" otherwise. +func mcpTool(pctx *pipeline.Context) string { + m := pctx.Extensions.MCP + if m == nil || m.Method != "tools/call" || m.Params == nil { + return "" + } + if name, ok := m.Params["name"].(string); ok { + return name + } + return "" +} + +// serviceLabel reduces a SPIFFE ID to its last path segment, or returns +// selfID as-is if it is not a SPIFFE URI. Used for the lineage.self.id fact +// and span names. +// +// "spiffe://trust-domain/ns/team1/sa/weather-service" → "weather-service" +// "weather-service" → "weather-service" +// +// selfID is never empty at the only call site: Init refuses to start without +// a resolved identity (v1.3). There is deliberately no empty-string fallback — +// inventing a label is the guess that rule exists to forbid. +func serviceLabel(selfID string) string { + parts := strings.Split(selfID, "/") + for i := len(parts) - 1; i >= 0; i-- { + if parts[i] != "" { + return parts[i] + } + } + return selfID +} + +// ioInputValue returns the input.value for a request span: the parsed request +// content for *protocol* — the hop's lineage.protocol fact — or "" if that +// parser produced nothing meaningful. Only that protocol's extension is read: +// parsers are precedence-ordered, not mutually exclusive (mcp-parser matches +// any JSON-RPC body, including every a2a exchange), so falling through to +// another parser's output would attach a mislabeled protocol envelope. A hop +// whose own parser yields nothing keeps a NULL payload — the contract's +// "interactions are independent of payloads". +func ioInputValue(pctx *pipeline.Context, protocol string) string { + ext := pctx.Extensions + switch { + case protocol == "a2a" && ext.A2A != nil && len(ext.A2A.Parts) > 0: + // Collect all text parts; fall back to JSON if non-text parts present. + var texts []string + for _, p := range ext.A2A.Parts { + if p.Content != "" { + texts = append(texts, p.Content) + } + } + if len(texts) > 0 { + return strings.Join(texts, "\n") + } + if b, err := json.Marshal(ext.A2A.Parts); err == nil { + return string(b) + } + case protocol == "inference" && ext.Inference != nil && len(ext.Inference.Messages) > 0: + if b, err := json.Marshal(ext.Inference.Messages); err == nil { + return string(b) + } + case protocol == "mcp" && ext.MCP != nil && ext.MCP.Params != nil: + // For tools/call, surface just the arguments (the semantically + // meaningful part) rather than the full {"name":…,"arguments":…} wrapper. + if ext.MCP.Method == "tools/call" { + if args, ok := ext.MCP.Params["arguments"]; ok { + if b, err := json.Marshal(args); err == nil { + return string(b) + } + } + } + if b, err := json.Marshal(ext.MCP.Params); err == nil { + return string(b) + } + } + return "" +} + +// isA2AProtocolEvent returns true when s is a JSON object carrying an A2A +// transport-level "kind" field (status-update, task-status-update, etc.) +// rather than actual content. Used to avoid surfacing protocol metadata +// as output.value when the a2a-parser captures a protocol event as the +// artifact instead of the real agent response text. +func isA2AProtocolEvent(s string) bool { + var obj map[string]json.RawMessage + if json.Unmarshal([]byte(s), &obj) != nil { + return false + } + var kind string + if raw, ok := obj["kind"]; ok { + _ = json.Unmarshal(raw, &kind) + } + return strings.Contains(kind, "status") || strings.Contains(kind, "artifact-update") || + strings.Contains(kind, "Status") || kind == "working" || kind == "canceled" +} + +// ioOutputValue returns the output.value for a response span: the parsed +// response content for *protocol* — the REQUEST span's lineage.protocol fact — +// or "" if that parser produced nothing. Only that protocol's extension is +// read, for the same reason as ioInputValue: mcp-parser also parses every a2a +// response (any JSON-RPC body), and falling through to it would emit the raw +// JSON-RPC envelope — including the protocol events isA2AProtocolEvent exists +// to suppress — as an a2a hop's payload. +func ioOutputValue(pctx *pipeline.Context, protocol string) string { + ext := pctx.Extensions + switch { + case protocol == "a2a" && ext.A2A != nil && ext.A2A.Artifact != "" && !isA2AProtocolEvent(ext.A2A.Artifact): + return ext.A2A.Artifact + case protocol == "a2a" && ext.A2A != nil && ext.A2A.ErrorMessage != "": + return ext.A2A.ErrorMessage + case protocol == "inference" && ext.Inference != nil && ext.Inference.Completion != "": + return ext.Inference.Completion + case protocol == "inference" && ext.Inference != nil && len(ext.Inference.ToolCalls) > 0: + if b, err := json.Marshal(ext.Inference.ToolCalls); err == nil { + return string(b) + } + case protocol == "mcp" && ext.MCP != nil && ext.MCP.Result != nil: + // For tools/call results, extract the text content from the MCP + // content array rather than returning the full {"content":[…],"_meta":…} + // envelope, so the output matches what Phoenix shows for the tool span. + if ext.MCP.Method == "tools/call" { + if content, ok := ext.MCP.Result["content"]; ok { + if items, ok := content.([]any); ok { + var texts []string + for _, item := range items { + if m, ok := item.(map[string]any); ok { + if m["type"] == "text" { + if t, ok := m["text"].(string); ok && t != "" { + texts = append(texts, t) + } + } + } + } + if len(texts) > 0 { + return strings.Join(texts, "\n") + } + } + } + } + if b, err := json.Marshal(ext.MCP.Result); err == nil { + return string(b) + } + case protocol == "mcp" && ext.MCP != nil && ext.MCP.Err != nil: + if b, err := json.Marshal(ext.MCP.Err); err == nil { + return string(b) + } + } + return "" +} + +// Compile-time interface assertions. +var ( + _ pipeline.Plugin = (*LineageTelemetry)(nil) + _ pipeline.Configurable = (*LineageTelemetry)(nil) + _ pipeline.Initializer = (*LineageTelemetry)(nil) + _ pipeline.Shutdowner = (*LineageTelemetry)(nil) + _ pipeline.Finisher = (*LineageTelemetry)(nil) + _ pipeline.Readier = (*LineageTelemetry)(nil) +) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go new file mode 100644 index 000000000..59de55f05 --- /dev/null +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -0,0 +1,912 @@ +package lineage + +import ( + "context" + "encoding/json" + "maps" + "net/http" + "os" + "strings" + "testing" + "time" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" + + "github.com/rossoctl/cortex/authbridge/authlib/pipeline" +) + +// newTestPlugin creates a LineageTelemetry wired to an in-memory span exporter +// (synchronous, so a span appears the instant it is ended) and marks it ready +// so Init is not needed. +func newTestPlugin(t *testing.T) (*LineageTelemetry, *tracetest.InMemoryExporter) { + t.Helper() + exp := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) + p := NewLineageTelemetry() + p.tp = tp + p.tracer = tp.Tracer("test") + p.selfID = "weather-service" + p.ready.Store(true) + return p, exp +} + +// run drives a full exchange (request pass + finish) through a single-plugin +// pipeline. Spans are read from the caller's exporter. +func run(t *testing.T, p *LineageTelemetry, pctx *pipeline.Context, outcome pipeline.Outcome) { + t.Helper() + pl, err := pipeline.New([]pipeline.Plugin{p}) + if err != nil { + t.Fatalf("pipeline.New: %v", err) + } + pl.Run(context.Background(), pctx) + pl.RunFinish(context.Background(), pctx, outcome) +} + +// allow is the ordinary success outcome. +func allow(status int) pipeline.Outcome { + return pipeline.Outcome{FinalAction: pipeline.OutcomeAllow, StatusCode: status} +} + +// fakeContext mirrors what the real listeners supply. Method is populated +// because every listener now supplies it (reverseproxy/forwardproxy from +// r.Method, ext_proc from the :method pseudo-header) — if a listener stops, +// the fixture must change with it rather than keep asserting a fiction. +func fakeContext(dir pipeline.Direction, headers http.Header) *pipeline.Context { + return &pipeline.Context{ + Direction: dir, + Method: "POST", + Host: "test-service:8000", + Path: "/test", + Headers: headers, + } +} + +// traceparent builds a header carrier naming traceID/spanID as the wire parent. +func traceparent(traceID, spanID string) http.Header { + h := http.Header{} + h.Set("traceparent", "00-"+traceID+"-"+spanID+"-01") + return h +} + +// extractParent decodes the span context named by the headers' traceparent. +func extractParent(h http.Header) trace.SpanContext { + ctx := propagation.TraceContext{}.Extract(context.Background(), propagation.HeaderCarrier(h)) + return trace.SpanContextFromContext(ctx) +} + +// roleSplit returns the request and response spans from an exported set, +// asserting exactly one of each. +func roleSplit(t *testing.T, spans tracetest.SpanStubs) (req, resp tracetest.SpanStub) { + t.Helper() + var gotReq, gotResp bool + for _, s := range spans { + switch attrStr(s, "lineage.role") { + case "request": + if gotReq { + t.Fatal("more than one request span") + } + req, gotReq = s, true + case "response": + if gotResp { + t.Fatal("more than one response span") + } + resp, gotResp = s, true + default: + t.Fatalf("span %q has no lineage.role", s.Name) + } + } + if !gotReq || !gotResp { + t.Fatalf("want one request + one response span, got %d spans (req=%v resp=%v)", len(spans), gotReq, gotResp) + } + return req, resp +} + +// ---- identifiers, pairing, parenting ---- + +func TestExchange_TwoSpansPairedAndParented(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + + pl, err := pipeline.New([]pipeline.Plugin{p}) + if err != nil { + t.Fatalf("pipeline.New: %v", err) + } + + // Emit on sight: the request span exists after the request pass, before finish. + pl.Run(context.Background(), pctx) + if got := len(exp.GetSpans()); got != 1 { + t.Fatalf("after request pass: want 1 span (request), got %d", got) + } + + pl.RunFinish(context.Background(), pctx, allow(200)) + spans := exp.GetSpans() + if len(spans) != 2 { + t.Fatalf("after finish: want 2 spans, got %d", len(spans)) + } + req, resp := roleSplit(t, spans) + + // exchange.id == request span id, echoed on both. + wantID := req.SpanContext.SpanID().String() + if got := attrStr(req, "lineage.exchange.id"); got != wantID { + t.Errorf("request exchange.id = %q, want %q", got, wantID) + } + if got := attrStr(resp, "lineage.exchange.id"); got != wantID { + t.Errorf("response exchange.id = %q, want %q", got, wantID) + } + + // Response span's parent is the request span (same trace). + if resp.Parent.SpanID() != req.SpanContext.SpanID() { + t.Errorf("response parent span = %s, want request span %s", resp.Parent.SpanID(), req.SpanContext.SpanID()) + } + if resp.SpanContext.TraceID() != req.SpanContext.TraceID() { + t.Errorf("response trace = %s, want request trace %s", resp.SpanContext.TraceID(), req.SpanContext.TraceID()) + } + + // Both spans share the same SpanKind (SERVER for inbound). + if req.SpanKind != trace.SpanKindServer || resp.SpanKind != trace.SpanKindServer { + t.Errorf("span kinds = %v/%v, want server/server", req.SpanKind, resp.SpanKind) + } +} + +// ---- the stamp (single-channel parenting, wire contract v1.5) ---- + +// TestStamp_OutboundRewritesStampNotTraceparent: the outbound reads its +// parent from the inbound's stamp, then re-stamps the forwarded tracestate +// with its OWN request span id for the peer sidecar's inbound to read. The +// forwarded traceparent is NOT modified (v1.5 removed the splice) — an app +// chain riding traceparent toward its own backend stays intact. +func TestStamp_OutboundRewritesStampNotTraceparent(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID, wireParent = "4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7" + const inboundID = "1111111111111111" + h := traceparent(traceID, wireParent) + h.Set("tracestate", tracestateStampKey+"="+inboundID) + pctx := fakeContext(pipeline.Outbound, h) + pctx.Extensions.MCP = &pipeline.MCPExtension{Method: "tools/call", Params: map[string]any{"name": "get_weather"}} + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + // Parent comes from the inbound's stamp. + if got := req.Parent.SpanID().String(); got != inboundID { + t.Errorf("parent = %s, want stamped inbound %s", got, inboundID) + } + // The forwarded traceparent is untouched — still the wire parent. + forwarded := extractParent(pctx.Headers) + if got := forwarded.SpanID().String(); got != wireParent { + t.Errorf("forwarded traceparent parent = %s, want untouched wire parent %s", got, wireParent) + } + if got := forwarded.TraceID().String(); got != traceID { + t.Errorf("forwarded trace = %s, want %s", got, traceID) + } + // The forwarded tracestate now stamps THIS outbound request span, + // replacing the inbound's stamp it consumed. + want := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != want { + t.Errorf("tracestate = %q, want re-stamp %q", got, want) + } +} + +// TestStamp_InboundParentsOnPeerStamp is the cross-pod link: the caller +// sidecar's outbound stamped tracestate with its request span id, and this +// inbound must parent on that stamp — not on the wire traceparent, whose +// span id may belong to an app chain this pipeline never exports. +func TestStamp_InboundParentsOnPeerStamp(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID, wireParent = "4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7" + const peerOutbound = "2222222222222222" + h := traceparent(traceID, wireParent) + h.Set("tracestate", tracestateStampKey+"="+peerOutbound) + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + if got := req.Parent.SpanID().String(); got != peerOutbound { + t.Errorf("parent = %s, want peer outbound stamp %s", got, peerOutbound) + } + if got := attrStr(req, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } + // The forwarded stamp now names THIS inbound request span — the app's + // shim couriers it to exactly the outbound calls this inbound causes. + want := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != want { + t.Errorf("tracestate = %q, want re-stamp %q", got, want) + } +} + +func TestStamp_InboundHeadersUntouchedExceptStamp(t *testing.T) { + p, exp := newTestPlugin(t) + h := traceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7") + before := http.Header{} + maps.Copy(before, h) + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + // The ONLY inbound mutation is the tracestate stamp; traceparent and + // everything else are forwarded as they arrived. + req, _ := roleSplit(t, exp.GetSpans()) + want := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != want { + t.Errorf("tracestate = %q, want stamp %q", got, want) + } + after := http.Header{} + maps.Copy(after, pctx.Headers) + after.Del("tracestate") + if !headersEqual(before, after) { + t.Errorf("inbound headers beyond tracestate mutated: before=%v after=%v", before, after) + } + // No stamp arrived, so the parent is the wire traceparent — recorded as such. + if got := attrStr(req, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +func TestStamp_PreservesForeignTracestateMembers(t *testing.T) { + p, exp := newTestPlugin(t) + h := traceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7") + h.Set("tracestate", "vendor=abc") + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + got := pctx.Headers.Get("tracestate") + wantStamp := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if !strings.Contains(got, wantStamp) || !strings.Contains(got, "vendor=abc") { + t.Errorf("tracestate = %q, want both %q and vendor=abc", got, wantStamp) + } +} + +func TestStamp_NoWireTraceparentNoStamp(t *testing.T) { + p, _ := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + + run(t, p, pctx, allow(200)) + + if got := pctx.Headers.Get("tracestate"); got != "" { + t.Errorf("tracestate stamped without a wire traceparent: %q", got) + } +} + +// TestStamp_OutboundPrefersStampOverMap is the same-trace fan-in case in +// miniature: two concurrent inbound exchanges on ONE trace (the trace-keyed +// map can only hold the later one), then an outbound whose tracestate stamp +// names the EARLIER inbound. Without the stamp this outbound would collapse +// onto the map entry — the 1/N misattribution the fanin-test.sh e2e proves. +func TestStamp_OutboundUsesTheStampedInbound(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + // Two concurrent inbounds on the SAME trace — the case no trace-keyed + // structure can disambiguate, and the reason the stamp exists. + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "1111111111111111")), allow(200)) + in1, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "2222222222222222")), allow(200)) + in2, _ := roleSplit(t, exp.GetSpans()) + + // Outbound couriered in1's stamp back through the app. It must parent + // under in1 specifically — not in2, not the wire parent. + exp.Reset() + h := traceparent(traceID, "3333333333333333") + h.Set("tracestate", tracestateStampKey+"="+in1.SpanContext.SpanID().String()) + run(t, p, fakeContext(pipeline.Outbound, h), allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if outReq.Parent.SpanID() != in1.SpanContext.SpanID() { + t.Errorf("parent = %s, want stamped inbound %s (the other in-flight inbound was %s)", + outReq.Parent.SpanID(), in1.SpanContext.SpanID(), in2.SpanContext.SpanID()) + } + if got := attrStr(outReq, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } +} + +func TestStamp_MalformedFallsBackToWire(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const wireParent = "3333333333333333" + + // An inbound on this trace exists — and must NOT be used, because a + // malformed stamp means "unknown", not "guess for me". + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "1111111111111111")), allow(200)) + in1, _ := roleSplit(t, exp.GetSpans()) + + exp.Reset() + h := traceparent(traceID, wireParent) + h.Set("tracestate", tracestateStampKey+"=nothex") + run(t, p, fakeContext(pipeline.Outbound, h), allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if got := outReq.Parent.SpanID().String(); got != wireParent { + t.Errorf("parent = %s, want wire parent %s", got, wireParent) + } + if outReq.Parent.SpanID() == in1.SpanContext.SpanID() { + t.Error("malformed stamp silently inherited this pod's inbound span") + } + if got := attrStr(outReq, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +func TestStamp_ParentSourceWireWhenUnstamped(t *testing.T) { + p, exp := newTestPlugin(t) + out := fakeContext(pipeline.Outbound, traceparent("cccccccccccccccccccccccccccccccc", "1111111111111111")) + run(t, p, out, allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + if got := attrStr(outReq, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +// TestStamp_UnstampedOutboundNeverInheritsInbound is the regression guard for +// the removal of the trace-keyed map. An outbound with no stamp must fall to the +// wire parent EVEN WHEN this pod has an inbound span for the same trace. The old +// map answered such cases from "the last inbound seen", which is correct only +// while exactly one inbound is in flight — a precondition it never checked. A +// missing edge is recoverable; a confidently wrong one is not. +func TestStamp_UnstampedOutboundNeverInheritsInbound(t *testing.T) { + p, exp := newTestPlugin(t) + const traceID = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const wireParent = "3333333333333333" + + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceID, "2222222222222222")), allow(200)) + inReq, _ := roleSplit(t, exp.GetSpans()) + + exp.Reset() + run(t, p, fakeContext(pipeline.Outbound, traceparent(traceID, wireParent)), allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if outReq.Parent.SpanID() == inReq.SpanContext.SpanID() { + t.Fatal("un-stamped outbound inherited this pod's inbound span — the map is back") + } + if got := outReq.Parent.SpanID().String(); got != wireParent { + t.Errorf("parent = %s, want wire parent %s", got, wireParent) + } + if got := attrStr(outReq, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } +} + +func TestStamp_ConcurrentTracesNeverCross(t *testing.T) { + p, exp := newTestPlugin(t) + const traceA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const traceB = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + + // One inbound on each trace. + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceA, "1111111111111111")), allow(200)) + inA, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + run(t, p, fakeContext(pipeline.Inbound, traceparent(traceB, "2222222222222222")), allow(200)) + inB, _ := roleSplit(t, exp.GetSpans()) + + // Each outbound couriers its own trace's stamp back. + exp.Reset() + hA := traceparent(traceA, "3333333333333333") + hA.Set("tracestate", tracestateStampKey+"="+inA.SpanContext.SpanID().String()) + run(t, p, fakeContext(pipeline.Outbound, hA), allow(200)) + outA, _ := roleSplit(t, exp.GetSpans()) + if outA.Parent.SpanID() != inA.SpanContext.SpanID() { + t.Errorf("outbound A parent = %s, want inbound A %s", outA.Parent.SpanID(), inA.SpanContext.SpanID()) + } + if outA.Parent.SpanID() == inB.SpanContext.SpanID() { + t.Error("outbound A crossed into inbound B's span") + } + if outA.SpanContext.TraceID().String() != traceA { + t.Errorf("outbound A trace = %s, want %s", outA.SpanContext.TraceID(), traceA) + } + + exp.Reset() + hB := traceparent(traceB, "4444444444444444") + hB.Set("tracestate", tracestateStampKey+"="+inB.SpanContext.SpanID().String()) + run(t, p, fakeContext(pipeline.Outbound, hB), allow(200)) + outB, _ := roleSplit(t, exp.GetSpans()) + if outB.Parent.SpanID() != inB.SpanContext.SpanID() { + t.Errorf("outbound B parent = %s, want inbound B %s", outB.Parent.SpanID(), inB.SpanContext.SpanID()) + } +} + +// ---- bodyless / unparsed completeness ---- + +func TestBodyless_UnparsedNoCaptureStillEmitsBothSpans(t *testing.T) { + p, exp := newTestPlugin(t) + // capture_io defaults false; no parser extensions → protocol http. + pctx := fakeContext(pipeline.Outbound, http.Header{}) + + run(t, p, pctx, allow(200)) + + req, resp := roleSplit(t, exp.GetSpans()) + if got := attrStr(req, "lineage.protocol"); got != "http" { + t.Errorf("protocol = %q, want http", got) + } + // Complete: both carry the shared facts and the exchange is paired. + if attrStr(req, "lineage.exchange.id") == "" || attrStr(resp, "lineage.exchange.id") == "" { + t.Error("exchange.id missing on a bodyless span") + } + if got := attrStr(resp, "lineage.outcome"); got != "ok" { + t.Errorf("outcome = %q, want ok", got) + } + // No payloads captured. + if _, ok := findAttr(req, "input.value"); ok { + t.Error("input.value present with capture_io off") + } + if _, ok := findAttr(resp, "output.value"); ok { + t.Error("output.value present with capture_io off") + } +} + +// ---- outcomes ---- + +func TestOutcome_Denied(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + + run(t, p, pctx, pipeline.Outcome{ + FinalAction: pipeline.OutcomeDeny, + StatusCode: 401, + DenyingPlugin: "jwt-validation", + }) + + _, resp := roleSplit(t, exp.GetSpans()) + if got := attrStr(resp, "lineage.outcome"); got != "denied" { + t.Errorf("outcome = %q, want denied", got) + } + if got := attrStr(resp, "lineage.denied_by"); got != "jwt-validation" { + t.Errorf("denied_by = %q, want jwt-validation", got) + } + if got, ok := intAttr(resp, "http.status_code"); !ok || got != 401 { + t.Errorf("http.status_code = %d (ok=%v), want 401", got, ok) + } +} + +func TestOutcome_AbandonedHasNoStatus(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + + // Terminal error with no response written (upstream reset / disconnect). + run(t, p, pctx, pipeline.Outcome{FinalAction: pipeline.OutcomeError, StatusCode: 0}) + + _, resp := roleSplit(t, exp.GetSpans()) + if got := attrStr(resp, "lineage.outcome"); got != "abandoned" { + t.Errorf("outcome = %q, want abandoned", got) + } + if _, ok := findAttr(resp, "http.status_code"); ok { + t.Error("http.status_code present on an abandoned exchange (none was produced)") + } +} + +// ---- request facts + capture_io + span names ---- + +func TestRequestFacts_MCPWithCapture(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Host = "weather-tool-mcp.team1.svc:8000" + pctx.Path = "/mcp" + pctx.Scheme = "http" + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", + Params: map[string]any{"name": "get_weather", "arguments": map[string]any{"city": "Tokyo"}}, + Result: map[string]any{"content": []any{map[string]any{"type": "text", "text": "sunny"}}}, + } + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "lineage.protocol", "mcp") + checkAttr(t, req, "mcp.method", "tools/call") + checkAttr(t, req, "mcp.tool", "get_weather") + checkAttr(t, req, "http.method", "POST") + checkAttr(t, req, "url.path", "/mcp") + checkAttr(t, req, "url.scheme", "http") + checkAttr(t, req, "lineage.self.id", "weather-service") + checkAttr(t, req, "lineage.peer.host", "weather-tool-mcp.team1.svc:8000") + checkAttr(t, req, "lineage.direction", "outbound") + checkAttr(t, req, "input.value", `{"city":"Tokyo"}`) + checkAttr(t, resp, "output.value", "sunny") + + if req.Name != "weather-service mcp get_weather" { + t.Errorf("request span name = %q", req.Name) + } + if resp.Name != "weather-service mcp get_weather response" { + t.Errorf("response span name = %q", resp.Name) + } + if req.SpanKind != trace.SpanKindClient { + t.Errorf("outbound request kind = %v, want client", req.SpanKind) + } +} + +func TestRequestFacts_A2AAndInference(t *testing.T) { + p, exp := newTestPlugin(t) + // A2A. + a := fakeContext(pipeline.Outbound, http.Header{}) + a.Extensions.A2A = &pipeline.A2AExtension{Method: "message/send", SessionID: "sess-123"} + run(t, p, a, allow(200)) + areq, _ := roleSplit(t, exp.GetSpans()) + checkAttr(t, areq, "lineage.protocol", "a2a") + checkAttr(t, areq, "a2a.method", "message/send") + checkAttr(t, areq, "a2a.session_id", "sess-123") + if _, ok := findAttr(areq, "url.scheme"); ok { + t.Error("url.scheme present although the context carried no scheme") + } + if areq.Name != "weather-service a2a message/send" { + t.Errorf("a2a span name = %q", areq.Name) + } + + // Inference. + exp.Reset() + i := fakeContext(pipeline.Outbound, http.Header{}) + i.Extensions.Inference = &pipeline.InferenceExtension{Model: "qwen2.5:7b"} + run(t, p, i, allow(200)) + ireq, _ := roleSplit(t, exp.GetSpans()) + checkAttr(t, ireq, "lineage.protocol", "inference") + checkAttr(t, ireq, "inference.model", "qwen2.5:7b") + if ireq.Name != "weather-service inference qwen2.5:7b" { + t.Errorf("inference span name = %q", ireq.Name) + } +} + +// mcp-parser attaches to ANY JSON-RPC body — including every a2a exchange — +// so on an a2a hop both extensions are populated. The payload read is keyed by +// the protocol fact: when the a2a parser yields nothing (no text parts, a +// protocol-event artifact), the payload stays ABSENT rather than falling +// through to the co-populated MCP parse of the same bytes (which would emit +// the raw JSON-RPC envelope on an lineage.protocol=a2a span). +func TestCaptureIO_A2ANeverFallsThroughToCoPopulatedMCP(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Extensions.A2A = &pipeline.A2AExtension{ + Method: "message/send", + // A status-update captured as the artifact — a protocol event, filtered. + Artifact: `{"kind":"status-update","taskId":"t-1"}`, + } + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "message/send", + Params: map[string]any{"message": map[string]any{"role": "user"}}, + Result: map[string]any{"artifacts": []any{map[string]any{"artifactId": "a-1"}}}, + } + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "lineage.protocol", "a2a") + if v, ok := findAttr(req, "input.value"); ok { + t.Errorf("input.value = %q on an a2a hop with no a2a parts — leaked from the co-populated MCP parse", v.Emit()) + } + if v, ok := findAttr(resp, "output.value"); ok { + t.Errorf("output.value = %q on an a2a hop whose artifact is a protocol event — leaked from the co-populated MCP parse", v.Emit()) + } + // mcp.* facts belong to mcp hops only; the a2a label must keep them off. + if v, ok := findAttr(req, "mcp.method"); ok { + t.Errorf("mcp.method = %q emitted on an a2a hop", v.Emit()) + } +} + +func TestPrincipalFacts_InboundRequestOnly(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + pctx.Identity = fakeIdentity{sub: "alice", client: "weather-ui"} + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "lineage.principal.sub", "alice") + checkAttr(t, req, "lineage.principal.client", "weather-ui") + // Principal facts are request-only. + if _, ok := findAttr(resp, "lineage.principal.sub"); ok { + t.Error("lineage.principal.sub leaked onto the response span") + } +} + +func TestPrincipalFacts_OutboundNeverEmitsPrincipal(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Identity = fakeIdentity{sub: "alice", client: "weather-ui"} + + run(t, p, pctx, allow(200)) + req, _ := roleSplit(t, exp.GetSpans()) + if _, ok := findAttr(req, "lineage.principal.sub"); ok { + t.Error("outbound span carried a principal fact (inbound-only)") + } +} + +// ---- the forbidden-keys guard ---- + +// TestForbiddenKeysNeverEmitted scans every attribute of every span emitted +// across a spread of exchange shapes and asserts none carries a key from a +// removed vocabulary. The contract deleted these; this test is the tripwire +// that keeps them deleted. +func TestForbiddenKeysNeverEmitted(t *testing.T) { + forbidden := []string{"trust.", "lineage.hop.kind", "enduser.id", "openinference.", "source", "authbridge.proxy"} + + shapes := []func() *pipeline.Context{ + func() *pipeline.Context { + c := fakeContext(pipeline.Inbound, http.Header{}) + c.Identity = fakeIdentity{sub: "alice", client: "weather-ui"} + return c + }, + func() *pipeline.Context { + c := fakeContext(pipeline.Outbound, http.Header{}) + c.Extensions.MCP = &pipeline.MCPExtension{Method: "tools/call", Params: map[string]any{"name": "get_weather"}} + return c + }, + func() *pipeline.Context { + c := fakeContext(pipeline.Outbound, http.Header{}) + c.Extensions.A2A = &pipeline.A2AExtension{Method: "message/send"} + return c + }, + func() *pipeline.Context { + c := fakeContext(pipeline.Outbound, http.Header{}) + c.Extensions.Inference = &pipeline.InferenceExtension{Model: "qwen2.5:7b"} + return c + }, + } + + for _, mk := range shapes { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + run(t, p, mk(), allow(200)) + for _, s := range exp.GetSpans() { + for _, kv := range s.Attributes { + key := string(kv.Key) + for _, bad := range forbidden { + if key == bad || strings.HasPrefix(key, bad) { + t.Errorf("span %q emitted forbidden attribute %q", s.Name, key) + } + } + } + } + } +} + +// ---- robustness ---- + +func TestOnFinish_NoStateDoesNotPanicOrEmit(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, http.Header{}) + // OnFinish without OnRequest having run — no exchangeState stored. + p.OnFinish(context.Background(), pctx) + if got := len(exp.GetSpans()); got != 0 { + t.Errorf("OnFinish with no state emitted %d spans, want 0", got) + } +} + +func TestNotReady_SkipsSpan(t *testing.T) { + p := NewLineageTelemetry() + // Do NOT set ready — Init never called. + pctx := fakeContext(pipeline.Inbound, http.Header{}) + action := p.OnRequest(context.Background(), pctx) + if action.Type != pipeline.Continue { + t.Fatalf("expected Continue, got %v", action.Type) + } + if pipeline.GetState[exchangeState](pctx, pluginName) != nil { + t.Error("exchangeState should not be set when plugin is not ready") + } +} + +// ---- config ---- + +func TestConfigure_Defaults(t *testing.T) { + p := NewLineageTelemetry() + if err := p.Configure(nil); err != nil { + t.Fatalf("Configure(nil): %v", err) + } + if p.cfg.OTelEndpoint != "localhost:4317" { + t.Errorf("default endpoint = %q, want localhost:4317", p.cfg.OTelEndpoint) + } + if p.cfg.SelfIDFile != "/shared/client-id.txt" { + t.Errorf("default self_id_file = %q", p.cfg.SelfIDFile) + } +} + +func TestConfigure_DecodesKeptKeys(t *testing.T) { + p := NewLineageTelemetry() + raw := json.RawMessage(`{"otel_endpoint":"http://collector:4317","capture_io":true,"self_id":"weather-service"}`) + if err := p.Configure(raw); err != nil { + t.Fatalf("Configure: %v", err) + } + if p.cfg.OTelEndpoint != "collector:4317" { + t.Errorf("endpoint = %q, want collector:4317 (scheme stripped)", p.cfg.OTelEndpoint) + } + if !p.cfg.CaptureIO { + t.Error("capture_io should be true") + } + if p.cfg.SelfID != "weather-service" { + t.Errorf("self_id = %q", p.cfg.SelfID) + } +} + +// ---- helpers ---- + +type fakeIdentity struct { + sub, client string + scopes []string +} + +func (f fakeIdentity) Subject() string { return f.sub } +func (f fakeIdentity) ClientID() string { return f.client } +func (f fakeIdentity) Scopes() []string { return f.scopes } + +func findAttr(span tracetest.SpanStub, key string) (attribute.Value, bool) { + for _, kv := range span.Attributes { + if string(kv.Key) == key { + return kv.Value, true + } + } + return attribute.Value{}, false +} + +func attrStr(span tracetest.SpanStub, key string) string { + if v, ok := findAttr(span, key); ok { + return v.AsString() + } + return "" +} + +func intAttr(span tracetest.SpanStub, key string) (int64, bool) { + if v, ok := findAttr(span, key); ok { + return v.AsInt64(), true + } + return 0, false +} + +// checkAttr asserts a span contains attribute key with the given string value. +func checkAttr(t *testing.T, span tracetest.SpanStub, key, want string) { + t.Helper() + got, ok := findAttr(span, key) + if !ok { + t.Errorf("attribute %q not found in span %q", key, span.Name) + return + } + if got.AsString() != want { + t.Errorf("attr %q = %q, want %q", key, got.AsString(), want) + } +} + +func headersEqual(a, b http.Header) bool { + if len(a) != len(b) { + return false + } + for k, av := range a { + bv, ok := b[k] + if !ok || len(av) != len(bv) { + return false + } + for i := range av { + if av[i] != bv[i] { + return false + } + } + } + return true +} + +// TestInit_RefusesToStartWithoutIdentity locks the v1.3 rule at the identity +// boundary: a pod whose self identity cannot be resolved must fail at boot, +// never serve traffic under a plausible-but-wrong label (the old behavior +// emitted lineage.self.id="agent" from the empty-string serviceLabel). +func TestInit_RefusesToStartWithoutIdentity(t *testing.T) { + cases := []struct { + name string + cfg Config + wantErr bool + }{ + {"inline self_id starts", Config{OTelEndpoint: "localhost:4317", SelfID: "weather-service"}, false}, + {"missing self_id_file refuses", Config{OTelEndpoint: "localhost:4317", SelfIDFile: t.TempDir() + "/absent.txt"}, true}, + {"no identity source refuses", Config{OTelEndpoint: "localhost:4317"}, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = tc.cfg + err := p.Init(context.Background()) + if p.tp != nil { + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + _ = p.tp.Shutdown(ctx) + cancel() + } + if tc.wantErr && err == nil { + t.Fatal("Init succeeded without a resolvable identity") + } + if !tc.wantErr && err != nil { + t.Fatalf("Init failed with a valid inline self_id: %v", err) + } + if tc.wantErr && p.Ready() { + t.Error("plugin reports Ready after a refused Init") + } + }) + } +} + +// TestInit_ReadsSelfIDFile covers the operator-injected path (file, not inline). +func TestInit_ReadsSelfIDFile(t *testing.T) { + dir := t.TempDir() + path := dir + "/client-id.txt" + if err := os.WriteFile(path, []byte("weather-service\n"), 0o600); err != nil { + t.Fatal(err) + } + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "localhost:4317", SelfIDFile: path} + if err := p.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + _ = p.tp.Shutdown(ctx) + cancel() + if p.selfID != "weather-service" { + t.Errorf("selfID = %q, want trimmed file content", p.selfID) + } +} + +// TestConfig_UnknownKeysRefused: a typo'd knob must be a boot error, not a +// silent run-with-defaults. +func TestConfig_UnknownKeysRefused(t *testing.T) { + if _, err := decodeConfig([]byte(`{"capture-io": true}`)); err == nil { + t.Fatal("unknown config key accepted silently") + } + if _, err := decodeConfig([]byte(`{"capture_io": true, "self_id": "x"}`)); err != nil { + t.Fatalf("valid config rejected: %v", err) + } +} + +// ---- bypass config ---- +// The one failure mode of bypass_paths / bypass_hosts produces NO signal +// anywhere: a matched hop is simply absent from the graph. So both directions +// are pinned — a match emits nothing, a near-miss emits the full pair. + +func TestBypassPaths_PrefixMatchEmitsNothing(t *testing.T) { + cases := []struct { + name string + path string + spans int // spans expected from the exchange + }{ + {"prefix match skipped", "/health/live", 0}, + {"exact prefix skipped", "/health", 0}, + {"non-matching path emits", "/api/health-report", 2}, + {"prefix is anchored, not substring", "/v1/health", 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.BypassPaths = []string{"/health"} + pctx := fakeContext(pipeline.Inbound, http.Header{}) + pctx.Path = tc.path + run(t, p, pctx, allow(200)) + if got := len(exp.GetSpans()); got != tc.spans { + t.Fatalf("path %q: got %d spans, want %d", tc.path, got, tc.spans) + } + }) + } +} + +func TestBypassHosts_SubstringMatchEmitsNothing(t *testing.T) { + cases := []struct { + name string + host string + spans int + }{ + {"substring match skipped", "otel-collector.rossoctl-system:4317", 0}, + {"bare name match skipped", "otel-collector:4317", 0}, + {"unrelated host emits", "weather-tool:8000", 2}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.BypassHosts = []string{"otel-collector"} + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Host = tc.host + run(t, p, pctx, allow(200)) + if got := len(exp.GetSpans()); got != tc.spans { + t.Fatalf("host %q: got %d spans, want %d", tc.host, got, tc.spans) + } + }) + } +} From a6df3b00abc2f0d637b3554d8a654cbaec443e3c Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 14 Aug 2026 19:49:24 +0300 Subject: [PATCH 02/37] Feat: Register lineage-telemetry behind an exclude_plugin build tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follows the one-tag-file-per-plugin convention: five lines per binary in plugins_lineage.go, gated by //go:build !exclude_plugin_lineage, so main.go imports no plugin package directly. A build carrying the exclude tags links neither the plugin nor its OTel dependency subtree. go.mod changes are go mod tidy output. Four direct dependencies, three of them promotions of modules already present as indirect (otel, otel/sdk, otel/trace); the fourth is the OTLP/gRPC trace exporter. Five new indirect. Licences are Apache-2.0 for the OpenTelemetry modules and genproto, MIT for backoff/v5, BSD-3-Clause for grpc-gateway/v2. No go.sum change is needed — the existing sums already cover these modules. Signed-off-by: YehoshuaSagron --- authbridge/authlib/go.mod | 12 +++++++++--- authbridge/cmd/authbridge-envoy/go.mod | 6 ++++++ authbridge/cmd/authbridge-envoy/plugins_lineage.go | 5 +++++ authbridge/cmd/authbridge-proxy/go.mod | 6 ++++++ authbridge/cmd/authbridge-proxy/plugins_lineage.go | 5 +++++ 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 authbridge/cmd/authbridge-envoy/plugins_lineage.go create mode 100644 authbridge/cmd/authbridge-proxy/plugins_lineage.go diff --git a/authbridge/authlib/go.mod b/authbridge/authlib/go.mod index 191d9a8f4..bf63a3d18 100644 --- a/authbridge/authlib/go.mod +++ b/authbridge/authlib/go.mod @@ -16,6 +16,10 @@ require ( github.com/tidwall/sjson v1.2.5 golang.org/x/net v0.58.0 golang.org/x/sync v0.22.0 + go.opentelemetry.io/otel v1.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 + go.opentelemetry.io/otel/sdk v1.44.0 + go.opentelemetry.io/otel/trace v1.44.0 golang.org/x/sys v0.47.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a google.golang.org/grpc v1.83.2 @@ -47,6 +51,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect @@ -58,6 +63,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/go-clone v1.7.3 // indirect github.com/huandu/go-sqlbuilder v1.42.1 // indirect @@ -109,10 +115,9 @@ require ( github.com/yashtewari/glob-intersection v0.2.0 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/otel/sdk v1.44.0 // indirect - go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -120,6 +125,7 @@ require ( golang.org/x/crypto v0.55.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/ini.v1 v1.67.3 // indirect oras.land/oras-go/v2 v2.6.2 // indirect diff --git a/authbridge/cmd/authbridge-envoy/go.mod b/authbridge/cmd/authbridge-envoy/go.mod index 494878bf7..b6646efdf 100644 --- a/authbridge/cmd/authbridge-envoy/go.mod +++ b/authbridge/cmd/authbridge-envoy/go.mod @@ -39,6 +39,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect @@ -52,6 +53,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/go-clone v1.7.3 // indirect github.com/huandu/go-sqlbuilder v1.42.1 // indirect @@ -110,9 +112,12 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -124,6 +129,7 @@ require ( golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/ini.v1 v1.67.3 // indirect diff --git a/authbridge/cmd/authbridge-envoy/plugins_lineage.go b/authbridge/cmd/authbridge-envoy/plugins_lineage.go new file mode 100644 index 000000000..76ea1e644 --- /dev/null +++ b/authbridge/cmd/authbridge-envoy/plugins_lineage.go @@ -0,0 +1,5 @@ +//go:build !exclude_plugin_lineage + +package main + +import _ "github.com/rossoctl/cortex/authbridge/authlib/plugins/lineage" diff --git a/authbridge/cmd/authbridge-proxy/go.mod b/authbridge/cmd/authbridge-proxy/go.mod index 2de1aedfb..43ebeb376 100644 --- a/authbridge/cmd/authbridge-proxy/go.mod +++ b/authbridge/cmd/authbridge-proxy/go.mod @@ -32,6 +32,7 @@ require ( github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.1 // indirect github.com/bytedance/sonic/loader v0.5.1 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect @@ -43,6 +44,7 @@ require ( github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.6 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/huandu/go-clone v1.7.3 // indirect github.com/huandu/go-sqlbuilder v1.42.1 // indirect @@ -100,9 +102,12 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect + go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect @@ -114,6 +119,7 @@ require ( golang.org/x/sys v0.47.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect diff --git a/authbridge/cmd/authbridge-proxy/plugins_lineage.go b/authbridge/cmd/authbridge-proxy/plugins_lineage.go new file mode 100644 index 000000000..76ea1e644 --- /dev/null +++ b/authbridge/cmd/authbridge-proxy/plugins_lineage.go @@ -0,0 +1,5 @@ +//go:build !exclude_plugin_lineage + +package main + +import _ "github.com/rossoctl/cortex/authbridge/authlib/plugins/lineage" From 84f53c3dc9d321a483ce63587f09bc83b31e0e7a Mon Sep 17 00:00:00 2001 From: Igor Gokhman Date: Sun, 30 Aug 2026 12:30:50 +0300 Subject: [PATCH 03/37] Fix: Address lineage-telemetry review feedback + resolve go.mod conflict Resolve the authlib go.mod merge conflict (keep both the OTel exporter deps and x/net/x/sync; take the higher x/net v0.58.0) and apply the straightforward review fixes on PR #761: - Init: resolve self identity before allocating the gRPC client, OTLP exporter, and TracerProvider, so a refused identity leaks no exporter or batch-processor goroutine (CodeRabbit). - Init: close the gRPC conn when otlptracegrpc.New fails after the dial succeeded, instead of leaking it on that error path (clawgenti). - isA2AProtocolEvent: match the enumerated A2A protocol event kinds exactly rather than by substring, so an agent-defined artifact kind that merely contains "status" (e.g. "final-status-report") is no longer suppressed; drops the redundant mixed-case check (clawgenti). - config: parse a URL-form otel_endpoint with net/url and use its host, so a path (http://collector:4317/v1/traces) no longer produces an invalid gRPC dial target; dedupe the localhost:4317 literal into a defaultOTelEndpoint const (CodeRabbit). - test: replace deprecated attribute.Value.Emit() with Value.String() (SA1019), and reduce headersEqual to maps.EqualFunc + slices.Equal. go mod tidy on the two cmd modules was required, not cosmetic: a readonly build (as CI runs it, GOWORK=off) failed against the updated authlib with "updates to go.mod needed" until the transitive graph and go.sum were refreshed. Not addressed here (left for a maintainer decision): the #760 tracestate propagation dependency, the plaintext-OTLP/TLS exposure, the captured- payload size bound, and the external-contract-doc concern. Assisted-By: Claude (Anthropic AI) Signed-off-by: Igor Gokhman --- authbridge/authlib/plugins/lineage/config.go | 22 +++++-- authbridge/authlib/plugins/lineage/plugin.go | 63 +++++++++++-------- .../authlib/plugins/lineage/plugin_test.go | 23 ++----- 3 files changed, 60 insertions(+), 48 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index c030c7e05..fa7bd42e1 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -4,9 +4,14 @@ import ( "bytes" "encoding/json" "fmt" + "net/url" "strings" ) +// defaultOTelEndpoint is the OTLP gRPC target used when otel_endpoint is unset: +// an in-pod collector reached over plaintext loopback. +const defaultOTelEndpoint = "localhost:4317" + // Config holds the per-plugin configuration decoded from the pipeline YAML. type Config struct { // OTelEndpoint is the OTLP gRPC endpoint (host:port or http://host:port). @@ -50,7 +55,7 @@ type Config struct { func defaultConfig() Config { return Config{ - OTelEndpoint: "localhost:4317", + OTelEndpoint: defaultOTelEndpoint, BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, BypassHosts: []string{"otel-collector", "jaeger", "zipkin", "prometheus"}, SelfIDFile: "/shared/client-id.txt", @@ -70,10 +75,17 @@ func decodeConfig(raw json.RawMessage) (Config, error) { return Config{}, fmt.Errorf("lineage-telemetry config: %w", err) } if cfg.OTelEndpoint == "" { - cfg.OTelEndpoint = "localhost:4317" + cfg.OTelEndpoint = defaultOTelEndpoint + } + // gRPC NewClient expects host:port only, so reduce a URL form (e.g. + // http://collector:4317/v1/traces) to its host — TrimPrefix left any path + // behind and produced an invalid dial target. + if strings.Contains(cfg.OTelEndpoint, "://") { + u, err := url.Parse(cfg.OTelEndpoint) + if err != nil || u.Host == "" { + return Config{}, fmt.Errorf("lineage-telemetry config: invalid otel_endpoint %q", cfg.OTelEndpoint) + } + cfg.OTelEndpoint = u.Host } - // Strip http:// or https:// prefix — gRPC NewClient expects host:port only. - cfg.OTelEndpoint = strings.TrimPrefix(cfg.OTelEndpoint, "https://") - cfg.OTelEndpoint = strings.TrimPrefix(cfg.OTelEndpoint, "http://") return cfg, nil } diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index d31169f54..d3fbc96ae 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -154,6 +154,32 @@ func (p *LineageTelemetry) Configure(raw json.RawMessage) error { } func (p *LineageTelemetry) Init(ctx context.Context) error { + // Resolve self identity for the lineage.self.id fact FIRST, before any + // exporter or tracer resource is allocated. Every span this plugin emits is + // a claim of the form "X did Y"; with no X there is no claim to make, so an + // unresolvable identity refuses to start rather than serving traffic under a + // plausible-but-wrong label ("no mechanism may guess", contract v1.3). Note + // the asymmetry with this file's other unknowns: a missing status, payload + // or parent anchor is a missing PART of a fact and degrades honestly + // (abandoned / NULL / parent.source=wire). Identity is the fact's subject — + // it has no degraded form, and a shared placeholder would collapse every + // unidentified pod onto one entity row (entity id = uuid5("{kind}:{self.id}"), + // and entities is upsert-only). Resolving it up front also means a refused + // identity leaks nothing: the gRPC client and batch-span-processor goroutine + // below are never created on that path. + if p.cfg.SelfID != "" { + p.selfID = p.cfg.SelfID + } else if p.cfg.SelfIDFile != "" { + raw, err := os.ReadFile(p.cfg.SelfIDFile) + if err != nil { + return fmt.Errorf("lineage-telemetry: no inline self_id and self_id_file unreadable: %w", err) + } + p.selfID = strings.TrimSpace(string(raw)) + } + if p.selfID == "" { + return fmt.Errorf("lineage-telemetry: self identity unresolved (empty self_id and self_id_file %q)", p.cfg.SelfIDFile) + } + endpoint := p.cfg.OTelEndpoint conn, err := grpc.NewClient(endpoint, grpc.WithTransportCredentials(insecure.NewCredentials()), @@ -166,6 +192,9 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { otlptracegrpc.WithGRPCConn(conn), ) if err != nil { + // The dial succeeded but the exporter did not adopt the conn, so close + // it here rather than leaking the gRPC client on this error path. + _ = conn.Close() return fmt.Errorf("lineage-telemetry: OTLP exporter: %w", err) } @@ -186,29 +215,6 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { ) p.tracer = p.tp.Tracer("authbridge/" + pluginName) - // Resolve self identity for the lineage.self.id fact. Every span this - // plugin emits is a claim of the form "X did Y"; with no X there is no - // claim to make, so an unresolvable identity refuses to start rather - // than serving traffic under a plausible-but-wrong label ("no mechanism - // may guess", contract v1.3). Note the asymmetry with this file's other - // unknowns: a missing status, payload or parent anchor is a missing PART - // of a fact and degrades honestly (abandoned / NULL / parent.source=wire). - // Identity is the fact's subject — it has no degraded form, and a shared - // placeholder would collapse every unidentified pod onto one entity row - // (entity id = uuid5("{kind}:{self.id}"), and entities is upsert-only). - if p.cfg.SelfID != "" { - p.selfID = p.cfg.SelfID - } else if p.cfg.SelfIDFile != "" { - raw, err := os.ReadFile(p.cfg.SelfIDFile) - if err != nil { - return fmt.Errorf("lineage-telemetry: no inline self_id and self_id_file unreadable: %w", err) - } - p.selfID = strings.TrimSpace(string(raw)) - } - if p.selfID == "" { - return fmt.Errorf("lineage-telemetry: self identity unresolved (empty self_id and self_id_file %q)", p.cfg.SelfIDFile) - } - p.ready.Store(true) slog.Info("lineage-telemetry: initialized", "endpoint", endpoint, "self_id", p.selfID) return nil @@ -677,8 +683,15 @@ func isA2AProtocolEvent(s string) bool { if raw, ok := obj["kind"]; ok { _ = json.Unmarshal(raw, &kind) } - return strings.Contains(kind, "status") || strings.Contains(kind, "artifact-update") || - strings.Contains(kind, "Status") || kind == "working" || kind == "canceled" + // A2A protocol event kinds are enumerated and stable, so match exactly: + // a substring test would suppress a legitimate agent-defined artifact whose + // kind merely contains one of these words (e.g. "final-status-report"). + switch kind { + case "status-update", "task-status-update", "artifact-update", "working", "canceled": + return true + default: + return false + } } // ioOutputValue returns the output.value for a response span: the parsed diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 59de55f05..3e5e0784f 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -6,6 +6,7 @@ import ( "maps" "net/http" "os" + "slices" "strings" "testing" "time" @@ -579,14 +580,14 @@ func TestCaptureIO_A2ANeverFallsThroughToCoPopulatedMCP(t *testing.T) { checkAttr(t, req, "lineage.protocol", "a2a") if v, ok := findAttr(req, "input.value"); ok { - t.Errorf("input.value = %q on an a2a hop with no a2a parts — leaked from the co-populated MCP parse", v.Emit()) + t.Errorf("input.value = %q on an a2a hop with no a2a parts — leaked from the co-populated MCP parse", v.String()) } if v, ok := findAttr(resp, "output.value"); ok { - t.Errorf("output.value = %q on an a2a hop whose artifact is a protocol event — leaked from the co-populated MCP parse", v.Emit()) + t.Errorf("output.value = %q on an a2a hop whose artifact is a protocol event — leaked from the co-populated MCP parse", v.String()) } // mcp.* facts belong to mcp hops only; the a2a label must keep them off. if v, ok := findAttr(req, "mcp.method"); ok { - t.Errorf("mcp.method = %q emitted on an a2a hop", v.Emit()) + t.Errorf("mcp.method = %q emitted on an a2a hop", v.String()) } } @@ -772,21 +773,7 @@ func checkAttr(t *testing.T, span tracetest.SpanStub, key, want string) { } func headersEqual(a, b http.Header) bool { - if len(a) != len(b) { - return false - } - for k, av := range a { - bv, ok := b[k] - if !ok || len(av) != len(bv) { - return false - } - for i := range av { - if av[i] != bv[i] { - return false - } - } - } - return true + return maps.EqualFunc(a, b, slices.Equal[[]string]) } // TestInit_RefusesToStartWithoutIdentity locks the v1.3 rule at the identity From cf1db7c0f5ac671d237e86fa56909af6b322c0c6 Mon Sep 17 00:00:00 2001 From: Igor Gokhman Date: Mon, 31 Aug 2026 09:06:14 +0300 Subject: [PATCH 04/37] =?UTF-8?q?Fix:=20Address=20lineage-telemetry=20revi?= =?UTF-8?q?ew=20=E2=80=94=20conn=20close,=20TLS,=20payload=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blocking fixes: - Close the OTLP gRPC conn on Shutdown. WithGRPCConn leaves connection ownership with the caller and the exporter's Shutdown does not close it, so the conn is now stored on LineageTelemetry and closed after the tracer provider shuts down (errors joined). The existing exporter-error-path close is kept. - Honour TLS for the OTLP export instead of silently downgrading. Adds an otel_tls config key; an https:// endpoint turns it on, and an https:// endpoint with an explicit otel_tls:false is rejected as a contradiction (fail closed) rather than exporting principal facts / captured payloads in cleartext. Default stays insecure for the in-pod loopback collector. Should-fix / cleanup: - Cap captured input.value/output.value at max_payload_bytes (default 4096, the OTLP attribute-value limit) with a UTF-8-safe truncate + explicit marker, so an oversize payload is cut at the producer rather than silently dropped by the exporter. Negative disables the cap. - Add docstrings for the new/lifecycle functions; note the deliberately pre-v1.21 http.method/http.status_code semconv keys; move the OPTION-4 read-only-variant explanation from an inline marker into the package doc. Tests: Shutdown closes conn / is safe uninitialised; the TLS config matrix; oversize-payload truncation + the truncate helper boundaries. Verified in golang:1.26 (GOWORK=off): vet/build/test -race green across authlib + both cmd modules and the lite exclude_plugin_* variant; go mod tidy byte-clean; gofmt clean. go.mod untouched. Signed-off-by: Igor Gokhman --- authbridge/authlib/plugins/lineage/config.go | 76 +++++++++- authbridge/authlib/plugins/lineage/plugin.go | 95 +++++++++++-- .../authlib/plugins/lineage/plugin_test.go | 132 ++++++++++++++++++ 3 files changed, 282 insertions(+), 21 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index fa7bd42e1..cf1a4d1f5 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -12,12 +12,31 @@ import ( // an in-pod collector reached over plaintext loopback. const defaultOTelEndpoint = "localhost:4317" +// defaultMaxPayloadBytes bounds a captured input.value / output.value. It +// matches the OTLP SDK's default span-attribute-value length limit, so a +// payload that fits here also fits the exporter and rides the wire intact; +// anything longer is truncated with an explicit marker rather than silently +// dropped downstream. +const defaultMaxPayloadBytes = 4096 + // Config holds the per-plugin configuration decoded from the pipeline YAML. type Config struct { - // OTelEndpoint is the OTLP gRPC endpoint (host:port or http://host:port). + // OTelEndpoint is the OTLP gRPC endpoint (host:port, http://host:port, or + // https://host:port). An https:// scheme implies OTelTLS=true. // Default: "localhost:4317" OTelEndpoint string `json:"otel_endpoint"` + // OTelTLS selects the OTLP transport. False (the default) dials plaintext, + // which is correct for the in-pod loopback collector but sends spans — + // including lineage.principal.* on every inbound request, and full payloads + // under CaptureIO — in cleartext. Set true for any collector off-pod: it + // dials with TLS against the system root CAs. An https:// otel_endpoint + // turns this on automatically; a plaintext otel_endpoint with otel_tls:true + // is honoured (TLS to a host:port). The one rejected combination is an + // https:// endpoint with an explicit otel_tls:false (see decodeConfig): a + // contradiction that would otherwise silently downgrade to cleartext. + OTelTLS bool `json:"otel_tls"` + // CaptureIO when true attaches parsed request/response content as // input.value (request span) and output.value (response span) // attributes, enabling Phoenix to display message content inline. @@ -30,6 +49,18 @@ type Config struct { // OTel backend enforces appropriate access controls. CaptureIO bool `json:"capture_io"` + // MaxPayloadBytes caps the size of the input.value / output.value + // attributes attached under CaptureIO. A payload longer than this is cut on + // a UTF-8 boundary and suffixed with a truncation marker, making the loss + // explicit at the producer rather than silent at the exporter: the OTLP SDK + // drops an attribute value that exceeds its own span-attribute-value limit + // (4096 bytes by default), so an uncapped large payload would simply vanish + // from the span with no marker. Zero (or unset) uses defaultMaxPayloadBytes; + // a negative value disables the cap (attach whole — the exporter limit then + // governs). Ignored when CaptureIO is false. + // Default: 4096 + MaxPayloadBytes int `json:"max_payload_bytes"` + // BypassPaths lists URL path prefixes that should not generate lineage // hops. Useful for suppressing infrastructure polling (agent-card // discovery, health checks) that would otherwise flood the lineage graph. @@ -55,10 +86,11 @@ type Config struct { func defaultConfig() Config { return Config{ - OTelEndpoint: defaultOTelEndpoint, - BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, - BypassHosts: []string{"otel-collector", "jaeger", "zipkin", "prometheus"}, - SelfIDFile: "/shared/client-id.txt", + OTelEndpoint: defaultOTelEndpoint, + MaxPayloadBytes: defaultMaxPayloadBytes, + BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, + BypassHosts: []string{"otel-collector", "jaeger", "zipkin", "prometheus"}, + SelfIDFile: "/shared/client-id.txt", } } @@ -77,15 +109,47 @@ func decodeConfig(raw json.RawMessage) (Config, error) { if cfg.OTelEndpoint == "" { cfg.OTelEndpoint = defaultOTelEndpoint } + // Zero means "unset" → the safe default; a negative value is the explicit + // opt-out (no cap). This keeps an omitted key and an explicit 0 identical. + if cfg.MaxPayloadBytes == 0 { + cfg.MaxPayloadBytes = defaultMaxPayloadBytes + } // gRPC NewClient expects host:port only, so reduce a URL form (e.g. // http://collector:4317/v1/traces) to its host — TrimPrefix left any path - // behind and produced an invalid dial target. + // behind and produced an invalid dial target. A URL scheme also carries an + // intent about transport: https:// asks for TLS. Honour it (or fail on a + // contradiction) rather than silently dropping to cleartext. if strings.Contains(cfg.OTelEndpoint, "://") { u, err := url.Parse(cfg.OTelEndpoint) if err != nil || u.Host == "" { return Config{}, fmt.Errorf("lineage-telemetry config: invalid otel_endpoint %q", cfg.OTelEndpoint) } + if u.Scheme == "https" { + // An explicit otel_tls:false alongside an https:// endpoint is a + // contradiction: one asks for encryption, the other for cleartext. + // Fail closed rather than pick one, consistent with the + // DisallowUnknownFields fail-on-ambiguity choice this package makes. + if tlsExplicitlyFalse(raw) { + return Config{}, fmt.Errorf("lineage-telemetry config: otel_endpoint %q is https but otel_tls is false", cfg.OTelEndpoint) + } + cfg.OTelTLS = true + } cfg.OTelEndpoint = u.Host } return cfg, nil } + +// tlsExplicitlyFalse reports whether the raw config carries otel_tls set to a +// literal false, as opposed to being absent (whose decoded value is also false +// but carries no intent). Used only to reject the https:// + otel_tls:false +// contradiction; a decode failure here is treated as "not explicitly false" +// since the DisallowUnknownFields pass above already validated the shape. +func tlsExplicitlyFalse(raw json.RawMessage) bool { + var probe struct { + OTelTLS *bool `json:"otel_tls"` + } + if err := json.Unmarshal(raw, &probe); err != nil { + return false + } + return probe.OTelTLS != nil && !*probe.OTelTLS +} diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index d3fbc96ae..70ba1d3c2 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -26,16 +26,29 @@ // so an exchange denied by a gate BEFORE OnRequest ran emits NO spans at all — // it is invisible to lineage. Moving lineage ahead of the gates (spans for // denied traffic too) is a named follow-up, not current behavior. +// +// Read-only variant ("Option 4"). This producer writes one tracestate member +// (tracestateStampKey) onto forwarded requests so a downstream sidecar can +// parent its exchange on this one. A deployment that wants a pure observer — +// no header written, parenting on the wire context alone — is obtained by +// deleting exactly the selectParent and restampTracestate calls in OnRequest +// (and the lineage.parent.source fact); the span emit itself stays. The +// trade-off: without the stamp, two sidecarred pods can only be joined through +// the app's own propagation, so cross-pod parenting degrades to whatever the +// wire parent carries. The call site is marked so the choice stays locatable; +// the variant is not built here. package lineage import ( "context" "encoding/json" + "errors" "fmt" "log/slog" "os" "strings" "sync/atomic" + "unicode/utf8" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" @@ -45,6 +58,7 @@ import ( semconv "go.opentelemetry.io/otel/semconv/v1.26.0" "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "github.com/rossoctl/cortex/authbridge/authlib/pipeline" @@ -83,6 +97,9 @@ const pluginName = "lineage-telemetry" // A visibly missing edge is recoverable; a silently wrong one is not. const tracestateStampKey = "dg-parent" +// truncatedSuffix marks a captured payload that MaxPayloadBytes cut short. +const truncatedSuffix = "…[truncated]" + func init() { plugins.RegisterPlugin(pluginName, func() pipeline.Plugin { return NewLineageTelemetry() }) } @@ -112,6 +129,7 @@ type LineageTelemetry struct { cfg Config tp *sdktrace.TracerProvider tracer trace.Tracer + conn *grpc.ClientConn // OTLP gRPC connection; owned by us, closed on Shutdown ready atomic.Bool propagator propagation.TextMapPropagator selfID string // agent's own client ID for the lineage.self.id fact @@ -181,8 +199,18 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { } endpoint := p.cfg.OTelEndpoint + // Transport credentials: plaintext by default (the loopback in-pod + // collector), TLS when otel_tls is set — an https:// endpoint sets it in + // decodeConfig. Spans carry principal facts on every inbound request and, + // under capture_io, user messages and model output, so a remote collector + // must not receive them in cleartext. + creds := insecure.NewCredentials() + if p.cfg.OTelTLS { + // nil cert pool = system roots; empty serverName = derive from endpoint. + creds = credentials.NewClientTLSFromCert(nil, "") + } conn, err := grpc.NewClient(endpoint, - grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithTransportCredentials(creds), ) if err != nil { return fmt.Errorf("lineage-telemetry: gRPC dial %s: %w", endpoint, err) @@ -197,6 +225,10 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { _ = conn.Close() return fmt.Errorf("lineage-telemetry: OTLP exporter: %w", err) } + // WithGRPCConn leaves connection ownership with the caller: the exporter's + // Shutdown will not close conn, so we retain it and close it ourselves in + // Shutdown. Store it only now, past the error path above. + p.conn = conn res, err := resource.New(ctx, resource.WithAttributes( @@ -220,11 +252,20 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { return nil } +// Shutdown flushes and stops the tracer provider and then closes the OTLP gRPC +// connection. The exporter created with WithGRPCConn does not own conn, so +// closing it here is what actually releases the socket; both errors are joined +// so neither is lost. Safe to call after a failed Init (tp/conn may be nil). func (p *LineageTelemetry) Shutdown(ctx context.Context) error { - if p.tp == nil { - return nil + var tpErr error + if p.tp != nil { + tpErr = p.tp.Shutdown(ctx) } - return p.tp.Shutdown(ctx) + var connErr error + if p.conn != nil { + connErr = p.conn.Close() + } + return errors.Join(tpErr, connErr) } func (p *LineageTelemetry) Ready() bool { return p.ready.Load() } @@ -273,15 +314,9 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context reqAttrs = p.appendRequestFacts(reqAttrs, pctx, protocol) // (3) parent · (4) emit · (5) re-stamp — wire contract v1.5. The emit is - // unconditional; the two calls around it are the stamp machinery. - // - // >>> OPTION-4 DELETION POINT <<< - // A pure read-only sidecar deletes exactly the selectParent and - // restampTracestate calls below (and the parent.source fact), parenting on - // remoteCtx alone — wire-parent-only propagation. The emit stays. The - // trade-off to weigh first: without the stamp, two sidecarred pods can - // only be joined through the app's own propagation, so cross-pod - // parenting degrades to whatever the wire parent happens to carry. + // unconditional; the two calls around it are the stamp machinery. The + // read-only "Option 4" variant deletes exactly these two calls (and the + // parent.source fact) — see the package doc for the trade-off. parent, parentSource := selectParent(ctx, remoteCtx) reqAttrs = append(reqAttrs, attribute.String("lineage.parent.source", parentSource)) reqCtx := p.emitRequestSpan(parent, spanName, spanKind, reqAttrs) @@ -421,6 +456,12 @@ func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) attrs = append(attrs, state.common...) attrs = append(attrs, attribute.String("lineage.outcome", outcome)) if hasStatus { + // "http.status_code" (like "http.method" on the request span) is the + // pre-v1.21 OTel semconv key, kept deliberately: this producer's + // contract vocabulary is lineage.* + these two well-known HTTP keys, + // pinned to the wire contract, not the stable OTel names + // (http.response.status_code / http.request.method). Interop with + // generic OTel tooling is a non-goal here. attrs = append(attrs, attribute.Int("http.status_code", status)) } if deniedBy != "" { @@ -428,7 +469,7 @@ func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) } if p.cfg.CaptureIO { if v := ioOutputValue(pctx, state.protocol); v != "" { - attrs = append(attrs, attribute.String("output.value", v)) + attrs = append(attrs, attribute.String("output.value", truncate(v, p.cfg.MaxPayloadBytes))) } } @@ -509,6 +550,8 @@ func baseAttrs(pctx *pipeline.Context, self, protocol string) []attribute.KeyVal // extension pointer is non-nil. func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx *pipeline.Context, protocol string) []attribute.KeyValue { if pctx.Method != "" { + // "http.method" is the pre-v1.21 OTel semconv key, kept deliberately — + // see the http.status_code note on the response span. attrs = append(attrs, attribute.String("http.method", pctx.Method)) } if pctx.Path != "" { @@ -551,12 +594,34 @@ func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx * } if p.cfg.CaptureIO { if v := ioInputValue(pctx, protocol); v != "" { - attrs = append(attrs, attribute.String("input.value", v)) + attrs = append(attrs, attribute.String("input.value", truncate(v, p.cfg.MaxPayloadBytes))) } } return attrs } +// truncate bounds a captured payload to max bytes, cutting on a UTF-8 +// rune boundary and appending truncatedSuffix so the loss is explicit in the +// span rather than a silent drop at the OTLP exporter's attribute-length limit. +// A non-positive max disables the cap (the caller's explicit opt-out). The +// returned string, suffix included, never exceeds max bytes. +func truncate(s string, max int) string { + if max <= 0 || len(s) <= max { + return s + } + // Reserve room for the marker; if the marker alone would not fit, fall back + // to a hard byte cut so we still never exceed max. + budget := max - len(truncatedSuffix) + if budget <= 0 { + return s[:max] + } + // Back up to a rune boundary so we never split a multi-byte character. + for budget > 0 && !utf8.RuneStart(s[budget]) { + budget-- + } + return s[:budget] + truncatedSuffix +} + // requestSpanName builds "{self} {protocol} {op}", dropping the trailing op // when it is empty. The response span appends " response". func requestSpanName(self, protocol, op string) string { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 3e5e0784f..a9fe565e3 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" "time" + "unicode/utf8" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" @@ -897,3 +898,134 @@ func TestBypassHosts_SubstringMatchEmitsNothing(t *testing.T) { }) } } + +// ---- lifecycle: gRPC connection ownership ---- +// WithGRPCConn leaves the conn for the caller to close; the exporter's Shutdown +// does not. A real Init dials a (never-answered) localhost target, stores the +// conn, and Shutdown must both stop the provider and close that conn without +// error. We assert the observable contract — conn stored after Init, Shutdown +// returns nil — since the closed socket itself is not introspectable here. +func TestShutdown_ClosesConn(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "localhost:4317", SelfID: "weather-service"} + if err := p.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + if p.conn == nil { + t.Fatal("Init did not store the gRPC conn for Shutdown to close") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := p.Shutdown(ctx); err != nil { + t.Fatalf("Shutdown returned an error: %v", err) + } + // A second Shutdown after conn is already closed must not panic; it may + // return an error from re-closing, which the caller can ignore. + _ = p.Shutdown(ctx) +} + +// TestShutdown_NoInitIsSafe: Shutdown on a plugin that never Init'd (tp and +// conn both nil) is a no-op, not a nil-deref — the host may call it after a +// failed Init. +func TestShutdown_NoInitIsSafe(t *testing.T) { + p := NewLineageTelemetry() + if err := p.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown on an uninitialized plugin: %v", err) + } +} + +// ---- OTLP transport selection ---- +// The export defaults to plaintext (in-pod loopback) but must honour a request +// for TLS rather than silently downgrading it: an https:// endpoint turns TLS +// on, and the one contradiction (https:// with an explicit otel_tls:false) +// fails closed rather than sending principal facts / payloads in the clear. +func TestConfig_TLSFromScheme(t *testing.T) { + cases := []struct { + name string + raw string + wantErr bool + wantTLS bool + wantHost string + }{ + {"bare host:port stays insecure", `{"otel_endpoint":"collector:4317","self_id":"x"}`, false, false, "collector:4317"}, + {"http:// stays insecure, host reduced", `{"otel_endpoint":"http://collector:4317/v1/traces","self_id":"x"}`, false, false, "collector:4317"}, + {"https:// turns TLS on, host reduced", `{"otel_endpoint":"https://collector.example.com:4317","self_id":"x"}`, false, true, "collector.example.com:4317"}, + {"explicit otel_tls on a plaintext host is honoured", `{"otel_endpoint":"collector:4317","otel_tls":true,"self_id":"x"}`, false, true, "collector:4317"}, + {"https:// with otel_tls:false is a rejected contradiction", `{"otel_endpoint":"https://collector:4317","otel_tls":false,"self_id":"x"}`, true, false, ""}, + {"https:// with otel_tls:true is consistent", `{"otel_endpoint":"https://collector:4317","otel_tls":true,"self_id":"x"}`, false, true, "collector:4317"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if tc.wantErr { + if err == nil { + t.Fatal("expected a config error, got nil") + } + return + } + if err != nil { + t.Fatalf("decodeConfig: %v", err) + } + if cfg.OTelTLS != tc.wantTLS { + t.Errorf("OTelTLS = %v, want %v", cfg.OTelTLS, tc.wantTLS) + } + if cfg.OTelEndpoint != tc.wantHost { + t.Errorf("OTelEndpoint = %q, want %q", cfg.OTelEndpoint, tc.wantHost) + } + }) + } +} + +// ---- payload truncation ---- +// With capture_io on, a payload larger than max_payload_bytes must be cut at +// the producer with an explicit marker, not left whole to be silently dropped +// by the OTLP exporter's attribute-length limit. The cut is byte-bounded and +// UTF-8-safe. +func TestCaptureIO_TruncatesOversizePayload(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.CaptureIO = true + p.cfg.MaxPayloadBytes = 64 + big := strings.Repeat("x", 500) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Extensions.A2A = &pipeline.A2AExtension{ + Method: "message/send", + Parts: []pipeline.A2APart{{Kind: "text", Content: big}}, + } + run(t, p, pctx, allow(200)) + req, _ := roleSplit(t, exp.GetSpans()) + v, ok := findAttr(req, "input.value") + if !ok { + t.Fatal("input.value absent on a captured a2a hop with text parts") + } + got := v.AsString() + if len(got) > p.cfg.MaxPayloadBytes { + t.Errorf("input.value is %d bytes, exceeds cap %d", len(got), p.cfg.MaxPayloadBytes) + } + if !strings.HasSuffix(got, truncatedSuffix) { + t.Errorf("truncated input.value %q missing %q suffix", got, truncatedSuffix) + } +} + +// TestTruncate covers the helper's boundaries directly: within-cap passthrough, +// the opt-out, and a UTF-8-safe cut that never splits a multi-byte rune. +func TestTruncate(t *testing.T) { + if got := truncate("short", 64); got != "short" { + t.Errorf("within cap mutated: %q", got) + } + if got := truncate("anything", -1); got != "anything" { + t.Errorf("negative cap should disable truncation, got %q", got) + } + // A run of 3-byte runes ("世") cut to a byte budget that lands mid-rune: + // the result must be valid UTF-8 and within the cap. + s := strings.Repeat("世", 100) // 300 bytes + got := truncate(s, 40) + if len(got) > 40 { + t.Errorf("truncated to %d bytes, exceeds cap 40", len(got)) + } + if !utf8.ValidString(got) { + t.Errorf("truncation split a multi-byte rune: %q is not valid UTF-8", got) + } + if !strings.HasSuffix(got, truncatedSuffix) { + t.Errorf("missing truncation marker: %q", got) + } +} From b669d90e15f2f9d9a7411ed0ddc8735c39375659 Mon Sep 17 00:00:00 2001 From: Igor Gokhman Date: Mon, 31 Aug 2026 09:47:54 +0300 Subject: [PATCH 05/37] =?UTF-8?q?Fix:=20Address=20CodeRabbit=20follow-ups?= =?UTF-8?q?=20=E2=80=94=20UTF-8=20truncate,=20scheme=20allowlist,=20payloa?= =?UTF-8?q?d-cap=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - truncate(): the suffix-can't-fit fallback (budget<=0) did a hard s[:max] byte cut that could split a multi-byte rune (e.g. truncate("世",1)). Back up to a utf8.RuneStart boundary before slicing so the fallback is always valid UTF-8. Extend TestTruncate with a sub-suffix multi-byte case. - decodeConfig: reject any otel_endpoint URL scheme other than http/https. ftp:// / ftps:// were accepted, scheme-stripped, and dialed insecure, silently sending lineage.principal.* facts and payloads in cleartext. Fail closed, matching the package's DisallowUnknownFields posture. Add ftp:// and ftps:// rejection rows to TestConfig_TLSFromScheme. - Correct the payload-limit doc comments: the OTel SDK's default attribute-value length limit is unlimited (-1) and Init sets no SpanLimits, so an uncapped value is not dropped/truncated downstream. MaxPayloadBytes is a deliberate producer-side bound, not a mirror of an SDK limit. Doc-only. Signed-off-by: Igor Gokhman --- authbridge/authlib/plugins/lineage/config.go | 37 ++++++++++++------- authbridge/authlib/plugins/lineage/plugin.go | 11 ++++-- .../authlib/plugins/lineage/plugin_test.go | 17 ++++++++- 3 files changed, 48 insertions(+), 17 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index cf1a4d1f5..660acfcb6 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -12,17 +12,20 @@ import ( // an in-pod collector reached over plaintext loopback. const defaultOTelEndpoint = "localhost:4317" -// defaultMaxPayloadBytes bounds a captured input.value / output.value. It -// matches the OTLP SDK's default span-attribute-value length limit, so a -// payload that fits here also fits the exporter and rides the wire intact; -// anything longer is truncated with an explicit marker rather than silently -// dropped downstream. +// defaultMaxPayloadBytes bounds a captured input.value / output.value as a +// deliberate producer-side cap. It is NOT a mirror of any SDK limit: the OTel +// SDK's default attribute-value length limit is unlimited (-1) and Init sets no +// SpanLimits, so an oversized value is not dropped or truncated downstream. This +// bound is our own guard against unbounded spans (and against any backend value +// limit); anything longer is cut here with an explicit marker so the loss is +// visible in the span. 4096 is a conservative default, not a hard requirement. const defaultMaxPayloadBytes = 4096 // Config holds the per-plugin configuration decoded from the pipeline YAML. type Config struct { // OTelEndpoint is the OTLP gRPC endpoint (host:port, http://host:port, or - // https://host:port). An https:// scheme implies OTelTLS=true. + // https://host:port). An https:// scheme implies OTelTLS=true. Any other + // URL scheme is rejected at decode (see decodeConfig). // Default: "localhost:4317" OTelEndpoint string `json:"otel_endpoint"` @@ -51,13 +54,13 @@ type Config struct { // MaxPayloadBytes caps the size of the input.value / output.value // attributes attached under CaptureIO. A payload longer than this is cut on - // a UTF-8 boundary and suffixed with a truncation marker, making the loss - // explicit at the producer rather than silent at the exporter: the OTLP SDK - // drops an attribute value that exceeds its own span-attribute-value limit - // (4096 bytes by default), so an uncapped large payload would simply vanish - // from the span with no marker. Zero (or unset) uses defaultMaxPayloadBytes; - // a negative value disables the cap (attach whole — the exporter limit then - // governs). Ignored when CaptureIO is false. + // a UTF-8 boundary and suffixed with a truncation marker, so the loss is + // explicit in the span. This is a deliberate producer-side bound; the OTel + // SDK does not itself drop or truncate an oversized value (its default + // attribute-value limit is unlimited and Init sets no SpanLimits), so + // without this cap the whole payload would be emitted. Zero (or unset) uses + // defaultMaxPayloadBytes; a negative value disables the cap (attach whole). + // Ignored when CaptureIO is false. // Default: 4096 MaxPayloadBytes int `json:"max_payload_bytes"` @@ -124,6 +127,14 @@ func decodeConfig(raw json.RawMessage) (Config, error) { if err != nil || u.Host == "" { return Config{}, fmt.Errorf("lineage-telemetry config: invalid otel_endpoint %q", cfg.OTelEndpoint) } + // Only http/https carry a meaningful OTLP transport intent. Reject any + // other scheme (ftp://, ftps://, …) rather than strip it and dial the + // bare host:port insecurely — that would silently send principal facts + // and payloads in cleartext. Fail closed, matching this package's + // DisallowUnknownFields / https+otel_tls:false posture. + if u.Scheme != "http" && u.Scheme != "https" { + return Config{}, fmt.Errorf("lineage-telemetry config: unsupported otel_endpoint scheme %q (want http or https)", u.Scheme) + } if u.Scheme == "https" { // An explicit otel_tls:false alongside an https:// endpoint is a // contradiction: one asks for encryption, the other for cleartext. diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 70ba1d3c2..16778c009 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -609,11 +609,16 @@ func truncate(s string, max int) string { if max <= 0 || len(s) <= max { return s } - // Reserve room for the marker; if the marker alone would not fit, fall back - // to a hard byte cut so we still never exceed max. + // Reserve room for the marker; if the marker alone would not fit, drop it + // and return the prefix. Still back up to a rune boundary so the fallback + // never emits invalid UTF-8, and never exceed max. budget := max - len(truncatedSuffix) if budget <= 0 { - return s[:max] + budget = max + for budget > 0 && !utf8.RuneStart(s[budget]) { + budget-- + } + return s[:budget] } // Back up to a rune boundary so we never split a multi-byte character. for budget > 0 && !utf8.RuneStart(s[budget]) { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index a9fe565e3..579002633 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -938,7 +938,9 @@ func TestShutdown_NoInitIsSafe(t *testing.T) { // The export defaults to plaintext (in-pod loopback) but must honour a request // for TLS rather than silently downgrading it: an https:// endpoint turns TLS // on, and the one contradiction (https:// with an explicit otel_tls:false) -// fails closed rather than sending principal facts / payloads in the clear. +// fails closed rather than sending principal facts / payloads in the clear. A +// non-http(s) scheme (ftp://, ftps://) is rejected outright, not stripped and +// dialed insecure. func TestConfig_TLSFromScheme(t *testing.T) { cases := []struct { name string @@ -953,6 +955,8 @@ func TestConfig_TLSFromScheme(t *testing.T) { {"explicit otel_tls on a plaintext host is honoured", `{"otel_endpoint":"collector:4317","otel_tls":true,"self_id":"x"}`, false, true, "collector:4317"}, {"https:// with otel_tls:false is a rejected contradiction", `{"otel_endpoint":"https://collector:4317","otel_tls":false,"self_id":"x"}`, true, false, ""}, {"https:// with otel_tls:true is consistent", `{"otel_endpoint":"https://collector:4317","otel_tls":true,"self_id":"x"}`, false, true, "collector:4317"}, + {"ftp:// scheme is rejected", `{"otel_endpoint":"ftp://collector:4317","self_id":"x"}`, true, false, ""}, + {"ftps:// scheme is rejected", `{"otel_endpoint":"ftps://collector:4317","self_id":"x"}`, true, false, ""}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -1028,4 +1032,15 @@ func TestTruncate(t *testing.T) { if !strings.HasSuffix(got, truncatedSuffix) { t.Errorf("missing truncation marker: %q", got) } + // The suffix-can't-fit fallback: max below len(truncatedSuffix) with a + // multi-byte payload must still return valid UTF-8 within the cap (the + // marker is dropped, but a mid-rune byte cut is not). Guards the budget<=0 + // branch, which previously did a hard s[:max] byte cut. + tiny := truncate(strings.Repeat("世", 100), 4) // 4 < len(truncatedSuffix) + if len(tiny) > 4 { + t.Errorf("suffix-can't-fit cut to %d bytes, exceeds cap 4", len(tiny)) + } + if !utf8.ValidString(tiny) { + t.Errorf("suffix-can't-fit cut split a rune: %q is not valid UTF-8", tiny) + } } From ed42dc4a6f79769234da27d42467744a94803b54 Mon Sep 17 00:00:00 2001 From: Igor Gokhman Date: Mon, 31 Aug 2026 10:46:53 +0300 Subject: [PATCH 06/37] Fix: Reset lineage-telemetry readiness on Shutdown Shutdown() set p.ready.Store(false) as its first statement, so Ready() returns false after teardown even when Init failed (tp/conn nil) or their shutdown errors. This makes the lifecycle observable to a pipeline orchestrator checking readiness before routing, and mirrors the p.ready.Store(true) in Init. Adds TestShutdown_ClearsReady (ready after Init, not ready after Shutdown) and TestShutdown_ClearsReadyAfterFailedInit (readiness cleared unconditionally on the no-Init path). Signed-off-by: Igor Gokhman --- authbridge/authlib/plugins/lineage/plugin.go | 6 +++ .../authlib/plugins/lineage/plugin_test.go | 39 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 16778c009..7b3c41b99 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -256,7 +256,13 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { // connection. The exporter created with WithGRPCConn does not own conn, so // closing it here is what actually releases the socket; both errors are joined // so neither is lost. Safe to call after a failed Init (tp/conn may be nil). +// +// Readiness is cleared first, unconditionally: after Shutdown the plugin is no +// longer ready even if tp/conn are nil (post-failed-Init) or their shutdown +// errors, so a pipeline orchestrator checking Ready() before routing sees the +// lifecycle transition. This mirrors the p.ready.Store(true) in Init. func (p *LineageTelemetry) Shutdown(ctx context.Context) error { + p.ready.Store(false) var tpErr error if p.tp != nil { tpErr = p.tp.Shutdown(ctx) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 579002633..d12be48c8 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -934,6 +934,45 @@ func TestShutdown_NoInitIsSafe(t *testing.T) { } } +// TestShutdown_ClearsReady: a successful Init makes the plugin ready; Shutdown +// clears readiness, so a pipeline orchestrator polling Ready() before routing +// sees the plugin fall out of rotation rather than keep receiving traffic into +// a torn-down tracer provider. +func TestShutdown_ClearsReady(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "localhost:4317", SelfID: "weather-service"} + if err := p.Init(context.Background()); err != nil { + t.Fatalf("Init: %v", err) + } + if !p.Ready() { + t.Fatal("Ready() is false after a successful Init") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := p.Shutdown(ctx); err != nil { + t.Fatalf("Shutdown returned an error: %v", err) + } + if p.Ready() { + t.Fatal("Ready() is still true after Shutdown") + } +} + +// TestShutdown_ClearsReadyAfterFailedInit: readiness is cleared unconditionally, +// even when Init never set it — Shutdown flips it false regardless of whether +// tp/conn were ever built, keeping the false→false transition idempotent. +func TestShutdown_ClearsReadyAfterFailedInit(t *testing.T) { + p := NewLineageTelemetry() + if p.Ready() { + t.Fatal("a freshly constructed plugin should not be ready") + } + if err := p.Shutdown(context.Background()); err != nil { + t.Fatalf("Shutdown on an uninitialized plugin: %v", err) + } + if p.Ready() { + t.Fatal("Ready() is true after Shutdown on an uninitialized plugin") + } +} + // ---- OTLP transport selection ---- // The export defaults to plaintext (in-pod loopback) but must honour a request // for TLS rather than silently downgrading it: an https:// endpoint turns TLS From 3705988c2247d9ff90c3f8c7c8d45e67726579e7 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Wed, 2 Sep 2026 15:52:10 +0300 Subject: [PATCH 07/37] Chore: go mod tidy after rebase onto main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebase of the lineage-telemetry lane onto main (bifrost 1.7.15, genproto 2026-07-20) — the three module files re-tidied in golang:1.26 with GOWORK=off; vet/build/test green across authlib and both cmd modules. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/go.mod | 8 ++++---- authbridge/cmd/authbridge-envoy/go.mod | 2 +- authbridge/cmd/authbridge-proxy/go.mod | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/authbridge/authlib/go.mod b/authbridge/authlib/go.mod index bf63a3d18..ae4432628 100644 --- a/authbridge/authlib/go.mod +++ b/authbridge/authlib/go.mod @@ -14,12 +14,12 @@ require ( github.com/spiffe/go-spiffe/v2 v2.8.1 github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 - golang.org/x/net v0.58.0 - golang.org/x/sync v0.22.0 go.opentelemetry.io/otel v1.44.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.44.0 go.opentelemetry.io/otel/sdk v1.44.0 go.opentelemetry.io/otel/trace v1.44.0 + golang.org/x/net v0.58.0 + golang.org/x/sync v0.22.0 golang.org/x/sys v0.47.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20260720211330-0afa2a65878a google.golang.org/grpc v1.83.2 @@ -117,7 +117,7 @@ require ( go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.44.0 // indirect go.opentelemetry.io/otel/metric v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect @@ -125,7 +125,7 @@ require ( golang.org/x/crypto v0.55.0 // indirect golang.org/x/text v0.41.0 // indirect golang.org/x/time v0.15.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260720211330-0afa2a65878a // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/ini.v1 v1.67.3 // indirect oras.land/oras-go/v2 v2.6.2 // indirect diff --git a/authbridge/cmd/authbridge-envoy/go.mod b/authbridge/cmd/authbridge-envoy/go.mod index b6646efdf..07d5b8077 100644 --- a/authbridge/cmd/authbridge-envoy/go.mod +++ b/authbridge/cmd/authbridge-envoy/go.mod @@ -117,7 +117,7 @@ require ( go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect diff --git a/authbridge/cmd/authbridge-proxy/go.mod b/authbridge/cmd/authbridge-proxy/go.mod index 43ebeb376..a3ffb0d5e 100644 --- a/authbridge/cmd/authbridge-proxy/go.mod +++ b/authbridge/cmd/authbridge-proxy/go.mod @@ -107,7 +107,7 @@ require ( go.opentelemetry.io/otel/metric v1.44.0 // indirect go.opentelemetry.io/otel/sdk v1.44.0 // indirect go.opentelemetry.io/otel/trace v1.44.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect go.starlark.net v0.0.0-20260708150628-5395d018f003 // indirect go.uber.org/atomic v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect From ddc460e882ba6868f1df80eeb42e1446d0cff32d Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Wed, 2 Sep 2026 15:54:42 +0300 Subject: [PATCH 08/37] Feat: Forward a traceparent when the request arrived without one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A request that reaches the sidecar with no traceparent header at all leaves the plugin with nothing to stamp onto: W3C reads tracestate only alongside a valid traceparent, so the app's propagate-only shim roots a fresh trace of its own and the dg-parent stamp never leaves the pod. The entry exchange lands alone in its own trace and every call it caused derives as a parentless root. Measured live on 2026-09-02 with one traceparent-less turn through a four-pod fleet: 32 spans in two traces, 9 derived roots instead of 1 — the tree is not welded, only the trace id is shared. mintTraceparent, step (4b) in OnRequest and therefore both directions: when the request carried no traceparent, inject one naming this exchange's request span (the span the SDK just rooted), then re-stamp as usual. Strictly additive — a traceparent that is present, valid or malformed, is left exactly as it arrived, and a malformed one still fragments visibly (parent.source=wire, no stamp). Behind mint_traceparent (default true); false restores the pure-observer posture for a deployment that must not add a header the application would see. Tests: minted traceparent + stamp for both directions; disabled leaves the headers bare; a malformed traceparent is never rewritten; an outbound carrying the entry's forwarded headers parents on the entry request span in the entry's trace; config default/explicit decode. newTestPlugin now starts from defaultConfig() so tests see the shipped defaults rather than the zero Config. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 15 +++ authbridge/authlib/plugins/lineage/plugin.go | 74 ++++++++--- .../authlib/plugins/lineage/plugin_test.go | 116 +++++++++++++++++- 3 files changed, 186 insertions(+), 19 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index 660acfcb6..3d3059de2 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -64,6 +64,20 @@ type Config struct { // Default: 4096 MaxPayloadBytes int `json:"max_payload_bytes"` + // MintTraceparent — both directions — forwards a W3C traceparent naming + // this exchange's request span when the request arrived with NO + // traceparent header at all. Without one the next element has nothing to + // extract: an app's propagate-only shim roots a fresh trace of its own, + // and the tracestate stamp (which W3C reads only alongside a valid + // traceparent) never leaves this pod — so the entry exchange lands alone + // in its own trace and every call it caused derives as a parentless root. + // This is the one place the plugin ADDS a header the caller did not send; + // a traceparent that is present, valid or malformed, is never modified. + // Set false for a pure observer that must not add a header the + // application would see (the exchange then fragments, visibly). + // Default: true + MintTraceparent bool `json:"mint_traceparent"` + // BypassPaths lists URL path prefixes that should not generate lineage // hops. Useful for suppressing infrastructure polling (agent-card // discovery, health checks) that would otherwise flood the lineage graph. @@ -91,6 +105,7 @@ func defaultConfig() Config { return Config{ OTelEndpoint: defaultOTelEndpoint, MaxPayloadBytes: defaultMaxPayloadBytes, + MintTraceparent: true, BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, BypassHosts: []string{"otel-collector", "jaeger", "zipkin", "prometheus"}, SelfIDFile: "/shared/client-id.txt", diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 7b3c41b99..4d385a332 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -27,16 +27,22 @@ // it is invisible to lineage. Moving lineage ahead of the gates (spans for // denied traffic too) is a named follow-up, not current behavior. // -// Read-only variant ("Option 4"). This producer writes one tracestate member -// (tracestateStampKey) onto forwarded requests so a downstream sidecar can -// parent its exchange on this one. A deployment that wants a pure observer — -// no header written, parenting on the wire context alone — is obtained by -// deleting exactly the selectParent and restampTracestate calls in OnRequest -// (and the lineage.parent.source fact); the span emit itself stays. The -// trade-off: without the stamp, two sidecarred pods can only be joined through -// the app's own propagation, so cross-pod parenting degrades to whatever the -// wire parent carries. The call site is marked so the choice stays locatable; -// the variant is not built here. +// What this producer writes onto forwarded requests — two things, both +// directions. Always: one tracestate member (tracestateStampKey) so the next +// lineage element can parent its exchange on this one. Only when the request +// carried no traceparent at all: a traceparent naming this request span +// (mintTraceparent, config mint_traceparent, default on), because without one +// the next element has nothing to extract and the stamp has nothing to ride +// on. A traceparent that is present is never modified. +// +// Read-only variant ("Option 4"). A deployment that wants a pure observer — +// no header written, parenting on the wire context alone — sets +// mint_traceparent: false and deletes exactly the selectParent and +// restampTracestate calls in OnRequest (and the lineage.parent.source fact); +// the span emit itself stays. The trade-off: without the stamp, two sidecarred +// pods can only be joined through the app's own propagation, so cross-pod +// parenting degrades to whatever the wire parent carries. The call site is +// marked so the choice stays locatable; the variant is not built here. package lineage import ( @@ -81,6 +87,10 @@ const pluginName = "lineage-telemetry" // own spans keeps an intact traceparent chain toward its own backend while // the sidecar chain stays self-consistent in ours. (Until v1.4 the outbound // instead rewrote the forwarded traceparent — the splice; v1.5 removed it.) +// The one additive exception: a request that arrived with no traceparent at +// all is forwarded with one naming this request span (mintTraceparent), so +// this member has a header to ride on — W3C reads tracestate only alongside +// a valid traceparent. // // The key names the consuming data-governance system (W3C convention: the key // identifies the owner of the entry) and is deliberately platform-neutral — @@ -319,14 +329,17 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context reqAttrs = append(reqAttrs, base...) reqAttrs = p.appendRequestFacts(reqAttrs, pctx, protocol) - // (3) parent · (4) emit · (5) re-stamp — wire contract v1.5. The emit is - // unconditional; the two calls around it are the stamp machinery. The - // read-only "Option 4" variant deletes exactly these two calls (and the - // parent.source fact) — see the package doc for the trade-off. + // (3) parent · (4) emit · (4b) mint · (5) re-stamp — wire contract v1.5. + // The emit is unconditional; the calls around it are the header + // machinery, and (4b) only acts when no traceparent arrived at all. The + // read-only "Option 4" variant deletes selectParent and restampTracestate + // (and the parent.source fact) and sets mint_traceparent: false — see the + // package doc for the trade-off. parent, parentSource := selectParent(ctx, remoteCtx) reqAttrs = append(reqAttrs, attribute.String("lineage.parent.source", parentSource)) reqCtx := p.emitRequestSpan(parent, spanName, spanKind, reqAttrs) exchangeID := reqCtx.SpanID().String() + remoteCtx = p.mintTraceparent(ctx, pctx, remoteCtx, reqCtx) restampTracestate(pctx, remoteCtx, exchangeID) common := make([]attribute.KeyValue, 0, len(base)+1) @@ -382,15 +395,40 @@ func (p *LineageTelemetry) emitRequestSpan( return sc } +// mintTraceparent is step (4b), both directions: when the request arrived +// with NO traceparent header, forward one naming this request span, and +// return that context for the re-stamp to build on. Without it the next +// element has nothing to extract — an app's propagate-only shim roots a fresh +// trace of its own and the tracestate stamp never leaves this pod, because +// W3C reads tracestate only alongside a valid traceparent — so the entry +// exchange lands alone in its own trace and every call it caused derives as +// a parentless root (measured live 2026-09-02: one traceparent-less turn, 32 +// spans, 9 derived roots instead of 1). Strictly additive: a traceparent that +// is present, valid or malformed, is left exactly as it arrived (a malformed +// one still fragments visibly — parent.source=wire, no stamp). Disabled by +// mint_traceparent: false, for a pure observer. +func (p *LineageTelemetry) mintTraceparent(ctx context.Context, pctx *pipeline.Context, remoteCtx context.Context, reqCtx trace.SpanContext) context.Context { + if !p.cfg.MintTraceparent || trace.SpanContextFromContext(remoteCtx).IsValid() || pctx.Headers.Get("traceparent") != "" { + return remoteCtx + } + minted := trace.ContextWithRemoteSpanContext(ctx, reqCtx) + // TraceContext.Inject writes traceparent only; tracestate follows in + // restampTracestate (reqCtx carries an empty TraceState). + p.propagator.Inject(minted, propagation.HeaderCarrier(pctx.Headers)) + return minted +} + // restampTracestate is step (5): rewrite the forwarded request's tracestate // member with this exchange id — both directions. Inbound: the app's // propagate-only shim couriers it to exactly the outbound calls this inbound // caused. Outbound: the peer sidecar's inbound reads it as its parent. The // forwarded traceparent is never modified (see tracestateStampKey). A valid -// wire traceparent is required — without one the app's shim starts a fresh -// root trace and drops the tracestate anyway, so there is nothing to stamp. -// The listener is responsible for propagating this header mutation (ext_proc -// emits a SetHeaders diff). +// traceparent to ride on is required — the wire's, or the one mintTraceparent +// just forwarded; with neither (mint_traceparent off, or a malformed +// traceparent left untouched) the shim would root a fresh trace and drop the +// tracestate anyway, so there is nothing to stamp. The listener is +// responsible for propagating these header mutations (ext_proc emits a +// SetHeaders diff). func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchangeID string) { rsc := trace.SpanContextFromContext(remoteCtx) if !rsc.IsValid() { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index d12be48c8..ae5859421 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -29,6 +29,7 @@ func newTestPlugin(t *testing.T) (*LineageTelemetry, *tracetest.InMemoryExporter exp := tracetest.NewInMemoryExporter() tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) p := NewLineageTelemetry() + p.cfg = defaultConfig() // the shipped defaults, so a test sees what a deployment sees p.tp = tp p.tracer = tp.Tracer("test") p.selfID = "weather-service" @@ -266,17 +267,130 @@ func TestStamp_PreservesForeignTracestateMembers(t *testing.T) { } } -func TestStamp_NoWireTraceparentNoStamp(t *testing.T) { +// TestMint_NoTraceparentForwardsOwn is the traceparent-less entry, both +// directions: the request span roots a fresh trace, and the forwarded request +// now carries a traceparent naming that span PLUS the tracestate stamp — so +// the next element (the app's shim inbound, the peer's sidecar outbound) has +// a context to extract and the stamp has a header to ride on. +func TestMint_NoTraceparentForwardsOwn(t *testing.T) { + for _, dir := range []pipeline.Direction{pipeline.Inbound, pipeline.Outbound} { + t.Run(dir.String(), func(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(dir, http.Header{}) + + run(t, p, pctx, allow(200)) + + req, _ := roleSplit(t, exp.GetSpans()) + if req.Parent.IsValid() { + t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) + } + if got := attrStr(req, "lineage.parent.source"); got != "wire" { + t.Errorf("lineage.parent.source = %q, want wire", got) + } + want := "00-" + req.SpanContext.TraceID().String() + "-" + req.SpanContext.SpanID().String() + "-01" + if got := pctx.Headers.Get("traceparent"); got != want { + t.Errorf("forwarded traceparent = %q, want minted %q", got, want) + } + wantStamp := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != wantStamp { + t.Errorf("tracestate = %q, want stamp %q", got, wantStamp) + } + }) + } +} + +// TestMint_Disabled is the pure-observer posture: with mint_traceparent off a +// traceparent-less request is forwarded exactly as it arrived — no +// traceparent, and therefore no stamp either. +func TestMint_Disabled(t *testing.T) { p, _ := newTestPlugin(t) + p.cfg.MintTraceparent = false pctx := fakeContext(pipeline.Inbound, http.Header{}) run(t, p, pctx, allow(200)) + if got := pctx.Headers.Get("traceparent"); got != "" { + t.Errorf("traceparent minted with mint_traceparent off: %q", got) + } if got := pctx.Headers.Get("tracestate"); got != "" { t.Errorf("tracestate stamped without a wire traceparent: %q", got) } } +// TestMint_NeverRewritesPresentTraceparent: minting is additive only. A +// traceparent that is present but unparseable extracts as no context — the +// exact situation minting exists for — and is still left untouched, with no +// stamp: the plugin never modifies a header the caller sent. +func TestMint_NeverRewritesPresentTraceparent(t *testing.T) { + p, exp := newTestPlugin(t) + h := http.Header{} + h.Set("traceparent", "not-a-traceparent") + pctx := fakeContext(pipeline.Inbound, h) + + run(t, p, pctx, allow(200)) + + if got := pctx.Headers.Get("traceparent"); got != "not-a-traceparent" { + t.Errorf("malformed traceparent rewritten to %q", got) + } + if got := pctx.Headers.Get("tracestate"); got != "" { + t.Errorf("tracestate stamped onto a malformed traceparent: %q", got) + } + req, _ := roleSplit(t, exp.GetSpans()) + if req.Parent.IsValid() { + t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) + } +} + +// TestMint_ChainsThroughEntry is the whole entry mechanism end to end in +// miniature: a traceparent-less inbound (the entry), then an outbound that +// arrives carrying exactly the headers the inbound forwarded — as a +// propagate-only shim would courier them — parents on the entry's request +// span via the stamp, in the entry's own trace. One tree, one root. +func TestMint_ChainsThroughEntry(t *testing.T) { + p, exp := newTestPlugin(t) + entry := fakeContext(pipeline.Inbound, http.Header{}) + run(t, p, entry, allow(200)) + entryReq, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + + out := fakeContext(pipeline.Outbound, entry.Headers.Clone()) + run(t, p, out, allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + + if outReq.SpanContext.TraceID() != entryReq.SpanContext.TraceID() { + t.Errorf("outbound trace %s, want the entry's %s", outReq.SpanContext.TraceID(), entryReq.SpanContext.TraceID()) + } + if outReq.Parent.SpanID() != entryReq.SpanContext.SpanID() { + t.Errorf("outbound parent %s, want the entry request span %s", outReq.Parent.SpanID(), entryReq.SpanContext.SpanID()) + } + if got := attrStr(outReq, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) + } +} + +func TestConfig_MintTraceparent(t *testing.T) { + cases := []struct { + name string + raw string + want bool + }{ + {"default on", `{"self_id":"x"}`, true}, + {"explicit true", `{"mint_traceparent":true,"self_id":"x"}`, true}, + {"explicit false", `{"mint_traceparent":false,"self_id":"x"}`, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if err != nil { + t.Fatalf("decodeConfig: %v", err) + } + if cfg.MintTraceparent != tc.want { + t.Errorf("MintTraceparent = %v, want %v", cfg.MintTraceparent, tc.want) + } + }) + } +} + // TestStamp_OutboundPrefersStampOverMap is the same-trace fan-in case in // miniature: two concurrent inbound exchanges on ONE trace (the trace-keyed // map can only hold the later one), then an outbound whose tracestate stamp From cf4343592e81484b4056bc25a2e62c6fc3d2d4b7 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Wed, 2 Sep 2026 17:08:34 +0300 Subject: [PATCH 09/37] Feat: Record parent.source=none when nothing valid was on the wire MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit selectParent labelled a request span that had NO valid wire context as "wire" — the same word used for a real wire parent this pipeline did not export. The two are different facts: one is a parent that exists somewhere, the other is no parent at all — the span roots a trace. With mint_traceparent that root is now the ordinary entry of a sidecar-entered trace, so the label matters for anyone auditing attribution. lineage.parent.source gains a third value, "none": nothing valid on the wire (no traceparent, or a malformed one left untouched). Precedence is unchanged — stamp, else wire parent, else none — and guessing is still not among the options. Wire contract v1.6 (with the traceparent forwarded-when-absent rule); the capability description cites it. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 35 +++++++++++-------- .../authlib/plugins/lineage/plugin_test.go | 7 ++-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 4d385a332..445645f74 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -80,8 +80,9 @@ const pluginName = "lineage-telemetry" // carries tracestate through its per-request causal chain (contextvars), so // the member surfaces on exactly the outbound calls that inbound caused. // Outbound re-stamps the request it forwards to the peer, whose inbound -// sidecar reads it as its parent. Parent precedence is stamp > wire parent in -// BOTH directions, and the chosen source is recorded as the +// sidecar reads it as its parent. Parent precedence is stamp > wire parent > +// none (nothing valid on the wire: the span roots a trace) in BOTH +// directions, and the chosen source is recorded as the // lineage.parent.source fact. The forwarded traceparent is never modified: // the sidecar chain lives entirely in this member, so an app that emits its // own spans keeps an intact traceparent chain toward its own backend while @@ -168,7 +169,7 @@ func (p *LineageTelemetry) Capabilities() pipeline.PluginCapabilities { // The contract is cited major.minor only, deliberately: patch // revisions (v1.5.x) clarify prose and never change span semantics, // so a patch bump must not imply a producer change. - Description: "Emits two facts-only lineage spans per HTTP exchange (wire contract v1.5).", + Description: "Emits two facts-only lineage spans per HTTP exchange (wire contract v1.6).", } } @@ -189,7 +190,7 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { // plausible-but-wrong label ("no mechanism may guess", contract v1.3). Note // the asymmetry with this file's other unknowns: a missing status, payload // or parent anchor is a missing PART of a fact and degrades honestly - // (abandoned / NULL / parent.source=wire). Identity is the fact's subject — + // (abandoned / NULL / parent.source=wire or none). Identity is the fact's subject — // it has no degraded form, and a shared placeholder would collapse every // unidentified pod onto one entity row (entity id = uuid5("{kind}:{self.id}"), // and entities is upsert-only). Resolving it up front also means a refused @@ -329,7 +330,7 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context reqAttrs = append(reqAttrs, base...) reqAttrs = p.appendRequestFacts(reqAttrs, pctx, protocol) - // (3) parent · (4) emit · (4b) mint · (5) re-stamp — wire contract v1.5. + // (3) parent · (4) emit · (4b) mint · (5) re-stamp — wire contract v1.6. // The emit is unconditional; the calls around it are the header // machinery, and (4b) only acts when no traceparent arrived at all. The // read-only "Option 4" variant deletes selectParent and restampTracestate @@ -358,18 +359,22 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context } // selectParent is step (3) of the single-channel parenting mechanism (wire -// contract v1.5): the parent is the tracestate stamp — the previous sidecar +// contract v1.6): the parent is the tracestate stamp — the previous sidecar // element in the chain (the caller's outbound for an inbound, this pod's -// inbound for an outbound) — else the wire parent. Same precedence in both -// directions. There is deliberately no third option: guessing an attribution -// is worse than declining to give one. Returns the parent context and the -// source label the caller emits as the lineage.parent.source fact. +// inbound for an outbound) — else the wire parent; and when nothing valid is +// on the wire at all, no parent: the request span roots a trace and the fact +// says so ("none") rather than claiming a wire parent that was never there. +// Same precedence in both directions. Guessing an attribution is deliberately +// not among the options: it is worse than declining to give one. Returns the +// parent context and the source label the caller emits as the +// lineage.parent.source fact. func selectParent(ctx, remoteCtx context.Context) (context.Context, string) { rsc := trace.SpanContextFromContext(remoteCtx) - if rsc.IsValid() { - if psc, ok := stampedParent(rsc); ok { - return trace.ContextWithRemoteSpanContext(ctx, psc), "tracestate" - } + if !rsc.IsValid() { + return remoteCtx, "none" + } + if psc, ok := stampedParent(rsc); ok { + return trace.ContextWithRemoteSpanContext(ctx, psc), "tracestate" } return remoteCtx, "wire" } @@ -405,7 +410,7 @@ func (p *LineageTelemetry) emitRequestSpan( // a parentless root (measured live 2026-09-02: one traceparent-less turn, 32 // spans, 9 derived roots instead of 1). Strictly additive: a traceparent that // is present, valid or malformed, is left exactly as it arrived (a malformed -// one still fragments visibly — parent.source=wire, no stamp). Disabled by +// one still fragments visibly — parent.source=none, no stamp). Disabled by // mint_traceparent: false, for a pure observer. func (p *LineageTelemetry) mintTraceparent(ctx context.Context, pctx *pipeline.Context, remoteCtx context.Context, reqCtx trace.SpanContext) context.Context { if !p.cfg.MintTraceparent || trace.SpanContextFromContext(remoteCtx).IsValid() || pctx.Headers.Get("traceparent") != "" { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index ae5859421..60574eeb7 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -284,8 +284,8 @@ func TestMint_NoTraceparentForwardsOwn(t *testing.T) { if req.Parent.IsValid() { t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) } - if got := attrStr(req, "lineage.parent.source"); got != "wire" { - t.Errorf("lineage.parent.source = %q, want wire", got) + if got := attrStr(req, "lineage.parent.source"); got != "none" { + t.Errorf("lineage.parent.source = %q, want none", got) } want := "00-" + req.SpanContext.TraceID().String() + "-" + req.SpanContext.SpanID().String() + "-01" if got := pctx.Headers.Get("traceparent"); got != want { @@ -339,6 +339,9 @@ func TestMint_NeverRewritesPresentTraceparent(t *testing.T) { if req.Parent.IsValid() { t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) } + if got := attrStr(req, "lineage.parent.source"); got != "none" { + t.Errorf("lineage.parent.source = %q, want none", got) + } } // TestMint_ChainsThroughEntry is the whole entry mechanism end to end in From c8747d4eccbb00e0335f2178b5f6b6f69b3cdc4d Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Wed, 2 Sep 2026 17:28:17 +0300 Subject: [PATCH 10/37] Fix: Restart the trace on an invalid traceparent; test the chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two preflight findings on the mint change, resolved by conforming to W3C rather than documenting a divergence, plus one note. mintTraceparent gated on the header's presence (Get != ""), which had two problems: an empty header was minted over while a malformed one was left alone, and the malformed case was inconsistent with selectParent, which already records parent.source=none for it. Gate on the propagator's validity verdict instead — the same IsValid() the parent choice uses, no parsing of our own. Absent, empty and malformed traceparents are now all restarted: a new traceparent naming this request span, and the caller's tracestate dropped because the minted context carries an empty TraceState. That is W3C Trace Context's processing model for an unparseable traceparent. A valid traceparent is never modified (TestStamp_InboundHeadersUntouchedExceptStamp). TestMint_RestartsInvalidTraceparent covers malformed, empty and version-ff values with a foreign tracestate riding along, and asserts the restarted header, the stamp alone, the root, and parent.source=none. The outbound-minted → peer-inbound chain was implied by the shared code path but not proven; TestMint_OutboundChainsIntoPeerInbound drives it across two plugin instances with the forwarded headers, the twin of TestMint_ChainsThroughEntry. Init: a comment at the TracerProvider names the sampler the minted traceparent's flag comes from (SDK default ParentBased(AlwaysSample), OTEL_TRACES_SAMPLER-overridable) and why a ratio sampler would un-sample whole chains rather than one pod's spans. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 9 ++- authbridge/authlib/plugins/lineage/plugin.go | 51 +++++++----- .../authlib/plugins/lineage/plugin_test.go | 79 ++++++++++++++----- 3 files changed, 96 insertions(+), 43 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index 3d3059de2..d09b71201 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -65,14 +65,15 @@ type Config struct { MaxPayloadBytes int `json:"max_payload_bytes"` // MintTraceparent — both directions — forwards a W3C traceparent naming - // this exchange's request span when the request arrived with NO - // traceparent header at all. Without one the next element has nothing to + // this exchange's request span when the request arrived with no + // valid traceparent. Without one the next element has nothing to // extract: an app's propagate-only shim roots a fresh trace of its own, // and the tracestate stamp (which W3C reads only alongside a valid // traceparent) never leaves this pod — so the entry exchange lands alone // in its own trace and every call it caused derives as a parentless root. - // This is the one place the plugin ADDS a header the caller did not send; - // a traceparent that is present, valid or malformed, is never modified. + // Absent, empty and malformed traceparents are all restarted, which is + // W3C's processing model for an unparseable one; a valid traceparent is + // never modified. This is the one place the plugin writes a traceparent. // Set false for a pure observer that must not add a header the // application would see (the exchange then fragments, visibly). // Default: true diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 445645f74..7b39cc9b0 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -33,7 +33,8 @@ // carried no traceparent at all: a traceparent naming this request span // (mintTraceparent, config mint_traceparent, default on), because without one // the next element has nothing to extract and the stamp has nothing to ride -// on. A traceparent that is present is never modified. +// on. A valid traceparent is never modified; an invalid one is replaced, which +// is W3C's processing model for it (restart the trace, drop tracestate). // // Read-only variant ("Option 4"). A deployment that wants a pure observer — // no header written, parenting on the wire context alone — sets @@ -88,10 +89,10 @@ const pluginName = "lineage-telemetry" // own spans keeps an intact traceparent chain toward its own backend while // the sidecar chain stays self-consistent in ours. (Until v1.4 the outbound // instead rewrote the forwarded traceparent — the splice; v1.5 removed it.) -// The one additive exception: a request that arrived with no traceparent at -// all is forwarded with one naming this request span (mintTraceparent), so -// this member has a header to ride on — W3C reads tracestate only alongside -// a valid traceparent. +// The one exception: a request that arrived with no valid traceparent is +// forwarded with one naming this request span (mintTraceparent), so this +// member has a header to ride on — W3C reads tracestate only alongside a +// valid traceparent. // // The key names the consuming data-governance system (W3C convention: the key // identifies the owner of the entry) and is deliberately platform-neutral — @@ -252,6 +253,11 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { res = resource.Default() } + // No sampler is set, so the SDK default applies: ParentBased(AlwaysSample), + // overridable through OTEL_TRACES_SAMPLER. The sampling flag of a root + // span is what a minted traceparent carries downstream (mintTraceparent), + // and every peer sidecar is ParentBased too — a ratio sampler here would + // silently un-sample whole chains, not just this pod's spans. p.tp = sdktrace.NewTracerProvider( sdktrace.WithBatcher(exporter), sdktrace.WithResource(res), @@ -401,19 +407,22 @@ func (p *LineageTelemetry) emitRequestSpan( } // mintTraceparent is step (4b), both directions: when the request arrived -// with NO traceparent header, forward one naming this request span, and -// return that context for the re-stamp to build on. Without it the next -// element has nothing to extract — an app's propagate-only shim roots a fresh -// trace of its own and the tracestate stamp never leaves this pod, because -// W3C reads tracestate only alongside a valid traceparent — so the entry -// exchange lands alone in its own trace and every call it caused derives as -// a parentless root (measured live 2026-09-02: one traceparent-less turn, 32 -// spans, 9 derived roots instead of 1). Strictly additive: a traceparent that -// is present, valid or malformed, is left exactly as it arrived (a malformed -// one still fragments visibly — parent.source=none, no stamp). Disabled by -// mint_traceparent: false, for a pure observer. +// with NO VALID traceparent, forward one naming this request span, and return +// that context for the re-stamp to build on. Without it the next element has +// nothing to extract — an app's propagate-only shim roots a fresh trace of +// its own and the tracestate stamp never leaves this pod, because W3C reads +// tracestate only alongside a valid traceparent — so the entry exchange lands +// alone in its own trace and every call it caused derives as a parentless +// root (measured live 2026-09-02: one traceparent-less turn, 32 spans, 9 +// derived roots instead of 1). Validity is the propagator's verdict, the same +// one selectParent used to record parent.source=none: absent, empty and +// malformed all extract as no context, and all three are restarted here — +// exactly W3C's processing model for an unparseable traceparent (new +// traceparent, tracestate dropped; the re-stamp then writes ours alone). A +// valid traceparent is never modified. Disabled by mint_traceparent: false, +// for a pure observer. func (p *LineageTelemetry) mintTraceparent(ctx context.Context, pctx *pipeline.Context, remoteCtx context.Context, reqCtx trace.SpanContext) context.Context { - if !p.cfg.MintTraceparent || trace.SpanContextFromContext(remoteCtx).IsValid() || pctx.Headers.Get("traceparent") != "" { + if !p.cfg.MintTraceparent || trace.SpanContextFromContext(remoteCtx).IsValid() { return remoteCtx } minted := trace.ContextWithRemoteSpanContext(ctx, reqCtx) @@ -429,9 +438,11 @@ func (p *LineageTelemetry) mintTraceparent(ctx context.Context, pctx *pipeline.C // caused. Outbound: the peer sidecar's inbound reads it as its parent. The // forwarded traceparent is never modified (see tracestateStampKey). A valid // traceparent to ride on is required — the wire's, or the one mintTraceparent -// just forwarded; with neither (mint_traceparent off, or a malformed -// traceparent left untouched) the shim would root a fresh trace and drop the -// tracestate anyway, so there is nothing to stamp. The listener is +// just forwarded; with neither (mint_traceparent off) the shim would root a +// fresh trace and drop the tracestate anyway, so there is nothing to stamp. On +// a minted context the base TraceState is empty, so the caller's tracestate, +// if any rode in with an invalid traceparent, is dropped — W3C's restart +// semantics, not an accident. The listener is // responsible for propagating these header mutations (ext_proc emits a // SetHeaders diff). func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchangeID string) { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 60574eeb7..b9944db18 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -3,6 +3,7 @@ package lineage import ( "context" "encoding/json" + "fmt" "maps" "net/http" "os" @@ -317,30 +318,70 @@ func TestMint_Disabled(t *testing.T) { } } -// TestMint_NeverRewritesPresentTraceparent: minting is additive only. A -// traceparent that is present but unparseable extracts as no context — the -// exact situation minting exists for — and is still left untouched, with no -// stamp: the plugin never modifies a header the caller sent. -func TestMint_NeverRewritesPresentTraceparent(t *testing.T) { - p, exp := newTestPlugin(t) - h := http.Header{} - h.Set("traceparent", "not-a-traceparent") - pctx := fakeContext(pipeline.Inbound, h) +// TestMint_RestartsInvalidTraceparent: W3C's processing model for an +// unparseable traceparent is to restart the trace — a new traceparent, the +// tracestate dropped. The propagator's verdict decides: a malformed value, an +// empty one, and a version-ff one all extract as no context, so all three are +// restarted exactly like an absent header, and a foreign tracestate that rode +// in with them does not survive. (A VALID traceparent is never touched: +// TestStamp_InboundHeadersUntouchedExceptStamp.) +func TestMint_RestartsInvalidTraceparent(t *testing.T) { + for _, sent := range []string{"not-a-traceparent", "", "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} { + t.Run(fmt.Sprintf("traceparent=%q", sent), func(t *testing.T) { + p, exp := newTestPlugin(t) + h := http.Header{} + h.Set("traceparent", sent) + h.Set("tracestate", "vendor=abc") + pctx := fakeContext(pipeline.Inbound, h) - run(t, p, pctx, allow(200)) + run(t, p, pctx, allow(200)) - if got := pctx.Headers.Get("traceparent"); got != "not-a-traceparent" { - t.Errorf("malformed traceparent rewritten to %q", got) + req, _ := roleSplit(t, exp.GetSpans()) + if req.Parent.IsValid() { + t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) + } + if got := attrStr(req, "lineage.parent.source"); got != "none" { + t.Errorf("lineage.parent.source = %q, want none", got) + } + want := "00-" + req.SpanContext.TraceID().String() + "-" + req.SpanContext.SpanID().String() + "-01" + if got := pctx.Headers.Values("traceparent"); len(got) != 1 || got[0] != want { + t.Errorf("forwarded traceparent = %v, want restarted %q", got, want) + } + wantStamp := tracestateStampKey + "=" + req.SpanContext.SpanID().String() + if got := pctx.Headers.Get("tracestate"); got != wantStamp { + t.Errorf("tracestate = %q, want the stamp alone (caller's dropped on restart) %q", got, wantStamp) + } + }) } - if got := pctx.Headers.Get("tracestate"); got != "" { - t.Errorf("tracestate stamped onto a malformed traceparent: %q", got) +} + +// TestMint_OutboundChainsIntoPeerInbound is the cross-pod twin of +// TestMint_ChainsThroughEntry: an app with no context of its own calls out +// bare, this pod's outbound mints and stamps, and the peer's inbound — +// receiving exactly the forwarded headers — parents on that outbound via the +// stamp, in the outbound's trace. Same code path, both directions, proven +// rather than assumed. +func TestMint_OutboundChainsIntoPeerInbound(t *testing.T) { + p, exp := newTestPlugin(t) + out := fakeContext(pipeline.Outbound, http.Header{}) + run(t, p, out, allow(200)) + outReq, _ := roleSplit(t, exp.GetSpans()) + exp.Reset() + + peer, peerExp := newTestPlugin(t) + peer.selfID = "weather-tool" + in := fakeContext(pipeline.Inbound, out.Headers.Clone()) + run(t, peer, in, allow(200)) + inReq, _ := roleSplit(t, peerExp.GetSpans()) + + if inReq.SpanContext.TraceID() != outReq.SpanContext.TraceID() { + t.Errorf("peer inbound trace %s, want the outbound's %s", inReq.SpanContext.TraceID(), outReq.SpanContext.TraceID()) } - req, _ := roleSplit(t, exp.GetSpans()) - if req.Parent.IsValid() { - t.Errorf("request span has parent %s, want a root", req.Parent.SpanID()) + if inReq.Parent.SpanID() != outReq.SpanContext.SpanID() { + t.Errorf("peer inbound parent %s, want the outbound request span %s", inReq.Parent.SpanID(), outReq.SpanContext.SpanID()) } - if got := attrStr(req, "lineage.parent.source"); got != "none" { - t.Errorf("lineage.parent.source = %q, want none", got) + if got := attrStr(inReq, "lineage.parent.source"); got != "tracestate" { + t.Errorf("lineage.parent.source = %q, want tracestate", got) } } From 5a4514eea56d2957bd0428d2009a7355fb5f486d Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 11:40:55 +0300 Subject: [PATCH 11/37] Refactor: Rename the tracestate stamp key to lineage-parent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The member carries the sidecar chain's own parent link — inbound stamps it toward its app, outbound re-stamps it toward the peer — and names no consumer. dg-parent named one (the data-governance system it was first built for), which the #761 review flagged as a spec owned elsewhere leaking onto the wire. lineage-parent names the producer: this plugin is lineage-telemetry and every fact it emits is lineage.*. Wire-only: the key never lands in stored data. Every sidecar on a hop must run the same key, so it changes in one release; wire contract v1.6.0 carries it. Tests reference the constant and are unchanged. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 7b39cc9b0..4ccf136e1 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -94,10 +94,12 @@ const pluginName = "lineage-telemetry" // member has a header to ride on — W3C reads tracestate only alongside a // valid traceparent. // -// The key names the consuming data-governance system (W3C convention: the key -// identifies the owner of the entry) and is deliberately platform-neutral — -// it was `kglin` until 2026-08-04; the name never lands in stored data, so -// renaming is wire-only. +// The key is producer-owned and names the lineage domain (W3C convention: the +// key identifies the owner of the entry): the member carries the sidecar +// chain's own parent link and names no consumer. It was `kglin` until +// 2026-08-04 and `dg-parent` until 2026-09-03; the name never lands in stored +// data, so renaming is wire-only — but every sidecar on a hop must run the +// same key, so it changes in one release. // // A trace-keyed map (one entry per trace, "the last inbound seen") used to sit // between the two. It was removed: its answer is correct only while exactly one @@ -107,7 +109,7 @@ const pluginName = "lineage-telemetry" // parent, which is an app-internal span this pipeline never exported: the // interaction still derives in full, but as a trace entry rather than a child. // A visibly missing edge is recoverable; a silently wrong one is not. -const tracestateStampKey = "dg-parent" +const tracestateStampKey = "lineage-parent" // truncatedSuffix marks a captured payload that MaxPayloadBytes cut short. const truncatedSuffix = "…[truncated]" From 3879ea2a6c415106910a36e20f4eaf7a0f57ebdd Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 11:56:53 +0300 Subject: [PATCH 12/37] Docs: Vendor the lineage wire contract (v1.6.0) into authbridge/docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #761 review asked for the normative spec of this plugin's output to be reviewable from this repository: the attribute set, the parenting rule and the tracestate member were specified in a document the consumer maintains, and could change without a signal here. authbridge/docs/lineage-wire-contract.md is that document, kept byte-identical with the consumer's copy (lab-data-governance docs/sidecar-wire-contract.md); the version in its title is the pin and a change to it is a PR to both repositories. It is written as a current-state specification — principles, span model, trace context on the wire (the stamp, parent precedence, the traceparent rule, what the producer writes, un-stamped traffic by case), attributes, payloads, configuration, consumer commitments, retired names — with a version ladder as its history and no dates. Every statement was checked against plugin.go on this branch. The plugin's package doc now points at the in-repo copy. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 6 +- authbridge/docs/lineage-wire-contract.md | 287 +++++++++++++++++++ 2 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 authbridge/docs/lineage-wire-contract.md diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 4ccf136e1..ac8b9b91d 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -1,7 +1,9 @@ // Package lineage provides the lineage-telemetry authbridge plugin. // -// Two-span model (see docs/sidecar-wire-contract.md in the lab-data-governance -// repo, the consumer side — the law this file implements). Each HTTP exchange through the sidecar produces TWO OTLP spans: +// Two-span model (see authbridge/docs/lineage-wire-contract.md — the wire +// contract this file implements, kept byte-identical with the consumer's copy +// in the lab-data-governance repo). Each HTTP exchange through the sidecar +// produces TWO OTLP spans: // // - a request span, emitted as soon as the request has been seen and // forwarded, carrying caller-side facts + input.value; and diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md new file mode 100644 index 000000000..924d01f08 --- /dev/null +++ b/authbridge/docs/lineage-wire-contract.md @@ -0,0 +1,287 @@ +# Lineage wire contract — two-span sidecar lineage (v1.6.0) + +What the AuthBridge `lineage-telemetry` plugin emits, what it writes onto the wire, and what the +data-governance `sidecar` interactions algorithm (ADR-0030) commits to when consuming it. + +- Producer: `cortex/authbridge/authlib/plugins/lineage/` (repo `rossoctl/cortex`). +- Consumer: `data_governance/processors/interactions/sidecar.py`; vocabulary in + `data_governance/sidecar_facts.py` (repo `rossoctl/lab-data-governance`). + +This document is kept **byte-identical in both repositories** — `cortex/authbridge/docs/lineage-wire-contract.md` +and `lab-data-governance/docs/sidecar-wire-contract.md`. The version in the title is the pin: a +minor bump means producer behaviour or vocabulary changed; a patch bump means prose only. A change +is a pull request to both repositories. + +## 1. Principles + +- **Facts, not meaning.** The producer emits what it observed on the wire plus parsed protocol + facts. All vocabulary — hop kinds, entity kinds, caller/callee — lives in the consumer's + `classify()`. +- **Emit on sight.** Two spans per exchange, each emitted and ended as soon as its half has been + seen. No span is held open across the wait and no body is buffered for the exchange's lifetime. +- **One channel for the sidecar chain.** The sidecar parent chain lives entirely in one W3C + `tracestate` member, `lineage-parent`. A valid forwarded `traceparent` is never modified. The + sidecar's spans and an application's own spans land in different backends, so a chain that + pointed across the two would always dangle somewhere; the member keeps the sidecar chain + self-consistent in this store while an instrumented application keeps its own `traceparent` chain + intact toward its own backend. +- **No mechanism may guess.** A mechanism whose correctness depends on a precondition it cannot + verify at runtime does not belong in the producer. When attribution is unknown the producer says + so — `lineage.parent.source` is `wire` or `none` — and the edge is visibly absent. A missing edge + is recoverable downstream; a confidently wrong one is not, because it is indistinguishable from a + true one. +- **Parsers reduce payloads; interactions do not depend on them.** `input.value` and + `output.value` are semantic content produced by the protocol parsers, not raw bytes, and they are + enrichment only. Every exchange the sidecar saw becomes a complete interaction — kind, endpoints, + timing, status — whether or not a body could be read. + +## 2. Span model + +One HTTP exchange through the sidecar produces two OTLP spans. + +| | request span | response span | +|---|---|---| +| emitted | when the request (headers and body) has been seen and forwarded | when the response has been fully relayed, or the stream ends or errors | +| SpanKind | SERVER for inbound, CLIENT for outbound | same as its request span | +| parent | see §3.2 | its request span | +| carries | caller-side facts and `input.value` | outcome and status facts and `output.value` | + +- `lineage.exchange.id` is the request span's own span id, echoed on both spans. No new identifier + is minted; the response span names its request twin. Exchange duration is computed downstream as + response end minus request start. +- The response span is emitted at stream end **even when no response was produced** — client + disconnect, upstream reset, plugin denial. It then carries `lineage.outcome` and whatever status + exists, so the row completes as failed instead of dangling. +- A lone request span means one of three things: the sidecar died mid-exchange; the plugin + recovered a panic while emitting the response span (a WARN is logged); or the response span was + emitted but lost — the two halves enter a batching exporter an exchange apart, so a response can + be lost after its request has flushed. The consumer renders it as in-flight, never as a wrong + pairing. A response span whose `lineage.outcome` is absent derives with `error` NULL (honest + unknown), never `false`. +- **Scope of `denied`.** The lineage plugin runs after the gate plugins and the pipeline + short-circuits on a request-phase denial, so an exchange a gate rejects **before** the request + span exists emits **no spans at all** and is invisible to lineage. `denied` appears only for + denials after the request span exists: response-phase denials, or gates ordered after lineage. + Spans for gate-denied traffic are a named follow-up, not current behaviour. +- **Bypass.** Requests whose path starts with a `bypass_paths` prefix, or whose host contains a + `bypass_hosts` substring, produce no spans (defaults in §6). + +## 3. Trace context on the wire + +### 3.1 The stamp + +Each lineage element — inbound and outbound alike — re-stamps one W3C `tracestate` member on the +request it forwards: + +``` +tracestate: lineage-parent= +``` + +Inbound stamps toward its own application: an application that propagates W3C context carries +`tracestate` through its per-request causal chain, so the member surfaces on exactly the outbound +calls that inbound caused. Outbound stamps toward the peer, whose inbound sidecar reads it as its +parent. Foreign `tracestate` members are preserved. The key is producer-owned and names the lineage +domain; it never lands in stored data, so it is wire-only, and every sidecar on a hop must run the +same key. + +The stamp needs a valid `traceparent` to ride on: a W3C reader takes `tracestate` only alongside a +valid `traceparent`. That is why §3.3 exists. + +### 3.2 Parent selection + +The request span's parent is chosen by the first rule that applies, in both directions: + +| precedence | parent | `lineage.parent.source` | +|---|---|---| +| 1 | the `lineage-parent` stamp, if the wire context is valid and the member parses as a span id in that trace | `tracestate` | +| 2 | the wire `traceparent`'s parent span, if the wire context is valid | `wire` | +| 3 | none — the request span roots a new trace | `none` | + +There is no fourth option. A malformed stamp falls through to the wire parent silently. Under +precedence 1 the parent is the previous lineage element: the caller sidecar's outbound request span +for an inbound, this pod's inbound request span for an outbound. Under precedence 2 the parent is +usually a span this pipeline never exported (an application-internal span, or an un-sidecared +caller's); the exchange still derives as a complete interaction, but as a trace entry rather than a +child. + +### 3.3 The `traceparent` rule + +- **Valid → never modified.** A `traceparent` the W3C propagator accepts is forwarded byte for + byte. +- **Invalid → restarted.** When the request carries no valid `traceparent` — absent, empty or + malformed, as the propagator judges it, the same verdict that yields `parent.source=none` — the + producer forwards a new one naming its own request span, and the caller's `tracestate` is + dropped; the stamp is then written alone. This is W3C Trace Context's processing model for an + unparseable `traceparent`. Without it the next element has nothing to extract: a propagating + application roots a trace of its own and discards `tracestate`, the stamp never leaves the pod, + and every call the application makes derives as a separate root. +- Controlled by `mint_traceparent` (default on). Off, the producer writes no `traceparent` at all + and an entry without a valid one fragments as described. + +Consequences for the stored trace: a trace entered through a sidecar with no valid `traceparent` +has a real, exported root (the entry request span, `parent.source=none`). A trace entered with a +foreign valid `traceparent` — an un-sidecared driver or UI that propagates — keeps one dangling +parent at the trace edge, by design. + +### 3.4 What the producer writes onto a forwarded request + +| header | when | value | +|---|---|---| +| `tracestate` | every exchange, both directions, whenever a valid context exists after §3.3 | the caller's members with `lineage-parent` set to this request span id | +| `traceparent` | only when the request carried no valid one and `mint_traceparent` is on | `00---` | + +Nothing else is written. The listener is responsible for carrying these header mutations to the +wire. + +### 3.5 Un-stamped traffic + +An application with no context propagation, one that strips `tracestate`, or a caller with no +sidecar yields precedence 2 or 3 at the next element. The trace fragments at that pod, visibly, +instead of being welded by a guess. The consequences per case: + +| case | inbound entry | that pod's outbound calls | +|---|---|---| +| caller propagates, application propagates | stamp or wire | stamp | +| caller propagates, application does not | wire | wire: each call a trace of its own, restarted by its outbound sidecar | +| caller sends no valid context, application propagates | none (restarted) | stamp: one tree under the entry | +| caller sends no valid context, application does not | none (restarted) | wire: each call a trace of its own | + +## 4. Attributes + +Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-telemetry`. + +| key | on | example | notes | +|---|---|---|---| +| `lineage.exchange.id` | both | `00f067aa0ba902b7` | the request span id, hex | +| `lineage.role` | both | `request` \| `response` | which half this span is | +| `lineage.direction` | both | `inbound` \| `outbound` | | +| `lineage.self.id` | both | `weather-service` | this workload's identity, from `self_id` or `self_id_file`; the producer refuses to start without one | +| `lineage.peer.host` | both, when present | `weather-tool-mcp.team1.svc:8000` | the Host/authority header. Outbound: the service being called. Inbound: the address this workload was reached on | +| `lineage.protocol` | both | `a2a` \| `mcp` \| `inference` \| `http` | which parser matched; `http` = none | +| `lineage.parent.source` | request | `tracestate` \| `wire` \| `none` | which precedence in §3.2 chose the parent. An audit fact; the consumer derives nothing from it | +| `http.method` | request, when the listener supplies it | `POST` | all listeners do | +| `url.path` | request, when present | `/mcp` | | +| `url.scheme` | request, when present | `http` | the listener's observed scheme. Optional: the consumer composes `scheme://peer.host + url.path` only when all three exist | +| `a2a.method`, `a2a.session_id` | request, a2a | `message/send` | parsed facts | +| `mcp.method`, `mcp.tool` | request, mcp | `tools/call`, `get_weather` | `mcp.tool` only for `tools/call` | +| `inference.model` | request, inference | `qwen2.5:7b` | from the parsed request body | +| `lineage.principal.sub`, `lineage.principal.client` | request, inbound, only when a gate plugin validated a JWT | `alice` | raw identity facts, never inferred from a network address | +| `input.value` | request, with `capture_io` | `{"city":"Tokyo"}` | see §5 | +| `output.value` | response, with `capture_io` | `{...}` | see §5; absent when unparsed or streamed | +| `http.status_code` | response, when a status was produced | `200` | | +| `lineage.outcome` | response | `ok` \| `denied` \| `error` \| `abandoned` | how the exchange ended as the proxy saw it; `abandoned` = no status was ever produced | +| `lineage.denied_by` | response, denials | `jwt-validation` | the plugin that denied | + +`http.method` and `http.status_code` are the pre-1.21 OpenTelemetry semantic-convention keys, kept +deliberately: this producer's vocabulary is `lineage.*` plus these two well-known keys, and +interoperability with generic OpenTelemetry tooling is not a goal. + +Span names: request = `{self.id} {protocol} {op}`, where op is `mcp.tool` (else `mcp.method`), +`a2a.method`, or `inference.model`, falling back to `url.path`, and is omitted when empty; +response = the request name + ` response`. + +## 5. Payloads + +- `input.value` and `output.value` are the parsers' semantic reduction of the request and + response: for a2a the message or artifact text, for mcp the tool arguments and the text content + of a result, for inference the messages and the completion or tool calls. +- Two heuristics live in that reduction, affecting payloads only, never interactions: the a2a + parser falls back to the status-message text when a result carries no artifact; and the lineage + plugin suppresses an A2A protocol event from `output.value` when the reduced value is a JSON + object whose `kind` is exactly one of `status-update`, `task-status-update`, `artifact-update`, + `working`, `canceled`. Either can mislabel an unusual payload; since payload absence is legal, the + failure mode is a missing or imprecise `output.value`, never a wrong interaction. +- A value longer than `max_payload_bytes` is cut on a UTF-8 boundary and suffixed `…[truncated]`, + so the loss is visible in the span. A truncated value no longer parses as JSON; the consumer then + stores it as a string. Deployments that want whole prompts set `max_payload_bytes: -1` or raise it + (LLM chat prompts on the reference fleet reach 14 KB; a third exceed the 4096 default). +- TLS-passthrough connections bypass the HTTP pipeline entirely and produce no exchange. That is a + capture gap, not a derivation rule: once such a connection is seen, the same rules apply. + +## 6. Producer configuration + +| key | default | meaning | +|---|---|---| +| `otel_endpoint` | `localhost:4317` | OTLP gRPC target; `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused | +| `otel_tls` | `false` | TLS with system roots to the collector; an `https://` endpoint implies it, and `https://` with `otel_tls: false` is refused | +| `capture_io` | `false` | attach `input.value` / `output.value` | +| `max_payload_bytes` | `4096` | producer-side cap on those two values; `-1` attaches whole | +| `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | +| `bypass_paths` | `/.well-known/`, `/healthz`, `/readyz`, `/health` | path prefixes that produce no spans | +| `bypass_hosts` | `otel-collector`, `jaeger`, `zipkin`, `prometheus` | host substrings that produce no spans | +| `self_id` | — | this workload's identity | +| `self_id_file` | `/shared/client-id.txt` | read when `self_id` is empty; the producer refuses to start if neither yields an identity | + +Unknown keys are a boot error. + +## 7. Consumer commitments + +- Interaction id = `uuid5(NS_INTERACTION, f"{trace_id}/{exchange.id}")`. The request half fills + caller, callee, request payload hash and started-at; the response half fills response payload + hash, ended-at and error. +- **Whole-trace reconcile, not per-half upsert.** Every arriving span re-derives its entire trace + from all stored spans of that trace: idempotent, order-independent, authoritative. Rows no longer + justified by the current span set are deleted, trace-scoped; `entities` is global and never + deleted. A half arriving alone still produces its row, so in-flight stays visible. The wanted set + can shrink — an inbound request is a real interaction until its outbound ancestor arrives, then it + demotes to the callee-side echo and its row is removed — which a per-half upsert cannot express. + See ADR-0030. +- Anchors: `role=request` and either `direction=outbound`, or `direction=inbound` with no stored + anchor ancestor (the trace entry). Entry detection tolerates both a NULL parent and a dangling + wire parent, since an un-sidecared caller's root span is never exported. +- Response spans are never anchors; they attach by `exchange.id`. +- Kinds and entity identity come from the facts only. `classify()` never requires `input.value` or + `output.value`; bodyless exchanges produce complete, first-class interaction rows with NULL payload + hashes, and the UI renders them like any other row. +- The consumer never welds a fragmented trace: an anchor whose parent is not a stored anchor derives + as a root. +- `content_kind` vocabulary stays ADR-0014-compatible; the classification processor consumes + `interaction_payloads` as a stream. +- Drain loop = the shared `data_governance/processors/_driver.py` StreamSpec. + +## 8. Retired names + +The producer must not emit these, and the consumer reads nothing from them. + +| retired | replaced by | +|---|---| +| `lineage.hop.kind`, `trust.hop_kind` | consumer `classify()` over (direction, protocol, mcp.method) | +| `lineage.source.id`, `lineage.target.id`, `trust.source_id`, `trust.target_id` | `lineage.self.id` + `lineage.peer.host` + `lineage.direction`; caller and callee computed downstream | +| `enduser.id`, `trust.principal_id` | `lineage.principal.*` | +| `source=sidecar` | the resource `service.name=authbridge` | +| `openinference.span.kind` | not a producer attribute; a display backend derives it from `lineage.protocol` in a collector transform | +| `lineage.peer.addr` | nothing; it was never producible in ext_proc mode. Anonymous inbound callers derive as `client:(unknown)` | +| config `is_principal`, `emit_body_hash` | nothing | +| tracestate keys `kglin`, `dg-parent` | `lineage-parent` | + +## 9. History + +Version ladder, newest first. Each line is what changed on the wire or in the vocabulary; the +mechanisms named as removed are not to be reintroduced. + +- **v1.6** — an invalid or absent `traceparent` is restarted per W3C (`mint_traceparent`); + `lineage.parent.source` gains `none`; the stamp key becomes `lineage-parent`; `otel_tls` and + `max_payload_bytes` added; the document is vendored into the producer repository. Motivation: + one traceparent-less turn through a four-pod fleet derived nine roots instead of one — the + application's shim minted the trace, but with no `traceparent` to ride on the entry's stamp never + reached the application's outbound calls. +- **v1.5** — the single channel: both directions parent stamp-first; the outbound `traceparent` + rewrite ("the splice", v1.2–v1.4) removed; `lineage.peer.host` read on both directions; + `url.scheme` added. +- **v1.4** — `lineage.peer.addr` removed; the listener header diff made live on all handler + paths (until then the stamp never reached the wire in ext_proc mode, the mechanical cause of the + phantom-rooted per-pod trees stored before it). +- **v1.3** — the trace-keyed inbound map removed: it answered "the last inbound seen for this + trace", correct only while exactly one inbound of that trace is in flight, a precondition it could + not verify, and under same-trace concurrency it produced a real, exported, untrue parent with no + signal. A census before removal found zero spans attributed by it. `lineage.parent.source` added. +- **v1.2** — the `tracestate` stamp introduced (then keyed `kglin`), proven under same-trace + fan-in: six concurrent same-trace turns through a mid-chain agent paired 1/6 by the map and 6/6 + by the stamp. + +Naming decisions, not to be re-litigated: span attributes stay `lineage.*` — they are this +producer's own telemetry and name the domain, not a product. The `tracestate` key is a shared +channel every intermediary must preserve, so it carries a producer-owned, consumer-neutral name. + +Open item: the response-span name suffix (` response`) is cosmetic, for trace-viewer legibility +only. From 0c5ac75a962655f7eaee3490498d8394b9ae5698 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 12:58:15 +0300 Subject: [PATCH 13/37] Docs: Add lineage-telemetry to the plugin catalog The catalog lists every plugin that calls plugins.RegisterPlugin(); this plugin was missing. One table row and one section in the catalog's own shape: what it emits, where to place it in the chain, and the nine configuration keys with their defaults, matching config.go and the vendored wire contract. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/docs/plugin-catalog.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index bcfa16db4..4cfdb767c 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -28,6 +28,7 @@ AuthBridge pipeline YAML, not whether it is compiled into the binary | [`ibac`](#ibac) | LLM-judge intent-based access control for outbound tool calls. | Alpha | Outbound | No | | [`inference-parser`](#inference-parser) | Parses LLM completions into `pctx.Extensions.Inference`. | Alpha | Outbound | No | | [`jwt-validation`](#jwt-validation) | Inbound JWT validation (signature, issuer, audience) against JWKS. | Ready | Inbound | YES | +| [`lineage-telemetry`](#lineage-telemetry) | Emits two facts-only OTel lineage spans per HTTP exchange, parented across pods through one `tracestate` member. | Alpha | Both | No | | [`litellm-budget-track`](#litellm-budget-track) | Tracks `x-litellm-response-cost` (with `-original` fallback) and enforces a daily budget limit. Place on whichever chain carries LLM traffic — inbound when fronting the LLM endpoint, outbound when hosting an agent via `authbridge exec`. | Alpha | Both | No | | [`mcp-parser`](#mcp-parser) | Parses MCP tool calls/results into `pctx.Extensions.MCP`. | Beta | Outbound | No | | [`opa`](#opa) | [OPA](https://www.openpolicyagent.org/docs) policy enforcement for inbound and outbound requests. | Alpha | Both | No | @@ -128,6 +129,30 @@ Validates inbound JWTs: signature via JWKS, issuer, and audience. - `placeholder_mode` (bool) — replace the validated inbound token with an opaque placeholder before forwarding, for later outbound resolution. Default `false`. - `placeholder_ttl` (string) — how long the real token is retained. Default `1h`. +## `lineage-telemetry` + +Emits two facts-only OpenTelemetry spans per HTTP exchange — a request +span on sight and a response span at stream end, paired by +`lineage.exchange.id` — carrying direction, protocol, endpoints, outcome +and, optionally, the parsed payload. Cross-pod parenting rides one +`tracestate` member, `lineage-parent`; a request that arrives with no +valid `traceparent` is forwarded with one naming the request span, and a +valid one is never modified. The wire format is +[lineage-wire-contract.md](./lineage-wire-contract.md). Place it after +the protocol parsers (declared in `RequiresAny`) and after +`jwt-validation` when the principal facts are wanted; a request-phase +denial by a plugin ordered before it emits no spans. + +- `otel_endpoint` (string) — OTLP gRPC target: `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused. Default `localhost:4317`. +- `otel_tls` (bool) — dial the collector with TLS against the system roots. An `https://` endpoint implies it; `https://` with `otel_tls: false` is refused. Default `false`. +- `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. +- `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `-1` attaches whole. Default `4096`. +- `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. +- `bypass_paths` (`[]string`) — path prefixes that produce no spans. Default `/.well-known/`, `/healthz`, `/readyz`, `/health`. +- `bypass_hosts` (`[]string`) — host substrings that produce no spans. Default `otel-collector`, `jaeger`, `zipkin`, `prometheus`. +- `self_id` (string) — this workload's identity, emitted as `lineage.self.id`. +- `self_id_file` (string) — read when `self_id` is empty; the plugin refuses to start if neither yields an identity. Default `/shared/client-id.txt`. + ## `litellm-budget-track` Tracks the `x-litellm-response-cost` response header and enforces a From 24139abe4178082e6b97a0addf2528778c4eec6b Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 12:58:53 +0300 Subject: [PATCH 14/37] Docs: Package comments gate on a valid traceparent, like the code Two comments still said the traceparent is minted only when none arrived at all; since the restart-on-invalid change the gate is the propagator's validity verdict (absent, empty or malformed). Comments only. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index ac8b9b91d..0ab965609 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -32,7 +32,8 @@ // What this producer writes onto forwarded requests — two things, both // directions. Always: one tracestate member (tracestateStampKey) so the next // lineage element can parent its exchange on this one. Only when the request -// carried no traceparent at all: a traceparent naming this request span +// carried no VALID traceparent — absent, empty or malformed, as the W3C +// propagator judges it: a traceparent naming this request span // (mintTraceparent, config mint_traceparent, default on), because without one // the next element has nothing to extract and the stamp has nothing to ride // on. A valid traceparent is never modified; an invalid one is replaced, which @@ -342,7 +343,7 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context // (3) parent · (4) emit · (4b) mint · (5) re-stamp — wire contract v1.6. // The emit is unconditional; the calls around it are the header - // machinery, and (4b) only acts when no traceparent arrived at all. The + // machinery, and (4b) only acts when no valid traceparent arrived. The // read-only "Option 4" variant deletes selectParent and restampTracestate // (and the parent.source fact) and sets mint_traceparent: false — see the // package doc for the trade-off. From 79a9093c6b8bcfbe044f51252a619ab071825379 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 19:51:26 +0300 Subject: [PATCH 15/37] Docs: State the self.id last-segment reduction in the contract (v1.6.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serviceLabel reduces a self_id containing "/" to its last non-empty segment before it is emitted, so a SPIFFE ID emits its final element and two identities that differ only above it emit the same lineage.self.id. The code is deliberate; the vendored contract said the value was emitted as configured. §4 now states the reduction and its consequence for entity identity, §6 points at it, and the SelfID comment in config.go says the same. Prose only, so v1.6.1; the consumer's copy is updated to the identical bytes. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 5 ++++- authbridge/docs/lineage-wire-contract.md | 8 +++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index d09b71201..352c4b0ea 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -93,7 +93,10 @@ type Config struct { // SelfID is the agent's own stable identifier, emitted as the // lineage.self.id fact on every span. Typically the Keycloak client ID - // of this workload. If empty, SelfIDFile is consulted instead. + // of this workload. If empty, SelfIDFile is consulted instead. A value + // containing "/" (a SPIFFE ID) is reduced to its last non-empty path + // segment before emission — see serviceLabel — so two identities that + // differ only above that segment emit the same lineage.self.id. SelfID string `json:"self_id"` // SelfIDFile is the path to a file containing the agent's own client ID. diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 924d01f08..ffc102b99 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -1,4 +1,4 @@ -# Lineage wire contract — two-span sidecar lineage (v1.6.0) +# Lineage wire contract — two-span sidecar lineage (v1.6.1) What the AuthBridge `lineage-telemetry` plugin emits, what it writes onto the wire, and what the data-governance `sidecar` interactions algorithm (ADR-0030) commits to when consuming it. @@ -155,7 +155,7 @@ Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-te | `lineage.exchange.id` | both | `00f067aa0ba902b7` | the request span id, hex | | `lineage.role` | both | `request` \| `response` | which half this span is | | `lineage.direction` | both | `inbound` \| `outbound` | | -| `lineage.self.id` | both | `weather-service` | this workload's identity, from `self_id` or `self_id_file`; the producer refuses to start without one | +| `lineage.self.id` | both | `weather-service` | this workload's identity, from `self_id` or `self_id_file`, **reduced to its last non-empty `/`-segment**: a SPIFFE ID `spiffe://td/ns/team1/sa/agent` emits `agent`, and two identities that differ only above that segment emit the same value — the consumer keys entity identity on it (§7). The producer refuses to start without an identity | | `lineage.peer.host` | both, when present | `weather-tool-mcp.team1.svc:8000` | the Host/authority header. Outbound: the service being called. Inbound: the address this workload was reached on | | `lineage.protocol` | both | `a2a` \| `mcp` \| `inference` \| `http` | which parser matched; `http` = none | | `lineage.parent.source` | request | `tracestate` \| `wire` \| `none` | which precedence in §3.2 chose the parent. An audit fact; the consumer derives nothing from it | @@ -209,7 +209,7 @@ response = the request name + ` response`. | `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | | `bypass_paths` | `/.well-known/`, `/healthz`, `/readyz`, `/health` | path prefixes that produce no spans | | `bypass_hosts` | `otel-collector`, `jaeger`, `zipkin`, `prometheus` | host substrings that produce no spans | -| `self_id` | — | this workload's identity | +| `self_id` | — | this workload's identity (§4: reduced to its last `/`-segment) | | `self_id_file` | `/shared/client-id.txt` | read when `self_id` is empty; the producer refuses to start if neither yields an identity | Unknown keys are a boot error. @@ -259,6 +259,8 @@ The producer must not emit these, and the consumer reads nothing from them. Version ladder, newest first. Each line is what changed on the wire or in the vocabulary; the mechanisms named as removed are not to be reintroduced. +- **v1.6.1** — prose only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its + last `/`-segment before emission, which the producer has always done. - **v1.6** — an invalid or absent `traceparent` is restarted per W3C (`mint_traceparent`); `lineage.parent.source` gains `none`; the stamp key becomes `lineage-parent`; `otel_tls` and `max_payload_bytes` added; the document is vendored into the producer repository. Motivation: From a5075ee3c27186e659cd85609857d8d0455eaa67 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 20:12:41 +0300 Subject: [PATCH 16/37] Feat: Add otel_ca_file so a collector under a private CA can be verified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otel_tls verified the collector against the system roots only, and because the exporter is built on a caller-supplied gRPC conn the OTel SDK's OTEL_EXPORTER_OTLP_CERTIFICATE is never consulted — so the documented recommendation (otel_tls for any collector off-pod) could not export to an in-cluster collector whose certificate cert-manager issued. otel_ca_file names a PEM bundle; Init reads it into an x509.CertPool and dials with NewClientTLSFromCert against that pool. Setting it implies otel_tls, as an https:// endpoint already does. Contradictions are refused at decode rather than resolved silently: otel_ca_file with an explicit otel_tls:false, and an http:// endpoint with otel_tls:true or otel_ca_file (the mirror image of the existing https:// + otel_tls:false refusal). Init keys on the file alone, not on otel_tls, so a Config built without the decoder still cannot dial cleartext with a CA configured; an unreadable file, or one with no certificate in it, refuses to start rather than falling back to the system roots. Tests: the implication and the three contradictions at the decode boundary; Init against a self-signed CA generated in the test (starts, ready), a missing file and a certificate-less file (refuse, not ready). Contract §6 and the catalog carry the key. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 38 +++++- authbridge/authlib/plugins/lineage/plugin.go | 28 ++++- .../authlib/plugins/lineage/plugin_test.go | 111 ++++++++++++++++++ authbridge/docs/lineage-wire-contract.md | 8 +- authbridge/docs/plugin-catalog.md | 3 +- 5 files changed, 174 insertions(+), 14 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index 352c4b0ea..c50a689d3 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -33,13 +33,27 @@ type Config struct { // which is correct for the in-pod loopback collector but sends spans — // including lineage.principal.* on every inbound request, and full payloads // under CaptureIO — in cleartext. Set true for any collector off-pod: it - // dials with TLS against the system root CAs. An https:// otel_endpoint - // turns this on automatically; a plaintext otel_endpoint with otel_tls:true - // is honoured (TLS to a host:port). The one rejected combination is an - // https:// endpoint with an explicit otel_tls:false (see decodeConfig): a - // contradiction that would otherwise silently downgrade to cleartext. + // dials with TLS and verifies the collector against the system root CAs, + // or against OTelCAFile when set. An https:// otel_endpoint turns this on + // automatically; a bare host:port with otel_tls:true is honoured (TLS to + // that host:port). Two combinations are refused at decode as + // contradictions rather than silently resolved: an https:// endpoint with + // an explicit otel_tls:false, and an http:// endpoint with otel_tls:true + // or OTelCAFile — the scheme states a transport intent, and the knobs + // must agree with it. OTelTLS bool `json:"otel_tls"` + // OTelCAFile is a PEM bundle of CA certificates to verify the collector's + // serving certificate against, for a collector whose certificate is not + // signed by a system root — an in-cluster collector with a cert-manager + // issued certificate, typically (mount its TLS Secret's ca.crt). Setting it + // implies OTelTLS=true; an explicit otel_tls:false alongside it, or an + // http:// endpoint, is a refused contradiction. Read at Init into the cert + // pool the dial verifies against: an unreadable file, or one with no + // certificate in it, refuses to start rather than falling back to the + // system roots. Empty (the default) verifies against the system roots. + OTelCAFile string `json:"otel_ca_file"` + // CaptureIO when true attaches parsed request/response content as // input.value (request span) and output.value (response span) // attributes, enabling Phoenix to display message content inline. @@ -164,8 +178,22 @@ func decodeConfig(raw json.RawMessage) (Config, error) { } cfg.OTelTLS = true } + // The mirror image: an http:// endpoint asks for cleartext, so a TLS + // knob beside it is the same contradiction the other way round. + if u.Scheme == "http" && (cfg.OTelTLS || cfg.OTelCAFile != "") { + return Config{}, fmt.Errorf("lineage-telemetry config: otel_endpoint %q is http but otel_tls or otel_ca_file asks for TLS", cfg.OTelEndpoint) + } cfg.OTelEndpoint = u.Host } + // A CA file is only meaningful for a TLS dial: it implies otel_tls, and + // pairing it with an explicit otel_tls:false is the same contradiction as + // https:// + otel_tls:false above. + if cfg.OTelCAFile != "" { + if tlsExplicitlyFalse(raw) { + return Config{}, fmt.Errorf("lineage-telemetry config: otel_ca_file is set but otel_tls is false") + } + cfg.OTelTLS = true + } return cfg, nil } diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 0ab965609..ab0f04418 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -51,6 +51,7 @@ package lineage import ( "context" + "crypto/x509" "encoding/json" "errors" "fmt" @@ -217,12 +218,29 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { endpoint := p.cfg.OTelEndpoint // Transport credentials: plaintext by default (the loopback in-pod - // collector), TLS when otel_tls is set — an https:// endpoint sets it in - // decodeConfig. Spans carry principal facts on every inbound request and, - // under capture_io, user messages and model output, so a remote collector - // must not receive them in cleartext. + // collector), TLS when otel_tls is set — an https:// endpoint or an + // otel_ca_file sets it in decodeConfig. Spans carry principal facts on + // every inbound request and, under capture_io, user messages and model + // output, so a remote collector must not receive them in cleartext. creds := insecure.NewCredentials() - if p.cfg.OTelTLS { + switch { + case p.cfg.OTelCAFile != "": + // A private CA (a cert-manager issued in-cluster collector, typically): + // the dial verifies against this bundle only. Keyed on the file alone, + // not on OTelTLS, so a Config built without the decoder still cannot + // dial cleartext with a CA configured. Fails closed — an unreadable + // file or one with no certificate refuses to start rather than falling + // back to the system roots. Empty serverName = derive from endpoint. + pemBytes, err := os.ReadFile(p.cfg.OTelCAFile) + if err != nil { + return fmt.Errorf("lineage-telemetry: otel_ca_file %q: %w", p.cfg.OTelCAFile, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pemBytes) { + return fmt.Errorf("lineage-telemetry: otel_ca_file %q: no CA certificate found in PEM", p.cfg.OTelCAFile) + } + creds = credentials.NewClientTLSFromCert(pool, "") + case p.cfg.OTelTLS: // nil cert pool = system roots; empty serverName = derive from endpoint. creds = credentials.NewClientTLSFromCert(nil, "") } diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index b9944db18..c51a19b2d 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -2,9 +2,16 @@ package lineage import ( "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" "maps" + "math/big" "net/http" "os" "slices" @@ -1241,3 +1248,107 @@ func TestTruncate(t *testing.T) { t.Errorf("suffix-can't-fit cut split a rune: %q is not valid UTF-8", tiny) } } + +// ---- otel_ca_file: a private CA for the collector ---- + +// TestConfig_CAFileImpliesTLS: a CA bundle is only meaningful for a TLS dial, +// so otel_ca_file turns otel_tls on. The contradictions are refused at decode +// like https:// + otel_tls:false: an explicit otel_tls:false beside the file, +// and an http:// endpoint beside either TLS knob. +func TestConfig_CAFileImpliesTLS(t *testing.T) { + cfg, err := decodeConfig([]byte(`{"self_id":"x","otel_endpoint":"collector.ns:4317","otel_ca_file":"/etc/lineage/ca.pem"}`)) + if err != nil { + t.Fatalf("decodeConfig: %v", err) + } + if !cfg.OTelTLS { + t.Error("otel_ca_file did not imply otel_tls") + } + if cfg.OTelCAFile != "/etc/lineage/ca.pem" { + t.Errorf("OTelCAFile = %q", cfg.OTelCAFile) + } + for name, raw := range map[string]string{ + "otel_ca_file with an explicit otel_tls:false": `{"self_id":"x","otel_ca_file":"/etc/lineage/ca.pem","otel_tls":false}`, + "http:// endpoint with otel_ca_file": `{"self_id":"x","otel_endpoint":"http://collector:4317","otel_ca_file":"/etc/lineage/ca.pem"}`, + "http:// endpoint with otel_tls:true": `{"self_id":"x","otel_endpoint":"http://collector:4317","otel_tls":true}`, + } { + if _, err := decodeConfig([]byte(raw)); err == nil { + t.Errorf("%s: accepted, want a refused contradiction", name) + } + } +} + +// selfSignedCAPEM returns a PEM-encoded self-signed CA certificate, enough +// for Init to build a cert pool from. +func selfSignedCAPEM(t *testing.T) []byte { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "lineage-test-ca"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageCertSign, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) +} + +// TestInit_CAFile: a private-CA bundle is loaded at Init and a bad one refuses +// to start (fail closed — never a silent fallback to the system roots). The +// dial is lazy, so a valid bundle lets Init succeed without a collector. The +// Config is built directly, with OTelTLS left false, to pin that Init keys +// on the file alone. +func TestInit_CAFile(t *testing.T) { + dir := t.TempDir() + good := dir + "/ca.pem" + if err := os.WriteFile(good, selfSignedCAPEM(t), 0o600); err != nil { + t.Fatal(err) + } + garbage := dir + "/garbage.pem" + if err := os.WriteFile(garbage, []byte("not a certificate"), 0o600); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + caFile string + wantErr bool + }{ + {"valid bundle starts", good, false}, + {"missing file refuses", dir + "/absent.pem", true}, + {"file with no certificate refuses", garbage, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := NewLineageTelemetry() + p.cfg = Config{OTelEndpoint: "collector.ns:4317", OTelCAFile: tc.caFile, SelfID: "x"} + err := p.Init(context.Background()) + if tc.wantErr { + if err == nil { + _ = p.Shutdown(context.Background()) + t.Fatal("Init succeeded with an unusable otel_ca_file") + } + if p.Ready() { + t.Error("plugin ready after a refused Init") + } + return + } + if err != nil { + t.Fatalf("Init: %v", err) + } + if !p.Ready() { + t.Error("plugin not ready after a successful Init") + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = p.Shutdown(ctx) + }) + } +} diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index ffc102b99..6c15c0f09 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -203,7 +203,8 @@ response = the request name + ` response`. | key | default | meaning | |---|---|---| | `otel_endpoint` | `localhost:4317` | OTLP gRPC target; `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused | -| `otel_tls` | `false` | TLS with system roots to the collector; an `https://` endpoint implies it, and `https://` with `otel_tls: false` is refused | +| `otel_tls` | `false` | TLS to the collector, verified against the system roots or `otel_ca_file`; an `https://` endpoint implies it. Refused contradictions: `https://` with `otel_tls: false`, `http://` with `otel_tls: true` or `otel_ca_file` | +| `otel_ca_file` | — | PEM bundle to verify the collector's certificate against, for a private CA; implies `otel_tls`, and `otel_ca_file` with `otel_tls: false` is refused. An unreadable file, or one with no certificate, refuses to start | | `capture_io` | `false` | attach `input.value` / `output.value` | | `max_payload_bytes` | `4096` | producer-side cap on those two values; `-1` attaches whole | | `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | @@ -259,8 +260,9 @@ The producer must not emit these, and the consumer reads nothing from them. Version ladder, newest first. Each line is what changed on the wire or in the vocabulary; the mechanisms named as removed are not to be reintroduced. -- **v1.6.1** — prose only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its - last `/`-segment before emission, which the producer has always done. +- **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented + as reduced to its last `/`-segment before emission, which the producer has always done; + `otel_ca_file` added for a collector under a private CA. - **v1.6** — an invalid or absent `traceparent` is restarted per W3C (`mint_traceparent`); `lineage.parent.source` gains `none`; the stamp key becomes `lineage-parent`; `otel_tls` and `max_payload_bytes` added; the document is vendored into the producer repository. Motivation: diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index 4cfdb767c..554bb0787 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -144,7 +144,8 @@ the protocol parsers (declared in `RequiresAny`) and after denial by a plugin ordered before it emits no spans. - `otel_endpoint` (string) — OTLP gRPC target: `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused. Default `localhost:4317`. -- `otel_tls` (bool) — dial the collector with TLS against the system roots. An `https://` endpoint implies it; `https://` with `otel_tls: false` is refused. Default `false`. +- `otel_tls` (bool) — dial the collector with TLS, verified against the system roots or `otel_ca_file`. An `https://` endpoint implies it; `https://` with `otel_tls: false`, and `http://` with `otel_tls: true` or `otel_ca_file`, are refused. Default `false`. +- `otel_ca_file` (string) — PEM bundle to verify the collector's certificate against (a private CA, e.g. cert-manager issued). Implies `otel_tls`; with an explicit `otel_tls: false` it is refused; an unreadable file or one with no certificate refuses to start. Default: system roots. - `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. - `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `-1` attaches whole. Default `4096`. - `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. From 724b344edc83bf7ad4feabda511dd63ee7ecbd06 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 22:23:40 +0300 Subject: [PATCH 17/37] Feat: Surface export failures from the plugin; say what Ready does and does not claim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit grpc.NewClient dials lazily and the batch processor exports on its own goroutine, so nothing in Init can prove the collector is reachable or that its TLS chain verifies, and a failed export surfaced only through the OTel SDK's default error handler on stderr. The exporter is now wrapped: a refused or undeliverable batch increments a counter and logs a plugin-namespaced WARN carrying the batch size, the running total and the error, throttled to the 1st, 2nd, 4th, 8th… failure so a dead collector costs log lines in proportion to log2 of the outage rather than one line per export interval. The error is returned unchanged, so the SDK's retry and drop behaviour is untouched. Readiness deliberately does not follow the collector: an unready plugin skips OnRequest, which is where the tracestate stamp and the minted traceparent are written, so an outage would fragment every trace on the wire instead of merely delaying export. Ready's doc comment now states exactly what it claims. The counter is to be exposed through pipeline.MetricsProvider once this branch is rebased onto main, where that interface now lives. Tests: the observer counts and passes through a refused batch, directly and through the SDK; the throttle logs exactly the powers of two. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 46 +++++++++++++++- .../authlib/plugins/lineage/plugin_test.go | 53 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index ab0f04418..0ce460f63 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -151,6 +151,12 @@ type LineageTelemetry struct { ready atomic.Bool propagator propagation.TextMapPropagator selfID string // agent's own client ID for the lineage.self.id fact + // exportFailures counts batches the collector refused or never received. + // Export is asynchronous and the dial is lazy, so this — with the WARN + // exportObserver logs — is how an unreachable collector or a TLS chain + // that does not verify becomes visible; see Ready for why readiness + // deliberately does not follow it. + exportFailures atomic.Uint64 } // NewLineageTelemetry constructs an unconfigured plugin. Configure + Init must @@ -282,7 +288,7 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { // and every peer sidecar is ParentBased too — a ratio sampler here would // silently un-sample whole chains, not just this pod's spans. p.tp = sdktrace.NewTracerProvider( - sdktrace.WithBatcher(exporter), + sdktrace.WithBatcher(&exportObserver{SpanExporter: exporter, failures: &p.exportFailures}), sdktrace.WithResource(res), ) p.tracer = p.tp.Tracer("authbridge/" + pluginName) @@ -314,8 +320,46 @@ func (p *LineageTelemetry) Shutdown(ctx context.Context) error { return errors.Join(tpErr, connErr) } +// Ready reports that the plugin is configured, has an identity and can +// record spans — not that the collector is reachable. grpc.NewClient dials +// lazily and the batch processor exports asynchronously, so no point in Init +// can prove the collector or its TLS chain; and readiness must not follow the +// collector anyway: an unready plugin skips OnRequest, which is where the +// tracestate stamp and the minted traceparent are written, so a collector +// outage would fragment every trace on the wire instead of merely delaying +// its export. A collector that cannot be reached surfaces through +// exportObserver instead: a plugin-namespaced WARN and the exportFailures +// counter. func (p *LineageTelemetry) Ready() bool { return p.ready.Load() } +// exportObserver wraps the OTLP exporter so a failed export is visible from +// this plugin — a plugin-namespaced WARN and a counter — instead of only +// through the OTel SDK's default error handler on stderr. The error is +// returned unchanged; the batch processor's retry/drop behaviour is untouched. +// The WARN is throttled to the 1st, 2nd, 4th, 8th… failure: a dead collector +// fails a batch every export interval, and one line per failure would bury +// the logs exactly when they matter. The running total is in every line. +type exportObserver struct { + sdktrace.SpanExporter + failures *atomic.Uint64 +} + +func (e *exportObserver) ExportSpans(ctx context.Context, spans []sdktrace.ReadOnlySpan) error { + err := e.SpanExporter.ExportSpans(ctx, spans) + if err != nil { + if n := e.failures.Add(1); logExportFailure(n) { + slog.Warn("lineage-telemetry: span export failed; the collector is unreachable or refused the batch", + "spans", len(spans), "failures", n, "error", err) + } + } + return err +} + +// logExportFailure reports whether the n-th consecutive-count failure is one +// of the logged ones: powers of two, so the volume grows with the log of the +// outage length rather than with the outage length. +func logExportFailure(n uint64) bool { return n&(n-1) == 0 } + func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context) pipeline.Action { if !p.ready.Load() { pctx.Skip("not_ready") diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index c51a19b2d..37193eb73 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -9,6 +9,7 @@ import ( "crypto/x509/pkix" "encoding/json" "encoding/pem" + "errors" "fmt" "maps" "math/big" @@ -16,6 +17,7 @@ import ( "os" "slices" "strings" + "sync/atomic" "testing" "time" "unicode/utf8" @@ -1352,3 +1354,54 @@ func TestInit_CAFile(t *testing.T) { }) } } + +// ---- export failure visibility ---- + +type failingExporter struct{ calls atomic.Int32 } + +func (f *failingExporter) ExportSpans(context.Context, []sdktrace.ReadOnlySpan) error { + f.calls.Add(1) + return errors.New("collector unreachable") +} +func (f *failingExporter) Shutdown(context.Context) error { return nil } + +// TestExportObserver_CountsFailures: a refused batch is counted (and logged +// under the plugin's name) and the error still reaches the SDK unchanged, so +// the batch processor's own handling is not altered. +func TestExportObserver_CountsFailures(t *testing.T) { + fe := &failingExporter{} + p := NewLineageTelemetry() + obs := &exportObserver{SpanExporter: fe, failures: &p.exportFailures} + if err := obs.ExportSpans(context.Background(), nil); err == nil { + t.Fatal("observer swallowed the export error") + } + if got := p.exportFailures.Load(); got != 1 { + t.Fatalf("exportFailures = %d after one refused batch, want 1", got) + } + // Reachable through the SDK: a span ended on a provider that exports + // through the observer increments the counter again. + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(obs)) + _, span := tp.Tracer("t").Start(context.Background(), "s") + span.End() + _ = tp.Shutdown(context.Background()) + if got := p.exportFailures.Load(); got != 2 { + t.Errorf("exportFailures = %d after a span through the SDK, want 2", got) + } + if fe.calls.Load() != 2 { + t.Errorf("underlying exporter called %d times, want 2", fe.calls.Load()) + } +} + +// TestLogExportFailure pins the throttle: powers of two are logged, nothing +// else is, so a long outage costs log lines in proportion to log2(length). +func TestLogExportFailure(t *testing.T) { + var logged []uint64 + for n := uint64(1); n <= 40; n++ { + if logExportFailure(n) { + logged = append(logged, n) + } + } + if !slices.Equal(logged, []uint64{1, 2, 4, 8, 16, 32}) { + t.Errorf("logged failures = %v, want the powers of two up to 32", logged) + } +} From fd4646b3d54d4cd9a0c6bf3598c5a7c3065246c0 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 23:15:46 +0300 Subject: [PATCH 18/37] Fix: Anchor bypass_hosts to outbound host globs and validate both lists bypass_hosts was an unanchored strings.Contains against bare-word defaults, so a legitimate workload at prometheus-metrics-agent.team1.svc silently left the lineage graph, and a tenant could opt out of being graphed at all by naming a service to contain one of the default words. An entry is now a path.Match glob checked with the port stripped and case folded, which is the convention ibac, sparc and cpex already use for the key of the same name; the defaults carry both the short and the dotted form because in-cluster short-name calls are ordinary. The list is now honoured on the outbound phase only. On the inbound phase Host is the caller's own header, so a bypass driven by it is an opt-out from being recorded that needs no service name at all. Both bypass lists are validated at decode. An entry that matches everything - empty or whitespace-only, "/" for a path, "*" for a host - disabled the plugin with no signal anywhere: every exchange took the skip, no span was ever emitted, and Ready() still reported true. It is now a boot error, and a host entry that is not valid glob syntax is refused too. Entries are trimmed rather than left to never match. Setting either key replaces the default list rather than extending it. That was already the behaviour, and is the convention the sibling plugins share, but nothing said so next to a field documented as "Default: [...]" - so it is now stated on both keys, in the contract key table and in the catalog. Contract goes to v1.6.1 wording only; spans and wire are unchanged. Tests: the glob matrix pins both false positives that used to be skipped, the case fold, the optional port and an IPv6 literal; an inbound Host matching the list still emits its pair; every refused config shape fails at decode and a trimmed one round-trips; and setting one key is proven to leave the other's defaults intact. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 72 ++++++++++++++-- authbridge/authlib/plugins/lineage/plugin.go | 49 +++++++++-- .../authlib/plugins/lineage/plugin_test.go | 85 ++++++++++++++++++- authbridge/docs/lineage-wire-contract.md | 18 +++- authbridge/docs/plugin-catalog.md | 4 +- 5 files changed, 206 insertions(+), 22 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index c50a689d3..40759326b 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/url" + "path" "strings" ) @@ -96,13 +97,34 @@ type Config struct { // BypassPaths lists URL path prefixes that should not generate lineage // hops. Useful for suppressing infrastructure polling (agent-card // discovery, health checks) that would otherwise flood the lineage graph. + // Prefixes, not globs — a path is bypassed when it starts with an entry. + // + // Setting the key REPLACES this list rather than extending it, the same + // convention ibac, sparc and cpex use for their bypass keys: an operator + // who adds one prefix must restate the defaults they want to keep. + // Entries are trimmed of surrounding whitespace; one that is empty, or + // "/", is refused at decode because it would match every path and + // silently turn the plugin off. // Default: ["/.well-known/", "/healthz", "/readyz", "/health"] BypassPaths []string `json:"bypass_paths"` - // BypassHosts lists target host substrings (matched against pctx.Host) - // that should not generate lineage hops. Useful for suppressing - // infrastructure outbound calls such as OTel trace exports. - // Default: ["otel-collector", "jaeger", "zipkin", "prometheus"] + // BypassHosts lists host globs whose exchanges should not generate lineage + // hops. Useful for suppressing infrastructure outbound calls such as OTel + // trace exports. Matched with path.Match against the request Host with the + // port stripped and case folded — see matchesAnyHost — so "otel-collector" + // matches only that exact name and "otel-collector.*" matches + // otel-collector.rossoctl-system.svc. This is the glob convention ibac, + // sparc and cpex already use for the key of the same name; the defaults + // carry both forms because in-cluster short-name calls are ordinary. + // + // Honoured on the outbound phase only: an inbound Host is the caller's own + // header, and a bypass driven by it would be an opt-out from being graphed. + // + // Setting the key REPLACES this list rather than extending it, as with + // BypassPaths. Entries are trimmed of surrounding whitespace; one that is + // empty, "*", or not valid path.Match syntax is refused at decode. + // Default: ["otel-collector", "otel-collector.*", "jaeger", "jaeger.*", + // "zipkin", "zipkin.*", "prometheus", "prometheus.*"] BypassHosts []string `json:"bypass_hosts"` // SelfID is the agent's own stable identifier, emitted as the @@ -125,8 +147,13 @@ func defaultConfig() Config { MaxPayloadBytes: defaultMaxPayloadBytes, MintTraceparent: true, BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, - BypassHosts: []string{"otel-collector", "jaeger", "zipkin", "prometheus"}, - SelfIDFile: "/shared/client-id.txt", + BypassHosts: []string{ + "otel-collector", "otel-collector.*", + "jaeger", "jaeger.*", + "zipkin", "zipkin.*", + "prometheus", "prometheus.*", + }, + SelfIDFile: "/shared/client-id.txt", } } @@ -194,9 +221,42 @@ func decodeConfig(raw json.RawMessage) (Config, error) { } cfg.OTelTLS = true } + if err := validateBypass(&cfg); err != nil { + return Config{}, fmt.Errorf("lineage-telemetry config: %w", err) + } return cfg, nil } +// validateBypass trims and checks both bypass lists in place. An entry that +// matches everything disables the plugin silently — every exchange takes the +// skip, no span is ever emitted, and Ready() still reports true — so it is a +// boot error rather than a runtime surprise. ibac, sparc and cpex each refuse +// the same shapes with the same reasoning; the wording of the error mirrors +// theirs, including the advice to remove the plugin from the pipeline if +// disabling it is what was meant. +func validateBypass(cfg *Config) error { + for i, entry := range cfg.BypassPaths { + entry = strings.TrimSpace(entry) + if entry == "" || entry == "/" { + return fmt.Errorf("bypass_paths entry %q matches every path; "+ + "to disable lineage-telemetry, remove it from the pipeline instead", cfg.BypassPaths[i]) + } + cfg.BypassPaths[i] = entry + } + for i, entry := range cfg.BypassHosts { + entry = strings.TrimSpace(entry) + if _, err := path.Match(entry, ""); err != nil { + return fmt.Errorf("invalid bypass_hosts glob %q: %w", cfg.BypassHosts[i], err) + } + if entry == "" || entry == "*" { + return fmt.Errorf("bypass_hosts entry %q matches every host; "+ + "to disable lineage-telemetry, remove it from the pipeline instead", cfg.BypassHosts[i]) + } + cfg.BypassHosts[i] = entry + } + return nil +} + // tlsExplicitlyFalse reports whether the raw config carries otel_tls set to a // literal false, as opposed to being absent (whose decoded value is also false // but carries no intent). Used only to reject the https:// + otel_tls:false diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 0ce460f63..f6ca14b2b 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -56,7 +56,9 @@ import ( "errors" "fmt" "log/slog" + "net" "os" + "path" "strings" "sync/atomic" "unicode/utf8" @@ -374,12 +376,15 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context } } - // Skip infrastructure outbound targets (OTel exporters, metrics scrapers, etc.) - for _, substr := range p.cfg.BypassHosts { - if strings.Contains(pctx.Host, substr) { - pctx.Skip("bypass_host") - return pipeline.Action{Type: pipeline.Continue} - } + // Skip infrastructure outbound targets (OTel exporters, metrics scrapers, etc.). + // Outbound only: on the inbound phase Host is the caller's own header, so + // honouring the list there would let any caller suppress its own exchange + // by sending "Host: otel-collector". cpex reached the same conclusion for + // the same reason (its bypass_hosts is outbound-only for an attacker- + // controlled inbound Host). + if pctx.Direction == pipeline.Outbound && matchesAnyHost(p.cfg.BypassHosts, pctx.Host) { + pctx.Skip("bypass_host") + return pipeline.Action{Type: pipeline.Continue} } // Extract remote trace context from the incoming W3C traceparent header. @@ -529,6 +534,38 @@ func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchan pctx.Headers.Set("tracestate", ts.String()) } +// matchesAnyHost reports whether host matches any configured bypass_hosts +// glob. Semantics follow the convention ibac, sparc and cpex already share +// for this key: path.Match against the host with its port stripped, so +// "otel-collector.*" matches otel-collector.rossoctl-system.svc:4317. +// +// The anchoring is the point. The earlier strings.Contains match was +// unanchored against bare-word defaults, so a legitimate workload at +// prometheus-metrics-agent.team1.svc silently left the lineage graph, and a +// tenant could opt out of being graphed at all by naming a service to contain +// one of the default words. A glob has to match from the first character. +// +// Two deliberate differences from the siblings, both strict improvements: +// the port is split with net.SplitHostPort so an IPv6 literal ([::1]:4317) +// survives, and matching is case-folded because an authority is +// case-insensitive (RFC 3986) — a target spelled OTel-Collector would +// otherwise be recorded while otel-collector was skipped. +func matchesAnyHost(patterns []string, host string) bool { + if host == "" { + return false + } + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.ToLower(host) + for _, pattern := range patterns { + if matched, _ := path.Match(strings.ToLower(pattern), host); matched { + return true + } + } + return false +} + // stampedParent resolves the tracestate stamp on an outbound wire context: // the inbound exchange id this pod's sidecar wrote into tracestate on the // forwarded request, carried back by the app's shim. Returns ok=false when diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 37193eb73..f675683f6 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -1042,20 +1042,29 @@ func TestBypassPaths_PrefixMatchEmitsNothing(t *testing.T) { } } -func TestBypassHosts_SubstringMatchEmitsNothing(t *testing.T) { +// TestBypassHosts_GlobMatchEmitsNothing pins the anchored matcher. The +// unanchored strings.Contains it replaced skipped every host in the third and +// fourth rows: a real workload leaving the graph, and a tenant opting out of +// being graphed by choosing its own name. +func TestBypassHosts_GlobMatchEmitsNothing(t *testing.T) { cases := []struct { name string host string spans int }{ - {"substring match skipped", "otel-collector.rossoctl-system:4317", 0}, - {"bare name match skipped", "otel-collector:4317", 0}, + {"bare name skipped", "otel-collector:4317", 0}, + {"fqdn skipped by the .* form", "otel-collector.rossoctl-system.svc:4317", 0}, + {"prefixed workload is NOT skipped", "prometheus-metrics-agent.team1.svc:9090", 2}, + {"suffixed workload is NOT skipped", "my-otel-collector:4317", 2}, {"unrelated host emits", "weather-tool:8000", 2}, + {"case is folded", "OTel-Collector:4317", 0}, + {"port is optional", "otel-collector", 0}, + {"ipv6 literal keeps its host", "[::1]:4317", 2}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { p, exp := newTestPlugin(t) - p.cfg.BypassHosts = []string{"otel-collector"} + p.cfg = defaultConfig() pctx := fakeContext(pipeline.Outbound, http.Header{}) pctx.Host = tc.host run(t, p, pctx, allow(200)) @@ -1066,6 +1075,74 @@ func TestBypassHosts_SubstringMatchEmitsNothing(t *testing.T) { } } +// TestBypassHosts_InboundIgnoresHost: an inbound Host is the caller's own +// header. Honouring bypass_hosts there would let any caller suppress the +// record of its own request by naming the target it claims to be calling. +func TestBypassHosts_InboundIgnoresHost(t *testing.T) { + p, exp := newTestPlugin(t) + p.cfg.BypassHosts = []string{"otel-collector"} + pctx := fakeContext(pipeline.Inbound, http.Header{}) + pctx.Host = "otel-collector:4317" + run(t, p, pctx, allow(200)) + if got := len(exp.GetSpans()); got != 2 { + t.Fatalf("inbound Host bypass honoured: got %d spans, want 2", got) + } +} + +// TestConfig_BypassEntriesValidated: an entry that matches everything turns +// the plugin off with no signal at all — Ready() stays true and no span is +// ever emitted — so it has to fail at boot. +func TestConfig_BypassEntriesValidated(t *testing.T) { + refused := []struct { + name string + raw string + }{ + {"empty path", `{"bypass_paths": ["/healthz", ""]}`}, + {"whitespace-only path", `{"bypass_paths": [" "]}`}, + {"root path", `{"bypass_paths": ["/"]}`}, + {"empty host", `{"bypass_hosts": ["jaeger", ""]}`}, + {"whitespace-only host", `{"bypass_hosts": [" "]}`}, + {"star host", `{"bypass_hosts": ["*"]}`}, + {"invalid glob", `{"bypass_hosts": ["[unclosed"]}`}, + } + for _, tc := range refused { + t.Run(tc.name, func(t *testing.T) { + if _, err := decodeConfig([]byte(tc.raw)); err == nil { + t.Fatalf("%s accepted; it disables the plugin silently", tc.raw) + } + }) + } + + cfg, err := decodeConfig([]byte(`{"bypass_hosts": [" jaeger.* "], "bypass_paths": [" /healthz "]}`)) + if err != nil { + t.Fatalf("valid bypass config rejected: %v", err) + } + // Surrounding whitespace is trimmed rather than silently never matching. + if got := cfg.BypassHosts; len(got) != 1 || got[0] != "jaeger.*" { + t.Fatalf("bypass_hosts = %q, want [\"jaeger.*\"]", got) + } + if got := cfg.BypassPaths; len(got) != 1 || got[0] != "/healthz" { + t.Fatalf("bypass_paths = %q, want [\"/healthz\"]", got) + } +} + +// TestConfig_BypassReplacesDefaults: setting either key replaces the default +// list rather than extending it — the convention ibac, sparc and cpex share. +// Undocumented until now, and the reason it is now stated on both keys. +func TestConfig_BypassReplacesDefaults(t *testing.T) { + cfg, err := decodeConfig([]byte(`{"bypass_hosts": ["my-metrics-thing"]}`)) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(cfg.BypassHosts) != 1 || cfg.BypassHosts[0] != "my-metrics-thing" { + t.Fatalf("bypass_hosts = %q, want the operator list alone", cfg.BypassHosts) + } + // The untouched key keeps its defaults. + if len(cfg.BypassPaths) != len(defaultConfig().BypassPaths) { + t.Fatalf("bypass_paths = %q, want the defaults", cfg.BypassPaths) + } +} + // ---- lifecycle: gRPC connection ownership ---- // WithGRPCConn leaves the conn for the caller to close; the exporter's Shutdown // does not. A real Init dials a (never-answered) localhost target, stores the diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 6c15c0f09..72ec846cf 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -63,8 +63,10 @@ One HTTP exchange through the sidecar produces two OTLP spans. span exists emits **no spans at all** and is invisible to lineage. `denied` appears only for denials after the request span exists: response-phase denials, or gates ordered after lineage. Spans for gate-denied traffic are a named follow-up, not current behaviour. -- **Bypass.** Requests whose path starts with a `bypass_paths` prefix, or whose host contains a - `bypass_hosts` substring, produce no spans (defaults in §6). +- **Bypass.** Requests whose path starts with a `bypass_paths` prefix, and outbound requests whose + host matches a `bypass_hosts` glob (`path.Match`, port stripped, case folded), produce no spans + (defaults in §6). `bypass_hosts` is outbound-only: an inbound `Host` is the caller's own header, + so honouring it there would let a caller suppress the record of its own request. ## 3. Trace context on the wire @@ -209,10 +211,16 @@ response = the request name + ` response`. | `max_payload_bytes` | `4096` | producer-side cap on those two values; `-1` attaches whole | | `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | | `bypass_paths` | `/.well-known/`, `/healthz`, `/readyz`, `/health` | path prefixes that produce no spans | -| `bypass_hosts` | `otel-collector`, `jaeger`, `zipkin`, `prometheus` | host substrings that produce no spans | +| `bypass_hosts` | `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*` | outbound host globs that produce no spans | | `self_id` | — | this workload's identity (§4: reduced to its last `/`-segment) | | `self_id_file` | `/shared/client-id.txt` | read when `self_id` is empty; the producer refuses to start if neither yields an identity | +Setting `bypass_paths` or `bypass_hosts` **replaces** the default list rather than extending it — +the convention the `ibac`, `sparc` and `cpex` plugins use for their keys of the same name. An +operator who adds one entry must restate the defaults they want kept. An entry that would match +everything (empty, whitespace-only, `/` for a path, `*` for a host) is refused at start, as is a +host entry that is not valid `path.Match` syntax. + Unknown keys are a boot error. ## 7. Consumer commitments @@ -262,7 +270,9 @@ mechanisms named as removed are not to be reintroduced. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; - `otel_ca_file` added for a collector under a private CA. + `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only + `path.Match` glob list (it was an unanchored substring match on both directions) and both bypass + lists are validated at start. - **v1.6** — an invalid or absent `traceparent` is restarted per W3C (`mint_traceparent`); `lineage.parent.source` gains `none`; the stamp key becomes `lineage-parent`; `otel_tls` and `max_payload_bytes` added; the document is vendored into the producer repository. Motivation: diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index 554bb0787..1f3ecaf0b 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -149,8 +149,8 @@ denial by a plugin ordered before it emits no spans. - `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. - `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `-1` attaches whole. Default `4096`. - `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. -- `bypass_paths` (`[]string`) — path prefixes that produce no spans. Default `/.well-known/`, `/healthz`, `/readyz`, `/health`. -- `bypass_hosts` (`[]string`) — host substrings that produce no spans. Default `otel-collector`, `jaeger`, `zipkin`, `prometheus`. +- `bypass_paths` (`[]string`) — path prefixes that produce no spans. Default `/.well-known/`, `/healthz`, `/readyz`, `/health`. Setting either bypass key replaces its default list rather than extending it, as in `ibac` / `sparc` / `cpex`; an entry matching everything is refused at start. +- `bypass_hosts` (`[]string`) — outbound host globs (`path.Match`, port stripped, case folded) that produce no spans; ignored inbound, where `Host` is caller-controlled. Default `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*`. - `self_id` (string) — this workload's identity, emitted as `lineage.self.id`. - `self_id_file` (string) — read when `self_id` is empty; the plugin refuses to start if neither yields an identity. Default `/shared/client-id.txt`. From 676a521343571b566ad95e6655adeb21f207ed3f Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Thu, 3 Sep 2026 23:55:06 +0300 Subject: [PATCH 19/37] Test: Prove the capture_io default on a fixture where it can fail The only capture_io-off assertion in the file could not fail. It ran on a fixture with no parser extensions, so lineage.protocol was http - and ioInputValue/ioOutputValue dispatch on a2a, mcp and inference only, with no http arm, because the plugin attaches parsed content and never reads a raw body. Both returned "" for either value of the flag, so the absence of input.value/output.value proved nothing about the privacy default it was named after. The default is now proven on an MCP tools/call fixture whose arguments and result both yield a non-empty value, so the flag is the only thing that can suppress them. The second half of the test flips the flag on against a fresh fixture and asserts both values appear, which is what makes the first half non-vacuous; the fixture has to be rebuilt because a pipeline.Context carries its own finished state and a second RunFinish on it is dropped. Verified by mutation: with both capture_io guards forced open, the new test fails on each value while the old assertions still passed. TestBodyless_UnparsedNoCaptureStillEmitsBothSpans keeps the three assertions it is named for - protocol, a paired exchange.id and outcome=ok on an unparsed exchange - and carries a comment saying why it deliberately makes no capture_io claim. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- .../authlib/plugins/lineage/plugin_test.go | 53 +++++++++++++++++-- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index f675683f6..429c56ebc 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -602,13 +602,56 @@ func TestBodyless_UnparsedNoCaptureStillEmitsBothSpans(t *testing.T) { if got := attrStr(resp, "lineage.outcome"); got != "ok" { t.Errorf("outcome = %q, want ok", got) } - // No payloads captured. - if _, ok := findAttr(req, "input.value"); ok { - t.Error("input.value present with capture_io off") + // Deliberately no capture_io assertion here: protocol http has no payload + // extractor (ioInputValue/ioOutputValue dispatch on a2a/mcp/inference + // only), so this fixture yields "" whatever the flag says. The default is + // proven in TestCaptureIO_OffByDefaultEmitsNoPayload, on a fixture where + // the flag is the only thing that can suppress the payload. +} + +// TestCaptureIO_OffByDefaultEmitsNoPayload proves the privacy default: with +// capture_io unset, no parsed content leaves the pod. It runs on an MCP +// tools/call fixture whose arguments and result both yield a non-empty value, +// so the absence of input.value/output.value can only be the flag — the same +// assertion on an http fixture cannot fail. The second half flips the flag on +// and asserts both appear, which is what makes the first half non-vacuous. +// +// The two halves need separate fixtures: a pipeline.Context carries its own +// finished state, so a second RunFinish on the same one is dropped. +func TestCaptureIO_OffByDefaultEmitsNoPayload(t *testing.T) { + mcpContext := func() *pipeline.Context { + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Host = "weather-tool-mcp.team1.svc:8000" + pctx.Path = "/mcp" + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", + Params: map[string]any{"name": "get_weather", "arguments": map[string]any{"city": "Tokyo"}}, + Result: map[string]any{"content": []any{map[string]any{"type": "text", "text": "sunny"}}}, + } + return pctx } - if _, ok := findAttr(resp, "output.value"); ok { - t.Error("output.value present with capture_io off") + + off, offExp := newTestPlugin(t) + if off.cfg.CaptureIO { + t.Fatal("capture_io must default to false") + } + run(t, off, mcpContext(), allow(200)) + req, resp := roleSplit(t, offExp.GetSpans()) + if v, ok := findAttr(req, "input.value"); ok { + t.Errorf("input.value = %v present with capture_io off", v) } + if v, ok := findAttr(resp, "output.value"); ok { + t.Errorf("output.value = %v present with capture_io off", v) + } + + // Same fixture, flag on: both values appear, so the assertions above are + // measuring the flag rather than an absent extractor. + on, onExp := newTestPlugin(t) + on.cfg.CaptureIO = true + run(t, on, mcpContext(), allow(200)) + req, resp = roleSplit(t, onExp.GetSpans()) + checkAttr(t, req, "input.value", `{"city":"Tokyo"}`) + checkAttr(t, resp, "output.value", "sunny") } // ---- outcomes ---- From 669b81d548bd7a42d76a6480c986f88814e91a01 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 4 Sep 2026 00:41:11 +0300 Subject: [PATCH 20/37] Test: Give the forbidden-key tripwire a floor TestForbiddenKeysNeverEmitted ranged over exp.GetSpans() with no assertion that any span existed, so the scan was vacuous whenever the plugin emitted nothing and a regression that silenced it entirely passed green. A tripwire that cannot fail is worse than none, because it reads as coverage - and what it covers is the contract's hardest rule, the retired vocabulary that must never come back. It now takes the pair through roleSplit, the helper every other test in the file already uses. roleSplit fatals unless the exchange produced exactly one request and one response span, and rejects any span outside that pair, so the scan can neither run on an empty set nor miss a span. Verified by mutation: with OnRequest forced to skip every exchange, the test now fails where it previously passed. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 429c56ebc..edb22f5b0 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -866,7 +866,11 @@ func TestForbiddenKeysNeverEmitted(t *testing.T) { p, exp := newTestPlugin(t) p.cfg.CaptureIO = true run(t, p, mk(), allow(200)) - for _, s := range exp.GetSpans() { + // roleSplit fatals unless the exchange produced exactly one request + // and one response span, so the scan below can never run on an empty + // set and report green on a plugin that emitted nothing. + req, resp := roleSplit(t, exp.GetSpans()) + for _, s := range []tracetest.SpanStub{req, resp} { for _, kv := range s.Attributes { key := string(kv.Key) for _, bad := range forbidden { From 431876c584bff50fbd25b390b669d037ae9879b5 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 4 Sep 2026 00:49:45 +0300 Subject: [PATCH 21/37] Fix: Make -1 the only unbounded payload sentinel truncate treats every non-positive max as unbounded, so max_payload_bytes accepted -2 and below and quietly attached whole payloads - with capture_io on, unbounded prompts and tool arguments leaving the pod on a typo. The contract and the catalog both name -1 as the opt-out, so anything below it is now refused at decode and the sentinel has a name. The decode boundary also had no test. The cap was only ever set straight onto the struct, which skips the remap that makes an explicit 0 mean "unset" - and that remap is the only thing standing between max_payload_bytes: 0 and uncapped payloads. Deleting it would have been a silent privacy regression with every test still green. One table now pins all six shapes an operator can write. Contract v1.6.1 wording only; spans and wire unchanged. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 10 +++++- authbridge/authlib/plugins/lineage/plugin.go | 2 +- .../authlib/plugins/lineage/plugin_test.go | 36 +++++++++++++++++++ authbridge/docs/lineage-wire-contract.md | 5 +-- authbridge/docs/plugin-catalog.md | 2 +- 5 files changed, 50 insertions(+), 5 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index 40759326b..ac88b80b3 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -22,6 +22,10 @@ const defaultOTelEndpoint = "localhost:4317" // visible in the span. 4096 is a conservative default, not a hard requirement. const defaultMaxPayloadBytes = 4096 +// unboundedPayload is the sole opt-out from the cap. Any other negative is +// refused at decode, so a typo cannot quietly attach whole payloads. +const unboundedPayload = -1 + // Config holds the per-plugin configuration decoded from the pipeline YAML. type Config struct { // OTelEndpoint is the OTLP gRPC endpoint (host:port, http://host:port, or @@ -74,7 +78,8 @@ type Config struct { // SDK does not itself drop or truncate an oversized value (its default // attribute-value limit is unlimited and Init sets no SpanLimits), so // without this cap the whole payload would be emitted. Zero (or unset) uses - // defaultMaxPayloadBytes; a negative value disables the cap (attach whole). + // defaultMaxPayloadBytes; unboundedPayload (-1) attaches the whole value, + // and any other negative is refused at decode. // Ignored when CaptureIO is false. // Default: 4096 MaxPayloadBytes int `json:"max_payload_bytes"` @@ -177,6 +182,9 @@ func decodeConfig(raw json.RawMessage) (Config, error) { if cfg.MaxPayloadBytes == 0 { cfg.MaxPayloadBytes = defaultMaxPayloadBytes } + if cfg.MaxPayloadBytes < unboundedPayload { + return Config{}, fmt.Errorf("lineage-telemetry config: max_payload_bytes %d is invalid; use -1 to attach whole values or a positive byte cap", cfg.MaxPayloadBytes) + } // gRPC NewClient expects host:port only, so reduce a URL form (e.g. // http://collector:4317/v1/traces) to its host — TrimPrefix left any path // behind and produced an invalid dial target. A URL scheme also carries an diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index f6ca14b2b..7fba0fba8 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -767,7 +767,7 @@ func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx * // truncate bounds a captured payload to max bytes, cutting on a UTF-8 // rune boundary and appending truncatedSuffix so the loss is explicit in the // span rather than a silent drop at the OTLP exporter's attribute-length limit. -// A non-positive max disables the cap (the caller's explicit opt-out). The +// A non-positive max disables the cap; decode narrows that to exactly -1. The // returned string, suffix included, never exceeds max bytes. func truncate(s string, max int) string { if max <= 0 || len(s) <= max { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index edb22f5b0..967c3b636 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -883,6 +883,42 @@ func TestForbiddenKeysNeverEmitted(t *testing.T) { } } +// TestConfig_MaxPayloadBytes pins the decode boundary: the cap was only ever +// set straight onto the struct, so the remap that makes an explicit 0 mean +// "unset" - the only thing between max_payload_bytes: 0 and uncapped payloads, +// since truncate treats every non-positive max as unbounded - went untested. +func TestConfig_MaxPayloadBytes(t *testing.T) { + cases := []struct { + raw string + want int + wantErr bool + }{ + {raw: `{}`, want: defaultMaxPayloadBytes}, + {raw: `{"max_payload_bytes": 0}`, want: defaultMaxPayloadBytes}, + {raw: `{"max_payload_bytes": 128}`, want: 128}, + {raw: `{"max_payload_bytes": -1}`, want: unboundedPayload}, + {raw: `{"max_payload_bytes": -2}`, wantErr: true}, + {raw: `{"max_payload_bytes": -4096}`, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.raw, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if tc.wantErr { + if err == nil { + t.Fatal("accepted; it removes the cap silently") + } + return + } + if err != nil { + t.Fatalf("decode: %v", err) + } + if cfg.MaxPayloadBytes != tc.want { + t.Fatalf("max_payload_bytes = %d, want %d", cfg.MaxPayloadBytes, tc.want) + } + }) + } +} + // ---- robustness ---- func TestOnFinish_NoStateDoesNotPanicOrEmit(t *testing.T) { diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 72ec846cf..5811059cb 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -208,7 +208,7 @@ response = the request name + ` response`. | `otel_tls` | `false` | TLS to the collector, verified against the system roots or `otel_ca_file`; an `https://` endpoint implies it. Refused contradictions: `https://` with `otel_tls: false`, `http://` with `otel_tls: true` or `otel_ca_file` | | `otel_ca_file` | — | PEM bundle to verify the collector's certificate against, for a private CA; implies `otel_tls`, and `otel_ca_file` with `otel_tls: false` is refused. An unreadable file, or one with no certificate, refuses to start | | `capture_io` | `false` | attach `input.value` / `output.value` | -| `max_payload_bytes` | `4096` | producer-side cap on those two values; `-1` attaches whole | +| `max_payload_bytes` | `4096` | producer-side cap on those two values; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start | | `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | | `bypass_paths` | `/.well-known/`, `/healthz`, `/readyz`, `/health` | path prefixes that produce no spans | | `bypass_hosts` | `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*` | outbound host globs that produce no spans | @@ -272,7 +272,8 @@ mechanisms named as removed are not to be reintroduced. as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only `path.Match` glob list (it was an unanchored substring match on both directions) and both bypass - lists are validated at start. + lists are validated at start; `-1` is stated as the only `max_payload_bytes` opt-out, with any + other negative refused rather than silently unbounded. - **v1.6** — an invalid or absent `traceparent` is restarted per W3C (`mint_traceparent`); `lineage.parent.source` gains `none`; the stamp key becomes `lineage-parent`; `otel_tls` and `max_payload_bytes` added; the document is vendored into the producer repository. Motivation: diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index 1f3ecaf0b..2df95f6cb 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -147,7 +147,7 @@ denial by a plugin ordered before it emits no spans. - `otel_tls` (bool) — dial the collector with TLS, verified against the system roots or `otel_ca_file`. An `https://` endpoint implies it; `https://` with `otel_tls: false`, and `http://` with `otel_tls: true` or `otel_ca_file`, are refused. Default `false`. - `otel_ca_file` (string) — PEM bundle to verify the collector's certificate against (a private CA, e.g. cert-manager issued). Implies `otel_tls`; with an explicit `otel_tls: false` it is refused; an unreadable file or one with no certificate refuses to start. Default: system roots. - `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. -- `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `-1` attaches whole. Default `4096`. +- `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start. Default `4096`. - `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. - `bypass_paths` (`[]string`) — path prefixes that produce no spans. Default `/.well-known/`, `/healthz`, `/readyz`, `/health`. Setting either bypass key replaces its default list rather than extending it, as in `ibac` / `sparc` / `cpex`; an entry matching everything is refused at start. - `bypass_hosts` (`[]string`) — outbound host globs (`path.Match`, port stripped, case folded) that produce no spans; ignored inbound, where `Host` is caller-controlled. Default `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*`. From 64e55fdd340a316e8ae7f7eb10f48d42b01410b7 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 4 Sep 2026 02:07:08 +0300 Subject: [PATCH 22/37] Refactor: Drop the second recover over OnFinish The Finisher contract states that OnFinish runs best-effort and that panics are recovered and logged, and dispatchFinish scopes that recover to a single plugin's dispatch. A recover of our own over the same phase caught nothing the framework would not have caught; it only relabelled the WARN. The canonical Finisher example in that same interface doc keeps no recover either. OnRequest is left as it is. No plugin in the repo recovers that phase, so making this one the first is a framework question rather than a lineage one - raised on the review thread. The contract's lone-request-span note said the plugin recovered such a panic. The pipeline does, and still logs a WARN, so the cause stands and only the actor changes. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 12 ++++-------- authbridge/docs/lineage-wire-contract.md | 8 ++++---- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 7fba0fba8..cd6d3b6f3 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -599,15 +599,11 @@ func (p *LineageTelemetry) OnResponse(_ context.Context, _ *pipeline.Context) pi // OnFinish emits the response span — the twin of the request span, parented // under it and echoing the same exchange.id — carrying outcome/status/output. // Always fires at stream end, so a bodyless or failed exchange still completes -// as a first-class pair. Runs under a recover so an unexpected state never -// crashes the pipeline. +// as a first-class pair. No recover here: the Finisher contract states that +// OnFinish runs best-effort and that panics are recovered and logged, and +// dispatchFinish scopes that recover to one plugin, so a second net would only +// hide the same panic under a different logger. func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) { - defer func() { - if r := recover(); r != nil { - slog.Warn("lineage-telemetry: OnFinish panic recovered", "recover", r) - } - }() - state := pipeline.GetState[exchangeState](pctx, pluginName) if state == nil || !state.reqCtx.IsValid() { return diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 5811059cb..4671f671c 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -52,10 +52,10 @@ One HTTP exchange through the sidecar produces two OTLP spans. - The response span is emitted at stream end **even when no response was produced** — client disconnect, upstream reset, plugin denial. It then carries `lineage.outcome` and whatever status exists, so the row completes as failed instead of dangling. -- A lone request span means one of three things: the sidecar died mid-exchange; the plugin - recovered a panic while emitting the response span (a WARN is logged); or the response span was - emitted but lost — the two halves enter a batching exporter an exchange apart, so a response can - be lost after its request has flushed. The consumer renders it as in-flight, never as a wrong +- A lone request span means one of three things: the sidecar died mid-exchange; a panic while + emitting the response span was recovered by the pipeline (a WARN is logged); or the response span + was emitted but lost — the two halves enter a batching exporter an exchange apart, so a response + can be lost after its request has flushed. The consumer renders it as in-flight, never as a wrong pairing. A response span whose `lineage.outcome` is absent derives with `error` NULL (honest unknown), never `false`. - **Scope of `denied`.** The lineage plugin runs after the gate plugins and the pipeline From 7f03060cd9373a58ed55c574c72dec0f9061ce9c Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 4 Sep 2026 11:09:50 +0300 Subject: [PATCH 23/37] Docs: State the rule lineage.outcome actually follows The contract said "abandoned = no status was ever produced", which reads as an iff and is not one. An allow or a deny with StatusCode 0 emits ok or denied with no status attached, so three of the four words can appear without one. abandoned is reached only for a nil Outcome, or an error that never wrote a status. Nothing downstream mis-derives - the consumer reads lineage.outcome as an emitted fact and never recomputes it from the status - so this is the document being wrong rather than the producer. The mapping function's own comment carried the same imprecision and is corrected with it. No behaviour change. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 8 +++++--- authbridge/docs/lineage-wire-contract.md | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index cd6d3b6f3..2f5ce626d 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -643,9 +643,11 @@ func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) // lineageOutcome maps the pipeline's 3-value Outcome (allow/deny/error, nil // outside OnFinish) onto the contract's lineage.outcome vocabulary -// (ok|denied|error|abandoned) plus the http.status_code fact. A terminal state -// with no status written (upstream reset, client disconnect, listener death) -// is "abandoned" — the row completes as in-flight-turned-failed rather than +// (ok|denied|error|abandoned) plus the http.status_code fact. "ok" and "denied" +// are the pipeline's own verdicts and carry a status only if one was written — +// an allow that produced none is still an allow. "abandoned" is a nil Outcome, +// or an error that never wrote a status (upstream reset, client disconnect, +// listener death) — the row completes as in-flight-turned-failed rather than // dangling. hasStatus is false when no status code was produced. func lineageOutcome(o *pipeline.Outcome) (outcome string, status int, hasStatus bool, deniedBy string) { if o == nil { diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 4671f671c..47ce281ab 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -171,7 +171,7 @@ Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-te | `input.value` | request, with `capture_io` | `{"city":"Tokyo"}` | see §5 | | `output.value` | response, with `capture_io` | `{...}` | see §5; absent when unparsed or streamed | | `http.status_code` | response, when a status was produced | `200` | | -| `lineage.outcome` | response | `ok` \| `denied` \| `error` \| `abandoned` | how the exchange ended as the proxy saw it; `abandoned` = no status was ever produced | +| `lineage.outcome` | response | `ok` \| `denied` \| `error` \| `abandoned` | how the exchange ended as the proxy saw it; `ok` and `denied` are the pipeline's verdicts, with or without a status; `abandoned` is a nil outcome, or an error that never produced a status | | `lineage.denied_by` | response, denials | `jwt-validation` | the plugin that denied | `http.method` and `http.status_code` are the pre-1.21 OpenTelemetry semantic-convention keys, kept From 08b3ccfd617e3a95d4091e16575ee2e958ac8b0a Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 4 Sep 2026 11:55:11 +0300 Subject: [PATCH 24/37] Docs: Say why the resource attributes are constant on every pod MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit service.name is authbridge on every workload, so a backend that groups by it - Phoenix and Jaeger both do - shows one merged service rather than one per pod. That is the intended split: the resource says what produced the span, the span says which workload it was beside. §4 now states it, and points at a collector transform for anyone who wants per-workload grouping, the remedy §8 already names for a display concern. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/docs/lineage-wire-contract.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 47ce281ab..e9e6eb33d 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -150,7 +150,12 @@ instead of being welded by a guess. The consequences per case: ## 4. Attributes -Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-telemetry`. +Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-telemetry`. Both are +constant on every pod, deliberately: the resource says what produced the span, and the span says +which workload it was beside (`lineage.self.id`). A backend that groups by `service.name` — +Phoenix and Jaeger both do — therefore shows one merged service. Operators wanting per-workload +grouping map `lineage.self.id` onto `service.name` in a collector transform, the same way §8 +handles `openinference.span.kind`. | key | on | example | notes | |---|---|---|---| From 8d9008f8647d1fe477a22c32e2b2324d8c520750 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Fri, 4 Sep 2026 12:35:05 +0300 Subject: [PATCH 25/37] Feat: Warn when spans leave the pod in cleartext Spans carry principal facts on every inbound request, and whole prompts and tool output under capture_io, so a plaintext dial deserves to be visible. Requiring TLS instead was considered and does not work here: a host:port does not say whether the collector is in-cluster or across the internet, and the platform's own collector listens on plaintext gRPC at 4317 with no TLS option, so a hard requirement would leave the plugin unusable on the deployment it ships in. Init now logs a WARN, once, when it dials plaintext to a non-loopback endpoint, naming the endpoint and the two knobs that encrypt it. Loopback is exempt: that traffic never leaves the network namespace, and localhost:4317 is this plugin's own default - a warning that fires on the default configuration is a warning nobody reads. With otel_ca_file, TLS to a cert-manager-issued in-cluster collector is now possible as well as advised. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 33 +++++++++++++++++-- .../authlib/plugins/lineage/plugin_test.go | 27 +++++++++++++++ authbridge/docs/lineage-wire-contract.md | 2 +- authbridge/docs/plugin-catalog.md | 2 +- 4 files changed, 59 insertions(+), 5 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 2f5ce626d..8d9d7be7f 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -230,7 +230,7 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { // otel_ca_file sets it in decodeConfig. Spans carry principal facts on // every inbound request and, under capture_io, user messages and model // output, so a remote collector must not receive them in cleartext. - creds := insecure.NewCredentials() + creds, plaintext := insecure.NewCredentials(), true switch { case p.cfg.OTelCAFile != "": // A private CA (a cert-manager issued in-cluster collector, typically): @@ -247,11 +247,23 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { if !pool.AppendCertsFromPEM(pemBytes) { return fmt.Errorf("lineage-telemetry: otel_ca_file %q: no CA certificate found in PEM", p.cfg.OTelCAFile) } - creds = credentials.NewClientTLSFromCert(pool, "") + creds, plaintext = credentials.NewClientTLSFromCert(pool, ""), false case p.cfg.OTelTLS: // nil cert pool = system roots; empty serverName = derive from endpoint. - creds = credentials.NewClientTLSFromCert(nil, "") + creds, plaintext = credentials.NewClientTLSFromCert(nil, ""), false } + // A plaintext dial to anything but loopback puts principal facts, and under + // capture_io whole prompts and tool output, on the network in the clear. + // Whether that hop leaves the cluster is not knowable from a host:port, and + // the platform's own collector is plaintext gRPC, so this is the operator's + // call to make rather than ours to refuse — but they get told at the one + // moment they can act on it. + if plaintext && !isLoopback(endpoint) { + slog.Warn("lineage-telemetry: exporting spans in cleartext to a non-loopback collector; "+ + "set otel_tls (or otel_ca_file for a private CA) to encrypt them", + "endpoint", endpoint) + } + conn, err := grpc.NewClient(endpoint, grpc.WithTransportCredentials(creds), ) @@ -300,6 +312,21 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { return nil } +// isLoopback reports whether endpoint (host:port, already reduced from any URL +// form by decodeConfig) names this pod. Loopback traffic never leaves the +// network namespace, so cleartext there carries no exposure and earns no WARN. +func isLoopback(endpoint string) bool { + host := endpoint + if h, _, err := net.SplitHostPort(endpoint); err == nil { + host = h + } + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + // Shutdown flushes and stops the tracer provider and then closes the OTLP gRPC // connection. The exporter created with WithGRPCConn does not own conn, so // closing it here is what actually releases the socket; both errors are joined diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 967c3b636..f5804aa0e 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -919,6 +919,33 @@ func TestConfig_MaxPayloadBytes(t *testing.T) { } } +// TestIsLoopback pins which endpoints earn the cleartext WARN. Only traffic +// that never leaves the pod's network namespace is exempt; an in-cluster +// service name is indistinguishable from a collector across the internet, so +// both are warned about. +func TestIsLoopback(t *testing.T) { + cases := []struct { + endpoint string + want bool + }{ + {"localhost:4317", true}, + {"LocalHost:4317", true}, + {"127.0.0.1:4317", true}, + {"127.9.9.9:4317", true}, + {"[::1]:4317", true}, + {"localhost", true}, + {"otel-collector.rossoctl-system.svc.cluster.local:4317", false}, + {"collector.vendor.example.com:4317", false}, + {"10.0.0.7:4317", false}, + {"", false}, + } + for _, tc := range cases { + if got := isLoopback(tc.endpoint); got != tc.want { + t.Errorf("isLoopback(%q) = %v, want %v", tc.endpoint, got, tc.want) + } + } +} + // ---- robustness ---- func TestOnFinish_NoStateDoesNotPanicOrEmit(t *testing.T) { diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index e9e6eb33d..4dbf4242f 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -210,7 +210,7 @@ response = the request name + ` response`. | key | default | meaning | |---|---|---| | `otel_endpoint` | `localhost:4317` | OTLP gRPC target; `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused | -| `otel_tls` | `false` | TLS to the collector, verified against the system roots or `otel_ca_file`; an `https://` endpoint implies it. Refused contradictions: `https://` with `otel_tls: false`, `http://` with `otel_tls: true` or `otel_ca_file` | +| `otel_tls` | `false` | TLS to the collector, verified against the system roots or `otel_ca_file`; an `https://` endpoint implies it. Refused contradictions: `https://` with `otel_tls: false`, `http://` with `otel_tls: true` or `otel_ca_file`. Plaintext to a non-loopback collector is allowed and logged as a WARN at start — spans carry principal facts, and payloads under `capture_io` | | `otel_ca_file` | — | PEM bundle to verify the collector's certificate against, for a private CA; implies `otel_tls`, and `otel_ca_file` with `otel_tls: false` is refused. An unreadable file, or one with no certificate, refuses to start | | `capture_io` | `false` | attach `input.value` / `output.value` | | `max_payload_bytes` | `4096` | producer-side cap on those two values; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start | diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index 2df95f6cb..76f455736 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -144,7 +144,7 @@ the protocol parsers (declared in `RequiresAny`) and after denial by a plugin ordered before it emits no spans. - `otel_endpoint` (string) — OTLP gRPC target: `host:port`, `http://host:port` or `https://host:port`; any other scheme is refused. Default `localhost:4317`. -- `otel_tls` (bool) — dial the collector with TLS, verified against the system roots or `otel_ca_file`. An `https://` endpoint implies it; `https://` with `otel_tls: false`, and `http://` with `otel_tls: true` or `otel_ca_file`, are refused. Default `false`. +- `otel_tls` (bool) — dial the collector with TLS, verified against the system roots or `otel_ca_file`. An `https://` endpoint implies it; `https://` with `otel_tls: false`, and `http://` with `otel_tls: true` or `otel_ca_file`, are refused. A plaintext dial to a non-loopback collector is allowed and logged as a WARN at start. Default `false`. - `otel_ca_file` (string) — PEM bundle to verify the collector's certificate against (a private CA, e.g. cert-manager issued). Implies `otel_tls`; with an explicit `otel_tls: false` it is refused; an unreadable file or one with no certificate refuses to start. Default: system roots. - `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. - `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start. Default `4096`. From b015f869845ed23b1072e91b8f524b9052233c9e Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 12:22:58 +0300 Subject: [PATCH 26/37] Fix: Strip query string from url.path and span names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The extproc listener populates pctx.Path from the raw :path pseudo-header, query string included; the proxy listeners use the parsed r.URL.Path, which excludes it. In envoy-sidecar mode the query therefore reached the url.path attribute and the span-name fallback regardless of capture_io — query strings can carry secrets, and OTel semconv defines url.path as query-free. Strip anything from '?' on at the plugin's two consumption points (same defensive pattern as inference-parser). Contract v1.6.2. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 17 ++++++++++++++--- .../authlib/plugins/lineage/plugin_test.go | 17 +++++++++++++++++ authbridge/docs/lineage-wire-contract.md | 8 ++++++-- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 8d9d7be7f..4181e60b6 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -743,8 +743,8 @@ func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx * // see the http.status_code note on the response span. attrs = append(attrs, attribute.String("http.method", pctx.Method)) } - if pctx.Path != "" { - attrs = append(attrs, attribute.String("url.path", pctx.Path)) + if path := urlPath(pctx); path != "" { + attrs = append(attrs, attribute.String("url.path", path)) } if pctx.Scheme != "" { attrs = append(attrs, attribute.String("url.scheme", pctx.Scheme)) @@ -845,11 +845,22 @@ func spanOp(pctx *pipeline.Context, protocol string) string { } } if op == "" { - op = pctx.Path + op = urlPath(pctx) } return op } +// urlPath returns pctx.Path without any query string. The extproc listener +// populates Path from the raw :path pseudo-header, query included; the proxy +// listeners use the parsed r.URL.Path, which excludes it. Stripping here keeps +// url.path and the span-name fallback query-free under every listener — a +// query can carry secrets that must not reach the trace store even with +// capture_io off. +func urlPath(pctx *pipeline.Context) string { + path, _, _ := strings.Cut(pctx.Path, "?") + return path +} + // mcpTool returns the tool name for an MCP tools/call, or "" otherwise. func mcpTool(pctx *pipeline.Context) string { m := pctx.Extensions.MCP diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index f5804aa0e..6f68c91e1 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -830,6 +830,23 @@ func TestPrincipalFacts_OutboundNeverEmitsPrincipal(t *testing.T) { } } +// The extproc listener populates pctx.Path from the raw :path pseudo-header, +// query string included; url.path and the span-name fallback must never +// carry it. +func TestQueryStringNeverEmitted(t *testing.T) { + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Path = "/api/search?token=sekret" + + run(t, p, pctx, allow(200)) + req, _ := roleSplit(t, exp.GetSpans()) + + checkAttr(t, req, "url.path", "/api/search") + if req.Name != "weather-service http /api/search" { + t.Errorf("request span name = %q, want query-free", req.Name) + } +} + // ---- the forbidden-keys guard ---- // TestForbiddenKeysNeverEmitted scans every attribute of every span emitted diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 4dbf4242f..429639eea 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -1,4 +1,4 @@ -# Lineage wire contract — two-span sidecar lineage (v1.6.1) +# Lineage wire contract — two-span sidecar lineage (v1.6.2) What the AuthBridge `lineage-telemetry` plugin emits, what it writes onto the wire, and what the data-governance `sidecar` interactions algorithm (ADR-0030) commits to when consuming it. @@ -167,7 +167,7 @@ handles `openinference.span.kind`. | `lineage.protocol` | both | `a2a` \| `mcp` \| `inference` \| `http` | which parser matched; `http` = none | | `lineage.parent.source` | request | `tracestate` \| `wire` \| `none` | which precedence in §3.2 chose the parent. An audit fact; the consumer derives nothing from it | | `http.method` | request, when the listener supplies it | `POST` | all listeners do | -| `url.path` | request, when present | `/mcp` | | +| `url.path` | request, when present | `/mcp` | query-free: anything from `?` on is stripped before emission (per OTel semconv; the query can carry secrets and is never captured) | | `url.scheme` | request, when present | `http` | the listener's observed scheme. Optional: the consumer composes `scheme://peer.host + url.path` only when all three exist | | `a2a.method`, `a2a.session_id` | request, a2a | `message/send` | parsed facts | | `mcp.method`, `mcp.tool` | request, mcp | `tools/call`, `get_weather` | `mcp.tool` only for `tools/call` | @@ -273,6 +273,10 @@ The producer must not emit these, and the consumer reads nothing from them. Version ladder, newest first. Each line is what changed on the wire or in the vocabulary; the mechanisms named as removed are not to be reintroduced. +- **v1.6.2** — `url.path` and the span-name fallback derived from it are query-free: the producer + strips anything from `?` on before emission. Until now the envoy-sidecar listener's raw `:path` + pseudo-header put the query string on the wire regardless of `capture_io`; the proxy listeners + never delivered it. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only From 598ba80898553429add52791be6ca30eeaa1492e Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 13:03:42 +0300 Subject: [PATCH 27/37] Fix: Bound variable string attributes and span names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit truncate() guarded only input.value/output.value, so every other string attribute and the span name were emitted uncapped — and the OTel SDK never truncates on its own (unlimited default, no SpanLimits set). Several of those values are caller-controlled: one request could put a 100 KB span name, a 100 KB url.path or a 50 KB mcp.tool (a params.name field from the request body) into the backend. New max_attr_bytes key (default 256, same 0/-1/negative semantics as max_payload_bytes) applied to every variable-content string attribute and the composed request span name; fixed-vocabulary facts and the hex ids are bounded by construction. Contract v1.6.2 and catalog updated. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 25 +++++++ authbridge/authlib/plugins/lineage/plugin.go | 44 +++++++----- .../authlib/plugins/lineage/plugin_test.go | 67 +++++++++++++++++++ authbridge/docs/lineage-wire-contract.md | 12 +++- authbridge/docs/plugin-catalog.md | 1 + 5 files changed, 132 insertions(+), 17 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index ac88b80b3..71e735a3d 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -26,6 +26,13 @@ const defaultMaxPayloadBytes = 4096 // refused at decode, so a typo cannot quietly attach whole payloads. const unboundedPayload = -1 +// defaultMaxAttrBytes bounds every variable-content string attribute and the +// span name. Several of those values are caller-controlled (url.path, Host, +// mcp.tool from the request body, a2a.session_id) and the SDK never truncates +// on its own — without this cap one request can put a 100 KB span name into +// the backend. 256 comfortably fits real paths, hosts and tool names. +const defaultMaxAttrBytes = 256 + // Config holds the per-plugin configuration decoded from the pipeline YAML. type Config struct { // OTelEndpoint is the OTLP gRPC endpoint (host:port, http://host:port, or @@ -84,6 +91,16 @@ type Config struct { // Default: 4096 MaxPayloadBytes int `json:"max_payload_bytes"` + // MaxAttrBytes caps every variable-content string attribute (url.path, + // lineage.peer.host, mcp.tool, a2a.session_id, …) and the span name, cut + // on a UTF-8 boundary with the same truncation marker as payloads. + // input.value / output.value keep their own MaxPayloadBytes cap, and + // fixed-vocabulary facts (lineage.role, lineage.outcome, …) are bounded by + // construction. Zero (or unset) uses defaultMaxAttrBytes; -1 removes the + // cap, and any other negative is refused at decode. + // Default: 256 + MaxAttrBytes int `json:"max_attr_bytes"` + // MintTraceparent — both directions — forwards a W3C traceparent naming // this exchange's request span when the request arrived with no // valid traceparent. Without one the next element has nothing to @@ -150,6 +167,7 @@ func defaultConfig() Config { return Config{ OTelEndpoint: defaultOTelEndpoint, MaxPayloadBytes: defaultMaxPayloadBytes, + MaxAttrBytes: defaultMaxAttrBytes, MintTraceparent: true, BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, BypassHosts: []string{ @@ -185,6 +203,13 @@ func decodeConfig(raw json.RawMessage) (Config, error) { if cfg.MaxPayloadBytes < unboundedPayload { return Config{}, fmt.Errorf("lineage-telemetry config: max_payload_bytes %d is invalid; use -1 to attach whole values or a positive byte cap", cfg.MaxPayloadBytes) } + // Same convention as max_payload_bytes: 0 = the default, -1 = no cap. + if cfg.MaxAttrBytes == 0 { + cfg.MaxAttrBytes = defaultMaxAttrBytes + } + if cfg.MaxAttrBytes < unboundedPayload { + return Config{}, fmt.Errorf("lineage-telemetry config: max_attr_bytes %d is invalid; use -1 to remove the cap or a positive byte cap", cfg.MaxAttrBytes) + } // gRPC NewClient expects host:port only, so reduce a URL form (e.g. // http://collector:4317/v1/traces) to its host — TrimPrefix left any path // behind and produced an invalid dial target. A URL scheme also carries an diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 4181e60b6..3ec27d493 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -423,11 +423,11 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context protocol := protocolOf(pctx) self := serviceLabel(p.selfID) spanKind := spanKindFor(pctx.Direction) - spanName := requestSpanName(self, protocol, spanOp(pctx, protocol)) + spanName := truncate(requestSpanName(self, protocol, spanOp(pctx, protocol)), p.cfg.MaxAttrBytes) // Facts shared by both spans (exchange.id is appended once the request // span exists, since it IS the request span id). - base := baseAttrs(pctx, self, protocol) + base := p.baseAttrs(pctx, self, protocol) // Request-span attributes: role + shared facts + request-only facts. reqAttrs := make([]attribute.KeyValue, 0, len(base)+8) @@ -652,7 +652,7 @@ func (p *LineageTelemetry) OnFinish(ctx context.Context, pctx *pipeline.Context) attrs = append(attrs, attribute.Int("http.status_code", status)) } if deniedBy != "" { - attrs = append(attrs, attribute.String("lineage.denied_by", deniedBy)) + attrs = append(attrs, p.capped("lineage.denied_by", deniedBy)) } if p.cfg.CaptureIO { if v := ioOutputValue(pctx, state.protocol); v != "" { @@ -721,18 +721,30 @@ func spanKindFor(dir pipeline.Direction) trace.SpanKind { // baseAttrs returns the facts carried on BOTH spans except exchange.id (added // once the request span id is known) and role (differs per span). -func baseAttrs(pctx *pipeline.Context, self, protocol string) []attribute.KeyValue { +func (p *LineageTelemetry) baseAttrs(pctx *pipeline.Context, self, protocol string) []attribute.KeyValue { attrs := []attribute.KeyValue{ attribute.String("lineage.direction", pctx.Direction.String()), - attribute.String("lineage.self.id", self), + p.capped("lineage.self.id", self), attribute.String("lineage.protocol", protocol), } if pctx.Host != "" { - attrs = append(attrs, attribute.String("lineage.peer.host", pctx.Host)) + attrs = append(attrs, p.capped("lineage.peer.host", pctx.Host)) } return attrs } +// capped returns a string span attribute whose value is bounded by +// max_attr_bytes. Every variable-content string attribute goes through here — +// several are caller-controlled (url.path, Host, mcp.tool from the request +// body, a2a.session_id) and the SDK never truncates on its own. The +// fixed-vocabulary facts (lineage.role, lineage.direction, lineage.protocol, +// lineage.parent.source, lineage.outcome) and the 16-hex exchange id are +// bounded by construction and skip it; input.value / output.value carry their +// own max_payload_bytes cap. +func (p *LineageTelemetry) capped(key, val string) attribute.KeyValue { + return attribute.String(key, truncate(val, p.cfg.MaxAttrBytes)) +} + // appendRequestFacts adds the request-only facts: HTTP method/path/scheme, the // protocol-specific parsed facts, validated-JWT principal (inbound only), and // input.value when capture_io is on. protocolOf guarantees the matching @@ -741,44 +753,44 @@ func (p *LineageTelemetry) appendRequestFacts(attrs []attribute.KeyValue, pctx * if pctx.Method != "" { // "http.method" is the pre-v1.21 OTel semconv key, kept deliberately — // see the http.status_code note on the response span. - attrs = append(attrs, attribute.String("http.method", pctx.Method)) + attrs = append(attrs, p.capped("http.method", pctx.Method)) } if path := urlPath(pctx); path != "" { - attrs = append(attrs, attribute.String("url.path", path)) + attrs = append(attrs, p.capped("url.path", path)) } if pctx.Scheme != "" { - attrs = append(attrs, attribute.String("url.scheme", pctx.Scheme)) + attrs = append(attrs, p.capped("url.scheme", pctx.Scheme)) } switch protocol { case "a2a": a := pctx.Extensions.A2A if a.Method != "" { - attrs = append(attrs, attribute.String("a2a.method", a.Method)) + attrs = append(attrs, p.capped("a2a.method", a.Method)) } if a.SessionID != "" { - attrs = append(attrs, attribute.String("a2a.session_id", a.SessionID)) + attrs = append(attrs, p.capped("a2a.session_id", a.SessionID)) } case "mcp": m := pctx.Extensions.MCP if m.Method != "" { - attrs = append(attrs, attribute.String("mcp.method", m.Method)) + attrs = append(attrs, p.capped("mcp.method", m.Method)) } if t := mcpTool(pctx); t != "" { - attrs = append(attrs, attribute.String("mcp.tool", t)) + attrs = append(attrs, p.capped("mcp.tool", t)) } case "inference": if model := pctx.Extensions.Inference.Model; model != "" { - attrs = append(attrs, attribute.String("inference.model", model)) + attrs = append(attrs, p.capped("inference.model", model)) } } // Principal facts: request span, inbound only, and only from a validated // JWT (pctx.Identity non-nil). if pctx.Direction == pipeline.Inbound && pctx.Identity != nil { if s := pctx.Identity.Subject(); s != "" { - attrs = append(attrs, attribute.String("lineage.principal.sub", s)) + attrs = append(attrs, p.capped("lineage.principal.sub", s)) } if c := pctx.Identity.ClientID(); c != "" { - attrs = append(attrs, attribute.String("lineage.principal.client", c)) + attrs = append(attrs, p.capped("lineage.principal.client", c)) } } if p.cfg.CaptureIO { diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 6f68c91e1..cff585c0d 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -847,6 +847,73 @@ func TestQueryStringNeverEmitted(t *testing.T) { } } +// Variable-content attributes and the span name are caller-controlled (a +// long path, a huge Host header, a tool name from the request body) and the +// SDK never truncates on its own; max_attr_bytes must bound them all. +func TestAttrBytesCapsCallerControlledValues(t *testing.T) { + long := strings.Repeat("x", 100_000) + p, exp := newTestPlugin(t) + pctx := fakeContext(pipeline.Outbound, http.Header{}) + pctx.Path = "/" + long + pctx.Host = "evil-" + long + ".example:8000" + pctx.Extensions.MCP = &pipeline.MCPExtension{ + Method: "tools/call", + Params: map[string]any{"name": "tool_" + long}, + } + + run(t, p, pctx, allow(200)) + req, resp := roleSplit(t, exp.GetSpans()) + + max := defaultMaxAttrBytes + for _, key := range []string{"url.path", "lineage.peer.host", "mcp.tool"} { + v := attrStr(req, key) + if len(v) > max { + t.Errorf("attr %q is %d bytes, cap is %d", key, len(v), max) + } + if !strings.HasSuffix(v, truncatedSuffix) { + t.Errorf("attr %q lost bytes without the truncation marker", key) + } + } + if len(req.Name) > max { + t.Errorf("request span name is %d bytes, cap is %d", len(req.Name), max) + } + // The response name is the capped request name + " response". + if want := req.Name + " response"; resp.Name != want { + t.Errorf("response span name = %q, want %q", resp.Name, want) + } +} + +func TestConfig_MaxAttrBytes(t *testing.T) { + cases := []struct { + raw string + want int + wantErr bool + }{ + {raw: `{}`, want: defaultMaxAttrBytes}, + {raw: `{"max_attr_bytes": 0}`, want: defaultMaxAttrBytes}, + {raw: `{"max_attr_bytes": 64}`, want: 64}, + {raw: `{"max_attr_bytes": -1}`, want: unboundedPayload}, + {raw: `{"max_attr_bytes": -2}`, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.raw, func(t *testing.T) { + cfg, err := decodeConfig([]byte(tc.raw)) + if tc.wantErr { + if err == nil { + t.Fatal("accepted; it removes the cap silently") + } + return + } + if err != nil { + t.Fatalf("decode: %v", err) + } + if cfg.MaxAttrBytes != tc.want { + t.Fatalf("max_attr_bytes = %d, want %d", cfg.MaxAttrBytes, tc.want) + } + }) + } +} + // ---- the forbidden-keys guard ---- // TestForbiddenKeysNeverEmitted scans every attribute of every span emitted diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 429639eea..d0363a569 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -187,6 +187,13 @@ Span names: request = `{self.id} {protocol} {op}`, where op is `mcp.tool` (else `a2a.method`, or `inference.model`, falling back to `url.path`, and is omitted when empty; response = the request name + ` response`. +Every variable-content string attribute above, and the request span name, is capped at +`max_attr_bytes` (default 256), cut on a UTF-8 boundary and suffixed `…[truncated]` — several of +these values are caller-controlled (`url.path`, `lineage.peer.host`, `mcp.tool`, +`a2a.session_id`) and the SDK never truncates on its own. `input.value` / `output.value` carry +their own `max_payload_bytes` cap (§5); the fixed-vocabulary facts and the hex ids are bounded by +construction. + ## 5. Payloads - `input.value` and `output.value` are the parsers' semantic reduction of the request and @@ -214,6 +221,7 @@ response = the request name + ` response`. | `otel_ca_file` | — | PEM bundle to verify the collector's certificate against, for a private CA; implies `otel_tls`, and `otel_ca_file` with `otel_tls: false` is refused. An unreadable file, or one with no certificate, refuses to start | | `capture_io` | `false` | attach `input.value` / `output.value` | | `max_payload_bytes` | `4096` | producer-side cap on those two values; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start | +| `max_attr_bytes` | `256` | cap on every variable-content string attribute and the span name (§4); same `0` / `-1` / negative semantics as `max_payload_bytes` | | `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | | `bypass_paths` | `/.well-known/`, `/healthz`, `/readyz`, `/health` | path prefixes that produce no spans | | `bypass_hosts` | `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*` | outbound host globs that produce no spans | @@ -276,7 +284,9 @@ mechanisms named as removed are not to be reintroduced. - **v1.6.2** — `url.path` and the span-name fallback derived from it are query-free: the producer strips anything from `?` on before emission. Until now the envoy-sidecar listener's raw `:path` pseudo-header put the query string on the wire regardless of `capture_io`; the proxy listeners - never delivered it. + never delivered it. And every variable-content string attribute plus the span name is capped at + `max_attr_bytes` (default 256) — until now only the two payload values were bounded, so one + request could put a 100 KB span name into the backend. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index 76f455736..bc0d06fd1 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -148,6 +148,7 @@ denial by a plugin ordered before it emits no spans. - `otel_ca_file` (string) — PEM bundle to verify the collector's certificate against (a private CA, e.g. cert-manager issued). Implies `otel_tls`; with an explicit `otel_tls: false` it is refused; an unreadable file or one with no certificate refuses to start. Default: system roots. - `capture_io` (bool) — attach the parsed request/response content as `input.value` / `output.value`. Default `false`. - `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start. Default `4096`. +- `max_attr_bytes` (int) — cap on every variable-content string attribute (`url.path`, `lineage.peer.host`, `mcp.tool`, …) and the span name; same `0` / `-1` / negative semantics as `max_payload_bytes`. Default `256`. - `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. - `bypass_paths` (`[]string`) — path prefixes that produce no spans. Default `/.well-known/`, `/healthz`, `/readyz`, `/health`. Setting either bypass key replaces its default list rather than extending it, as in `ibac` / `sparc` / `cpex`; an entry matching everything is refused at start. - `bypass_hosts` (`[]string`) — outbound host globs (`path.Match`, port stripped, case folded) that produce no spans; ignored inbound, where `Host` is caller-controlled. Default `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*`. From b1436ff28837b62529ac8b6a4ca4ef5f7af2c741 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 13:34:51 +0300 Subject: [PATCH 28/37] Docs: Correct passthrough and tracestate claims in the contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two false absolutes: 'TLS-passthrough … produce no exchange' holds only in envoy-sidecar mode — the proxy-sidecar forward proxy runs the outbound pipeline on CONNECT, so every HTTPS destination emits an ordinary span pair (http.method=CONNECT, url.scheme=tcp, no path, no payload); and §3.4's 'every exchange, both directions' missed the three cases where the stamp is not written (bypassed exchange, producer not ready, insert refused on a malformed member). §3.5 gains the bypassed-hop consequence. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/docs/lineage-wire-contract.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index d0363a569..617a49188 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -129,7 +129,7 @@ parent at the trace edge, by design. | header | when | value | |---|---|---| -| `tracestate` | every exchange, both directions, whenever a valid context exists after §3.3 | the caller's members with `lineage-parent` set to this request span id | +| `tracestate` | both directions, whenever a valid context exists after §3.3 — except a bypassed exchange (§6), a not-yet-ready producer, or an inbound `tracestate` that refuses the insert (malformed member; skipped with a WARN) | the caller's members with `lineage-parent` set to this request span id | | `traceparent` | only when the request carried no valid one and `mint_traceparent` is on | `00---` | Nothing else is written. The listener is responsible for carrying these header mutations to the @@ -148,6 +148,10 @@ instead of being welded by a guess. The consequences per case: | caller sends no valid context, application propagates | none (restarted) | stamp: one tree under the entry | | caller sends no valid context, application does not | none (restarted) | wire: each call a trace of its own | +A bypassed exchange (§6) is a fifth case: no spans at that element and the stamp passes through +unchanged, so the next element parents on the last element that did stamp — the bypassed hop is +simply absent from the chain. + ## 4. Attributes Resource attributes: `service.name=authbridge`, `authbridge.component=lineage-telemetry`. Both are @@ -168,7 +172,7 @@ handles `openinference.span.kind`. | `lineage.parent.source` | request | `tracestate` \| `wire` \| `none` | which precedence in §3.2 chose the parent. An audit fact; the consumer derives nothing from it | | `http.method` | request, when the listener supplies it | `POST` | all listeners do | | `url.path` | request, when present | `/mcp` | query-free: anything from `?` on is stripped before emission (per OTel semconv; the query can carry secrets and is never captured) | -| `url.scheme` | request, when present | `http` | the listener's observed scheme. Optional: the consumer composes `scheme://peer.host + url.path` only when all three exist | +| `url.scheme` | request, when present | `http` | the listener's observed scheme; `tcp` marks a proxy-sidecar CONNECT tunnel (§5 Limits). Optional: the consumer composes `scheme://peer.host + url.path` only when all three exist | | `a2a.method`, `a2a.session_id` | request, a2a | `message/send` | parsed facts | | `mcp.method`, `mcp.tool` | request, mcp | `tools/call`, `get_weather` | `mcp.tool` only for `tools/call` | | `inference.model` | request, inference | `qwen2.5:7b` | from the parsed request body | @@ -209,8 +213,12 @@ construction. so the loss is visible in the span. A truncated value no longer parses as JSON; the consumer then stores it as a string. Deployments that want whole prompts set `max_payload_bytes: -1` or raise it (LLM chat prompts on the reference fleet reach 14 KB; a third exceed the 4096 default). -- TLS-passthrough connections bypass the HTTP pipeline entirely and produce no exchange. That is a - capture gap, not a derivation rule: once such a connection is seen, the same rules apply. +- In envoy-sidecar mode, TLS-passthrough connections bypass Envoy's HTTP filter chain entirely and + produce no exchange — a capture gap. In proxy-sidecar mode an HTTPS destination is a CONNECT + tunnel, which IS an exchange: an ordinary span pair with `http.method=CONNECT`, + `url.scheme=tcp`, `lineage.peer.host` naming the dial target, no path and no payload — the bytes + inside the tunnel are opaque. The producer does not filter tunnel exchanges; what they mean is + the consumer's call, like every other fact. ## 6. Producer configuration From d82b941a75aa5933dc0995650737abbcd61601d0 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 13:45:19 +0300 Subject: [PATCH 29/37] Fix: Sample unconditionally so a caller cannot suppress spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A valid traceparent with the sampled-out flag (-00) exported zero spans: the SDK-default ParentBased sampler honors the caller's decision, and every peer sidecar inherits it, so one flag byte at the fleet entry silenced the whole chain — silently (a dropped span is non-recording; nothing logs). Lineage is an audit record, and a caller-chosen flag is not an opt-out from being graphed — the same posture that made bypass_hosts outbound-only. Set AlwaysSample explicitly (extracted into newTracerProvider so the test exercises the wiring Init installs). The forwarded traceparent keeps the caller's flags — a valid one is never modified; only what this producer exports ignores them. Contract v1.6.2 §2. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 26 +++++++++++------- .../authlib/plugins/lineage/plugin_test.go | 27 +++++++++++++++++++ authbridge/docs/lineage-wire-contract.md | 8 +++++- 3 files changed, 51 insertions(+), 10 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 3ec27d493..c585d19f2 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -296,15 +296,7 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { res = resource.Default() } - // No sampler is set, so the SDK default applies: ParentBased(AlwaysSample), - // overridable through OTEL_TRACES_SAMPLER. The sampling flag of a root - // span is what a minted traceparent carries downstream (mintTraceparent), - // and every peer sidecar is ParentBased too — a ratio sampler here would - // silently un-sample whole chains, not just this pod's spans. - p.tp = sdktrace.NewTracerProvider( - sdktrace.WithBatcher(&exportObserver{SpanExporter: exporter, failures: &p.exportFailures}), - sdktrace.WithResource(res), - ) + p.tp = newTracerProvider(&exportObserver{SpanExporter: exporter, failures: &p.exportFailures}, res) p.tracer = p.tp.Tracer("authbridge/" + pluginName) p.ready.Store(true) @@ -312,6 +304,22 @@ func (p *LineageTelemetry) Init(ctx context.Context) error { return nil } +// newTracerProvider builds the provider Init installs. AlwaysSample is +// explicit and deliberate: lineage is an audit record, and under the SDK +// default ParentBased sampler a caller sending a valid traceparent with the +// sampled-out flag (…-00) suppressed both spans — the same caller-controlled +// opt-out from being graphed that the inbound bypass_hosts decision refuses. +// The forwarded traceparent keeps the caller's flags (a valid one is never +// modified); only what this producer exports ignores them. An explicit +// sampler also means OTEL_TRACES_SAMPLER no longer overrides it. +func newTracerProvider(exporter sdktrace.SpanExporter, res *resource.Resource) *sdktrace.TracerProvider { + return sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + ) +} + // isLoopback reports whether endpoint (host:port, already reduced from any URL // form by decodeConfig) names this pod. Loopback traffic never leaves the // network namespace, so cleartext there carries no exposure and earns no WARN. diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index cff585c0d..bce47145a 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -24,6 +24,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" @@ -914,6 +915,32 @@ func TestConfig_MaxAttrBytes(t *testing.T) { } } +// A valid traceparent with the sampled-out flag (…-00) must not suppress the +// exchange: lineage is an audit record, and a caller-chosen flag is not an +// opt-out from being graphed. Built on the provider Init installs — the +// other tests' newTestPlugin provider has no sampler and would pass or fail +// on the SDK default instead. +func TestSampledOutParentStillExports(t *testing.T) { + exp := tracetest.NewInMemoryExporter() + tp := newTracerProvider(exp, resource.Empty()) + p := NewLineageTelemetry() + p.cfg = defaultConfig() + p.tp = tp + p.tracer = tp.Tracer("test") + p.selfID = "weather-service" + p.ready.Store(true) + + h := http.Header{} + h.Set("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00") + run(t, p, fakeContext(pipeline.Inbound, h), allow(200)) + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("ForceFlush: %v", err) + } + if got := len(exp.GetSpans()); got != 2 { + t.Fatalf("exported %d spans for a sampled-out parent, want 2", got) + } +} + // ---- the forbidden-keys guard ---- // TestForbiddenKeysNeverEmitted scans every attribute of every span emitted diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 617a49188..65c986715 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -52,6 +52,10 @@ One HTTP exchange through the sidecar produces two OTLP spans. - The response span is emitted at stream end **even when no response was produced** — client disconnect, upstream reset, plugin denial. It then carries `lineage.outcome` and whatever status exists, so the row completes as failed instead of dangling. +- The producer samples unconditionally: a valid caller `traceparent` with the sampled-out flag + (`…-00`) does not suppress the spans — lineage is an audit record, and a caller-chosen flag is + not an opt-out from being graphed. The forwarded `traceparent` keeps the caller's flags (§3.3: a + valid one is never modified); only what this producer exports ignores them. - A lone request span means one of three things: the sidecar died mid-exchange; a panic while emitting the response span was recovered by the pipeline (a WARN is logged); or the response span was emitted but lost — the two halves enter a batching exporter an exchange apart, so a response @@ -294,7 +298,9 @@ mechanisms named as removed are not to be reintroduced. pseudo-header put the query string on the wire regardless of `capture_io`; the proxy listeners never delivered it. And every variable-content string attribute plus the span name is capped at `max_attr_bytes` (default 256) — until now only the two payload values were bounded, so one - request could put a 100 KB span name into the backend. + request could put a 100 KB span name into the backend. And the producer samples unconditionally + (§2): under the SDK-default ParentBased sampler a caller's sampled-out `traceparent` (`…-00`) + exported zero spans for the whole chain. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only From 776b85721529969ae2b8814f9bfd3196270086f9 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 14:33:05 +0300 Subject: [PATCH 30/37] Feat: Expose the config schema for catalog tooling Eight sibling plugins implement pipeline.SchemaProvider; lineage was the only configurable plugin without it, so its eleven operator keys were invisible to /v1/plugins, /v1/pipeline and abctl. Add the one-line ConfigSchema() delegation and the description/default struct tags the siblings carry. The test pins the schema to Config's JSON keys so a future key added without a description fails. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 22 ++++++------- authbridge/authlib/plugins/lineage/plugin.go | 20 ++++++++---- .../authlib/plugins/lineage/plugin_test.go | 32 +++++++++++++++++++ 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index 71e735a3d..0366ecc6a 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -39,7 +39,7 @@ type Config struct { // https://host:port). An https:// scheme implies OTelTLS=true. Any other // URL scheme is rejected at decode (see decodeConfig). // Default: "localhost:4317" - OTelEndpoint string `json:"otel_endpoint"` + OTelEndpoint string `json:"otel_endpoint" description:"OTLP gRPC target: host:port, http://host:port or https://host:port; any other scheme is refused." default:"localhost:4317"` // OTelTLS selects the OTLP transport. False (the default) dials plaintext, // which is correct for the in-pod loopback collector but sends spans — @@ -53,7 +53,7 @@ type Config struct { // an explicit otel_tls:false, and an http:// endpoint with otel_tls:true // or OTelCAFile — the scheme states a transport intent, and the knobs // must agree with it. - OTelTLS bool `json:"otel_tls"` + OTelTLS bool `json:"otel_tls" description:"Dial the collector with TLS, verified against the system roots or otel_ca_file; an https:// endpoint implies it." default:"false"` // OTelCAFile is a PEM bundle of CA certificates to verify the collector's // serving certificate against, for a collector whose certificate is not @@ -64,7 +64,7 @@ type Config struct { // pool the dial verifies against: an unreadable file, or one with no // certificate in it, refuses to start rather than falling back to the // system roots. Empty (the default) verifies against the system roots. - OTelCAFile string `json:"otel_ca_file"` + OTelCAFile string `json:"otel_ca_file" description:"PEM bundle to verify the collector certificate against (a private CA); implies otel_tls."` // CaptureIO when true attaches parsed request/response content as // input.value (request span) and output.value (response span) @@ -76,7 +76,7 @@ type Config struct { // // Off by default — enable only if traces do not contain PII or the // OTel backend enforces appropriate access controls. - CaptureIO bool `json:"capture_io"` + CaptureIO bool `json:"capture_io" description:"Attach parsed request/response content as input.value / output.value." default:"false"` // MaxPayloadBytes caps the size of the input.value / output.value // attributes attached under CaptureIO. A payload longer than this is cut on @@ -89,7 +89,7 @@ type Config struct { // and any other negative is refused at decode. // Ignored when CaptureIO is false. // Default: 4096 - MaxPayloadBytes int `json:"max_payload_bytes"` + MaxPayloadBytes int `json:"max_payload_bytes" description:"Byte cap on input.value / output.value; 0 or unset takes the default, -1 attaches whole values." default:"4096"` // MaxAttrBytes caps every variable-content string attribute (url.path, // lineage.peer.host, mcp.tool, a2a.session_id, …) and the span name, cut @@ -99,7 +99,7 @@ type Config struct { // construction. Zero (or unset) uses defaultMaxAttrBytes; -1 removes the // cap, and any other negative is refused at decode. // Default: 256 - MaxAttrBytes int `json:"max_attr_bytes"` + MaxAttrBytes int `json:"max_attr_bytes" description:"Byte cap on every variable-content string attribute and the span name; 0 or unset takes the default, -1 removes the cap." default:"256"` // MintTraceparent — both directions — forwards a W3C traceparent naming // this exchange's request span when the request arrived with no @@ -114,7 +114,7 @@ type Config struct { // Set false for a pure observer that must not add a header the // application would see (the exchange then fragments, visibly). // Default: true - MintTraceparent bool `json:"mint_traceparent"` + MintTraceparent bool `json:"mint_traceparent" description:"Forward a traceparent naming this request span when no valid one arrived; false = a pure observer that writes no traceparent." default:"true"` // BypassPaths lists URL path prefixes that should not generate lineage // hops. Useful for suppressing infrastructure polling (agent-card @@ -128,7 +128,7 @@ type Config struct { // "/", is refused at decode because it would match every path and // silently turn the plugin off. // Default: ["/.well-known/", "/healthz", "/readyz", "/health"] - BypassPaths []string `json:"bypass_paths"` + BypassPaths []string `json:"bypass_paths" description:"URL path prefixes that produce no spans; setting the key replaces the default list." default:"/.well-known/, /healthz, /readyz, /health"` // BypassHosts lists host globs whose exchanges should not generate lineage // hops. Useful for suppressing infrastructure outbound calls such as OTel @@ -147,7 +147,7 @@ type Config struct { // empty, "*", or not valid path.Match syntax is refused at decode. // Default: ["otel-collector", "otel-collector.*", "jaeger", "jaeger.*", // "zipkin", "zipkin.*", "prometheus", "prometheus.*"] - BypassHosts []string `json:"bypass_hosts"` + BypassHosts []string `json:"bypass_hosts" description:"Outbound host globs (path.Match, port stripped, case folded) that produce no spans; ignored inbound; replaces the default list." default:"otel-collector, otel-collector.*, jaeger, jaeger.*, zipkin, zipkin.*, prometheus, prometheus.*"` // SelfID is the agent's own stable identifier, emitted as the // lineage.self.id fact on every span. Typically the Keycloak client ID @@ -155,12 +155,12 @@ type Config struct { // containing "/" (a SPIFFE ID) is reduced to its last non-empty path // segment before emission — see serviceLabel — so two identities that // differ only above that segment emit the same lineage.self.id. - SelfID string `json:"self_id"` + SelfID string `json:"self_id" description:"This workload identity, emitted as lineage.self.id; a SPIFFE ID is reduced to its last path segment."` // SelfIDFile is the path to a file containing the agent's own client ID. // Defaults to /shared/client-id.txt (the operator-mounted credential). // Ignored when SelfID is set. - SelfIDFile string `json:"self_id_file"` + SelfIDFile string `json:"self_id_file" description:"Read when self_id is empty; the plugin refuses to start if neither yields an identity." default:"/shared/client-id.txt"` } func defaultConfig() Config { diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index c585d19f2..86ef58af6 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -171,6 +171,13 @@ func NewLineageTelemetry() *LineageTelemetry { func (p *LineageTelemetry) Name() string { return pluginName } +// ConfigSchema exposes the Config fields for schema-aware tooling +// (/v1/plugins, /v1/pipeline, abctl edit templates), the same one-line +// delegation every configurable sibling plugin uses. +func (p *LineageTelemetry) ConfigSchema() []pipeline.FieldSchema { + return pipeline.SchemaOf(Config{}) +} + func (p *LineageTelemetry) Capabilities() pipeline.PluginCapabilities { return pipeline.PluginCapabilities{ // At least one protocol parser must be present and earlier in the @@ -1040,10 +1047,11 @@ func ioOutputValue(pctx *pipeline.Context, protocol string) string { // Compile-time interface assertions. var ( - _ pipeline.Plugin = (*LineageTelemetry)(nil) - _ pipeline.Configurable = (*LineageTelemetry)(nil) - _ pipeline.Initializer = (*LineageTelemetry)(nil) - _ pipeline.Shutdowner = (*LineageTelemetry)(nil) - _ pipeline.Finisher = (*LineageTelemetry)(nil) - _ pipeline.Readier = (*LineageTelemetry)(nil) + _ pipeline.Plugin = (*LineageTelemetry)(nil) + _ pipeline.Configurable = (*LineageTelemetry)(nil) + _ pipeline.Initializer = (*LineageTelemetry)(nil) + _ pipeline.Shutdowner = (*LineageTelemetry)(nil) + _ pipeline.Finisher = (*LineageTelemetry)(nil) + _ pipeline.Readier = (*LineageTelemetry)(nil) + _ pipeline.SchemaProvider = (*LineageTelemetry)(nil) ) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index bce47145a..c343f9769 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -15,6 +15,7 @@ import ( "math/big" "net/http" "os" + "reflect" "slices" "strings" "sync/atomic" @@ -941,6 +942,37 @@ func TestSampledOutParentStillExports(t *testing.T) { } } +// The schema must track Config exactly — every operator key present, each +// with a description — so /v1/plugins and abctl never render a blank field, +// and a twelfth key added without a description goes red here. +func TestConfigSchema_TracksConfig(t *testing.T) { + schema := NewLineageTelemetry().ConfigSchema() + byName := map[string]pipeline.FieldSchema{} + for _, f := range schema { + byName[f.Name] = f + } + cfgType := reflect.TypeOf(Config{}) + keys := 0 + for i := 0; i < cfgType.NumField(); i++ { + key, _, _ := strings.Cut(cfgType.Field(i).Tag.Get("json"), ",") + if key == "" || key == "-" { + continue + } + keys++ + f, ok := byName[key] + if !ok { + t.Errorf("schema missing config key %q", key) + continue + } + if f.Description == "" { + t.Errorf("config key %q has no description tag", key) + } + } + if len(schema) != keys { + t.Errorf("schema has %d fields, Config has %d json keys", len(schema), keys) + } +} + // ---- the forbidden-keys guard ---- // TestForbiddenKeysNeverEmitted scans every attribute of every span emitted From 4ae2caa559e89e408f0054d21a3d4cc19d144756 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 14:53:29 +0300 Subject: [PATCH 31/37] Fix: Record modify when the stamp rewrote the message MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OnRequest recorded observe while rewriting the forwarded tracestate on nearly every exchange — the repo's invocation vocabulary defines observe as attaching data without changing the message, and modify as mutating it (cpex, the other header-writing plugin, records modify). restampTracestate now reports whether it wrote, which is exactly whether the message was mutated: whenever mintTraceparent writes, the restamp that follows cannot fail (a minted context's TraceState is empty). A pure observer, or a refused Insert, still records observe — so the mint_traceparent knob's effect is visible in the abctl timeline. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 24 +++++++++++++++---- .../authlib/plugins/lineage/plugin_test.go | 24 +++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 86ef58af6..262cc2130 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -461,7 +461,7 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context reqCtx := p.emitRequestSpan(parent, spanName, spanKind, reqAttrs) exchangeID := reqCtx.SpanID().String() remoteCtx = p.mintTraceparent(ctx, pctx, remoteCtx, reqCtx) - restampTracestate(pctx, remoteCtx, exchangeID) + stamped := restampTracestate(pctx, remoteCtx, exchangeID) common := make([]attribute.KeyValue, 0, len(base)+1) common = append(common, base...) @@ -474,7 +474,15 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context spanName: spanName, protocol: protocol, }) - pctx.Observe("recorded_request") + // modify vs observe follows what actually happened to the message: the + // stamp (and any minted traceparent under it) is a header rewrite; when + // nothing was written — pure-observer config, or a refused Insert — the + // exchange was only recorded. + if stamped { + pctx.Modify("stamped_tracestate") + } else { + pctx.Observe("recorded_request") + } return pipeline.Action{Type: pipeline.Continue} } @@ -559,10 +567,15 @@ func (p *LineageTelemetry) mintTraceparent(ctx context.Context, pctx *pipeline.C // semantics, not an accident. The listener is // responsible for propagating these header mutations (ext_proc emits a // SetHeaders diff). -func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchangeID string) { +// +// Reports whether it wrote — which is exactly whether the plugin mutated the +// forwarded message: whenever mintTraceparent wrote, the restamp that follows +// succeeds too (a minted context's TraceState is empty, so Insert cannot +// fail), so a false here means no header of either kind was written. +func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchangeID string) bool { rsc := trace.SpanContextFromContext(remoteCtx) if !rsc.IsValid() { - return + return false } ts, err := rsc.TraceState().Insert(tracestateStampKey, exchangeID) if err != nil { @@ -571,9 +584,10 @@ func restampTracestate(pctx *pipeline.Context, remoteCtx context.Context, exchan // is indistinguishable from "app has no shim". slog.Warn("lineage-telemetry: tracestate stamp rejected; the next element will attribute as wire", "exchange_id", exchangeID, "error", err) - return + return false } pctx.Headers.Set("tracestate", ts.String()) + return true } // matchesAnyHost reports whether host matches any configured bypass_hosts diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index c343f9769..b467a3ab4 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -973,6 +973,30 @@ func TestConfigSchema_TracksConfig(t *testing.T) { } } +// The invocation action follows what happened to the message: the tracestate +// stamp is a header rewrite (modify); a pure observer that wrote nothing +// records observe. The repo's action vocabulary is operator-facing. +func TestInvocationAction_ModifyOnlyWhenStamped(t *testing.T) { + // Default config, valid inbound context: the stamp is written. + p, _ := newTestPlugin(t) + pctx := fakeContext(pipeline.Inbound, traceparent("4bf92f3577b34da6a3ce929d0e0e4736", "00f067aa0ba902b7")) + run(t, p, pctx, allow(200)) + inv := pctx.Extensions.Invocations.Inbound[0] + if string(inv.Action) != "modify" { + t.Errorf("stamped exchange action = %q, want modify", inv.Action) + } + + // Pure observer: mint off, nothing valid on the wire — nothing written. + p2, _ := newTestPlugin(t) + p2.cfg.MintTraceparent = false + pctx2 := fakeContext(pipeline.Inbound, http.Header{}) + run(t, p2, pctx2, allow(200)) + inv2 := pctx2.Extensions.Invocations.Inbound[0] + if string(inv2.Action) != "observe" { + t.Errorf("unstamped exchange action = %q, want observe", inv2.Action) + } +} + // ---- the forbidden-keys guard ---- // TestForbiddenKeysNeverEmitted scans every attribute of every span emitted From c65fd4ce613711bb58a97fd86c9b31ca140df5fa Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 17:13:14 +0300 Subject: [PATCH 32/37] Fix: Match bypass_paths as globs via the shared bypass matcher MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bypass_paths was a hand-rolled prefix match while the same key is a path.Match glob in ibac, sparc and cpex — and the repo already has the shared bypass package (built for jwt-validation) doing exactly this job, with boot-time validation, query stripping and path normalization. Two measured consequences of the divergence: the /health default prefix silently swallowed /health-records/... (real traffic exempt from being graphed), and a glob copied from a sibling config (/.well-known/*) could never match and was accepted without a word. Build a bypass.Matcher in Configure, the same wiring jwt-validation and sparc use; defaults become the glob shape. Contract v1.6.2 and catalog updated. This completes for paths the same convention move review round 4 made for bypass_hosts. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/config.go | 42 +++++++++--------- authbridge/authlib/plugins/lineage/plugin.go | 35 +++++++++------ .../authlib/plugins/lineage/plugin_test.go | 43 +++++++++++++++---- authbridge/docs/lineage-wire-contract.md | 15 ++++--- authbridge/docs/plugin-catalog.md | 2 +- 5 files changed, 88 insertions(+), 49 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/config.go b/authbridge/authlib/plugins/lineage/config.go index 0366ecc6a..8b32992e7 100644 --- a/authbridge/authlib/plugins/lineage/config.go +++ b/authbridge/authlib/plugins/lineage/config.go @@ -116,19 +116,25 @@ type Config struct { // Default: true MintTraceparent bool `json:"mint_traceparent" description:"Forward a traceparent naming this request span when no valid one arrived; false = a pure observer that writes no traceparent." default:"true"` - // BypassPaths lists URL path prefixes that should not generate lineage + // BypassPaths lists URL path globs that should not generate lineage // hops. Useful for suppressing infrastructure polling (agent-card // discovery, health checks) that would otherwise flood the lineage graph. - // Prefixes, not globs — a path is bypassed when it starts with an entry. + // Matched by the shared bypass package (path.Match, query stripped, path + // normalized) — the same package and semantics jwt-validation and sparc + // use for this key, so a pattern copied between plugins means the same + // thing. Note path.Match's "*" does not cross "/": "/.well-known/*" + // matches "/.well-known/agent.json" but not "/.well-known/a/b". An + // earlier prefix match here silently bypassed real traffic under the + // "/health" default ("/health-records/..."). // // Setting the key REPLACES this list rather than extending it, the same // convention ibac, sparc and cpex use for their bypass keys: an operator - // who adds one prefix must restate the defaults they want to keep. - // Entries are trimmed of surrounding whitespace; one that is empty, or - // "/", is refused at decode because it would match every path and - // silently turn the plugin off. - // Default: ["/.well-known/", "/healthz", "/readyz", "/health"] - BypassPaths []string `json:"bypass_paths" description:"URL path prefixes that produce no spans; setting the key replaces the default list." default:"/.well-known/, /healthz, /readyz, /health"` + // who adds one glob must restate the defaults they want to keep. + // Entries are trimmed of surrounding whitespace; bypass.NewMatcher + // refuses invalid path.Match syntax and match-everything patterns + // (empty, "*", "/*") at boot. + // Default: ["/.well-known/*", "/healthz", "/readyz", "/health"] + BypassPaths []string `json:"bypass_paths" description:"URL path globs (path.Match) that produce no spans; setting the key replaces the default list." default:"/.well-known/*, /healthz, /readyz, /health"` // BypassHosts lists host globs whose exchanges should not generate lineage // hops. Useful for suppressing infrastructure outbound calls such as OTel @@ -169,7 +175,7 @@ func defaultConfig() Config { MaxPayloadBytes: defaultMaxPayloadBytes, MaxAttrBytes: defaultMaxAttrBytes, MintTraceparent: true, - BypassPaths: []string{"/.well-known/", "/healthz", "/readyz", "/health"}, + BypassPaths: []string{"/.well-known/*", "/healthz", "/readyz", "/health"}, BypassHosts: []string{ "otel-collector", "otel-collector.*", "jaeger", "jaeger.*", @@ -260,21 +266,17 @@ func decodeConfig(raw json.RawMessage) (Config, error) { return cfg, nil } -// validateBypass trims and checks both bypass lists in place. An entry that +// validateBypass trims and checks the bypass lists in place. An entry that // matches everything disables the plugin silently — every exchange takes the // skip, no span is ever emitted, and Ready() still reports true — so it is a -// boot error rather than a runtime surprise. ibac, sparc and cpex each refuse -// the same shapes with the same reasoning; the wording of the error mirrors -// theirs, including the advice to remove the plugin from the pipeline if -// disabling it is what was meant. +// boot error rather than a runtime surprise. Paths are only trimmed here: +// their validation (invalid path.Match syntax, match-everything patterns) +// lives in bypass.NewMatcher, called at Configure. Hosts are checked here, +// mirroring ibac / sparc / cpex, including the advice to remove the plugin +// from the pipeline if disabling it is what was meant. func validateBypass(cfg *Config) error { for i, entry := range cfg.BypassPaths { - entry = strings.TrimSpace(entry) - if entry == "" || entry == "/" { - return fmt.Errorf("bypass_paths entry %q matches every path; "+ - "to disable lineage-telemetry, remove it from the pipeline instead", cfg.BypassPaths[i]) - } - cfg.BypassPaths[i] = entry + cfg.BypassPaths[i] = strings.TrimSpace(entry) } for i, entry := range cfg.BypassHosts { entry = strings.TrimSpace(entry) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 262cc2130..3aca35d1a 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -74,6 +74,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + "github.com/rossoctl/cortex/authbridge/authlib/bypass" "github.com/rossoctl/cortex/authbridge/authlib/pipeline" "github.com/rossoctl/cortex/authbridge/authlib/plugins" ) @@ -146,13 +147,14 @@ type exchangeState struct { // LineageTelemetry emits OTel spans for each request hop observed by authbridge. type LineageTelemetry struct { - cfg Config - tp *sdktrace.TracerProvider - tracer trace.Tracer - conn *grpc.ClientConn // OTLP gRPC connection; owned by us, closed on Shutdown - ready atomic.Bool - propagator propagation.TextMapPropagator - selfID string // agent's own client ID for the lineage.self.id fact + cfg Config + bypassPaths *bypass.Matcher // built in Configure from cfg.BypassPaths + tp *sdktrace.TracerProvider + tracer trace.Tracer + conn *grpc.ClientConn // OTLP gRPC connection; owned by us, closed on Shutdown + ready atomic.Bool + propagator propagation.TextMapPropagator + selfID string // agent's own client ID for the lineage.self.id fact // exportFailures counts batches the collector refused or never received. // Export is asynchronous and the dial is lazy, so this — with the WARN // exportObserver logs — is how an unreachable collector or a TLS chain @@ -200,7 +202,16 @@ func (p *LineageTelemetry) Configure(raw json.RawMessage) error { if err != nil { return err } + // The shared bypass matcher (path.Match globs, path normalization) — the + // same package and wiring jwt-validation and sparc use for this key. It + // validates at boot: invalid syntax and match-everything patterns refuse + // to start. + matcher, err := bypass.NewMatcher(cfg.BypassPaths) + if err != nil { + return fmt.Errorf("lineage-telemetry bypass_paths: %w", err) + } p.cfg = cfg + p.bypassPaths = matcher return nil } @@ -410,12 +421,10 @@ func (p *LineageTelemetry) OnRequest(ctx context.Context, pctx *pipeline.Context return pipeline.Action{Type: pipeline.Continue} } - // Skip infrastructure paths (health checks, agent-card discovery, etc.) - for _, prefix := range p.cfg.BypassPaths { - if strings.HasPrefix(pctx.Path, prefix) { - pctx.Skip("bypass_path") - return pipeline.Action{Type: pipeline.Continue} - } + // Skip infrastructure paths (health checks, agent-card discovery, etc.). + if p.bypassPaths.Match(pctx.Path) { + pctx.Skip("bypass_path") + return pipeline.Action{Type: pipeline.Continue} } // Skip infrastructure outbound targets (OTel exporters, metrics scrapers, etc.). diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index b467a3ab4..34bfab626 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -30,6 +30,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" + "github.com/rossoctl/cortex/authbridge/authlib/bypass" "github.com/rossoctl/cortex/authbridge/authlib/pipeline" ) @@ -42,6 +43,7 @@ func newTestPlugin(t *testing.T) (*LineageTelemetry, *tracetest.InMemoryExporter tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) p := NewLineageTelemetry() p.cfg = defaultConfig() // the shipped defaults, so a test sees what a deployment sees + p.bypassPaths, _ = bypass.NewMatcher(p.cfg.BypassPaths) p.tp = tp p.tracer = tp.Tracer("test") p.selfID = "weather-service" @@ -926,6 +928,7 @@ func TestSampledOutParentStillExports(t *testing.T) { tp := newTracerProvider(exp, resource.Empty()) p := NewLineageTelemetry() p.cfg = defaultConfig() + p.bypassPaths, _ = bypass.NewMatcher(p.cfg.BypassPaths) p.tp = tp p.tracer = tp.Tracer("test") p.selfID = "weather-service" @@ -1294,21 +1297,27 @@ func TestConfig_UnknownKeysRefused(t *testing.T) { // anywhere: a matched hop is simply absent from the graph. So both directions // are pinned — a match emits nothing, a near-miss emits the full pair. -func TestBypassPaths_PrefixMatchEmitsNothing(t *testing.T) { +func TestBypassPaths_GlobMatchEmitsNothing(t *testing.T) { cases := []struct { name string path string spans int // spans expected from the exchange }{ - {"prefix match skipped", "/health/live", 0}, - {"exact prefix skipped", "/health", 0}, + {"exact match skipped", "/health", 0}, + {"glob matches one level", "/.well-known/agent.json", 0}, + {"glob does not cross /", "/.well-known/a/b", 2}, + {"over-match fixed: not a prefix rule", "/health-records/patient/42", 2}, + {"non-canonical form normalized", "//health", 0}, {"non-matching path emits", "/api/health-report", 2}, - {"prefix is anchored, not substring", "/v1/health", 2}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { p, exp := newTestPlugin(t) - p.cfg.BypassPaths = []string{"/health"} + // Through Configure, so the test exercises the same NewMatcher + // wiring a deployment gets (jwt-validation and sparc shape). + if err := p.Configure(json.RawMessage(`{"bypass_paths": ["/.well-known/*", "/health"]}`)); err != nil { + t.Fatalf("Configure: %v", err) + } pctx := fakeContext(pipeline.Inbound, http.Header{}) pctx.Path = tc.path run(t, p, pctx, allow(200)) @@ -1374,13 +1383,10 @@ func TestConfig_BypassEntriesValidated(t *testing.T) { name string raw string }{ - {"empty path", `{"bypass_paths": ["/healthz", ""]}`}, - {"whitespace-only path", `{"bypass_paths": [" "]}`}, - {"root path", `{"bypass_paths": ["/"]}`}, {"empty host", `{"bypass_hosts": ["jaeger", ""]}`}, {"whitespace-only host", `{"bypass_hosts": [" "]}`}, {"star host", `{"bypass_hosts": ["*"]}`}, - {"invalid glob", `{"bypass_hosts": ["[unclosed"]}`}, + {"invalid host glob", `{"bypass_hosts": ["[unclosed"]}`}, } for _, tc := range refused { t.Run(tc.name, func(t *testing.T) { @@ -1389,6 +1395,25 @@ func TestConfig_BypassEntriesValidated(t *testing.T) { } }) } + // Path entries are validated by bypass.NewMatcher at Configure — the + // shared package jwt-validation and sparc validate with. + refusedPaths := []struct { + name string + raw string + }{ + {"empty path", `{"bypass_paths": ["/healthz", ""]}`}, + {"whitespace-only path", `{"bypass_paths": [" "]}`}, + {"star path", `{"bypass_paths": ["*"]}`}, + {"slash-star path", `{"bypass_paths": ["/*"]}`}, + {"invalid path glob", `{"bypass_paths": ["[unclosed"]}`}, + } + for _, tc := range refusedPaths { + t.Run(tc.name, func(t *testing.T) { + if err := NewLineageTelemetry().Configure(json.RawMessage(tc.raw)); err == nil { + t.Fatalf("%s accepted; it disables the plugin silently", tc.raw) + } + }) + } cfg, err := decodeConfig([]byte(`{"bypass_hosts": [" jaeger.* "], "bypass_paths": [" /healthz "]}`)) if err != nil { diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 65c986715..5e310b09f 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -67,8 +67,9 @@ One HTTP exchange through the sidecar produces two OTLP spans. span exists emits **no spans at all** and is invisible to lineage. `denied` appears only for denials after the request span exists: response-phase denials, or gates ordered after lineage. Spans for gate-denied traffic are a named follow-up, not current behaviour. -- **Bypass.** Requests whose path starts with a `bypass_paths` prefix, and outbound requests whose - host matches a `bypass_hosts` glob (`path.Match`, port stripped, case folded), produce no spans +- **Bypass.** Requests whose path matches a `bypass_paths` glob, and outbound requests whose + host matches a `bypass_hosts` glob (both `path.Match`; hosts port-stripped and case folded, + paths query-stripped and normalized), produce no spans (defaults in §6). `bypass_hosts` is outbound-only: an inbound `Host` is the caller's own header, so honouring it there would let a caller suppress the record of its own request. @@ -235,7 +236,7 @@ construction. | `max_payload_bytes` | `4096` | producer-side cap on those two values; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start | | `max_attr_bytes` | `256` | cap on every variable-content string attribute and the span name (§4); same `0` / `-1` / negative semantics as `max_payload_bytes` | | `mint_traceparent` | `true` | §3.3; `false` = a pure observer that never writes a `traceparent` | -| `bypass_paths` | `/.well-known/`, `/healthz`, `/readyz`, `/health` | path prefixes that produce no spans | +| `bypass_paths` | `/.well-known/*`, `/healthz`, `/readyz`, `/health` | path globs (`path.Match`; `*` does not cross `/`) that produce no spans, matched by the shared bypass package (query stripped, path normalized) — the same key and semantics as `jwt-validation` and `sparc` | | `bypass_hosts` | `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*` | outbound host globs that produce no spans | | `self_id` | — | this workload's identity (§4: reduced to its last `/`-segment) | | `self_id_file` | `/shared/client-id.txt` | read when `self_id` is empty; the producer refuses to start if neither yields an identity | @@ -243,8 +244,8 @@ construction. Setting `bypass_paths` or `bypass_hosts` **replaces** the default list rather than extending it — the convention the `ibac`, `sparc` and `cpex` plugins use for their keys of the same name. An operator who adds one entry must restate the defaults they want kept. An entry that would match -everything (empty, whitespace-only, `/` for a path, `*` for a host) is refused at start, as is a -host entry that is not valid `path.Match` syntax. +everything (empty, whitespace-only, `*`, `/*` for a path, `*` for a host) is refused at start, as +is an entry of either kind that is not valid `path.Match` syntax. Unknown keys are a boot error. @@ -300,7 +301,9 @@ mechanisms named as removed are not to be reintroduced. `max_attr_bytes` (default 256) — until now only the two payload values were bounded, so one request could put a 100 KB span name into the backend. And the producer samples unconditionally (§2): under the SDK-default ParentBased sampler a caller's sampled-out `traceparent` (`…-00`) - exported zero spans for the whole chain. + exported zero spans for the whole chain. And `bypass_paths` becomes a `path.Match` glob list via + the shared bypass matcher (it was a prefix match): the `/health` default no longer swallows + `/health-records/...`, and a pattern copied from a sibling plugin means the same thing here. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only diff --git a/authbridge/docs/plugin-catalog.md b/authbridge/docs/plugin-catalog.md index bc0d06fd1..a8864a02d 100644 --- a/authbridge/docs/plugin-catalog.md +++ b/authbridge/docs/plugin-catalog.md @@ -150,7 +150,7 @@ denial by a plugin ordered before it emits no spans. - `max_payload_bytes` (int) — cap on those two values, cut on a UTF-8 boundary with a `…[truncated]` marker; `0` or unset takes the default, `-1` attaches whole, any other negative is refused at start. Default `4096`. - `max_attr_bytes` (int) — cap on every variable-content string attribute (`url.path`, `lineage.peer.host`, `mcp.tool`, …) and the span name; same `0` / `-1` / negative semantics as `max_payload_bytes`. Default `256`. - `mint_traceparent` (bool) — forward a `traceparent` naming this request span when the request carried no valid one; `false` = a pure observer that writes no `traceparent`. Default `true`. -- `bypass_paths` (`[]string`) — path prefixes that produce no spans. Default `/.well-known/`, `/healthz`, `/readyz`, `/health`. Setting either bypass key replaces its default list rather than extending it, as in `ibac` / `sparc` / `cpex`; an entry matching everything is refused at start. +- `bypass_paths` (`[]string`) — path globs (`path.Match`, query stripped, path normalized — the shared bypass matcher `jwt-validation` and `sparc` use) that produce no spans. Default `/.well-known/*`, `/healthz`, `/readyz`, `/health`. Setting either bypass key replaces its default list rather than extending it, as in `ibac` / `sparc` / `cpex`; an entry matching everything is refused at start. - `bypass_hosts` (`[]string`) — outbound host globs (`path.Match`, port stripped, case folded) that produce no spans; ignored inbound, where `Host` is caller-controlled. Default `otel-collector`, `otel-collector.*`, `jaeger`, `jaeger.*`, `zipkin`, `zipkin.*`, `prometheus`, `prometheus.*`. - `self_id` (string) — this workload's identity, emitted as `lineage.self.id`. - `self_id_file` (string) — read when `self_id` is empty; the plugin refuses to start if neither yields an identity. Default `/shared/client-id.txt`. From 87019bb85164ad88de9374a3386cb9c2352d84d4 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 17:26:48 +0300 Subject: [PATCH 33/37] Docs: State the lineage.protocol precedence in the contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parsers are not mutually exclusive — mcp-parser attaches to any JSON-RPC body, including every a2a exchange — so the fixed precedence (a2a > mcp > inference) decides real classifications and keys the payload reduction, yet the contract's row read as if the label were unambiguous. Prose only; the behaviour is unchanged and as old as protocolOf's switch order. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/docs/lineage-wire-contract.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 5e310b09f..f9277f575 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -173,7 +173,7 @@ handles `openinference.span.kind`. | `lineage.direction` | both | `inbound` \| `outbound` | | | `lineage.self.id` | both | `weather-service` | this workload's identity, from `self_id` or `self_id_file`, **reduced to its last non-empty `/`-segment**: a SPIFFE ID `spiffe://td/ns/team1/sa/agent` emits `agent`, and two identities that differ only above that segment emit the same value — the consumer keys entity identity on it (§7). The producer refuses to start without an identity | | `lineage.peer.host` | both, when present | `weather-tool-mcp.team1.svc:8000` | the Host/authority header. Outbound: the service being called. Inbound: the address this workload was reached on | -| `lineage.protocol` | both | `a2a` \| `mcp` \| `inference` \| `http` | which parser matched; `http` = none | +| `lineage.protocol` | both | `a2a` \| `mcp` \| `inference` \| `http` | which parser matched, at fixed precedence `a2a` > `mcp` > `inference`; `http` = none. The precedence is load-bearing: the parsers are not mutually exclusive — `mcp-parser` attaches to any JSON-RPC body, including every a2a exchange — so an a2a hop is labeled `a2a`, never `mcp`. The payload reduction (§5) is keyed by this label, reading the same protocol's parser | | `lineage.parent.source` | request | `tracestate` \| `wire` \| `none` | which precedence in §3.2 chose the parent. An audit fact; the consumer derives nothing from it | | `http.method` | request, when the listener supplies it | `POST` | all listeners do | | `url.path` | request, when present | `/mcp` | query-free: anything from `?` on is stripped before emission (per OTel semconv; the query can carry secrets and is never captured) | @@ -304,6 +304,8 @@ mechanisms named as removed are not to be reintroduced. exported zero spans for the whole chain. And `bypass_paths` becomes a `path.Match` glob list via the shared bypass matcher (it was a prefix match): the `/health` default no longer swallows `/health-records/...`, and a pattern copied from a sibling plugin means the same thing here. + Prose: the `lineage.protocol` precedence (`a2a` > `mcp` > `inference`), always the producer's + behaviour, is stated in §4. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only From 3e9ae16083810db3010272ce22be0bd9b7bf5465 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 17:29:06 +0300 Subject: [PATCH 34/37] Docs: Add config hot-reload to the lone-request-span causes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Contract §2 called its lone-request-span list exhaustive with three causes, all crash-shaped. The fourth is routine: on a hot reload old pipelines stop a drain window (default 30 s) after the swap, and an exchange that outlives it — any SSE stream or slow LLM turn — emits its response span into the old, already-shut-down provider, where it is dropped. An operator following the list would hunt for a crash that never happened. Prose only; the consumer already renders the lone span as in-flight. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/docs/lineage-wire-contract.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index f9277f575..5e5ae1e91 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -56,10 +56,13 @@ One HTTP exchange through the sidecar produces two OTLP spans. (`…-00`) does not suppress the spans — lineage is an audit record, and a caller-chosen flag is not an opt-out from being graphed. The forwarded `traceparent` keeps the caller's flags (§3.3: a valid one is never modified); only what this producer exports ignores them. -- A lone request span means one of three things: the sidecar died mid-exchange; a panic while - emitting the response span was recovered by the pipeline (a WARN is logged); or the response span +- A lone request span means one of four things: the sidecar died mid-exchange; a panic while + emitting the response span was recovered by the pipeline (a WARN is logged); the response span was emitted but lost — the two halves enter a batching exporter an exchange apart, so a response - can be lost after its request has flushed. The consumer renders it as in-flight, never as a wrong + can be lost after its request has flushed; or the exchange outlived a config hot-reload — old + pipelines stop a drain window (default 30 s) after the swap, and a response span emitted into + the old, already-shut-down provider is dropped, which is routine for SSE or LLM exchanges + longer than the window. The consumer renders it as in-flight, never as a wrong pairing. A response span whose `lineage.outcome` is absent derives with `error` NULL (honest unknown), never `false`. - **Scope of `denied`.** The lineage plugin runs after the gate plugins and the pipeline @@ -305,7 +308,7 @@ mechanisms named as removed are not to be reintroduced. the shared bypass matcher (it was a prefix match): the `/health` default no longer swallows `/health-records/...`, and a pattern copied from a sibling plugin means the same thing here. Prose: the `lineage.protocol` precedence (`a2a` > `mcp` > `inference`), always the producer's - behaviour, is stated in §4. + behaviour, is stated in §4, and §2's lone-request-span causes gain config hot-reload. - **v1.6.1** — prose and configuration only; spans and wire unchanged. `lineage.self.id` is documented as reduced to its last `/`-segment before emission, which the producer has always done; `otel_ca_file` added for a collector under a private CA; `bypass_hosts` becomes an outbound-only From c59856dfbe42b6ee015358578c279dc565a4028a Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 17:55:00 +0300 Subject: [PATCH 35/37] Feat: Expose lineage.export_failures via MetricsProvider The export-failure counter is the operator signal for a collector outage (readiness deliberately does not follow the collector); surfacing it on /v1/pipeline was promised in review. Running total since Init; carries no request content, as that endpoint is unauthenticated. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/authlib/plugins/lineage/plugin.go | 28 ++++++++++++++----- .../authlib/plugins/lineage/plugin_test.go | 11 ++++++++ 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/authbridge/authlib/plugins/lineage/plugin.go b/authbridge/authlib/plugins/lineage/plugin.go index 3aca35d1a..4e40903a7 100644 --- a/authbridge/authlib/plugins/lineage/plugin.go +++ b/authbridge/authlib/plugins/lineage/plugin.go @@ -387,6 +387,19 @@ func (p *LineageTelemetry) Shutdown(ctx context.Context) error { // counter. func (p *LineageTelemetry) Ready() bool { return p.ready.Load() } +// Metrics exposes the export-failure counter through pipeline.MetricsProvider, +// so an operator can see a collector outage on /v1/pipeline rather than only in +// the logs. The counter is a running total since Init, not a rate. Name and +// Note carry no request or response content, as that endpoint is unauthenticated. +func (p *LineageTelemetry) Metrics() []pipeline.Metric { + return []pipeline.Metric{{ + Name: "lineage.export_failures", + Value: float64(p.exportFailures.Load()), + Unit: "count", + Note: "OTLP batches the collector refused or never received, since start", + }} +} + // exportObserver wraps the OTLP exporter so a failed export is visible from // this plugin — a plugin-namespaced WARN and a counter — instead of only // through the OTel SDK's default error handler on stderr. The error is @@ -1070,11 +1083,12 @@ func ioOutputValue(pctx *pipeline.Context, protocol string) string { // Compile-time interface assertions. var ( - _ pipeline.Plugin = (*LineageTelemetry)(nil) - _ pipeline.Configurable = (*LineageTelemetry)(nil) - _ pipeline.Initializer = (*LineageTelemetry)(nil) - _ pipeline.Shutdowner = (*LineageTelemetry)(nil) - _ pipeline.Finisher = (*LineageTelemetry)(nil) - _ pipeline.Readier = (*LineageTelemetry)(nil) - _ pipeline.SchemaProvider = (*LineageTelemetry)(nil) + _ pipeline.Plugin = (*LineageTelemetry)(nil) + _ pipeline.Configurable = (*LineageTelemetry)(nil) + _ pipeline.Initializer = (*LineageTelemetry)(nil) + _ pipeline.Shutdowner = (*LineageTelemetry)(nil) + _ pipeline.Finisher = (*LineageTelemetry)(nil) + _ pipeline.Readier = (*LineageTelemetry)(nil) + _ pipeline.SchemaProvider = (*LineageTelemetry)(nil) + _ pipeline.MetricsProvider = (*LineageTelemetry)(nil) ) diff --git a/authbridge/authlib/plugins/lineage/plugin_test.go b/authbridge/authlib/plugins/lineage/plugin_test.go index 34bfab626..f7efd33d9 100644 --- a/authbridge/authlib/plugins/lineage/plugin_test.go +++ b/authbridge/authlib/plugins/lineage/plugin_test.go @@ -1000,6 +1000,17 @@ func TestInvocationAction_ModifyOnlyWhenStamped(t *testing.T) { } } +// The metric name is a promise to operators (and was promised in review): +// the export-failure total is visible on /v1/pipeline as lineage.export_failures. +func TestMetrics_ExportFailures(t *testing.T) { + p := NewLineageTelemetry() + p.exportFailures.Store(7) + m := p.Metrics() + if len(m) != 1 || m[0].Name != "lineage.export_failures" || m[0].Value != 7 { + t.Fatalf("Metrics() = %+v, want one lineage.export_failures = 7", m) + } +} + // ---- the forbidden-keys guard ---- // TestForbiddenKeysNeverEmitted scans every attribute of every span emitted From df726ed1a11c4a3d4f880900ef2d29566996904d Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 18:50:16 +0300 Subject: [PATCH 36/37] Test: Acknowledge lineage in the lite-tags guard test TestDiscover_ExpectedTags exists to force acknowledgment when a new plugin changes the lite tag set; this branch's plugins_lineage.go makes lite-tags derive exclude_plugin_lineage, and the want string must say so. CI only go-runs the module, so the red test was invisible to the sweep. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/scripts/lite-tags/main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/authbridge/scripts/lite-tags/main_test.go b/authbridge/scripts/lite-tags/main_test.go index b68c258cb..81ecb0afd 100644 --- a/authbridge/scripts/lite-tags/main_test.go +++ b/authbridge/scripts/lite-tags/main_test.go @@ -16,7 +16,7 @@ func TestDiscover_ExpectedTags(t *testing.T) { t.Fatalf("discover: %v", err) } got := strings.Join(tags, ",") - want := "exclude_plugin_a2aparser,exclude_plugin_ibac,exclude_plugin_inferenceparser,exclude_plugin_mcpparser,exclude_plugin_opa,exclude_plugin_sparc,exclude_plugin_tokenbroker,exclude_plugin_toolprune" + want := "exclude_plugin_a2aparser,exclude_plugin_ibac,exclude_plugin_inferenceparser,exclude_plugin_lineage,exclude_plugin_mcpparser,exclude_plugin_opa,exclude_plugin_sparc,exclude_plugin_tokenbroker,exclude_plugin_toolprune" if got != want { t.Errorf("output changed\n got: %s\nwant: %s", got, want) } From 1ed8473ca2cb30d8906cbefd299cfce90b675141 Mon Sep 17 00:00:00 2001 From: YehoshuaSagron Date: Sun, 6 Sep 2026 18:50:16 +0300 Subject: [PATCH 37/37] Docs: State the bypass refusal check by its literal shape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The contract claimed a match-everything bypass entry is refused at boot; the code refuses only the literal shapes (empty, whitespace, '*', '/*'). An exotic glob such as '?*' matches every non-empty value and is accepted — deliberately: bypass config is operator-owned, the refusal is a typo guard rather than a boundary, and the siblings' keys behave identically. Say exactly that. Assisted-By: Claude (Anthropic AI) Signed-off-by: YehoshuaSagron --- authbridge/docs/lineage-wire-contract.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/authbridge/docs/lineage-wire-contract.md b/authbridge/docs/lineage-wire-contract.md index 5e5ae1e91..88eb7270c 100644 --- a/authbridge/docs/lineage-wire-contract.md +++ b/authbridge/docs/lineage-wire-contract.md @@ -247,8 +247,11 @@ construction. Setting `bypass_paths` or `bypass_hosts` **replaces** the default list rather than extending it — the convention the `ibac`, `sparc` and `cpex` plugins use for their keys of the same name. An operator who adds one entry must restate the defaults they want kept. An entry that would match -everything (empty, whitespace-only, `*`, `/*` for a path, `*` for a host) is refused at start, as -is an entry of either kind that is not valid `path.Match` syntax. +everything by its literal shape — empty, whitespace-only, `*` for a host, `*` or `/*` for a +path — is refused at start, as is an entry of either kind that is not valid `path.Match` syntax. +(The check is by shape, not by semantics: an exotic glob that happens to match every value, such +as `?*`, is the operator's own deliberate choice and is accepted — the same behaviour as the +sibling plugins' keys.) Unknown keys are a boot error.