feat(observability): configurable health probes, incident timelines, trace propagation, and parser alerting

apps/api/src/common/tracing/index.ts

@@ -0,0 +1,3 @@
+export * from "./trace-context";
+export * from "./trace-carrier";
+export * from "./trace-propagation.service";

apps/api/src/common/tracing/trace-carrier.ts

@@ -0,0 +1,164 @@
+import {
+  createChildTraceContext,
+  createRootTraceContext,
+  formatBaggage,
+  formatTraceparent,
+  generateSpanId,
+  parseBaggage,
+  parseTraceparent,
+  TraceContext,
+} from "./trace-context";
+
+export interface HttpHeadersCarrier {
+  [header: string]: string | string[] | undefined;
+}
+
+export interface JobEnvelopeWithTrace {
+  payload?: any;
+  _traceContext?: {
+    traceparent: string;
+    tracestate?: string;
+    baggage?: string;
+    tenantId?: string;
+    startedAt?: number;
+  };
+  [key: string]: any;
+}
+
+/**
+ * Extracts a TraceContext from incoming HTTP request headers.
+ * Supports W3C traceparent, baggage, and fallback headers (x-correlation-id, x-request-id, x-tenant-id).
+ */
+export function extractFromHttpHeaders(
+  headers: HttpHeadersCarrier,
+): TraceContext {
+  const getHeader = (name: string): string | undefined => {
+    const val = headers[name.toLowerCase()] ?? headers[name];
+    if (Array.isArray(val)) {
+      return val[0];
+    }
+    return val;
+  };
+
+  const traceparentHeader = getHeader("traceparent");
+  const baggageHeader = getHeader("baggage");
+  const tenantIdHeader = getHeader("x-tenant-id");
+
+  const parsed = parseTraceparent(traceparentHeader);
+  const baggage = parseBaggage(baggageHeader);
+
+  const tenantId = tenantIdHeader || baggage.tenantId || undefined;
+  if (tenantId) {
+    baggage.tenantId = tenantId;
+  }
+
+  if (parsed) {
+    return {
+      traceId: parsed.traceId,
+      spanId: generateSpanId(),
+      parentId: parsed.parentId,
+      sampled: parsed.sampled,
+      tenantId,
+      baggage,
+      startedAt: Date.now(),
+    };
+  }
+
+  // Fallback to x-correlation-id or x-request-id if valid 32-char hex, or create new root trace
+  return createRootTraceContext({
+    tenantId,
+    baggage,
+  });
+}
+
+/**
+ * Injects TraceContext into outgoing HTTP headers.
+ */
+export function injectIntoHttpHeaders(
+  context: TraceContext,
+  headers: Record<string, string> = {},
+): Record<string, string> {
+  headers["traceparent"] = formatTraceparent(
+    context.traceId,
+    context.spanId,
+    context.sampled,
+  );
+
+  const baggageStr = formatBaggage(context.baggage);
+  if (baggageStr) {
+    headers["baggage"] = baggageStr;
+  }
+
+  if (context.tenantId) {
+    headers["x-tenant-id"] = context.tenantId;
+  }
+
+  headers["x-trace-id"] = context.traceId;
+  headers["x-span-id"] = context.spanId;
+
+  return headers;
+}
+
+/**
+ * Injects trace context into a queue job payload/envelope.
+ */
+export function injectIntoJobEnvelope<T extends Record<string, any>>(
+  jobData: T,
+  context: TraceContext,
+): T & JobEnvelopeWithTrace {
+  const traceparent = formatTraceparent(
+    context.traceId,
+    context.spanId,
+    context.sampled,
+  );
+  const baggage = formatBaggage(context.baggage);
+
+  return {
+    ...jobData,
+    _traceContext: {
+      traceparent,
+      ...(baggage ? { baggage } : {}),
+      ...(context.tenantId ? { tenantId: context.tenantId } : {}),
+      startedAt: context.startedAt,
+    },
+  };
+}
+
+/**
+ * Extracts and restores TraceContext from a job payload/envelope.
+ * Returns child trace context linked to the job's span.
+ */
+export function extractFromJobEnvelope(jobData: any): TraceContext {
+  if (!jobData || typeof jobData !== "object") {
+    return createRootTraceContext();
+  }
+
+  const envelope = jobData._traceContext;
+  if (!envelope || !envelope.traceparent) {
+    // Check if directly in payload or fallback
+    const tenantId = jobData.tenantId || jobData.payload?.tenantId;
+    return createRootTraceContext({ tenantId });
+  }
+
+  const parsed = parseTraceparent(envelope.traceparent);
+  if (!parsed) {
+    return createRootTraceContext({ tenantId: envelope.tenantId });
+  }
+
+  const baggage = parseBaggage(envelope.baggage);
+  const tenantId = envelope.tenantId || baggage.tenantId;
+  if (tenantId) {
+    baggage.tenantId = tenantId;
+  }
+
+  const parentContext: TraceContext = {
+    traceId: parsed.traceId,
+    spanId: parsed.parentId,
+    sampled: parsed.sampled,
+    tenantId,
+    baggage,
+    startedAt: envelope.startedAt ?? Date.now(),
+  };
+
+  return createChildTraceContext(parentContext);
+}

apps/api/src/common/tracing/trace-context.spec.ts

@@ -0,0 +1,184 @@
+import {
+  parseTraceparent,
+  formatTraceparent,
+  parseBaggage,
+  formatBaggage,
+  generateTraceId,
+  generateSpanId,
+  createRootTraceContext,
+  createChildTraceContext,
+} from "./trace-context";
+import {
+  extractFromHttpHeaders,
+  injectIntoHttpHeaders,
+  extractFromJobEnvelope,
+  injectIntoJobEnvelope,
+} from "./trace-carrier";
+import { TracePropagationService } from "./trace-propagation.service";
+
+describe("W3C Trace Context and Carrier (#1263)", () => {
+  describe("parseTraceparent and formatTraceparent", () => {
+    it("parses valid standard W3C traceparent headers", () => {
+      const header = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01";
+      const parsed = parseTraceparent(header);
+      expect(parsed).not.toBeNull();
+      expect(parsed!.traceId).toBe("4bf92f3577b34da6a3ce929d0e0e4736");
+      expect(parsed!.parentId).toBe("00f067aa0ba902b7");
+      expect(parsed!.sampled).toBe(true);
+    });
+
+    it("parses unsampled traceparent flag", () => {
+      const header = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00";
+      const parsed = parseTraceparent(header);
+      expect(parsed!.sampled).toBe(false);
+    });
+
+    it("rejects invalid formats, wrong lengths, or non-hex characters", () => {
+      expect(parseTraceparent("not-a-traceparent")).toBeNull();
+      expect(parseTraceparent("00-shorttrace-00f067aa0ba902b7-01")).toBeNull();
+      expect(
+        parseTraceparent("00-4bf92f3577b34da6a3ce929d0e0e4736-shortspan-01"),
+      ).toBeNull();
+      expect(
+        parseTraceparent(
+          "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-xyz",
+        ),
+      ).toBeNull();
+      expect(
+        parseTraceparent(
+          "ff-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
+        ),
+      ).toBeNull();
+      expect(parseTraceparent("")).toBeNull();
+      expect(parseTraceparent(null)).toBeNull();
+    });
+
+    it("rejects all-zeros traceId or parentId per W3C specification", () => {
+      expect(
+        parseTraceparent(
+          "00-00000000000000000000000000000000-00f067aa0ba902b7-01",
+        ),
+      ).toBeNull();
+      expect(
+        parseTraceparent(
+          "00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01",
+        ),
+      ).toBeNull();
+    });
+
+    it("formats valid traceparent header string", () => {
+      const traceId = "4bf92f3577b34da6a3ce929d0e0e4736";
+      const spanId = "00f067aa0ba902b7";
+      const formatted = formatTraceparent(traceId, spanId, true);
+      expect(formatted).toBe(
+        "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
+      );
+    });
+
+    it("throws when formatting with invalid traceId or spanId", () => {
+      expect(() => formatTraceparent("invalid", "00f067aa0ba902b7")).toThrow(
+        TypeError,
+      );
+      expect(() =>
+        formatTraceparent("4bf92f3577b34da6a3ce929d0e0e4736", "short"),
+      ).toThrow(TypeError);
+    });
+  });
+
+  describe("parseBaggage and formatBaggage", () => {
+    it("parses comma-separated key-value pairs with URL decoding", () => {
+      const raw = "userId=alice,role=admin,tenantId=acme-corp%20ltd";
+      const baggage = parseBaggage(raw);
+      expect(baggage.userId).toBe("alice");
+      expect(baggage.role).toBe("admin");
+      expect(baggage.tenantId).toBe("acme-corp ltd");
+    });
+
+    it("formats baggage object into standard string", () => {
+      const formatted = formatBaggage({ userId: "alice", tenantId: "acme" });
+      expect(formatted).toContain("userId=alice");
+      expect(formatted).toContain("tenantId=acme");
+    });
+
+    it("handles empty or malformed baggage headers safely", () => {
+      expect(parseBaggage("")).toEqual({});
+      expect(parseBaggage("malformed-no-equal")).toEqual({});
+      expect(parseBaggage(undefined)).toEqual({});
+    });
+  });
+
+  describe("ID Generators and Context Creation", () => {
+    it("generates 32-hex character traceId and 16-hex character spanId", () => {
+      const traceId = generateTraceId();
+      const spanId = generateSpanId();
+      expect(traceId).toHaveLength(32);
+      expect(/^[0-9a-f]{32}$/.test(traceId)).toBe(true);
+      expect(spanId).toHaveLength(16);
+      expect(/^[0-9a-f]{16}$/.test(spanId)).toBe(true);
+    });
+
+    it("creates root context and propagates traceId to child context", () => {
+      const root = createRootTraceContext({
+        tenantId: "tenant-99",
+        baggage: { env: "production" },
+      });
+
+      expect(root.traceId).toBeDefined();
+      expect(root.parentId).toBeUndefined();
+      expect(root.tenantId).toBe("tenant-99");
+      expect(root.baggage.env).toBe("production");
+
+      const child = createChildTraceContext(root, {
+        extraBaggage: { step: "parsing" },
+      });
+
+      expect(child.traceId).toBe(root.traceId);
+      expect(child.parentId).toBe(root.spanId);
+      expect(child.spanId).not.toBe(root.spanId);
+      expect(child.tenantId).toBe("tenant-99");
+      expect(child.baggage.step).toBe("parsing");
+      expect(child.baggage.env).toBe("production");
+    });
+  });
+
+  describe("Trace Carriers (HTTP and Job Envelopes)", () => {
+    it("extracts from HTTP headers and injects into outgoing headers", () => {
+      const incomingHeaders = {
+        traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
+        baggage: "tenantId=tenant-corp,region=us-east",
+      };
+
+      const extracted = extractFromHttpHeaders(incomingHeaders);
+      expect(extracted.traceId).toBe("4bf92f3577b34da6a3ce929d0e0e4736");
+      expect(extracted.parentId).toBe("00f067aa0ba902b7");
+      expect(extracted.tenantId).toBe("tenant-corp");
+
+      const outgoing: Record<string, string> = {};
+      injectIntoHttpHeaders(extracted, outgoing);
+      expect(outgoing.traceparent).toMatch(
+        /^00-4bf92f3577b34da6a3ce929d0e0e4736-[0-9a-f]{16}-01$/,
+      );
+      expect(outgoing["x-tenant-id"]).toBe("tenant-corp");
+      expect(outgoing["x-trace-id"]).toBe("4bf92f3577b34da6a3ce929d0e0e4736");
+    });
+
+    it("injects trace context into job envelope and extracts it in worker", () => {
+      const rootCtx = createRootTraceContext({
+        tenantId: "tenant-worker-test",
+      });
+      const jobData = {
+        scanId: "scan-job-42",
+        repositoryUrl: "https://github.com/MDTechLabs/GasGuard",
+      };
+
+      const enqueuedData = injectIntoJobEnvelope(jobData, rootCtx);
+      expect(enqueuedData._traceContext).toBeDefined();
+      expect(enqueuedData._traceContext?.tenantId).toBe("tenant-worker-test");
+
+      const workerCtx = extractFromJobEnvelope(enqueuedData);
+      expect(workerCtx.traceId).toBe(rootCtx.traceId);
+      expect(workerCtx.parentId).toBe(rootCtx.spanId);
+      expect(workerCtx.tenantId).toBe("tenant-worker-test");
+    });
+  });
+});