diff --git a/src/lib/navigation.ts b/src/lib/navigation.ts index 27bd52296..8b7dcf83f 100644 --- a/src/lib/navigation.ts +++ b/src/lib/navigation.ts @@ -639,6 +639,7 @@ export const tabNavigation: NavTab[] = [ title: 'Reference', items: [ { title: 'Filters', href: '/docs/observe/reference/filters' }, + { title: 'Export and endpoints', href: '/docs/observe/reference/export-formats' }, { title: 'traceAI', href: '/docs/observe/concepts/traceai' }, ] }, @@ -647,6 +648,7 @@ export const tabNavigation: NavTab[] = [ items: [ { title: 'No traces appear', href: '/docs/observe/troubleshooting/no-traces-appearing' }, { title: 'Missing spans or fields', href: '/docs/observe/troubleshooting/missing-attributes' }, + { title: 'Traces are noisy or incomplete', href: '/docs/observe/troubleshooting/noisy-or-incomplete-traces' }, { title: 'Dashboard numbers look wrong', href: '/docs/observe/troubleshooting/dashboard-numbers-look-wrong' }, { title: 'Alerts not firing', href: '/docs/observe/troubleshooting/alerts-did-not-fire' }, ] diff --git a/src/pages/docs/cookbook/observe-langgraph-agent-and-obtain-insights.mdx b/src/pages/docs/cookbook/observe-langgraph-agent-and-obtain-insights.mdx index 56c0f05d1..4d01cea0e 100644 --- a/src/pages/docs/cookbook/observe-langgraph-agent-and-obtain-insights.mdx +++ b/src/pages/docs/cookbook/observe-langgraph-agent-and-obtain-insights.mdx @@ -210,7 +210,7 @@ No trace? It is almost always order or flush. `register()` and `.instrument()` m -Before any eval runs, the traces already answer questions you'd otherwise guess at. Open the `support-agent` project: every row is one turn, with its status, model, latency, tokens, and cost. Click the final-sale turn to see the ReAct loop as a span tree: the LLM call that decides to search, the `search_help_center` call with the exact policy it returned, and the LLM call that writes the answer. +Before any eval runs, the traces already answer questions you'd otherwise guess at. Open the `support-agent` project: every row is one turn, with its status and latency. The trace table's Model, Cost, and Tokens columns read the root span only, and `support_turn` never sets those attributes, so they stay blank on these rows; the LLM calls that do carry model and cost show up once you open the span tree. Click the final-sale turn to see the ReAct loop as a span tree: the LLM call that decides to search, the `search_help_center` call with the exact policy it returned, and the LLM call that writes the answer. Three things are readable before any score exists: diff --git a/src/pages/docs/integrations/traceai/langchain.mdx b/src/pages/docs/integrations/traceai/langchain.mdx index d65ed7c27..3cd42be93 100644 --- a/src/pages/docs/integrations/traceai/langchain.mdx +++ b/src/pages/docs/integrations/traceai/langchain.mdx @@ -76,6 +76,10 @@ const tracerProvider = register({ ## 4. Instrument your Project Initialize the LangChain Instrumentor to enable automatic tracing. This step ensures that all interactions with the LangChain are tracked and monitored. + +`LangChainInstrumentor().instrument(...)` patches LangChain's callback manager for the **whole process**, not just the code you call it from. Calling `.instrument()` again later (a second service, a background job, a vendored tool that also imports LangChain) is a no-op: the first `tracer_provider` you passed wins, and every LangChain call anywhere in that process is traced into that project. If your process runs more than one LangChain-based workflow and you want them in different projects, instrument each one with its own `tracer_provider` before any of them run, or keep unrelated LangChain code (linters, internal tools, vendored SDKs) in a separate process entirely. + + ```python Python diff --git a/src/pages/docs/observe/reference/export-formats.mdx b/src/pages/docs/observe/reference/export-formats.mdx index 9722fdc81..d7bd319d8 100644 --- a/src/pages/docs/observe/reference/export-formats.mdx +++ b/src/pages/docs/observe/reference/export-formats.mdx @@ -73,9 +73,28 @@ The ingestion endpoint is an **OTLP/traces** receiver — you don't call it dire | Auth | `FI_API_KEY` + `FI_SECRET_KEY` from the [keys page](https://app.futureagi.com/dashboard/keys), sent by the SDK on every export. Keys are workspace-scoped. | | Success | The exporter batches spans and sends them in the background; a successful export returns no payload. Spans appear in Observe within seconds. | | Errors | A `401` means the keys are wrong for this workspace; a `4xx` means a malformed/oversized batch; transient `5xx`/network errors are retried by the batch exporter. | -| Limits | Spans are sent by the **batch** span processor on an interval, so a short-lived process must call `trace_provider.force_flush()` before exit or the last batch is lost. Very large payloads (huge prompts/outputs) can be dropped — [mask or trim](/docs/sdk/tracing/mask-span-attributes) them at the SDK. | +| Limits | Spans are sent by the **batch** span processor on an interval, so a short-lived process must call `trace_provider.force_flush()` before exit or the last batch is lost. Very large payloads (huge prompts/outputs) can be dropped: [cap attribute size](/docs/sdk/tracing/trace-config#cap-attribute-size) or [mask](/docs/sdk/tracing/mask-span-attributes) them at the SDK. Future AGI Cloud accepts OTLP requests up to 16 MiB over HTTP or gRPC. See [Ingestion request limits](#ingestion-request-limits). | | Versioning | Pin `fi-instrumentation-otel` and each instrumentor to a tested version so a release can't change span shape under you; the wire format follows the OTLP version the SDK ships. | +### Ingestion request limits + +Future AGI Cloud enforces the same supported request limit for both OTLP transports: + +| Transport | Maximum request size | Error when exceeded | +|---|---:|---| +| OTLP/HTTP | 16 MiB | `413 Request Entity Too Large` | +| OTLP/gRPC | 16 MiB | `RESOURCE_EXHAUSTED` | + +An oversized request is rejected as a whole and its spans are not stored. If you see either error in your exporter logs, reduce the number of spans sent in each batch. OpenTelemetry's default maximum export batch size is 512 spans. Start with: + +```bash +OTEL_BSP_MAX_EXPORT_BATCH_SIZE=256 +``` + + + These limits apply to Future AGI Cloud. For a self-hosted deployment, the configured collector and proxy limits apply. + + Span input and output can carry customer data before they leave your process. Redact at the SDK with `TraceConfig` or the `FI_HIDE_*` variables — see [Mask span attributes](/docs/sdk/tracing/mask-span-attributes). diff --git a/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx b/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx index 6e5cf9711..b6ab37edf 100644 --- a/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx +++ b/src/pages/docs/observe/troubleshooting/no-traces-appearing.mdx @@ -44,6 +44,7 @@ The most common cause is a short-lived process that exited before its spans flus - The process **stays alive** long enough to export, or calls `force_flush()` before exiting. - `FI_API_KEY` and `FI_SECRET_KEY` are set to this workspace's keys. - `register()` is called with the correct `project_type` and `project_name`, **before** the framework client is created. +- The exporter logs do not contain HTTP `413 Request Entity Too Large` or gRPC `RESOURCE_EXHAUSTED` errors. - The date picker is widened to **Today** (not the default 7-day window) and **Auto refresh** is on. ## Causes and fixes @@ -54,6 +55,7 @@ The most common cause is a short-lived process that exited before its spans flus | Wrong `project_type` | App runs, keys are valid, but traces never land in the project you expect | Set `project_type=ProjectType.OBSERVE` (and the matching `project_name`) in `register()`. | | Missing `FI_API_KEY` / `FI_SECRET_KEY` | Export fails or is silently dropped; nothing reaches Observe | Set both env vars to this workspace's keys before the app starts. | | Instrumented after the client was created | Some or all spans never emit because the client wasn't wrapped | Call `register()` and the instrumentor **before** constructing the framework client. | +| Export request is too large | The exporter logs HTTP `413 Request Entity Too Large` or gRPC `RESOURCE_EXHAUSTED` | Set `OTEL_BSP_MAX_EXPORT_BATCH_SIZE=256` and restart the application. If the error continues, use `128`. This cannot fix a single span over 16 MiB; [cap attribute size](/docs/sdk/tracing/trace-config#cap-attribute-size) instead. See [ingestion request limits](/docs/observe/reference/export-formats#ingestion-request-limits). | | Date-picker window too narrow | The trace exists but is filtered out of the view | Widen the date range to **Today** and enable **Auto refresh**. | ## Diagnostic commands diff --git a/src/pages/docs/observe/troubleshooting/noisy-or-incomplete-traces.mdx b/src/pages/docs/observe/troubleshooting/noisy-or-incomplete-traces.mdx new file mode 100644 index 000000000..4b90f7c81 --- /dev/null +++ b/src/pages/docs/observe/troubleshooting/noisy-or-incomplete-traces.mdx @@ -0,0 +1,156 @@ +--- +title: "Traces Are Noisy or Incomplete" +description: "Traces you don't recognize show up in this project, or a trace's Type/Cost/Session/User is blank even though the LLM calls inside it ran. Usual causes: a process-wide instrumentor, a hand-built root span, or an oversized payload." +slug: "noisy-or-incomplete-traces" +page_type: "troubleshooting" +products: ["Observe"] +failure_surface: "sdk" +symptom: "unrelated traces mixed into a project, or an agent root missing type/cost/session/user" +audience: ["engineer"] +difficulty: "intermediate" +status: "review" +owner: "observability" +reviewers: ["observability-eng"] +last_tested: "2026-09-14" +last_screenshotted: "2026-09-14" +support_escalation: "support@futureagi.com" +schema_type: "TechArticle" +seo: + title: "Traces are noisy or incomplete in Observe" + description: "Traces you don't recognize show up in this project, or a trace's Type/Cost/Session/User is blank even though the LLM calls inside it ran. Usual causes: a process-wide instrumentor, a hand-built root span, or an oversized payload." + primary_keyword: "trace missing session user cost unknown type" + direct_answer: true +geo: + answer_target: "Why do unrelated traces show up in my project, or why is a trace's type, cost, session, or user blank?" + llm_summary: "Traces from code you don't own can land in a project because an auto-instrumentor patches its framework for the whole process, not just the code that called instrument(). A trace whose root span you created by hand can show Type unknown, Cost $0.00, and be missing from Sessions/Users because those fields are never set automatically on a manual span; set gen_ai.span.kind, session.id, and user.id on it directly. The trace table's Cost, Model, and Tokens columns always read the root span only, so even after fixing session/user, a manual root that never sets model/cost will show blank on the row while its child LLM spans carry the real values in the span tree." +canonical: "/docs/observe/troubleshooting/noisy-or-incomplete-traces" +related: + - "/docs/sdk/tracing/context-helpers" + - "/docs/integrations/traceai/langchain" + - "/docs/sdk/tracing/set-session-user-id" + - "/docs/sdk/tracing/trace-config" + - "/docs/observe/reference/filters" +--- + + +## Symptom + +A trace shows up, but something about it is wrong in one of two directions: either it's not yours, or it's missing the fields that make it useful. Both usually trace back to how the root span (the top row of the trace) was created, not to a bug in your data. + +- Rows appear in this project that don't match your own code's input/output shape: a different call signature, a different kind of prompt. +- A trace's **Type** shows "unknown", **Cost** shows $0.00, and it's missing from both the **Sessions** and **Users** tabs, even though you can see LLM calls with a real model and cost when you open the span tree underneath it. +- A trace is missing entirely, or your exporter logs a `413 Request Entity Too Large` or `RESOURCE_EXHAUSTED`. + +--- + +## Quick checks + +- Does more than one workflow using the same framework (e.g. LangChain) run inside the same process? Instrumenting one wires up all of them. +- Did you create the top-level span yourself (`tracer.start_as_current_span(...)`) instead of letting the framework's auto-instrumentor create the root? +- Does any single call send an unusually large prompt, tool result, or document as an attribute? + +## Causes and fixes + +| Cause | What you see | Fix | +|---|---|---| +| One instrumentor patches the whole process | Traces from code you don't own show up in this project | `SomeFrameworkInstrumentor().instrument(tracer_provider=...)` patches that framework's callback/client machinery for the **entire process**, not just the module that called it. A second `instrument()` call elsewhere in the same process is a no-op and reuses the first `tracer_provider`. Give unrelated workflows their own process, or their own `tracer_provider` registered before either one runs. | +| Suppressing drops spans, it doesn't redirect them | You suppressed the unrelated calls and now they don't show up *anywhere*, not just in this project | [`suppress_tracing()`](/docs/sdk/tracing/context-helpers#suppress-tracing) drops spans created in its block. It never moves them to another project. If that code still needs to be observable, give it its own `tracer_provider`/project instead of suppressing it. | +| Manual root span never gets the standard fields | Trace **Type** is "unknown", no **Session**/**User** | An auto-instrumented span picks up `session.id`/`user.id` from [`using_session`/`using_user`](/docs/sdk/tracing/set-session-user-id) automatically. A root span you create yourself doesn't get `gen_ai.span.kind`, `session.id`, or `user.id` for free: set them explicitly on it (see below). | +| The row's Cost/Model always reflect the root span | Cost and Model stay $0.00 / blank on the row even after Session and User are fixed | The trace table's Cost, Model, and Tokens columns read the **root span only**. A hand-built root that never sets those attributes shows blank on the row even though the LLM call underneath it has both; open the span tree to see them there. | +| Oversized payload | Exporter logs `413 Request Entity Too Large` / `RESOURCE_EXHAUSTED`, or a span silently never lands | A single span over 16 MiB is rejected outright. Cap attribute size with [`OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT` or `span_limits`](/docs/sdk/tracing/trace-config#cap-attribute-size) so no one attribute can grow that large. | + +## Fix it: set the standard fields on a manual root span + +```python +from fi_instrumentation import using_session, using_user +from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes + +with using_session(session_id), using_user(user_id): + with tracer.start_as_current_span("agent_turn") as span: + result = agent.invoke({"messages": [{"role": "user", "content": question}]}) + + span.set_attribute(SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.AGENT.value) + span.set_attribute(SpanAttributes.SESSION_ID, session_id) + span.set_attribute(SpanAttributes.USER_ID, user_id) +``` + +Set all three explicitly, even though `using_session`/`using_user` also tag a `FITracer` span automatically; a plain OpenTelemetry tracer does not get them, so setting them here works either way. + +## Fix it: keep an unrelated workflow out of this project + +```python +from fi_instrumentation import suppress_tracing + +with suppress_tracing(): + result = other_workflow.invoke(...) # never traced, anywhere, in any project +``` + +Only reach for this if that workflow doesn't need to be observable at all. If it does, register a second `tracer_provider` for it instead: see [Instrument your project](/docs/integrations/traceai/langchain#4-instrument-your-project). + +`async with suppress_tracing():` isn't supported: it raises a `TypeError` and can leave tracing suppressed past the block. Use the synchronous `with`, even inside an `async def`. + +## Fix it: cap oversized attributes + +```bash +export OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT=100000 +``` + +or in code, once at registration: + +```python +from fi_instrumentation import register, SpanLimits + +trace_provider = register( + project_name="my_project", + span_limits=SpanLimits(max_attribute_length=100000), +) +``` + +## Diagnostic commands + +Print each span's name and attributes as they're created, so you can see exactly which span has what before anything is exported: + +```python +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter + +class PrintAttributes(SpanExporter): + def export(self, spans): + for span in spans: + print(span.name, dict(span.attributes)) + return 0 + +trace_provider.add_span_processor(SimpleSpanProcessor(PrintAttributes())) +``` + +If the root span's print shows no `gen_ai.span.kind`/`session.id`/`user.id`, it never had them; that's the manual-root case, not a UI bug. If you see spans printed for code you don't recognize, that's the shared-instrumentor case. + +## Minimal smoke test + +Run one request through end to end, then check three things on that trace: it's the only new row in the project (no unrelated code's calls alongside it), its **Type**/**Session**/**User** are populated on the row, and it landed at all if the request included an unusually large input or output. + +## Escalate + +If a trace still looks wrong after the fixes above, contact support@futureagi.com with the `project_name`, the trace ID, and which of the three symptoms above you're still seeing. + +## Prevent recurrence + +- Give each distinct framework-based workflow its own process or its own `tracer_provider` before you run more than one in the same service. +- If you build a root span by hand, treat `gen_ai.span.kind`, `session.id`, and `user.id` as a fixed checklist to set on it every time, not a maybe. +- Set a size cap once at registration, rather than after you hit the 16 MiB limit in production. + +## Next steps + + + + What suppress_tracing does and doesn't do + + + Tagging spans with session and user + + + Stop one attribute from hitting the request-size limit + + + A different symptom: masking or a detached instrumentor + + diff --git a/src/pages/docs/sdk/tracing/context-helpers.mdx b/src/pages/docs/sdk/tracing/context-helpers.mdx index b4fbc33bd..4569a1b44 100644 --- a/src/pages/docs/sdk/tracing/context-helpers.mdx +++ b/src/pages/docs/sdk/tracing/context-helpers.mdx @@ -189,6 +189,19 @@ Attach metadata, tags, session IDs, and user IDs to spans. These apply to all sp Temporarily disable tracing for a block of code. Useful for health checks, internal calls, or operations you don't want in your traces. Available in Python and C# only - Java and TypeScript don't have this API. + +`suppress_tracing` **drops** spans created in its block, it does not move them anywhere else. If a LangChain call inside the block is one you still want traced, but into a different project (for example a linter or internal tool that shares a process with your main agent), suppressing it makes those calls disappear entirely rather than showing up somewhere else. Use a separate `tracer_provider`/project for that code path instead if you need it observable. + +In Python, only use the synchronous form, even inside an `async def`: + +```python +with suppress_tracing(): + result = await client.chat.completions.create(...) +``` + +`suppress_tracing` does not support `async with`. Its `__aenter__`/`__aexit__` methods are plain (non-`async def`), so `async with suppress_tracing():` raises a `TypeError` on entry. Because `__aenter__` already attaches the suppression context before that error, the `TypeError` also skips the matching `__aexit__`/detach, so the suppression can leak into code that runs after the failed block. + + ```python diff --git a/src/pages/docs/sdk/tracing/mask-span-attributes.mdx b/src/pages/docs/sdk/tracing/mask-span-attributes.mdx index 89c991163..0ed8d38d4 100644 --- a/src/pages/docs/sdk/tracing/mask-span-attributes.mdx +++ b/src/pages/docs/sdk/tracing/mask-span-attributes.mdx @@ -5,7 +5,7 @@ description: "Redact sensitive inputs, outputs, images, and embeddings from span ## About -Traces often contain sensitive data: user messages, API responses, PII, or large base64-encoded images. Sending all of this to a trace backend creates privacy, compliance, and payload size problems. Masking span attributes removes or truncates this data before it leaves the application. Configuration is available at two levels: environment variables for global defaults across all instrumentors, and `TraceConfig` in code for per-instrumentor control. +Traces often contain sensitive data: user messages, API responses, PII, or large base64-encoded images. Sending all of this to a trace backend creates privacy and compliance problems. Masking span attributes removes this data before it leaves the application: it only truncates for one field, `base64_image_max_length`, which caps the length of base64-encoded images. Everything else is either sent in full or hidden completely; there is no general truncation. To cap the size of arbitrary attribute values (large prompts, tool outputs), see [Cap attribute size](/docs/sdk/tracing/trace-config#cap-attribute-size) instead. Configuration is available at two levels: environment variables for global defaults across all instrumentors, and `TraceConfig` in code for per-instrumentor control. --- diff --git a/src/pages/docs/sdk/tracing/trace-config.mdx b/src/pages/docs/sdk/tracing/trace-config.mdx index eeecdb282..ae92a8e71 100644 --- a/src/pages/docs/sdk/tracing/trace-config.mdx +++ b/src/pages/docs/sdk/tracing/trace-config.mdx @@ -95,6 +95,33 @@ Control what data gets captured. Useful for privacy compliance, reducing payload Each field maps to an environment variable with the `FI_` prefix (e.g. `hide_inputs` -> `FI_HIDE_INPUTS`). +## Cap attribute size + +`TraceConfig`'s `hide_*` fields redact or drop values; only `base64_image_max_length` actually truncates, and only for base64 images. Everything else, an oversized input, output, or tool payload, is sent at full size and can be rejected outright (see [ingestion request limits](/docs/observe/reference/export-formats#ingestion-request-limits)). + +To cap the size of every attribute value before it is exported, set an OpenTelemetry span limit instead. This applies to all instrumentors and all attributes, not just images. + + + + ```bash + # Truncates every span attribute value to 100,000 characters + export OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT=100000 + ``` + + + ```python + from fi_instrumentation import register, SpanLimits + + trace_provider = register( + project_name="my_project", + span_limits=SpanLimits(max_attribute_length=100000), + ) + ``` + + + +`span_limits` is a `register()` argument; there is no per-instrumentor equivalent, so set it once at registration. + ## PII Redaction (Python) When `pii_redaction=True`, the SDK automatically detects and masks 6 types of personally identifiable information: