From e77caffa5bdf29798e0fac16b9daabd7a28cc11d Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Mon, 27 Jul 2026 11:31:33 -0700 Subject: [PATCH 01/12] Fix System Nexus tracing headers --- CHANGELOG.md | 7 +++ .../contrib/opentelemetry/_interceptor.py | 10 ++++ .../opentelemetry/_otel_interceptor.py | 10 ++++ temporalio/worker/__init__.py | 2 + temporalio/worker/_interceptor.py | 50 +++++++++++++++++++ temporalio/worker/_workflow_instance.py | 42 +++++++++++++++- 6 files changed, 120 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be3444177..4a66bb1a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -172,6 +172,9 @@ to include examples, links to docs, or any other relevant information. `temporalio.client`) instead of setting `payload_limits` on `DataConverter`. Config fields were renamed to `payloads_warn_size` and `memo_warn_size`, and the deprecated `PayloadSizeWarning` was removed. +- `WorkflowOutboundInterceptor.start_nexus_operation` no longer receives Temporal System Nexus + operations. Custom interceptors that need to observe or modify these operations must implement + `start_system_nexus_operation` instead. ### Fixed @@ -179,6 +182,10 @@ to include examples, links to docs, or any other relevant information. task instead of leaving the update unresolved. - Marked system Nexus envelope payloads so nested payloads can be detected and visited after the envelope is already stored as a payload. +- Fixed OpenTelemetry context propagation when a workflow uses signal-with-start. Trace context is + now added to the called workflow's headers instead of the System Nexus transport headers. + +### Security ## [1.30.0] - 2026-07-01 diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index eb22f8be6..8bb8564f0 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -830,6 +830,16 @@ async def start_nexus_operation( return await super().start_nexus_operation(input) + async def start_system_nexus_operation( + self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] + ) -> temporalio.workflow.NexusOperationHandle[Any]: + self.root._completed_span( + f"StartNexusOperation:{input.service}/{input.operation_name}", + kind=opentelemetry.trace.SpanKind.CLIENT, + add_to_outbound=input, + ) + return await super().start_system_nexus_operation(input) + def _carrier_to_nexus_headers( carrier: _CarrierDict, initial: Mapping[str, str] | None = None diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index c120fcd03..98875c849 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -600,3 +600,13 @@ async def start_nexus_operation( ): input.headers = _context_to_nexus_headers(input.headers or {}) return await super().start_nexus_operation(input) + + async def start_system_nexus_operation( + self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] + ) -> temporalio.workflow.NexusOperationHandle[Any]: + with self._workflow_maybe_span( + f"StartNexusOperation:{input.service}/{input.operation_name}", + kind=opentelemetry.trace.SpanKind.CLIENT, + ): + input.headers = _context_to_headers(input.headers) + return await super().start_system_nexus_operation(input) diff --git a/temporalio/worker/__init__.py b/temporalio/worker/__init__.py index 4f6efe68c..ada34ab71 100644 --- a/temporalio/worker/__init__.py +++ b/temporalio/worker/__init__.py @@ -21,6 +21,7 @@ StartChildWorkflowInput, StartLocalActivityInput, StartNexusOperationInput, + StartSystemNexusOperationInput, WorkflowInboundInterceptor, WorkflowInterceptorClassInput, WorkflowOutboundInterceptor, @@ -100,6 +101,7 @@ "StartChildWorkflowInput", "StartLocalActivityInput", "StartNexusOperationInput", + "StartSystemNexusOperationInput", "WorkflowInterceptorClassInput", "ExecuteNexusOperationStartInput", "ExecuteNexusOperationCancelInput", diff --git a/temporalio/worker/_interceptor.py b/temporalio/worker/_interceptor.py index 4acf3c5d1..866d06ea0 100644 --- a/temporalio/worker/_interceptor.py +++ b/temporalio/worker/_interceptor.py @@ -348,6 +348,50 @@ def operation_name(self) -> str: raise ValueError(f"Operation is not a Nexus operation: {self.operation}") +@dataclass +class StartSystemNexusOperationInput(Generic[InputT, OutputT]): + """Input for :py:meth:`WorkflowOutboundInterceptor.start_system_nexus_operation`.""" + + service: str + operation: nexusrpc.Operation[InputT, OutputT] | str | Callable[..., Any] + input: InputT + schedule_to_close_timeout: timedelta | None + schedule_to_start_timeout: timedelta | None + start_to_close_timeout: timedelta | None + cancellation_type: temporalio.workflow.NexusOperationCancellationType + headers: Mapping[str, temporalio.api.common.v1.Payload] + summary: str | None + output_type: type[OutputT] | None = None + + def __post_init__(self) -> None: + """Initialize operation-specific attributes after dataclass creation.""" + if isinstance(self.operation, nexusrpc.Operation): + self.output_type = self.operation.output_type + elif callable(self.operation): + _, op = temporalio.nexus._util.get_operation_factory(self.operation) + if isinstance(op, nexusrpc.Operation): + self.output_type = op.output_type + else: + raise ValueError( + f"Operation callable is not a Nexus operation: {self.operation}" + ) + elif not isinstance(self.operation, str): + raise ValueError(f"Operation is not a Nexus operation: {self.operation}") + + @property + def operation_name(self) -> str: + """Get the name of the Nexus operation.""" + if isinstance(self.operation, nexusrpc.Operation): + return self.operation.name + elif isinstance(self.operation, str): + return self.operation + elif callable(self.operation): + _, op = temporalio.nexus._util.get_operation_factory(self.operation) + if isinstance(op, nexusrpc.Operation): + return op.name + raise ValueError(f"Operation is not a Nexus operation: {self.operation}") + + @dataclass class StartLocalActivityInput: """Input for :py:meth:`WorkflowOutboundInterceptor.start_local_activity`.""" @@ -481,6 +525,12 @@ async def start_nexus_operation( """Called for every :py:func:`temporalio.workflow.NexusClient.start_operation` call.""" return await self.next.start_nexus_operation(input) + async def start_system_nexus_operation( + self, input: StartSystemNexusOperationInput[InputT, OutputT] + ) -> temporalio.workflow.NexusOperationHandle[OutputT]: + """Called for every Temporal System Nexus operation started by a workflow.""" + return await self.next.start_system_nexus_operation(input) + @dataclass class ExecuteNexusOperationStartInput: diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 69b416f26..81765ab9f 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -76,6 +76,7 @@ StartChildWorkflowInput, StartLocalActivityInput, StartNexusOperationInput, + StartSystemNexusOperationInput, WorkflowInboundInterceptor, WorkflowOutboundInterceptor, ) @@ -1732,7 +1733,21 @@ async def workflow_start_nexus_operation( headers: Mapping[str, str] | None, summary: str | None, ) -> temporalio.workflow.NexusOperationHandle[OutputT]: - # start_nexus_operation + if temporalio.nexus.system.is_system_endpoint(endpoint): + return await self._outbound.start_system_nexus_operation( + StartSystemNexusOperationInput( + service=service, + operation=operation, + input=input, + output_type=output_type, + schedule_to_close_timeout=schedule_to_close_timeout, + schedule_to_start_timeout=schedule_to_start_timeout, + start_to_close_timeout=start_to_close_timeout, + cancellation_type=cancellation_type, + headers={}, + summary=summary, + ) + ) return await self._outbound.start_nexus_operation( StartNexusOperationInput( endpoint=endpoint, @@ -2179,6 +2194,26 @@ async def operation_handle_fn() -> OutputT: ) return handle + async def _outbound_start_system_nexus_operation( + self, input: StartSystemNexusOperationInput[Any, OutputT] + ) -> _NexusOperationHandle[OutputT]: + temporalio.nexus.system._apply_headers_to_request(input.input, input.headers) + return await self._outbound_start_nexus_operation( + StartNexusOperationInput( + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + service=input.service, + operation=input.operation, + input=input.input, + output_type=input.output_type, + schedule_to_close_timeout=input.schedule_to_close_timeout, + schedule_to_start_timeout=input.schedule_to_start_timeout, + start_to_close_timeout=input.start_to_close_timeout, + cancellation_type=input.cancellation_type, + headers=None, + summary=input.summary, + ) + ) + #### Miscellaneous helpers #### # These are in alphabetical order. @@ -3165,6 +3200,11 @@ async def start_nexus_operation( ) -> _NexusOperationHandle[OutputT]: return await self._instance._outbound_start_nexus_operation(input) + async def start_system_nexus_operation( + self, input: StartSystemNexusOperationInput[Any, OutputT] + ) -> _NexusOperationHandle[OutputT]: + return await self._instance._outbound_start_system_nexus_operation(input) + def start_local_activity( self, input: StartLocalActivityInput ) -> temporalio.workflow.ActivityHandle[Any]: From 1cf783ed2fbf9d8d22d7e0a51714aaf7c895a1ff Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Mon, 27 Jul 2026 12:35:59 -0700 Subject: [PATCH 02/12] Silence System Nexus helper type warning --- temporalio/nexus/system/__init__.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index 730dd4258..d68d51ee2 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -8,10 +8,12 @@ import contextlib import contextvars -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass from typing import Any +from google.protobuf.message import Message + import temporalio.api.common.v1 import temporalio.common import temporalio.converter @@ -132,13 +134,23 @@ def is_system_endpoint(endpoint: str) -> bool: return endpoint == TEMPORAL_SYSTEM_ENDPOINT +def _apply_headers_to_request( + request: Message, + headers: Mapping[str, temporalio.api.common.v1.Payload], +) -> None: + """Apply headers to a system request when it supports Temporal headers.""" + if not headers or "header" not in request.DESCRIPTOR.fields_by_name: + return + request_header = getattr(request, "header") + for key, payload in headers.items(): + request_header.fields[key].CopyFrom(payload) + + def _is_system_payload(payload: temporalio.api.common.v1.Payload) -> bool: return ( payload.metadata.get(_SYSTEM_PAYLOAD_METADATA_KEY) == _SYSTEM_PAYLOAD_METADATA_VALUE ) - - async def maybe_visit_payload( payload: temporalio.api.common.v1.Payload, visitor_functions: VisitorFunctions, From d1818d6a23899826891d59e5cae91b5240355608 Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Mon, 27 Jul 2026 12:39:06 -0700 Subject: [PATCH 03/12] Simplify System Nexus interceptor input --- temporalio/worker/_interceptor.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/temporalio/worker/_interceptor.py b/temporalio/worker/_interceptor.py index 866d06ea0..76209ab56 100644 --- a/temporalio/worker/_interceptor.py +++ b/temporalio/worker/_interceptor.py @@ -363,21 +363,6 @@ class StartSystemNexusOperationInput(Generic[InputT, OutputT]): summary: str | None output_type: type[OutputT] | None = None - def __post_init__(self) -> None: - """Initialize operation-specific attributes after dataclass creation.""" - if isinstance(self.operation, nexusrpc.Operation): - self.output_type = self.operation.output_type - elif callable(self.operation): - _, op = temporalio.nexus._util.get_operation_factory(self.operation) - if isinstance(op, nexusrpc.Operation): - self.output_type = op.output_type - else: - raise ValueError( - f"Operation callable is not a Nexus operation: {self.operation}" - ) - elif not isinstance(self.operation, str): - raise ValueError(f"Operation is not a Nexus operation: {self.operation}") - @property def operation_name(self) -> str: """Get the name of the Nexus operation.""" From a7467ba3b2602f805f275cfabf682ed8763d6df2 Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Thu, 20 Aug 2026 09:51:30 -0700 Subject: [PATCH 04/12] Adapt System Nexus tracing to generated request types --- temporalio/converter/_payload_converter.py | 14 ++++- temporalio/nexus/system/__init__.py | 31 ++++------ temporalio/worker/_workflow_instance.py | 57 +++++++++++++----- .../opentelemetry/test_opentelemetry.py | 60 +++++++++++++++++++ .../test_opentelemetry_plugin.py | 57 ++++++++++++++++++ tests/nexus/test_temporal_system_nexus.py | 11 +++- 6 files changed, 195 insertions(+), 35 deletions(-) diff --git a/temporalio/converter/_payload_converter.py b/temporalio/converter/_payload_converter.py index a8bc35e28..ac1a921a9 100644 --- a/temporalio/converter/_payload_converter.py +++ b/temporalio/converter/_payload_converter.py @@ -613,7 +613,9 @@ def wrap(payload_converter: PayloadConverter) -> PayloadConverter: return _TemporalTransferTypePayloadConverter(payload_converter) def to_payloads( - self, values: Sequence[Any] + self, + values: Sequence[Any], + headers: Mapping[str, temporalio.api.common.v1.Payload] | None = None, ) -> list[temporalio.api.common.v1.Payload]: """See base class.""" transfer_type_values: list[Any] = [] @@ -621,6 +623,16 @@ def to_payloads( converter = _get_transfer_type_converter(type(value)) if converter is not None: value = converter.to_transfer_type(value) + if ( + headers + and isinstance(value, google.protobuf.message.Message) + and "header" in value.DESCRIPTOR.fields_by_name + ): + # System Nexus starts with generated models, so headers can only be + # applied after conversion to a request protobuf and before encoding. + temporalio.common._apply_headers( + headers, getattr(value, "header").fields + ) transfer_type_values.append(value) return self._inner_payload_converter.to_payloads(transfer_type_values) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index d68d51ee2..ddf223b68 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -12,8 +12,6 @@ from dataclasses import dataclass from typing import Any -from google.protobuf.message import Message - import temporalio.api.common.v1 import temporalio.common import temporalio.converter @@ -93,27 +91,31 @@ class _SystemNexusPayloadConverter(temporalio.converter.PayloadConverter): """Payload converter for system Nexus outer envelopes.""" _user_converters: _SystemNexusUserConverters - _outer_payload_converter: temporalio.converter.PayloadConverter + _outer_payload_converter: _TemporalTransferTypePayloadConverter + _headers: Mapping[str, temporalio.api.common.v1.Payload] | None def __init__( self, user_payload_converter: temporalio.converter.PayloadConverter, user_failure_converter: temporalio.converter.FailureConverter, + headers: Mapping[str, temporalio.api.common.v1.Payload] | None = None, ) -> None: """Create a payload converter for system Nexus outer envelopes.""" self._user_converters = _SystemNexusUserConverters( user_payload_converter, user_failure_converter ) - self._outer_payload_converter = _TemporalTransferTypePayloadConverter.wrap( + + self._outer_payload_converter = _TemporalTransferTypePayloadConverter( _SystemNexusOuterPayloadConverter() ) + self._headers = headers def to_payloads( self, values: Sequence[Any] ) -> list[temporalio.api.common.v1.Payload]: """See base class.""" with _user_converter_context(self._user_converters): - return self._outer_payload_converter.to_payloads(values) + return self._outer_payload_converter.to_payloads(values, self._headers) def from_payloads( self, @@ -134,23 +136,13 @@ def is_system_endpoint(endpoint: str) -> bool: return endpoint == TEMPORAL_SYSTEM_ENDPOINT -def _apply_headers_to_request( - request: Message, - headers: Mapping[str, temporalio.api.common.v1.Payload], -) -> None: - """Apply headers to a system request when it supports Temporal headers.""" - if not headers or "header" not in request.DESCRIPTOR.fields_by_name: - return - request_header = getattr(request, "header") - for key, payload in headers.items(): - request_header.fields[key].CopyFrom(payload) - - def _is_system_payload(payload: temporalio.api.common.v1.Payload) -> bool: return ( payload.metadata.get(_SYSTEM_PAYLOAD_METADATA_KEY) == _SYSTEM_PAYLOAD_METADATA_VALUE ) + + async def maybe_visit_payload( payload: temporalio.api.common.v1.Payload, visitor_functions: VisitorFunctions, @@ -175,9 +167,12 @@ async def maybe_visit_payload( def _get_payload_converter( # pyright: ignore[reportUnusedFunction] user_payload_converter: temporalio.converter.PayloadConverter, user_failure_converter: temporalio.converter.FailureConverter, + headers: Mapping[str, temporalio.api.common.v1.Payload] | None = None, ) -> temporalio.converter.PayloadConverter: """Return the fixed payload converter for system Nexus outer envelopes.""" - return _SystemNexusPayloadConverter(user_payload_converter, user_failure_converter) + return _SystemNexusPayloadConverter( + user_payload_converter, user_failure_converter, headers + ) def _get_serialization_context( # pyright: ignore[reportUnusedFunction] diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 81765ab9f..18ad2228b 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -2197,22 +2197,51 @@ async def operation_handle_fn() -> OutputT: async def _outbound_start_system_nexus_operation( self, input: StartSystemNexusOperationInput[Any, OutputT] ) -> _NexusOperationHandle[OutputT]: - temporalio.nexus.system._apply_headers_to_request(input.input, input.headers) - return await self._outbound_start_nexus_operation( - StartNexusOperationInput( - endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, - service=input.service, - operation=input.operation, - input=input.input, - output_type=input.output_type, - schedule_to_close_timeout=input.schedule_to_close_timeout, - schedule_to_start_timeout=input.schedule_to_start_timeout, - start_to_close_timeout=input.start_to_close_timeout, - cancellation_type=input.cancellation_type, - headers=None, - summary=input.summary, + nexus_input = StartNexusOperationInput( + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + service=input.service, + operation=input.operation, + input=input.input, + output_type=input.output_type, + schedule_to_close_timeout=input.schedule_to_close_timeout, + schedule_to_start_timeout=input.schedule_to_start_timeout, + start_to_close_timeout=input.start_to_close_timeout, + cancellation_type=input.cancellation_type, + headers=None, + summary=input.summary, + ) + handle: _NexusOperationHandle[OutputT] + + async def operation_handle_fn() -> OutputT: + return cast( + OutputT, + await self._await_temporal_operation( + handle._result_fut, + lambda _err, command: handle._apply_cancel_command(command), + ), ) + + payload_converter = temporalio.nexus.system._get_payload_converter( + self._workflow_context_payload_converter, + self._workflow_context_failure_converter, + input.headers, ) + handle = _NexusOperationHandle( + self, + self._next_seq("nexus_operation"), + nexus_input, + operation_handle_fn(), + payload_converter, + ) + handle._apply_schedule_command() + self._pending_nexus_operations[handle._seq] = handle + + await self._await_temporal_operation( + handle._start_fut, + lambda _err, command: handle._apply_cancel_command(command), + reraise_on_workflow_cancellation=True, + ) + return handle #### Miscellaneous helpers #### # These are in alphabetical order. diff --git a/tests/contrib/opentelemetry/test_opentelemetry.py b/tests/contrib/opentelemetry/test_opentelemetry.py index 1bab931ac..55e652925 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry.py +++ b/tests/contrib/opentelemetry/test_opentelemetry.py @@ -88,6 +88,34 @@ class TracingWorkflowActionActivity: fail_on_non_replay_before_complete: bool = False +@workflow.defn +class LegacySignalWithStartHeaderWorkflow: + def __init__(self) -> None: + self._signaled = False + + @workflow.run + async def run(self) -> bool: + await workflow.wait_condition(lambda: self._signaled) + return "_tracer-data" in workflow.info().headers + + @workflow.signal + def notify(self) -> None: + self._signaled = True + + +@workflow.defn +class LegacySignalWithStartCallerWorkflow: + @workflow.run + async def run(self, target_id: str, task_queue: str) -> str: + handle = await workflow.signal_with_start_workflow( + LegacySignalWithStartHeaderWorkflow.run, + id=target_id, + task_queue=task_queue, + signal=LegacySignalWithStartHeaderWorkflow.notify, + ) + return handle.id + + @dataclass class TracingWorkflowActionContinueAsNew: param: TracingWorkflowParam @@ -229,6 +257,38 @@ def update_validator(self) -> None: pass +async def test_legacy_otel_workflow_signal_with_start_propagates_trace_headers( + client: Client, env: WorkflowEnvironment +): + if env.supports_time_skipping: + pytest.skip("Nexus tests don't work with the Java test server") + provider = TracerProvider() + tracer = provider.get_tracer(__name__) + config = client.config() + config["interceptors"] = [TracingInterceptor(tracer)] + client = Client(**config) + + async with Worker( + client, + task_queue=f"signal-with-start-{uuid.uuid4()}", + workflows=[ + LegacySignalWithStartCallerWorkflow, + LegacySignalWithStartHeaderWorkflow, + ], + workflow_runner=UnsandboxedWorkflowRunner(), + ) as worker: + target_id = f"signal-with-start-target-{uuid.uuid4()}" + with tracer.start_as_current_span("signal-with-start"): + caller = await client.start_workflow( + LegacySignalWithStartCallerWorkflow.run, + args=[target_id, worker.task_queue], + id=f"signal-with-start-caller-{uuid.uuid4()}", + task_queue=worker.task_queue, + ) + assert await caller.result() == target_id + assert await client.get_workflow_handle(target_id).result() is True + + async def test_opentelemetry_tracing(client: Client, env: WorkflowEnvironment): # TODO(cretz): Fix if env.supports_time_skipping: diff --git a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py index 12d0c5972..6cf8132ec 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py +++ b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py @@ -122,6 +122,34 @@ async def run(self): return +@workflow.defn +class SignalWithStartHeaderWorkflow: + def __init__(self) -> None: + self._signaled = False + + @workflow.run + async def run(self) -> bool: + await workflow.wait_condition(lambda: self._signaled) + return "_tracer-data" in workflow.info().headers + + @workflow.signal + def notify(self) -> None: + self._signaled = True + + +@workflow.defn +class SignalWithStartCallerWorkflow: + @workflow.run + async def run(self, target_id: str, task_queue: str) -> str: + handle = await workflow.signal_with_start_workflow( + SignalWithStartHeaderWorkflow.run, + id=target_id, + task_queue=task_queue, + signal=SignalWithStartHeaderWorkflow.notify, + ) + return handle.id + + async def test_otel_tracing_basic(client: Client, reset_otel_tracer_provider: Any): # type: ignore[reportUnusedParameter] exporter = InMemorySpanExporter() provider = create_tracer_provider() @@ -169,6 +197,35 @@ async def test_otel_tracing_basic(client: Client, reset_otel_tracer_provider: An ) +async def test_otel_workflow_signal_with_start_propagates_trace_headers( + client: Client, + env: WorkflowEnvironment, + reset_otel_tracer_provider: Any, # type: ignore[reportUnusedParameter] +): + if env.supports_time_skipping: + pytest.skip("Nexus tests don't work with the Java test server") + provider = create_tracer_provider() + opentelemetry.trace.set_tracer_provider(provider) + config = client.config() + config["plugins"] = [OpenTelemetryPlugin()] + client = Client(**config) + + async with new_worker( + client, SignalWithStartCallerWorkflow, SignalWithStartHeaderWorkflow + ) as worker: + target_id = f"signal-with-start-target-{uuid.uuid4()}" + with get_tracer(__name__).start_as_current_span("signal-with-start"): + caller = await client.start_workflow( + SignalWithStartCallerWorkflow.run, + args=[target_id, worker.task_queue], + id=f"signal-with-start-caller-{uuid.uuid4()}", + task_queue=worker.task_queue, + execution_timeout=timedelta(seconds=3), + ) + assert await caller.result() == target_id + assert await client.get_workflow_handle(target_id).result() is True + + @workflow.defn class ComprehensiveWorkflow: def __init__(self) -> None: diff --git a/tests/nexus/test_temporal_system_nexus.py b/tests/nexus/test_temporal_system_nexus.py index 6a9bc9959..f1fb47ea1 100644 --- a/tests/nexus/test_temporal_system_nexus.py +++ b/tests/nexus/test_temporal_system_nexus.py @@ -37,6 +37,7 @@ from temporalio.worker import ( Interceptor, StartNexusOperationInput, + StartSystemNexusOperationInput, Worker, WorkflowInboundInterceptor, WorkflowInterceptorClassInput, @@ -257,6 +258,12 @@ async def start_nexus_operation( interceptor_traces.append(("workflow.start_nexus_operation", input)) return await super().start_nexus_operation(input) + async def start_system_nexus_operation( + self, input: StartSystemNexusOperationInput[Any, Any] + ) -> workflow.NexusOperationHandle[Any]: + interceptor_traces.append(("workflow.start_system_nexus_operation", input)) + return await super().start_system_nexus_operation(input) + def _assert_stored_payloads_include( driver: InMemoryTestDriver, expected_payload_data: set[bytes] @@ -273,8 +280,8 @@ def _assert_stored_payloads_include( def _assert_start_nexus_operation_interceptor_trace() -> None: assert len(interceptor_traces) == 1 trace_name, trace_value = interceptor_traces.pop() - assert trace_name == "workflow.start_nexus_operation" - trace_input = cast(StartNexusOperationInput[Any, Any], trace_value) + assert trace_name == "workflow.start_system_nexus_operation" + trace_input = cast(StartSystemNexusOperationInput[Any, Any], trace_value) request = trace_input.input assert request.id == "system-nexus-workflow-id" assert request.signal == "test-signal" From 32782c49ab97833a633a3194033d8f2252248228 Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Thu, 20 Aug 2026 10:45:49 -0700 Subject: [PATCH 05/12] Propagate tracing in system Nexus request headers --- .../contrib/opentelemetry/_interceptor.py | 12 ++-- .../opentelemetry/_otel_interceptor.py | 3 +- temporalio/converter/_payload_converter.py | 14 +--- temporalio/nexus/system/__init__.py | 16 ++--- temporalio/worker/_interceptor.py | 6 -- temporalio/worker/_workflow_instance.py | 65 +++++-------------- .../opentelemetry/test_opentelemetry.py | 2 + .../test_opentelemetry_plugin.py | 2 + 8 files changed, 36 insertions(+), 84 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index 8bb8564f0..38321a2bc 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -833,11 +833,13 @@ async def start_nexus_operation( async def start_system_nexus_operation( self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] ) -> temporalio.workflow.NexusOperationHandle[Any]: - self.root._completed_span( - f"StartNexusOperation:{input.service}/{input.operation_name}", - kind=opentelemetry.trace.SpanKind.CLIENT, - add_to_outbound=input, - ) + if hasattr(input.input, "headers"): + input.input.headers = input.input.headers or {} + self.root._completed_span( + f"StartNexusOperation:{input.service}/{input.operation_name}", + kind=opentelemetry.trace.SpanKind.CLIENT, + add_to_outbound=cast(_InputWithHeaders, input.input), + ) return await super().start_system_nexus_operation(input) diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index 98875c849..457ff982d 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -608,5 +608,6 @@ async def start_system_nexus_operation( f"StartNexusOperation:{input.service}/{input.operation_name}", kind=opentelemetry.trace.SpanKind.CLIENT, ): - input.headers = _context_to_headers(input.headers) + if hasattr(input.input, "headers"): + input.input.headers = _context_to_headers(input.input.headers or {}) return await super().start_system_nexus_operation(input) diff --git a/temporalio/converter/_payload_converter.py b/temporalio/converter/_payload_converter.py index ac1a921a9..a8bc35e28 100644 --- a/temporalio/converter/_payload_converter.py +++ b/temporalio/converter/_payload_converter.py @@ -613,9 +613,7 @@ def wrap(payload_converter: PayloadConverter) -> PayloadConverter: return _TemporalTransferTypePayloadConverter(payload_converter) def to_payloads( - self, - values: Sequence[Any], - headers: Mapping[str, temporalio.api.common.v1.Payload] | None = None, + self, values: Sequence[Any] ) -> list[temporalio.api.common.v1.Payload]: """See base class.""" transfer_type_values: list[Any] = [] @@ -623,16 +621,6 @@ def to_payloads( converter = _get_transfer_type_converter(type(value)) if converter is not None: value = converter.to_transfer_type(value) - if ( - headers - and isinstance(value, google.protobuf.message.Message) - and "header" in value.DESCRIPTOR.fields_by_name - ): - # System Nexus starts with generated models, so headers can only be - # applied after conversion to a request protobuf and before encoding. - temporalio.common._apply_headers( - headers, getattr(value, "header").fields - ) transfer_type_values.append(value) return self._inner_payload_converter.to_payloads(transfer_type_values) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index ddf223b68..60843aa1e 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -8,7 +8,7 @@ import contextlib import contextvars -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Iterator, Sequence from dataclasses import dataclass from typing import Any @@ -91,31 +91,28 @@ class _SystemNexusPayloadConverter(temporalio.converter.PayloadConverter): """Payload converter for system Nexus outer envelopes.""" _user_converters: _SystemNexusUserConverters - _outer_payload_converter: _TemporalTransferTypePayloadConverter - _headers: Mapping[str, temporalio.api.common.v1.Payload] | None + _outer_payload_converter: temporalio.converter.PayloadConverter def __init__( self, user_payload_converter: temporalio.converter.PayloadConverter, user_failure_converter: temporalio.converter.FailureConverter, - headers: Mapping[str, temporalio.api.common.v1.Payload] | None = None, ) -> None: """Create a payload converter for system Nexus outer envelopes.""" self._user_converters = _SystemNexusUserConverters( user_payload_converter, user_failure_converter ) - self._outer_payload_converter = _TemporalTransferTypePayloadConverter( + self._outer_payload_converter = _TemporalTransferTypePayloadConverter.wrap( _SystemNexusOuterPayloadConverter() ) - self._headers = headers def to_payloads( self, values: Sequence[Any] ) -> list[temporalio.api.common.v1.Payload]: """See base class.""" with _user_converter_context(self._user_converters): - return self._outer_payload_converter.to_payloads(values, self._headers) + return self._outer_payload_converter.to_payloads(values) def from_payloads( self, @@ -167,12 +164,9 @@ async def maybe_visit_payload( def _get_payload_converter( # pyright: ignore[reportUnusedFunction] user_payload_converter: temporalio.converter.PayloadConverter, user_failure_converter: temporalio.converter.FailureConverter, - headers: Mapping[str, temporalio.api.common.v1.Payload] | None = None, ) -> temporalio.converter.PayloadConverter: """Return the fixed payload converter for system Nexus outer envelopes.""" - return _SystemNexusPayloadConverter( - user_payload_converter, user_failure_converter, headers - ) + return _SystemNexusPayloadConverter(user_payload_converter, user_failure_converter) def _get_serialization_context( # pyright: ignore[reportUnusedFunction] diff --git a/temporalio/worker/_interceptor.py b/temporalio/worker/_interceptor.py index 76209ab56..b886a59b6 100644 --- a/temporalio/worker/_interceptor.py +++ b/temporalio/worker/_interceptor.py @@ -355,12 +355,6 @@ class StartSystemNexusOperationInput(Generic[InputT, OutputT]): service: str operation: nexusrpc.Operation[InputT, OutputT] | str | Callable[..., Any] input: InputT - schedule_to_close_timeout: timedelta | None - schedule_to_start_timeout: timedelta | None - start_to_close_timeout: timedelta | None - cancellation_type: temporalio.workflow.NexusOperationCancellationType - headers: Mapping[str, temporalio.api.common.v1.Payload] - summary: str | None output_type: type[OutputT] | None = None @property diff --git a/temporalio/worker/_workflow_instance.py b/temporalio/worker/_workflow_instance.py index 18ad2228b..a98d0d4af 100644 --- a/temporalio/worker/_workflow_instance.py +++ b/temporalio/worker/_workflow_instance.py @@ -1734,18 +1734,15 @@ async def workflow_start_nexus_operation( summary: str | None, ) -> temporalio.workflow.NexusOperationHandle[OutputT]: if temporalio.nexus.system.is_system_endpoint(endpoint): + # System operations have no caller-configurable Nexus options. Do not + # expose the normal operation's scheduling, cancellation, headers, or + # summary arguments to System Nexus interceptors. return await self._outbound.start_system_nexus_operation( StartSystemNexusOperationInput( service=service, operation=operation, input=input, output_type=output_type, - schedule_to_close_timeout=schedule_to_close_timeout, - schedule_to_start_timeout=schedule_to_start_timeout, - start_to_close_timeout=start_to_close_timeout, - cancellation_type=cancellation_type, - headers={}, - summary=summary, ) ) return await self._outbound.start_nexus_operation( @@ -2197,51 +2194,23 @@ async def operation_handle_fn() -> OutputT: async def _outbound_start_system_nexus_operation( self, input: StartSystemNexusOperationInput[Any, OutputT] ) -> _NexusOperationHandle[OutputT]: - nexus_input = StartNexusOperationInput( - endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, - service=input.service, - operation=input.operation, - input=input.input, - output_type=input.output_type, - schedule_to_close_timeout=input.schedule_to_close_timeout, - schedule_to_start_timeout=input.schedule_to_start_timeout, - start_to_close_timeout=input.start_to_close_timeout, - cancellation_type=input.cancellation_type, - headers=None, - summary=input.summary, - ) - handle: _NexusOperationHandle[OutputT] - - async def operation_handle_fn() -> OutputT: - return cast( - OutputT, - await self._await_temporal_operation( - handle._result_fut, - lambda _err, command: handle._apply_cancel_command(command), + return await self._outbound_start_nexus_operation( + StartNexusOperationInput( + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + service=input.service, + operation=input.operation, + input=input.input, + output_type=input.output_type, + schedule_to_close_timeout=None, + schedule_to_start_timeout=None, + start_to_close_timeout=None, + cancellation_type=( + temporalio.workflow.NexusOperationCancellationType.WAIT_COMPLETED ), + headers=None, + summary=None, ) - - payload_converter = temporalio.nexus.system._get_payload_converter( - self._workflow_context_payload_converter, - self._workflow_context_failure_converter, - input.headers, - ) - handle = _NexusOperationHandle( - self, - self._next_seq("nexus_operation"), - nexus_input, - operation_handle_fn(), - payload_converter, ) - handle._apply_schedule_command() - self._pending_nexus_operations[handle._seq] = handle - - await self._await_temporal_operation( - handle._start_fut, - lambda _err, command: handle._apply_cancel_command(command), - reraise_on_workflow_cancellation=True, - ) - return handle #### Miscellaneous helpers #### # These are in alphabetical order. diff --git a/tests/contrib/opentelemetry/test_opentelemetry.py b/tests/contrib/opentelemetry/test_opentelemetry.py index 55e652925..3253f4f2b 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry.py +++ b/tests/contrib/opentelemetry/test_opentelemetry.py @@ -257,6 +257,8 @@ def update_validator(self) -> None: pass +# Cloud namespaces created by CI do not have the System Nexus dynamic config. +@pytest.mark.requires_local_server async def test_legacy_otel_workflow_signal_with_start_propagates_trace_headers( client: Client, env: WorkflowEnvironment ): diff --git a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py index 6cf8132ec..8d922aeaa 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py +++ b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py @@ -197,6 +197,8 @@ async def test_otel_tracing_basic(client: Client, reset_otel_tracer_provider: An ) +# Cloud namespaces created by CI do not have the System Nexus dynamic config. +@pytest.mark.requires_local_server async def test_otel_workflow_signal_with_start_propagates_trace_headers( client: Client, env: WorkflowEnvironment, From 836a912d5d1e46ea75376dc0c006fa07880ebbd7 Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Mon, 24 Aug 2026 11:07:31 -0700 Subject: [PATCH 06/12] Name system Nexus tracing spans by operation --- temporalio/contrib/opentelemetry/_interceptor.py | 10 +++++++++- temporalio/contrib/opentelemetry/_otel_interceptor.py | 10 +++++++++- tests/contrib/opentelemetry/test_opentelemetry.py | 3 +++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index 38321a2bc..15c8e51d6 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -58,6 +58,14 @@ _ContextT = TypeVar("_ContextT", bound=nexusrpc.handler.OperationContext) +def _system_nexus_operation_span_name(service: str, operation: str) -> str: + match operation: + case "SignalWithStartWorkflowExecution": + return "SignalWithStart" + case _: + return f"StartSystemNexusOperation:{service}/{operation}" + + class TracingInterceptor(temporalio.client.Interceptor, temporalio.worker.Interceptor): """Interceptor that supports client and worker OpenTelemetry span creation and propagation. @@ -836,7 +844,7 @@ async def start_system_nexus_operation( if hasattr(input.input, "headers"): input.input.headers = input.input.headers or {} self.root._completed_span( - f"StartNexusOperation:{input.service}/{input.operation_name}", + _system_nexus_operation_span_name(input.service, input.operation_name), kind=opentelemetry.trace.SpanKind.CLIENT, add_to_outbound=cast(_InputWithHeaders, input.input), ) diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index 457ff982d..eed9acc07 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -58,6 +58,14 @@ _CarrierDict: TypeAlias = dict[str, opentelemetry.propagators.textmap.CarrierValT] +def _system_nexus_operation_span_name(service: str, operation: str) -> str: + match operation: + case "SignalWithStartWorkflowExecution": + return "SignalWithStart" + case _: + return f"StartSystemNexusOperation:{service}/{operation}" + + def _context_to_headers( headers: Mapping[str, temporalio.api.common.v1.Payload], ) -> Mapping[str, temporalio.api.common.v1.Payload]: @@ -605,7 +613,7 @@ async def start_system_nexus_operation( self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] ) -> temporalio.workflow.NexusOperationHandle[Any]: with self._workflow_maybe_span( - f"StartNexusOperation:{input.service}/{input.operation_name}", + _system_nexus_operation_span_name(input.service, input.operation_name), kind=opentelemetry.trace.SpanKind.CLIENT, ): if hasattr(input.input, "headers"): diff --git a/tests/contrib/opentelemetry/test_opentelemetry.py b/tests/contrib/opentelemetry/test_opentelemetry.py index 3253f4f2b..7bb6a5bf0 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry.py +++ b/tests/contrib/opentelemetry/test_opentelemetry.py @@ -265,6 +265,8 @@ async def test_legacy_otel_workflow_signal_with_start_propagates_trace_headers( if env.supports_time_skipping: pytest.skip("Nexus tests don't work with the Java test server") provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) tracer = provider.get_tracer(__name__) config = client.config() config["interceptors"] = [TracingInterceptor(tracer)] @@ -289,6 +291,7 @@ async def test_legacy_otel_workflow_signal_with_start_propagates_trace_headers( ) assert await caller.result() == target_id assert await client.get_workflow_handle(target_id).result() is True + assert any(span.name == "SignalWithStart" for span in exporter.get_finished_spans()) async def test_opentelemetry_tracing(client: Client, env: WorkflowEnvironment): From 412c98970b77dfb0976d3eee7a8b54a497e633fa Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Mon, 24 Aug 2026 11:08:18 -0700 Subject: [PATCH 07/12] Cover system Nexus tracing span names --- tests/contrib/opentelemetry/test_opentelemetry_plugin.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py index 8d922aeaa..55434c169 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py +++ b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py @@ -206,10 +206,12 @@ async def test_otel_workflow_signal_with_start_propagates_trace_headers( ): if env.supports_time_skipping: pytest.skip("Nexus tests don't work with the Java test server") + exporter = InMemorySpanExporter() provider = create_tracer_provider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) opentelemetry.trace.set_tracer_provider(provider) config = client.config() - config["plugins"] = [OpenTelemetryPlugin()] + config["plugins"] = [OpenTelemetryPlugin(add_temporal_spans=True)] client = Client(**config) async with new_worker( @@ -226,6 +228,7 @@ async def test_otel_workflow_signal_with_start_propagates_trace_headers( ) assert await caller.result() == target_id assert await client.get_workflow_handle(target_id).result() is True + assert any(span.name == "SignalWithStart" for span in exporter.get_finished_spans()) @workflow.defn From 769a1d1894cb6863af0b39b5ca9f931fccb3db28 Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Tue, 25 Aug 2026 13:18:30 -0700 Subject: [PATCH 08/12] Share System Nexus span names --- temporalio/contrib/opentelemetry/_interceptor.py | 13 ++++--------- .../contrib/opentelemetry/_otel_interceptor.py | 13 ++++--------- temporalio/nexus/system/__init__.py | 8 ++++++++ 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index 15c8e51d6..139e6e05d 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -33,6 +33,7 @@ import temporalio.client import temporalio.converter import temporalio.exceptions +import temporalio.nexus.system import temporalio.worker import temporalio.workflow from temporalio.exceptions import ApplicationError, ApplicationErrorCategory @@ -58,14 +59,6 @@ _ContextT = TypeVar("_ContextT", bound=nexusrpc.handler.OperationContext) -def _system_nexus_operation_span_name(service: str, operation: str) -> str: - match operation: - case "SignalWithStartWorkflowExecution": - return "SignalWithStart" - case _: - return f"StartSystemNexusOperation:{service}/{operation}" - - class TracingInterceptor(temporalio.client.Interceptor, temporalio.worker.Interceptor): """Interceptor that supports client and worker OpenTelemetry span creation and propagation. @@ -844,7 +837,9 @@ async def start_system_nexus_operation( if hasattr(input.input, "headers"): input.input.headers = input.input.headers or {} self.root._completed_span( - _system_nexus_operation_span_name(input.service, input.operation_name), + temporalio.nexus.system._system_nexus_operation_span_name( + input.service, input.operation_name + ), kind=opentelemetry.trace.SpanKind.CLIENT, add_to_outbound=cast(_InputWithHeaders, input.input), ) diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index eed9acc07..0f638a7fe 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -32,6 +32,7 @@ import temporalio.api.common.v1 import temporalio.client import temporalio.converter +import temporalio.nexus.system import temporalio.worker import temporalio.workflow from temporalio.contrib.opentelemetry._tracer_provider import ( @@ -58,14 +59,6 @@ _CarrierDict: TypeAlias = dict[str, opentelemetry.propagators.textmap.CarrierValT] -def _system_nexus_operation_span_name(service: str, operation: str) -> str: - match operation: - case "SignalWithStartWorkflowExecution": - return "SignalWithStart" - case _: - return f"StartSystemNexusOperation:{service}/{operation}" - - def _context_to_headers( headers: Mapping[str, temporalio.api.common.v1.Payload], ) -> Mapping[str, temporalio.api.common.v1.Payload]: @@ -613,7 +606,9 @@ async def start_system_nexus_operation( self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] ) -> temporalio.workflow.NexusOperationHandle[Any]: with self._workflow_maybe_span( - _system_nexus_operation_span_name(input.service, input.operation_name), + temporalio.nexus.system._system_nexus_operation_span_name( + input.service, input.operation_name + ), kind=opentelemetry.trace.SpanKind.CLIENT, ): if hasattr(input.input, "headers"): diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index 60843aa1e..481f384b3 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -24,6 +24,14 @@ TEMPORAL_SYSTEM_ENDPOINT = "__temporal_system" +def _system_nexus_operation_span_name(service: str, operation: str) -> str: + match operation: + case "SignalWithStartWorkflowExecution": + return "SignalWithStart" + case _: + return f"StartSystemNexusOperation:{service}/{operation}" + + @dataclass(frozen=True) class _SystemNexusUserConverters: payload_converter: temporalio.converter.PayloadConverter From c99748e2402a05aa36d4c1f3428ea58e376b71fe Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Tue, 25 Aug 2026 13:22:49 -0700 Subject: [PATCH 09/12] Name signal-with-start workflow spans --- temporalio/nexus/system/__init__.py | 2 +- tests/contrib/opentelemetry/test_opentelemetry.py | 5 ++++- tests/contrib/opentelemetry/test_opentelemetry_plugin.py | 5 ++++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index 481f384b3..c572d3940 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -27,7 +27,7 @@ def _system_nexus_operation_span_name(service: str, operation: str) -> str: match operation: case "SignalWithStartWorkflowExecution": - return "SignalWithStart" + return "SignalWithStartWorkflow" case _: return f"StartSystemNexusOperation:{service}/{operation}" diff --git a/tests/contrib/opentelemetry/test_opentelemetry.py b/tests/contrib/opentelemetry/test_opentelemetry.py index 7bb6a5bf0..331a9d99e 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry.py +++ b/tests/contrib/opentelemetry/test_opentelemetry.py @@ -291,7 +291,10 @@ async def test_legacy_otel_workflow_signal_with_start_propagates_trace_headers( ) assert await caller.result() == target_id assert await client.get_workflow_handle(target_id).result() is True - assert any(span.name == "SignalWithStart" for span in exporter.get_finished_spans()) + assert any( + span.name == "SignalWithStartWorkflow" + for span in exporter.get_finished_spans() + ) async def test_opentelemetry_tracing(client: Client, env: WorkflowEnvironment): diff --git a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py index 55434c169..3166a2614 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py +++ b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py @@ -228,7 +228,10 @@ async def test_otel_workflow_signal_with_start_propagates_trace_headers( ) assert await caller.result() == target_id assert await client.get_workflow_handle(target_id).result() is True - assert any(span.name == "SignalWithStart" for span in exporter.get_finished_spans()) + assert any( + span.name == "SignalWithStartWorkflow" + for span in exporter.get_finished_spans() + ) @workflow.defn From b7a5f3e6024a8e2c098a8276fd750c49b8036e2c Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Tue, 25 Aug 2026 13:34:08 -0700 Subject: [PATCH 10/12] Check System Nexus header types --- .../contrib/opentelemetry/_interceptor.py | 3 +-- .../opentelemetry/_otel_interceptor.py | 4 ++-- temporalio/nexus/system/__init__.py | 23 +++++++++++++++++-- .../operations/signal_with_start_workflow.py | 1 + .../opentelemetry/test_opentelemetry.py | 3 +-- .../test_opentelemetry_plugin.py | 3 +-- tests/nexus/test_temporal_system_nexus.py | 20 +++++++++++++++- 7 files changed, 46 insertions(+), 11 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index 139e6e05d..670f61f33 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -834,8 +834,7 @@ async def start_nexus_operation( async def start_system_nexus_operation( self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] ) -> temporalio.workflow.NexusOperationHandle[Any]: - if hasattr(input.input, "headers"): - input.input.headers = input.input.headers or {} + if temporalio.nexus.system._has_payload_headers(input.input): self.root._completed_span( temporalio.nexus.system._system_nexus_operation_span_name( input.service, input.operation_name diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index 0f638a7fe..9eadf3a00 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -611,6 +611,6 @@ async def start_system_nexus_operation( ), kind=opentelemetry.trace.SpanKind.CLIENT, ): - if hasattr(input.input, "headers"): - input.input.headers = _context_to_headers(input.input.headers or {}) + if temporalio.nexus.system._has_payload_headers(input.input): + input.input.headers = _context_to_headers(input.input.headers) return await super().start_system_nexus_operation(input) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index c572d3940..384cd5c50 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -8,9 +8,9 @@ import contextlib import contextvars -from collections.abc import Iterator, Sequence +from collections.abc import Iterator, Mapping, Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, get_args, get_origin, get_type_hints import temporalio.api.common.v1 import temporalio.common @@ -24,6 +24,25 @@ TEMPORAL_SYSTEM_ENDPOINT = "__temporal_system" +def _has_payload_headers(value: Any) -> bool: + headers = getattr(value, "headers", None) + if not isinstance(headers, Mapping): + return False + header_type = get_type_hints(type(value)).get("headers") + for possible_type in get_args(header_type) or (header_type,): + if get_origin(possible_type) is Mapping: + key_type, value_type = get_args(possible_type) + if key_type is str and ( + value_type is Any + or ( + isinstance(value_type, type) + and issubclass(temporalio.api.common.v1.Payload, value_type) + ) + ): + return True + return False + + def _system_nexus_operation_span_name(service: str, operation: str) -> str: match operation: case "SignalWithStartWorkflowExecution": diff --git a/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py b/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py index 97eb8159a..25f5257c6 100644 --- a/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py +++ b/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py @@ -650,5 +650,6 @@ async def signal_with_start_workflow( versioning_override=versioning_override, start_delay=start_delay, user_metadata=user_metadata, + headers={}, ) return await _signal_with_start_workflow(request) diff --git a/tests/contrib/opentelemetry/test_opentelemetry.py b/tests/contrib/opentelemetry/test_opentelemetry.py index 331a9d99e..1d47fea5d 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry.py +++ b/tests/contrib/opentelemetry/test_opentelemetry.py @@ -292,8 +292,7 @@ async def test_legacy_otel_workflow_signal_with_start_propagates_trace_headers( assert await caller.result() == target_id assert await client.get_workflow_handle(target_id).result() is True assert any( - span.name == "SignalWithStartWorkflow" - for span in exporter.get_finished_spans() + span.name == "SignalWithStartWorkflow" for span in exporter.get_finished_spans() ) diff --git a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py index 3166a2614..b34e677f0 100644 --- a/tests/contrib/opentelemetry/test_opentelemetry_plugin.py +++ b/tests/contrib/opentelemetry/test_opentelemetry_plugin.py @@ -229,8 +229,7 @@ async def test_otel_workflow_signal_with_start_propagates_trace_headers( assert await caller.result() == target_id assert await client.get_workflow_handle(target_id).result() is True assert any( - span.name == "SignalWithStartWorkflow" - for span in exporter.get_finished_spans() + span.name == "SignalWithStartWorkflow" for span in exporter.get_finished_spans() ) diff --git a/tests/nexus/test_temporal_system_nexus.py b/tests/nexus/test_temporal_system_nexus.py index f1fb47ea1..d806ec15a 100644 --- a/tests/nexus/test_temporal_system_nexus.py +++ b/tests/nexus/test_temporal_system_nexus.py @@ -2,7 +2,7 @@ import dataclasses import uuid -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import timedelta from typing import Any, cast @@ -147,6 +147,7 @@ def test_signal_with_start_serialization_context() -> None: task_queue="target-task-queue", signal="test-signal", namespace="target-namespace", + headers={}, ) operation_info = workflow_service.__nexus_operation_registry__[ ( @@ -166,6 +167,23 @@ def test_signal_with_start_serialization_context() -> None: assert context.workflow_id == "target-workflow-id" +def test_system_nexus_payload_header_detection() -> None: + class NexusInput: + headers: Mapping[str, str] = {} + + request = workflow_service_models.SignalWithStartWorkflowRequest( + workflow="test-workflow", + id="target-workflow-id", + task_queue="target-task-queue", + signal="test-signal", + namespace="target-namespace", + headers={}, + ) + + assert nexus_system._has_payload_headers(request) + assert not nexus_system._has_payload_headers(NexusInput()) + + class RejectOuterSystemNexusCodec(PayloadCodec): def __init__(self) -> None: self.encode_count = 0 From f1bbf5891521f7220019981d5c08ad91491e5154 Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Tue, 25 Aug 2026 15:02:26 -0700 Subject: [PATCH 11/12] Silence System Nexus helper lint warnings --- temporalio/nexus/system/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index 384cd5c50..79ca9eb50 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -24,7 +24,7 @@ TEMPORAL_SYSTEM_ENDPOINT = "__temporal_system" -def _has_payload_headers(value: Any) -> bool: +def _has_payload_headers(value: Any) -> bool: # pyright: ignore[reportUnusedFunction] headers = getattr(value, "headers", None) if not isinstance(headers, Mapping): return False @@ -43,7 +43,9 @@ def _has_payload_headers(value: Any) -> bool: return False -def _system_nexus_operation_span_name(service: str, operation: str) -> str: +def _system_nexus_operation_span_name( # pyright: ignore[reportUnusedFunction] + service: str, operation: str +) -> str: match operation: case "SignalWithStartWorkflowExecution": return "SignalWithStartWorkflow" From 3b7578b3e503d85a446f6604690ec663444f276e Mon Sep 17 00:00:00 2001 From: Tim Conley Date: Tue, 25 Aug 2026 15:10:04 -0700 Subject: [PATCH 12/12] Initialize payload headers in tracing interceptors --- temporalio/contrib/opentelemetry/_interceptor.py | 1 + temporalio/contrib/opentelemetry/_otel_interceptor.py | 2 +- temporalio/nexus/system/__init__.py | 3 --- .../workflow_service/operations/signal_with_start_workflow.py | 1 - tests/nexus/test_temporal_system_nexus.py | 2 -- 5 files changed, 2 insertions(+), 7 deletions(-) diff --git a/temporalio/contrib/opentelemetry/_interceptor.py b/temporalio/contrib/opentelemetry/_interceptor.py index 670f61f33..6cf01abff 100644 --- a/temporalio/contrib/opentelemetry/_interceptor.py +++ b/temporalio/contrib/opentelemetry/_interceptor.py @@ -835,6 +835,7 @@ async def start_system_nexus_operation( self, input: temporalio.worker.StartSystemNexusOperationInput[Any, Any] ) -> temporalio.workflow.NexusOperationHandle[Any]: if temporalio.nexus.system._has_payload_headers(input.input): + input.input.headers = input.input.headers or {} self.root._completed_span( temporalio.nexus.system._system_nexus_operation_span_name( input.service, input.operation_name diff --git a/temporalio/contrib/opentelemetry/_otel_interceptor.py b/temporalio/contrib/opentelemetry/_otel_interceptor.py index 9eadf3a00..37dc2a54e 100644 --- a/temporalio/contrib/opentelemetry/_otel_interceptor.py +++ b/temporalio/contrib/opentelemetry/_otel_interceptor.py @@ -612,5 +612,5 @@ async def start_system_nexus_operation( kind=opentelemetry.trace.SpanKind.CLIENT, ): if temporalio.nexus.system._has_payload_headers(input.input): - input.input.headers = _context_to_headers(input.input.headers) + input.input.headers = _context_to_headers(input.input.headers or {}) return await super().start_system_nexus_operation(input) diff --git a/temporalio/nexus/system/__init__.py b/temporalio/nexus/system/__init__.py index 79ca9eb50..bf360ea14 100644 --- a/temporalio/nexus/system/__init__.py +++ b/temporalio/nexus/system/__init__.py @@ -25,9 +25,6 @@ def _has_payload_headers(value: Any) -> bool: # pyright: ignore[reportUnusedFunction] - headers = getattr(value, "headers", None) - if not isinstance(headers, Mapping): - return False header_type = get_type_hints(type(value)).get("headers") for possible_type in get_args(header_type) or (header_type,): if get_origin(possible_type) is Mapping: diff --git a/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py b/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py index 25f5257c6..97eb8159a 100644 --- a/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py +++ b/temporalio/nexus/system/workflow_service/operations/signal_with_start_workflow.py @@ -650,6 +650,5 @@ async def signal_with_start_workflow( versioning_override=versioning_override, start_delay=start_delay, user_metadata=user_metadata, - headers={}, ) return await _signal_with_start_workflow(request) diff --git a/tests/nexus/test_temporal_system_nexus.py b/tests/nexus/test_temporal_system_nexus.py index d806ec15a..029f8a7ca 100644 --- a/tests/nexus/test_temporal_system_nexus.py +++ b/tests/nexus/test_temporal_system_nexus.py @@ -147,7 +147,6 @@ def test_signal_with_start_serialization_context() -> None: task_queue="target-task-queue", signal="test-signal", namespace="target-namespace", - headers={}, ) operation_info = workflow_service.__nexus_operation_registry__[ ( @@ -177,7 +176,6 @@ class NexusInput: task_queue="target-task-queue", signal="test-signal", namespace="target-namespace", - headers={}, ) assert nexus_system._has_payload_headers(request)