Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions crates/pantograph-diagnostics-ledger/src/tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6485,8 +6485,8 @@ fn sample_inference_execution_diagnostic_event() -> DiagnosticEventAppendRequest
privacy_class: DiagnosticEventPrivacyClass::SystemMetadata,
retention_class: DiagnosticEventRetentionClass::AuditMetadata,
payload_ref: None,
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(
Box::new(InferenceExecutionDiagnosticObservedPayload {
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(Box::new(
InferenceExecutionDiagnosticObservedPayload {
request_id: "req-a".to_string(),
task_id: "text_generation".to_string(),
lifecycle_phase: Some("task_validation".to_string()),
Expand Down Expand Up @@ -6552,8 +6552,8 @@ fn sample_inference_execution_diagnostic_event() -> DiagnosticEventAppendRequest
message: Some("not mapped by this backend boundary".to_string()),
},
],
}),
),
},
)),
}
}

Expand Down
70 changes: 36 additions & 34 deletions crates/pantograph-embedded-runtime/src/node_execution_ledger.rs
Original file line number Diff line number Diff line change
Expand Up @@ -532,28 +532,30 @@ impl NodeExecutionWorkflowLedgerSink {
privacy_class: artifact.privacy_class,
retention_class: artifact.retention_class,
payload_ref: artifact.payload_ref,
payload: DiagnosticEventPayload::IoArtifactObserved(Box::new(IoArtifactObservedPayload {
artifact_fact_id: Some(artifact.artifact_fact_id),
payload_artifact_id: Some(artifact.payload_artifact_id),
artifact_id: artifact.artifact_id,
artifact_role,
logical_payload_lineage_id: Some(artifact.logical_payload_lineage_id),
producer_node_id,
producer_port_id,
consumer_node_id,
consumer_port_id,
media_type: artifact.media_type,
size_bytes: artifact.size_bytes,
content_hash: artifact.content_hash,
retention_state: Some(artifact.retention_state),
retention_reason: artifact.retention_reason,
payload_kind: artifact.payload_kind,
lifecycle_state: artifact.lifecycle_state,
access_modes: artifact.access_modes,
read_handle: artifact.read_handle,
stream_handle: None,
format: artifact.format,
})),
payload: DiagnosticEventPayload::IoArtifactObserved(Box::new(
IoArtifactObservedPayload {
artifact_fact_id: Some(artifact.artifact_fact_id),
payload_artifact_id: Some(artifact.payload_artifact_id),
artifact_id: artifact.artifact_id,
artifact_role,
logical_payload_lineage_id: Some(artifact.logical_payload_lineage_id),
producer_node_id,
producer_port_id,
consumer_node_id,
consumer_port_id,
media_type: artifact.media_type,
size_bytes: artifact.size_bytes,
content_hash: artifact.content_hash,
retention_state: Some(artifact.retention_state),
retention_reason: artifact.retention_reason,
payload_kind: artifact.payload_kind,
lifecycle_state: artifact.lifecycle_state,
access_modes: artifact.access_modes,
read_handle: artifact.read_handle,
stream_handle: None,
format: artifact.format,
},
)),
};

self.workflow_service
Expand Down Expand Up @@ -951,8 +953,8 @@ fn build_kv_cache_diagnostic_event_ledger_append_request(
privacy_class: DiagnosticEventPrivacyClass::SystemMetadata,
retention_class: DiagnosticEventRetentionClass::AuditMetadata,
payload_ref: None,
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(
Box::new(InferenceExecutionDiagnosticObservedPayload {
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(Box::new(
InferenceExecutionDiagnosticObservedPayload {
request_id: format!("{task_id}:kv_cache"),
task_id: "kv_cache".to_string(),
lifecycle_phase: Some("kv_cache".to_string()),
Expand Down Expand Up @@ -981,8 +983,8 @@ fn build_kv_cache_diagnostic_event_ledger_append_request(
.take(MAX_INFERENCE_OPTION_DIAGNOSTICS)
.map(kv_cache_option_diagnostic_summary)
.collect(),
}),
),
},
)),
})
}

Expand Down Expand Up @@ -1047,8 +1049,8 @@ fn build_runtime_settings_diagnostic_event_ledger_append_request(
privacy_class: DiagnosticEventPrivacyClass::SystemMetadata,
retention_class: DiagnosticEventRetentionClass::AuditMetadata,
payload_ref: None,
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(
Box::new(InferenceExecutionDiagnosticObservedPayload {
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(Box::new(
InferenceExecutionDiagnosticObservedPayload {
request_id: format!("{task_id}:runtime_settings"),
task_id: "runtime_settings".to_string(),
lifecycle_phase: Some("runtime_settings".to_string()),
Expand All @@ -1075,8 +1077,8 @@ fn build_runtime_settings_diagnostic_event_ledger_append_request(
compatibility_issues: Vec::new(),
option_support_counts: InferenceOptionSupportCounts::default(),
option_diagnostics: Vec::new(),
}),
),
},
)),
})
}

Expand Down Expand Up @@ -1146,8 +1148,8 @@ fn build_inference_diagnostic_event_ledger_append_request(
privacy_class: DiagnosticEventPrivacyClass::SystemMetadata,
retention_class: DiagnosticEventRetentionClass::AuditMetadata,
payload_ref: None,
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(
Box::new(InferenceExecutionDiagnosticObservedPayload {
payload: DiagnosticEventPayload::InferenceExecutionDiagnosticObserved(Box::new(
InferenceExecutionDiagnosticObservedPayload {
request_id: event
.request_id
.clone()
Expand Down Expand Up @@ -1196,8 +1198,8 @@ fn build_inference_diagnostic_event_ledger_append_request(
.take(MAX_INFERENCE_OPTION_DIAGNOSTICS)
.map(option_diagnostic_summary)
.collect(),
}),
),
},
)),
})
}

Expand Down
21 changes: 12 additions & 9 deletions crates/pantograph-runtime-registry/src/technical_fit_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2563,15 +2563,18 @@ fn selector_reports_resource_accounting_overflow_before_selection() {

assert_eq!(decision.selected_candidate_id, None);
assert_eq!(decision.device_diagnostics.len(), 1);
assert_eq!(serde_json::to_value(&decision.device_diagnostics[0]).unwrap(), serde_json::json!({
"code": "resource_accounting_overflow",
"severity": "error",
"message": "technical-fit resource accounting overflowed while summing active ram_bytes claims for runtime 'runtime-overflow'",
"runtime_id": "runtime-overflow",
"backend_key": "pytorch",
"evidence_key": "ram_bytes",
"requested_runtime_key": "pytorch"
}));
assert_eq!(
serde_json::to_value(&decision.device_diagnostics[0]).unwrap(),
serde_json::json!({
"code": "resource_accounting_overflow",
"severity": "error",
"message": "technical-fit resource accounting overflowed while summing active ram_bytes claims for runtime 'runtime-overflow'",
"runtime_id": "runtime-overflow",
"backend_key": "pytorch",
"evidence_key": "ram_bytes",
"requested_runtime_key": "pytorch"
})
);
assert_eq!(
decision.device_diagnostics[0].code,
RuntimeTechnicalFitDeviceDiagnosticCode::ResourceAccountingOverflow
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1467,8 +1467,8 @@ impl CurrentInferenceValidationNodeRecord {
message: format!("dependency requirements proof is not current: {error:?}"),
},
)?;
return Ok(WorkflowSchedulerInferenceTaskProjection::Ready(
Box::new(WorkflowSchedulerReadyInferenceTaskProjection {
return Ok(WorkflowSchedulerInferenceTaskProjection::Ready(Box::new(
WorkflowSchedulerReadyInferenceTaskProjection {
node_id: pantograph_scheduler::SchedulerNodeId::parse(self.node_id.as_str())
.map_err(|error| {
CurrentInferenceSchedulerProjectionError::IncompleteNodeState {
Expand Down Expand Up @@ -1532,8 +1532,8 @@ impl CurrentInferenceValidationNodeRecord {
.dependency_override_fingerprint
.clone(),
},
}),
));
},
)));
}

Ok(WorkflowSchedulerInferenceTaskProjection::Blocked(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2115,9 +2115,9 @@ fn dispatch_selected_handoff_from_selection(
) -> Result<SchedulerRuntimeHandoff, WorkflowSchedulerTaskOrchestratorError> {
if selection.state != SchedulerDispatchSelectionState::Selected {
return Err(
WorkflowSchedulerTaskOrchestratorError::RuntimeDispatchSelectionNoSelection(
Box::new(selection),
),
WorkflowSchedulerTaskOrchestratorError::RuntimeDispatchSelectionNoSelection(Box::new(
selection,
)),
);
}
let Some(dispatch_decision) = selection.dispatch_decision else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -491,8 +491,8 @@ impl WorkflowExecutableValidationSnapshotNode {
}
})?;

Ok(WorkflowSchedulerInferenceTaskProjection::Ready(
Box::new(WorkflowSchedulerReadyInferenceTaskProjection {
Ok(WorkflowSchedulerInferenceTaskProjection::Ready(Box::new(
WorkflowSchedulerReadyInferenceTaskProjection {
node_id: scheduler_node_id,
descriptor_fingerprint: self.descriptor_fingerprint.clone(),
task_type,
Expand All @@ -504,8 +504,8 @@ impl WorkflowExecutableValidationSnapshotNode {
dependency_readiness_source: workflow_scheduler_dependency_readiness_source(
snapshot, self,
)?,
}),
))
},
)))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1353,8 +1353,8 @@ impl WorkflowService {
privacy_class: metadata.privacy_class,
retention_class: metadata.retention_class,
payload_ref: metadata.payload_ref.clone(),
payload: DiagnosticEventPayload::IoArtifactObserved(
Box::new(IoArtifactObservedPayload {
payload: DiagnosticEventPayload::IoArtifactObserved(Box::new(
IoArtifactObservedPayload {
artifact_fact_id: Some(metadata.artifact_fact_id),
payload_artifact_id: Some(metadata.payload_artifact_id),
artifact_id: metadata.artifact_id,
Expand Down Expand Up @@ -1391,8 +1391,8 @@ impl WorkflowService {
read_handle: metadata.read_handle,
stream_handle: metadata.stream_handle,
format: metadata.format,
}),
),
},
)),
},
)?;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -513,16 +513,18 @@ impl<'a> WorkflowSchedulerSessionRunner<'a> {
"scheduler non-runtime task completion failed: {error}"
))
})?;
self.record_scheduler_task_attempt_terminal(WorkflowSchedulerTaskAttemptTerminalInput {
task: started.task(),
attempt_id: started.attempt_id().as_str(),
started_at_ms: started.started_at_ms(),
transition: SchedulerTaskAttemptLifecycleTransition::Completed,
reason: "scheduler task attempt completed",
error_summary: None,
selected_dispatch: None,
terminal_mutation: None,
})?;
self.record_scheduler_task_attempt_terminal(
WorkflowSchedulerTaskAttemptTerminalInput {
task: started.task(),
attempt_id: started.attempt_id().as_str(),
started_at_ms: started.started_at_ms(),
transition: SchedulerTaskAttemptLifecycleTransition::Completed,
reason: "scheduler task attempt completed",
error_summary: None,
selected_dispatch: None,
terminal_mutation: None,
},
)?;
}
Err(
crate::scheduler::WorkflowSchedulerTaskOrchestratorError::NonRuntimeTaskAdapter(
Expand All @@ -541,16 +543,18 @@ impl<'a> WorkflowSchedulerSessionRunner<'a> {
&error,
);
if failed.is_ok() {
self.record_scheduler_task_attempt_terminal(WorkflowSchedulerTaskAttemptTerminalInput {
task: started.task(),
attempt_id: started.attempt_id().as_str(),
started_at_ms: started.started_at_ms(),
transition: SchedulerTaskAttemptLifecycleTransition::Failed,
reason: "scheduler non-runtime task execution failed",
error_summary: Some(error.to_string()),
selected_dispatch: None,
terminal_mutation: None,
})?;
self.record_scheduler_task_attempt_terminal(
WorkflowSchedulerTaskAttemptTerminalInput {
task: started.task(),
attempt_id: started.attempt_id().as_str(),
started_at_ms: started.started_at_ms(),
transition: SchedulerTaskAttemptLifecycleTransition::Failed,
reason: "scheduler non-runtime task execution failed",
error_summary: Some(error.to_string()),
selected_dispatch: None,
terminal_mutation: None,
},
)?;
}
return Err(WorkflowServiceError::InvalidRequest(format!(
"scheduler non-runtime task execution failed: {error}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1124,12 +1124,14 @@ impl WorkflowTaskExecutionWorkerOutcome {
response: WorkflowRunResponse,
diagnostics: Vec<WorkflowTaskExecutionWorkerDiagnostic>,
) -> Self {
Self::RuntimeBranchCompleted(Box::new(WorkflowTaskExecutionWorkerRuntimeBranchCompletedOutcome {
session_id: command.session_id.clone(),
workflow_run_id: command.workflow_run_id.clone(),
response,
diagnostics,
}))
Self::RuntimeBranchCompleted(Box::new(
WorkflowTaskExecutionWorkerRuntimeBranchCompletedOutcome {
session_id: command.session_id.clone(),
workflow_run_id: command.workflow_run_id.clone(),
response,
diagnostics,
},
))
}

pub(super) fn runtime_branch_failed(
Expand Down Expand Up @@ -1369,14 +1371,14 @@ async fn claim_and_execute_runtime_branch_event(
)
.await
{
Ok(response) => WorkflowTaskExecutionWorkerOutcome::RuntimeBranchCompleted(
Box::new(WorkflowTaskExecutionWorkerRuntimeBranchCompletedOutcome {
Ok(response) => WorkflowTaskExecutionWorkerOutcome::RuntimeBranchCompleted(Box::new(
WorkflowTaskExecutionWorkerRuntimeBranchCompletedOutcome {
session_id: command.session_id.clone(),
workflow_run_id: command.workflow_run_id.clone(),
response,
diagnostics: Vec::new(),
}),
),
},
)),
Err(error) => WorkflowTaskExecutionWorkerOutcome::runtime_branch_failed(
command,
error.to_string(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ fn workflow_run_id() -> WorkflowRunId {

fn inference_projection() -> WorkflowSchedulerInferenceTaskProjections {
WorkflowSchedulerInferenceTaskProjections::from_records(vec![
WorkflowSchedulerInferenceTaskProjection::Ready(
Box::new(WorkflowSchedulerReadyInferenceTaskProjection {
WorkflowSchedulerInferenceTaskProjection::Ready(Box::new(
WorkflowSchedulerReadyInferenceTaskProjection {
node_id: SchedulerNodeId::parse("infer").expect("node id"),
descriptor_fingerprint: InferenceInterfaceFingerprint::parse("iface.binding.v1")
.expect("fingerprint"),
Expand All @@ -50,8 +50,8 @@ fn inference_projection() -> WorkflowSchedulerInferenceTaskProjections {
trait_settings: Vec::new(),
estimate_hints: resource_estimate_hints(),
dependency_readiness_source: dependency_readiness_source("iface.binding.v1"),
}),
),
},
)),
])
.expect("projection")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,8 @@ fn ready_inference_projection(
estimate_hints: Vec<SchedulerEstimateHint>,
) -> WorkflowSchedulerInferenceTaskProjections {
WorkflowSchedulerInferenceTaskProjections::from_records(vec![
WorkflowSchedulerInferenceTaskProjection::Ready(
Box::new(WorkflowSchedulerReadyInferenceTaskProjection {
WorkflowSchedulerInferenceTaskProjection::Ready(Box::new(
WorkflowSchedulerReadyInferenceTaskProjection {
node_id: SchedulerNodeId::parse("infer").expect("node id"),
descriptor_fingerprint: InferenceInterfaceFingerprint::parse("iface.test.v1")
.expect("fingerprint"),
Expand All @@ -70,8 +70,8 @@ fn ready_inference_projection(
}],
estimate_hints,
dependency_readiness_source: dependency_readiness_source("iface.test.v1"),
}),
),
},
)),
])
.expect("projection")
}
Expand Down
Loading
Loading