Skip to content

Commit c81ebde

Browse files
ninthreezyclaude
andcommitted
0.8.1: hand each step its place in the record
Every action a runner executes receives DATATORCH_PROJECT_ID, DATATORCH_RUN_ID, DATATORCH_PIPELINE_ID and DATATORCH_STEP_ID, set by the runner from the dispatch and never inherited from the outer process. The same facts are available to an action's own templates as run.*. An action that works for the project it runs in no longer needs a project input, which is what lets a pipeline fork into another project unchanged. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
1 parent a7e1b1e commit c81ebde

6 files changed

Lines changed: 118 additions & 2 deletions

File tree

‎datatorch/core/env.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,12 @@
77
RUNNER_TOKEN = "DATATORCH_RUNNER_TOKEN"
88
API_URL = "DATATORCH_API_URL"
99
CONFIG_DIR = "DATATORCH_DIR"
10+
# The step's place in the record, handed to every action a runner runs
11+
# (0.8.1): the project the run belongs to, the run, the pipeline and the
12+
# step. An action that works "for this project" reads PROJECT_ID instead
13+
# of carrying a project input, so a pipeline forked into another project
14+
# works there unchanged.
15+
PROJECT_ID = "DATATORCH_PROJECT_ID"
16+
RUN_ID = "DATATORCH_RUN_ID"
17+
PIPELINE_ID = "DATATORCH_PIPELINE_ID"
18+
STEP_ID = "DATATORCH_STEP_ID"

‎datatorch/runner/pipelines/runner/runner.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,9 @@ def get_command(self, key: str, default=None):
9191
def runner_env(self) -> dict:
9292
"""What an action needs to reach the server this runner is on:
9393
the API url and the runner's own token (as DATATORCH_API_URL and
94-
DATATORCH_RUNNER_TOKEN). The SDK's client picks them up, so an
94+
DATATORCH_RUNNER_TOKEN), and where the step sits in the record
95+
(DATATORCH_PROJECT_ID, DATATORCH_RUN_ID, DATATORCH_PIPELINE_ID,
96+
DATATORCH_STEP_ID). The SDK's client picks them up, so an
9597
action talks to the same server as the runner, as the runner,
9698
instead of the machine's stored login. Anything already set in
9799
the environment wins (DATATORCH_API_KEY included, since the SDK
@@ -105,6 +107,21 @@ def runner_env(self) -> dict:
105107
values[env_names.API_URL] = settings.api_url
106108
if settings.runner_token and not os.environ.get(env_names.RUNNER_TOKEN):
107109
values[env_names.RUNNER_TOKEN] = settings.runner_token
110+
# The step's place in the record (0.8.1), so an action can work
111+
# for "this project" without a project input. Always set by the
112+
# runner, never inherited: a value left in the environment by an
113+
# outer process must not describe another run.
114+
sections = getattr(getattr(self, "variables", None), "variables", None) or {}
115+
run = sections.get("run") or {}
116+
step = sections.get("step") or {}
117+
for name, value in (
118+
(env_names.PROJECT_ID, run.get("projectId")),
119+
(env_names.RUN_ID, run.get("id")),
120+
(env_names.PIPELINE_ID, run.get("pipelineId")),
121+
(env_names.STEP_ID, step.get("id")),
122+
):
123+
if value:
124+
values[name] = str(value)
108125
return values
109126

110127
def input_env(self) -> dict:

‎datatorch/runner/pipelines/step/step.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ def from_dispatch(cls, dispatch: "AgentStepDispatch", api: "AgentApiClient" = No
3232
name=dispatch.get("stepName") or "",
3333
inputs=dispatch.get("input") or {},
3434
run_id=dispatch.get("runId"),
35+
run_number=dispatch.get("runNumber"),
36+
project_id=dispatch.get("projectId"),
37+
pipeline_id=dispatch.get("pipelineId"),
38+
job_id=dispatch.get("jobId"),
3539
api=api,
3640
)
3741

@@ -54,6 +58,10 @@ def __init__(
5458
cacheable: Union[bool, None] = None,
5559
inputs: dict = {},
5660
run_id: str = None,
61+
run_number: int = None,
62+
project_id: str = None,
63+
pipeline_id: str = None,
64+
job_id: str = None,
5765
api: "AgentApiClient" = None,
5866
):
5967
self._action = ActionConfig(action)
@@ -63,6 +71,10 @@ def __init__(
6371
self.inputs = inputs
6472
self.cacheable = cacheable
6573
self.run_id = run_id
74+
self.run_number = run_number
75+
self.project_id = project_id
76+
self.pipeline_id = pipeline_id
77+
self.job_id = job_id
6678
self.api = api
6779
self.rendered_inputs: Optional[dict] = None
6880
self.logger = logging.getLogger(f"datatorch.runner.[{self._action.full_name}]")

‎datatorch/runner/pipelines/template.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,8 @@ class Variables(object):
8686
8787
- ``agent``, ``directory``, ``machine``, ``python`` — machine facts
8888
- ``step``, ``action`` — the step being executed
89+
- ``run`` — the run the step belongs to (id, number, projectId,
90+
pipelineId, jobId); empty values in local mode
8991
- ``input`` / ``variable`` — the step's own (already-resolved) inputs,
9092
for use inside the action's own templates (``runs.command`` etc.).
9193
``variable`` is a legacy alias of ``input``; both are scoped to the
@@ -99,6 +101,19 @@ def __init__(self):
99101

100102
def set_step(self, step: "Step"):
101103
self.set("step", {"id": step.id, "name": step.name})
104+
# Where the step sits in the record (server mode; empty when run
105+
# locally). `${{ run.projectId }}` in an action's own templates, and
106+
# DATATORCH_PROJECT_ID / DATATORCH_RUN_ID in its environment.
107+
self.set(
108+
"run",
109+
{
110+
"id": getattr(step, "run_id", None),
111+
"number": getattr(step, "run_number", None),
112+
"projectId": getattr(step, "project_id", None),
113+
"pipelineId": getattr(step, "pipeline_id", None),
114+
"jobId": getattr(step, "job_id", None),
115+
},
116+
)
102117

103118
def set_action(self, action: "Action"):
104119
self.set(

‎setup.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333

3434
setup(
3535
name="datatorch",
36-
version="0.8.0",
36+
version="0.8.1",
3737
description="A CLI and library for interacting with DataTorch.",
3838
author="DataTorch",
3939
author_email="support@datatorch.io",
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
"""A dispatched step tells its action where it sits in the record (0.8.1):
2+
`${{ run.* }}` in the action's templates and DATATORCH_PROJECT_ID /
3+
DATATORCH_RUN_ID / DATATORCH_PIPELINE_ID / DATATORCH_STEP_ID in the
4+
process environment of whatever the action runs."""
5+
import os
6+
7+
from datatorch.core import env as env_names
8+
from datatorch.runner.pipelines.step.step import Step
9+
from datatorch.runner.pipelines.template import Variables
10+
from datatorch.runner.pipelines.runner.runner import Runner
11+
12+
13+
DISPATCH = {
14+
"stepId": "step-1",
15+
"runnerId": "runner-1",
16+
"action": "datatorch-actions/claude-chat@v4",
17+
"stepName": "Turn",
18+
"stepIndex": 0,
19+
"input": {"message": "hi"},
20+
"jobId": "job-1",
21+
"jobName": "chat",
22+
"runId": "run-1",
23+
"runNumber": 7,
24+
"pipelineId": "pipe-1",
25+
"projectId": "proj-1",
26+
}
27+
28+
29+
def test_run_namespace_from_dispatch():
30+
step = Step.from_dispatch(DISPATCH)
31+
variables = Variables()
32+
variables.set_step(step)
33+
assert variables.render("${{ run.projectId }}") == "proj-1"
34+
assert variables.render("${{ run.id }}/${{ run.number }}") == "run-1/7"
35+
assert variables.render("${{ run.pipelineId }}") == "pipe-1"
36+
assert variables.render("${{ step.id }}") == "step-1"
37+
38+
39+
def test_run_namespace_empty_in_local_mode():
40+
step = Step.from_dict({"id": "local", "action": "x/y@v1", "name": "n"})
41+
variables = Variables()
42+
variables.set_step(step)
43+
assert variables.variables["run"]["projectId"] is None
44+
45+
46+
def test_runner_env_carries_identity(monkeypatch):
47+
for name in (env_names.PROJECT_ID, env_names.RUN_ID, env_names.PIPELINE_ID, env_names.STEP_ID):
48+
monkeypatch.setenv(name, "stale") # never inherited from the outer process
49+
step = Step.from_dispatch(DISPATCH)
50+
variables = Variables()
51+
variables.set_step(step)
52+
runner = Runner({}, action=None)
53+
runner.variables = variables
54+
env = runner.runner_env()
55+
assert env[env_names.PROJECT_ID] == "proj-1"
56+
assert env[env_names.RUN_ID] == "run-1"
57+
assert env[env_names.PIPELINE_ID] == "pipe-1"
58+
assert env[env_names.STEP_ID] == "step-1"
59+
60+
61+
def test_runner_env_without_variables_sets_nothing():
62+
env = Runner({}, action=None).runner_env()
63+
assert env_names.PROJECT_ID not in env

0 commit comments

Comments
 (0)