Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions control_plane/merge_train_controller_run_once.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
MergeTrainGitHubStaleHeadError,
MergeTrainGitHubTransport,
UrllibMergeTrainGitHubTransport,
merge_train_construction_ref,
)
from control_plane.merge_train_stack_collapse import (
MergeTrainStackCollapsePlanRecordStore,
Expand Down Expand Up @@ -1826,6 +1827,15 @@ def _advance_active_candidate_record(
return reflow_result

candidate_build_error: MergeTrainGitHubStaleHeadError | None = None
construction_evidence = (
{
"construction_ref": merge_train_construction_ref(
active_candidate_record.candidate.candidate_ref
)
}
if active_candidate_record.ordinary_job_binding is None
else {}
)
if active_candidate_record.candidate.status in {"planned", "building"}:
controller_action = "build_candidate"
if request.mutate:
Expand Down Expand Up @@ -1858,6 +1868,7 @@ def checkpoint_candidate_progress(
"candidate_ref": progress_candidate.candidate_ref,
"candidate_sha": progress_candidate.candidate_sha,
"completed_entry_count": (int(phase.split(":", 1)[1]) if ":" in phase else 0),
**construction_evidence,
},
)

Expand Down Expand Up @@ -1931,6 +1942,7 @@ def checkpoint_candidate_progress(
result["details"] = {
"github_status_code": candidate_build_error.status_code,
"failed_pull_request_number": lease.record.active_pull_request_number,
**construction_evidence,
}
if request.mutate:
if candidate.status == "failed":
Expand All @@ -1957,6 +1969,7 @@ def checkpoint_candidate_progress(
"candidate_ref": candidate.candidate_ref,
"candidate_sha": candidate.candidate_sha,
"candidate_status": candidate.status,
**construction_evidence,
},
)
result["candidate"] = candidate.model_dump(mode="json")
Expand Down
89 changes: 84 additions & 5 deletions control_plane/merge_train_github.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import json
from hashlib import sha256
import logging
from time import sleep
from typing import TYPE_CHECKING, Callable, Literal, Protocol, TypeVar
from urllib.error import HTTPError, URLError
Expand Down Expand Up @@ -50,6 +52,8 @@
from control_plane.merge_train import MergeTrainPullRequestState
from control_plane.merge_admission import GuardedMergeAdmission, MergeAdmissionDeniedError

logger = logging.getLogger(__name__)

if TYPE_CHECKING:
from control_plane.tenant_admission_controller import TenantAdmissionTechnicalChecks

Expand Down Expand Up @@ -379,22 +383,23 @@ def build_batch_candidate(
) -> MergeTrainBatchCandidate:
resolved_effect_executor = effect_executor or self.semantic_effect_executor
repository_path = _repository_path(candidate.repository)
candidate_branch = _branch_name_from_ref(candidate.candidate_ref)
construction_ref = merge_train_construction_ref(candidate.candidate_ref)
candidate_branch = _branch_name_from_ref(construction_ref)
if checkpoint is not None:
checkpoint(candidate, None, "reset_candidate_ref")
checkpoint(candidate, None, "reset_construction_ref")
resolved_effect_executor.prepare_candidate_ref(
CandidateRefPrepareEffect(
lineage=MergeTrainEffectLineage(
repository=candidate.repository,
base_branch=candidate.base_branch,
batch_id=candidate.batch_id,
),
candidate_ref=candidate.candidate_ref,
candidate_ref=construction_ref,
base_sha=candidate.base_sha,
)
)
if checkpoint is not None:
checkpoint(candidate, None, "candidate_ref_ready")
checkpoint(candidate, None, "construction_ref_ready")
base_identity = _git_commit_identity(
transport=self.transport,
repository_path=repository_path,
Expand Down Expand Up @@ -426,7 +431,7 @@ def build_batch_candidate(
base_branch=candidate.base_branch,
batch_id=candidate.batch_id,
),
candidate_ref=candidate.candidate_ref,
candidate_ref=construction_ref,
rolling_parent_sha=parent_sha,
pull_request_number=entry.pull_request_number,
head_sha=entry.head_sha,
Expand Down Expand Up @@ -530,6 +535,47 @@ def build_batch_candidate(
f"candidate_entry_merged:{entry_index}",
)
candidate = progress_candidate
if checkpoint is not None:
checkpoint(candidate, None, "publish_candidate_ref")
resolved_effect_executor.prepare_candidate_ref(
CandidateRefPrepareEffect(
lineage=MergeTrainEffectLineage(
repository=candidate.repository,
base_branch=candidate.base_branch,
batch_id=candidate.batch_id,
),
candidate_ref=candidate.candidate_ref,
# Publication points at the completed candidate, never an intermediate base.
base_sha=candidate_sha,
)
)
_verify_candidate_publication(
transport=self.transport,
repository_path=repository_path,
candidate_ref=candidate.candidate_ref,
expected_sha=candidate_sha,
)
if checkpoint is not None:
checkpoint(candidate, None, "candidate_ref_published")
try:
resolved_effect_executor.delete_candidate_ref(
CandidateRefDeleteEffect(
lineage=MergeTrainEffectLineage(
repository=candidate.repository,
base_branch=candidate.base_branch,
batch_id=candidate.batch_id,
),
candidate_ref=construction_ref,
)
)
except MergeTrainGitHubError as error:
# Rebuilding here would replace a verified publication and start duplicate CI.
logger.warning(
"Published candidate retained; construction ref cleanup failed for %s "
"(GitHub status %s).",
construction_ref,
error.status_code,
)
return _validated_model_update(candidate, status="ready_for_checks")

def observe_batch_candidate_checks(
Expand Down Expand Up @@ -2351,6 +2397,39 @@ def _base_branch_sha(
return _required_text(commit.get("sha"), "GitHub branch commit requires sha.")


def merge_train_construction_ref(candidate_ref: str) -> str:
"""Locate native construction evidence from the canonical candidate identity."""
return "refs/heads/launchplane/construct/" + sha256(candidate_ref.encode("utf-8")).hexdigest()


def _verify_candidate_publication(
*,
transport: MergeTrainGitHubTransport,
repository_path: str,
candidate_ref: str,
expected_sha: str,
) -> None:
for delay_seconds in (0.0, *MERGE_REF_READ_DELAYS_SECONDS):
if delay_seconds:
sleep(delay_seconds)
try:
observed_sha = _base_branch_sha(
transport=transport,
repository_path=repository_path,
base_branch=_branch_name_from_ref(candidate_ref),
)
except MergeTrainGitHubError as error:
if error.status_code != 404:
raise
continue
if observed_sha == expected_sha:
return
raise MergeTrainGitHubStaleHeadError(
"GitHub published candidate ref did not resolve to the completed candidate.",
status_code=409,
)


def _wait_for_branch_sha(
*,
transport: MergeTrainGitHubTransport,
Expand Down
57 changes: 42 additions & 15 deletions docs/merge-train-policy.md
Original file line number Diff line number Diff line change
Expand Up @@ -285,12 +285,30 @@ A batch candidate represents:
base branch + queued PR #1 + queued PR #2 + ... + queued PR #N
```

The candidate is built in deterministic queue order. If any pull request cannot
be applied cleanly, candidate construction stops at that pull request and the
worker records a blocker. The first implementation should use an explicit
temporary candidate ref or branch so GitHub Actions can run checks against a
real commit SHA. The exact ref naming and cleanup policy are part of the batch
train implementation, not the repository policy TOML.
The candidate is built in deterministic queue order on a temporary
`launchplane/construct/<digest>` branch. The digest is the SHA-256 of the
canonical candidate ref, so retries use the same construction branch. Only
after every entry's rolling commit and tree are verified does the native
GitHub adapter publish `launchplane/train/**` at the completed candidate SHA.
Base and intermediate construction pushes therefore do not start required
workflows. The canonical candidate ref, persisted candidate identity, and
rolling provenance remain the inputs to checks and landing.

Publication has a bounded exact-SHA readback, including temporary 404s while a
new branch becomes visible. A failed or interrupted publication never returns
`ready_for_checks`. Retrying a build resets the construction branch and
reconstructs the candidate; a crash after publication but before persistence
can still require a new publication and checks.

After verified publication, the adapter deletes the construction ref through
the semantic effect executor. An already missing ref is clean. Other cleanup
failures are logged with the ref and HTTP status without discarding the verified
candidate or restarting its CI; the retained ref has no landing authority.
Failed or interrupted builds retain their construction ref as recovery evidence.
The native controller checkpoint records its exact `construction_ref`, and a
failed build returns that locator with the provider status. A ref locator is
not proof that the ref still exists.
Ref naming is an implementation detail, not mutable repository policy.

After GitHub creates a candidate merge commit, Launchplane performs a bounded
read-after-write convergence check before declaring the candidate ref stale.
Expand All @@ -306,8 +324,11 @@ Launchplane must fail closed when candidate check evidence is missing, pending,
failed, stale, or attached to a different commit SHA.

Repositories using batch candidates must run their required workflows for
pushes to `launchplane/train/**`. Aggregate required-check jobs must also run on
those push events and treat the candidate as same-repository work when no pull
pushes to `launchplane/train/**` and exclude `launchplane/construct/**` from
those triggers. A workflow matching every branch would still run intermediate
checks and could supply check evidence before the final train push registers.
Aggregate required-check jobs must also run on those push events and treat the
candidate as same-repository work when no pull
request payload exists. Otherwise the candidate has no exact-SHA check evidence
and remains fail-closed in `ready_for_checks`.

Expand All @@ -334,11 +355,12 @@ structural proof still blocks. This does not claim to observe GitHub's strict
setting and does not grant, change, or bypass provider protection; GitHub's
guarded merge endpoint continues to enforce its own policy.

Candidate-ref workflow concurrency must keep create/force-reset pushes separate
from normal construction pushes. Normal intermediate pushes cancel each other
for the same ref, while the reset run retains its own SHA-keyed group so a
cancelled duplicate does not replace the protected base commit's successful
required-check evidence. Candidate-specific cancellation must not broaden a
Candidate-ref workflow concurrency keeps create/force-reset pushes separate
from ordinary ref updates. Native construction now publishes only the completed
candidate; existing concurrency rules remain for publication retries and
previously created refs. Create/reset runs retain their SHA-keyed group so a
cancelled duplicate cannot replace the protected base commit's successful
required-check evidence. Candidate-specific cancellation does not broaden a
workflow's cancellation policy for ordinary base-branch pushes.

### PR-Native Landing
Expand Down Expand Up @@ -467,12 +489,17 @@ the failed candidate batch lineage and plan a replacement candidate from the
fresh snapshot.

Candidate construction can fail before required checks run when GitHub rejects
one rolling merge entry as stale or conflicting. The controller persists that
candidate as `failed`, reports the exact pull request reached by the build, and
one rolling merge entry as stale or conflicting. Its partial state is retained
on the construction ref; no new canonical train ref is published. The controller
persists that candidate as `failed`, reports the exact pull request reached by the build, and
releases the controller lease without replaying the rejected merge. The same
queue-change rule then governs replacement planning; an unchanged queue remains
stopped for operator attention.

An exhausted final-publication readback also fails closed, with no individual
failed pull request: its checkpoint identifies the publication phase and the
retained construction ref.

## Example Policy Entries

The example below is documentation/import material only. It is not packaged as a
Expand Down
14 changes: 9 additions & 5 deletions docs/style/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,20 @@ For pushes to `main` and `launchplane/train/**`, the `verified-tree` job can
reuse a completed, successful GitHub Actions `ci-gate` on the exact pushed
commit. Its check suite must identify that same commit on main without a PR
merge context; conflicting pending or failed gates veto reuse. This avoids full
work on already-tested train base creation/reset without trusting a fork or
retargeted PR's merge-ref checks. New candidate commits still run full CI.
work when an all-no-op batch publishes the already-tested base, without trusting
a fork or retargeted PR's merge-ref checks. Native construction uses
`launchplane/construct/**`, outside the required workflows' push filters, and
publishes the canonical train ref only after every entry is verified. New
completed candidate commits still run full CI.

Main retains its existing PR-tree reuse, tightened to require that the base is
an ancestor of the PR head as well as matching trees and a successful gate.
The train does not extend that shortcut: a historical PR gate alone cannot
identify the merge-ref tree tested before a retarget. PR events, unrelated
branches, and missing API evidence run the full suite. Candidate construction,
concurrency and required checks are unchanged, and every final candidate SHA
still receives its own gate.
branches, and missing API evidence run the full suite. Construction branches
must remain outside required workflow triggers. Concurrency and required checks
are unchanged, and every published final candidate SHA still receives its own
gate.

Same-repo CI currently uses 12 unittest shards with a 20-test/30-second split
threshold to keep large app and service targets under the tool wall-clock
Expand Down
10 changes: 6 additions & 4 deletions tests/support/merge_train.py
Original file line number Diff line number Diff line change
Expand Up @@ -111,15 +111,17 @@ def build_batch_candidate(
) = None,
) -> MergeTrainBatchCandidate:
if checkpoint is not None:
checkpoint(candidate, None, "reset_candidate_ref")
checkpoint(candidate, None, "candidate_ref_ready")
checkpoint(candidate, None, "reset_construction_ref")
checkpoint(candidate, None, "construction_ref_ready")
for entry_index, entry in enumerate(candidate.entries, start=1):
checkpoint(candidate, entry, "merge_candidate_entry")
checkpoint(
candidate.model_copy(update={"candidate_sha": "candidate-built"}),
entry,
f"candidate_entry_merged:{entry_index}",
)
checkpoint(candidate, None, "publish_candidate_ref")
checkpoint(candidate, None, "candidate_ref_published")
return candidate.model_copy(
update={"candidate_sha": "candidate-built", "status": "ready_for_checks"}
)
Expand Down Expand Up @@ -280,8 +282,8 @@ def build_batch_candidate(
) = None,
) -> MergeTrainBatchCandidate:
if checkpoint is not None:
checkpoint(candidate, None, "reset_candidate_ref")
checkpoint(candidate, None, "candidate_ref_ready")
checkpoint(candidate, None, "reset_construction_ref")
checkpoint(candidate, None, "construction_ref_ready")
checkpoint(candidate, candidate.entries[0], "merge_candidate_entry")
raise MergeTrainGitHubStaleHeadError(
"Candidate entry conflicts with the rolling merge base.", status_code=409
Expand Down
11 changes: 10 additions & 1 deletion tests/test_http_app_merge_train.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from control_plane.merge_train_controller_run_once import MERGE_TRAIN_CONTROLLER_ACTIVE_ACTION
from control_plane.merge_train_github import MergeTrainGitHubMergeRejectedError
from control_plane.merge_train_github import MergeTrainGitHubStaleHeadError
from control_plane.merge_train_github import merge_train_construction_ref
from control_plane.service_auth import (
BearerIdentityConfig,
LaunchplaneAuthzPolicy,
Expand Down Expand Up @@ -3093,7 +3094,13 @@ async def test_reflows_candidate_after_build_stale_state(self) -> None:
)
self.assertEqual(
failed_payload["result"]["details"],
{"failed_pull_request_number": 1, "github_status_code": 409},
{
"failed_pull_request_number": 1,
"github_status_code": 409,
"construction_ref": merge_train_construction_ref(
failed_payload["result"]["candidate"]["candidate_ref"]
),
},
)
self.assertEqual(reflow_response.status_code, 202)
self.assertEqual(reflow_payload["result"]["controller_action"], "plan_candidate")
Expand Down Expand Up @@ -3875,6 +3882,8 @@ def capture_idempotency(record: object) -> object:
self.assertEqual(response.status_code, 202)
self.assertIn("merge_candidate_entry", observed_phases)
self.assertIn("candidate_entry_merged", observed_phases)
self.assertIn("publish_candidate_ref", observed_phases)
self.assertIn("candidate_ref_published", observed_phases)
self.assertEqual(idempotency_controller_statuses, ["running"])
self.assertEqual(final_state.status, "idle")

Expand Down
Loading
Loading