From a8359512a580aa25fb2f9c9e4cd7e0dd64fd970d Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 05:51:01 -0700 Subject: [PATCH 01/20] Add proposed acquisition and runtime adapter plans --- docs/README.md | 2 + docs/contracts/artifact-acquisition.md | 130 ++++++++++++ .../artifact-acquisition/execution-ledger.md | 13 ++ docs/plans/artifact-acquisition/issues.md | 22 ++ docs/plans/artifact-acquisition/plan.md | 145 +++++++++++++ .../reports/acceptance-matrix.md | 60 ++++++ .../reports/architecture-review.md | 89 ++++++++ .../reports/codebase-audit.md | 51 +++++ .../reports/dependency-gates.md | 59 ++++++ .../reports/standards-and-sources.md | 57 +++++ .../reports/write-sets.md | 52 +++++ .../execution-ledger.md | 13 ++ .../issues.md | 20 ++ .../plan.md | 194 ++++++++++++++++++ .../reports/acceptance-matrix.md | 88 ++++++++ .../reports/adapter-contract.md | 94 +++++++++ .../reports/architecture-review.md | 63 ++++++ .../reports/codebase-audit.md | 30 +++ .../reports/write-sets.md | 69 +++++++ 19 files changed, 1251 insertions(+) create mode 100644 docs/contracts/artifact-acquisition.md create mode 100644 docs/plans/artifact-acquisition/execution-ledger.md create mode 100644 docs/plans/artifact-acquisition/issues.md create mode 100644 docs/plans/artifact-acquisition/plan.md create mode 100644 docs/plans/artifact-acquisition/reports/acceptance-matrix.md create mode 100644 docs/plans/artifact-acquisition/reports/architecture-review.md create mode 100644 docs/plans/artifact-acquisition/reports/codebase-audit.md create mode 100644 docs/plans/artifact-acquisition/reports/dependency-gates.md create mode 100644 docs/plans/artifact-acquisition/reports/standards-and-sources.md create mode 100644 docs/plans/artifact-acquisition/reports/write-sets.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/execution-ledger.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/issues.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/plan.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/reports/acceptance-matrix.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/reports/adapter-contract.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/reports/architecture-review.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/reports/codebase-audit.md create mode 100644 docs/plans/runtime-installations-and-model-adapters/reports/write-sets.md diff --git a/docs/README.md b/docs/README.md index 9c1aba37..c0b888a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,8 @@ and remediation inputs, not current operating instructions. ## Active and Planned Work +- [Artifact acquisition](plans/artifact-acquisition/plan.md) — proposed plan for reusable HTTP, package, and S3 acquisition; implementation gates are not ready +- [Runtime installations and model adapters](plans/runtime-installations-and-model-adapters/plan.md) — proposed plan for installation identity, bound profiles, and model-specific adapters; implementation is gated by acquisition evidence - [2026-09-03 current-standards remediation program](plans/current-standards-remediation-2026-09-03/plan.md) - [Local intent API and transport-independent domain language](plans/local-intent-api-2026-09-12/plan.md) — complete within the recorded local scope; native resolution, acquisition, durable declarations and existing IPC/RPC projections accepted; nodes/fleets/new networking deferred until after the next release diff --git a/docs/contracts/artifact-acquisition.md b/docs/contracts/artifact-acquisition.md new file mode 100644 index 00000000..f575b1af --- /dev/null +++ b/docs/contracts/artifact-acquisition.md @@ -0,0 +1,130 @@ +# Artifact acquisition contract + +**Status:** Proposed for the paired acquisition/runtime plans; not an implemented public API. +**Canonical owner:** Pumas acquisition integration. +**Implementation authority:** [Acquisition plan](../plans/artifact-acquisition/plan.md). +**Consumer:** [Runtime installation and model-adapter plan](../plans/runtime-installations-and-model-adapters/plan.md), plus the existing model-library and native/package integrations. + +This document owns the semantic handoff. Names below describe required domain values and obligations, not a preapproved Rust signature, JSON schema, endpoint collection or permanent type per row. Choose the smallest API that preserves these facts. The first implementation supplies exact typed constructors/decoders and fixtures here or through linked canonical generated sources before integration. + +## 1. Authority and promise + +A consumer provides an accepted immutable artifact specification and current authority to obtain/store it. Acquisition produces ordinary local files matching the requested evidence and retains them for the agreed consumption lifetime. It does not import models, install packages, choose runtime builds, execute code, allocate model slots or publish runnable state. + +Three boundaries remain distinct: + +1. **Resolution:** choose exactly which files and source versions are required. +2. **Acquisition:** obtain and verify those files under owned transfer and storage custody. +3. **Consumption:** validate/import a model, or extract/install/probe a runtime, then publish that consumer's result. + +A consumer can fail after bytes are ready. That is not a failed HTTP transfer. The combined UI must preserve both states without marking the overall operation complete early. + +## 2. Domain values + +| Value | Required meaning | Excluded authority | +| --- | --- | --- | +| Artifact specification | Complete logical file set; source-object/revision evidence; sizes when known; required verification policy; non-sensitive provenance | Transient credentials, runtime readiness, package-solving logic | +| File identity | Selected representation, expected digest/algorithm when available, source-scoped immutable identity/validator, logical destination role/name | URL alone, filename alone, size alone | +| Source reference | Stable provider/configuration reference plus object locator and pinned version/evidence; refreshed access must preserve identity | Permission to change selection or execute a retrieved file | +| Authorization reference | Owned credential/policy handle and allowed destinations/redirect/retrieval behavior | A persisted bearer token, presigned URL in the artifact identity | +| Acquisition demand | The consumer's operation identity and retained request for these bytes, including policy and destination grant | Ownership of another consumer's demand | +| Attempt generation | Exact admitted execution generation; stale work cannot mutate a successor | A timestamp or arbitrary current process assumption | +| Workspace grant | Capability-backed authority to a bounded destination owned by the composition/consumer; mutation rights scoped to acquisition | An arbitrary caller path revived after restart | +| Verified file set | Validated files plus actual evidence, provenance and live use custody; optional per-file verified observations while set remains partial | A boolean promising the model or installation works | +| Consumer settlement | Explicit release or transfer of acquisition custody after consumer completion/failure and required cleanup | Automatic deletion of consumer-published files | + +The same file digest may satisfy multiple specifications only when the receiving policy permits it. RuntimeInstallationId and ModelRef remain consumer identities. Do not replace them with acquisition IDs or digests. A source revision is not necessarily a byte digest; retain the distinction in the evidence. + +## 3. Resolution and completeness + +HF selection pins one concrete repository commit for the selected file set and preserves current file/variant intent. A runtime driver selects the native archive/build. A package owner resolves distribution artifacts. S3 supports explicit object references and explicit multi-file manifests; bounded prefix enumeration must complete and pin each selected object before it can claim completeness. Listings do not promise an atomic package snapshot without a source contract proving one. + +Reject duplicate/colliding local logical paths, incomplete source manifests, unsupported representation kinds, contradictory size/digest evidence and unsupported versions. Validate platform-specific filename collisions at materialization without changing the source key. Encode object keys according to the source protocol, not by treating them as filesystem paths. + +An object with only weak/no stable identity can be acquired as one complete transfer under an explicitly weaker caller policy, but cannot silently acquire strong resume/reproducibility or executable-origin guarantees. If the caller requires strong identity, return insufficient-evidence. Preserve current explicitly supported legacy policies without silently upgrading their claims or weakening strict new requests. + +## 4. Public surface and state + +A deep interface owns submit/acquire, observe/wait, pause/resume, cancel, open verified files, release/settle, and shutdown as necessary to current consumers. Reuse current operation/control handles and public domain facades where possible. No new public planning-session protocol is needed just because internal manifests are immutable. + +Validate inbound and outbound wire/persistence representations at their receiving/destination boundaries. In-process consumers retain constructed validated values rather than re-decoding the same JSON. Status/progress is a projection of one owner, not another mutable store. + +Illustrative lifecycle: + +`Admitted → Transferring ↔ Paused/WaitingForAccess → Verifying → FilesReady → Settled` + +Failures and cancellation retain their exact generation and move through owned cleanup or an explicit RecoveryRequired/Uncertain state. These labels do not replace existing domain states or mandate a single enum. Do not claim Paused until no worker still writes the affected destination. FilesReady requires complete selected content and required verification; a partly verified file set can only produce explicitly partial observations. + +A disconnected observer does not cancel durably admitted work. Explicit cancellation records the decision, prevents new effects, signals workers, and observes terminal/cleanup effects. Dropping a future or lease wrapper does not prove executor I/O stopped. Synchronous Drop can release a safe local handle or enqueue cleanup with a retained supervisor, not detach required cleanup. + +## 5. HTTP representation rules + +Source readers use established HTTP parsing/transport facilities and preserve status, headers and representation facts needed by acquisition. Request exact bytes (normally identity content encoding); decompression must not silently alter offsets or the representation being hashed. + +Resume requires verified local partial identity, an exact offset/length, and source identity evidence appropriate to that source. With strong HTTP validators, apply the relevant conditional request. Validate the returned range, total length where known, actual byte count and current validator. A `200` full-body response to a ranged request is a restart/replan outcome, never data to append. A `206` with wrong/missing range evidence is invalid. `416` is not proof of completion; inspect the selected length and verify the entire local file before any ready transition. `304`, short successful bodies, extra bytes and changed representations retain their owning result rather than becoming success. + +A source can explicitly decline resume. Starting a fresh transfer is a policy-authorized owned action that preserves/disposes old partial state safely, not a silent best-effort splice. Unknown size is represented as unknown and bounded by consumer capacity; percentage progress is not invented. Zero-byte files and selected file-set completeness have explicit tests. + +RFC authority: [HTTP Semantics, RFC 9110](https://www.rfc-editor.org/rfc/rfc9110.html), particularly conditional requests, Range, Content-Range and 206/416. These rules are an acquisition use of the protocol, not a new HTTP implementation. + +## 6. S3 and other source readers + +Source configuration includes endpoint, region where applicable, bucket, addressing style, TLS/approved local-development policy, and explicit credential-provider identity. Preserve object version IDs and conditional reads. Treat ETag as an opaque source validator unless its specific documented checksum semantics are established. A multipart or encrypted-object ETag must not be relabeled SHA-256/MD5 evidence. + +Use one range per S3 GetObject call and bounded parallel calls only when the supported reader and identity contract permit them. Pagination failures or capacity limits produce incomplete/unavailable manifests, not silently shortened file sets. Recheck the exact version/evidence when refreshing credentials or location. Capability differences of compatible endpoints produce explicit unsupported results. + +[AWS GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) and [Object metadata](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html) provide source semantics. A maintained SDK supplies signing and request mechanics. Pumas owns selected identity, allowed access, acquisition attempts, destination custody and verification. + +A new source implementation can register behind the internal source-reader interface without changing model-adapter IDs or installation-state variants. This plan does not enable untrusted executable source plugins or assume all providers obey identical APIs. + +## 7. Integrity, provenance and trust + +Record separately: requested origin and revision, actual retrieval location, expected evidence and its authority, observed evidence, and consumer approval. Compare expected digests using the existing hashing facilities. A locally computed hash alone proves neither publisher origin nor permission to install. + +Expiring locators are refreshable access material. Keep them in ephemeral/scoped storage rather than ordinary manifests/logs; when they cannot be recovered after restart, report WaitingForAccess. A refreshed credential/URL may access only the previously selected object, unless a new explicit selection creates a new request. Forward credentials across redirects only under the configured receiving-origin policy. Public progress identifies safe source labels, not query secrets or private object paths by default. + +Private MinIO/enterprise endpoints can be explicitly allowed. Untrusted model metadata cannot authorize arbitrary host access, turn off TLS verification, change registries or expand credential scope. Local-cache access follows its retained consumer/policy scope; remote token expiry is not an instruction to delete already authorized local content, and possession of shared cached bytes is not authorization for another consumer. + +Initial multi-location support means one selected source plus explicit complete-content reuse. A permitted mirror can satisfy the same expected digest, but matching ETag text from different origins is not equivalence. Cross-origin reuse of partial ranges is deferred until an independently verified chunk/range contract is implemented. Full re-acquisition may be offered explicitly instead. + +## 8. Filesystem and handoff lifetime + +The composition supplies trusted root capabilities; source metadata supplies only validated logical names. Maintain no-follow/containment and actual file/directory identity through effectful operations, including creation, replacement, deletion and reopening. Serialized paths/IDs are equality/context only, not filesystem authority. + +Write only into owned staging. Finish and flush owned writes and apply the required durability barrier before exposing a ready handoff. Verify the selected file set. Seal its mutation grant and issue a read/use capability; consumers may obtain ordinary paths while retaining that capability. Partial observation can expose verified auxiliary files to the existing model workflow without exposing the entire model as ready. + +Prefer same-filesystem staging/adoption for large models where the consumer can safely publish it. Provide ordinary copying when different storage or immutable-cache requirements demand it; support reflink only as an optimization. No unconditional cache-plus-full-model duplication, guaranteed zero-copy, or mandatory CAS. If a file/directory is adopted into a consumer's authority, record the transfer and exact identity before acquisition cleanup can reclaim the old workspace. A move that has unknown visibility remains an uncertainty state. + +An installer holds the handoff until package/extraction workers and their cleanup finish. Mutation of installed outputs cannot modify shared cached inputs. Eviction considers durable demands, active read leases and uncertain handoffs. Release of one demand does not release another. If coalescing is not implemented, do not add an idle global demand scheduler just to prepare for it. + +## 9. Persistence and consumer finalization + +Persist the selected manifest/specification identity, current demand/attempt, source-scoped nonsecret revision evidence, workspace identity, validated resume progress and lifecycle dispositions. Reuse the owned atomic store/publication machinery after separating model-specific records. Runtime artifacts must not require a fake repo/model/library UUID. One shared acquisition implementation can retain a supported legacy decoding boundary, but two stores must not both authorize the same transfer. + +Persisting progress is not the same as making the file bytes durable. After a crash, validate source identity and actual partial files under reopened authority; do not trust a saved byte counter as proof. Complete-file verification must re-establish readiness when prior verification cannot safely be reused. Unsupported recovery remains non-authorizing. + +Acquisition FilesReady and model/runtime publication are distinct commits. Consumers record their own idempotent finalization linked to the exact request/generation. If a crash occurs after consumer commit but before acquisition settlement, retain inputs conservatively and reconcile through the consumer's authoritative result; do not automatically repeat installation/import or delete final output. There is no assumed cross-store atomic transaction or exactly-once remote-I/O guarantee. + +Consumer settlement is idempotent for its exact generation. Old acknowledgements cannot release a successor. Durable unfinished consumption retains custody across restart even though in-memory RAII handles no longer exist. If confirmation is unavailable, surface recovery-required and bounded retained state, not deletion by age. + +## 10. Failure, retries, progress and shutdown + +Differentiate malformed request, unsupported source/representation, authorization-required/denied, source-changed, integrity mismatch, not found, network inconclusive/transient, storage full, busy/capacity, cancellation, consumer failure, and visibility/durability uncertainty. Project through existing public error contracts with bounded non-sensitive diagnostics; the list does not mandate a new cross-project universal error enum. + +The source reader reports retryability and required refresh, not a second outer retry loop. Acquisition owns the combined retry budget; respect source Retry-After only within authorized policy. Paused or access-blocked jobs release active transfer capacity while retaining resumable custody. Hashing/writes use governed async/blocking capacity and never hold bookkeeping locks through external code or blocking I/O. + +Progress separates logical verified bytes from wire bytes, retries and unknown totals; preserve current UI-visible domain progress. Bounded coalesced notifications have snapshot recovery and a reliable terminal/control path. Download 100% does not mean install complete. End-to-end cancellation follows the initiating domain's explicit action while retaining cleanup owners. + +Shutdown closes new admission, signals appropriate work, drains tracked async and blocking effects, publishes the true resumable/terminal disposition, then releases custody. Repeated shutdown shares the owned result. An elapsed deadline can produce incomplete shutdown, not fictional cleanup success. + +## 11. Package consumption + +The package owner supplies a version-checked accepted resolution and trusted immutable artifact set. Acquisition obtains files; package tooling installs them into an owned empty stage from exact local inputs. Preserve original URLs/digests as provenance and validate the installed set afterward. Use only supported public tooling. A report is evidence about a resolution, not by itself a consumable lock format. + +The decisive test denies network during the final installation leg and rejects hidden direct-URL retrieval, alternate same-name/version wheels, and missing closure members. Resolver metadata and managed-Python bootstrap traffic remain separately recorded; the claim is exact payload handoff, not interception of every package-tool request. Sources: [pip report](https://pip.pypa.io/en/stable/reference/installation-report/) and [pip install](https://pip.pypa.io/en/stable/cli/pip_install/). + +## 12. Compatibility and extension + +Internal coordinated DTOs, public Rust/IPC APIs, persisted formats, optional source implementations and consumer install/model records have different evolution obligations. Update actual generated consumers with their producer. New source support within this contract does not change model-adapter registration or runtime installation identity. Unknown schema/protocol versions are explicitly rejected, not decoded into weaker defaults. + +Before independently deploying a breaking contract, disposition every supported public client and retained data state. Gate status is owned by the acquisition plan, not by a version number in this document. This document remains proposed until the implemented producer/consumer evidence exists. diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md new file mode 100644 index 00000000..d4510567 --- /dev/null +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -0,0 +1,13 @@ +# Acquisition execution ledger + +## 2026-09-29 — coordinated planning delivery + +The user requested a separate acquisition prerequisite plan alongside the runtime/model-adapter plan. The attached r3 plan and integration review were read; current Pumas branch and standards refs were verified. Pumas remains at `04e7f156`; standards is now `39d55dc` (the inspected intervening commit changes test/evidence material, not the retained normative planning/architecture rules). + +The source review expanded into actual model-bound destination/recovery types, durable snapshots and task ownership. The design now places neutral acquisition below both consumer domains, with one proposed contract, existing HF/native and package consumers to prove it, and explicit HTTP/package/S3 gates. This avoids a cycle with the downstream new installation and adapter implementations. + +Created this acquisition plan, a proposed shared contract, gate record, source and composed-design reviews, acceptance/write-set/issue records, and an updated runtime plan at its existing intended path. The planning package uses two sibling execution plans, not a third master plan. No production source, repository state, live store, installed runtime, model or release was changed. + +**Evidence:** source/docs inspection and mechanical delivery checks only. All AC production claims and AQ gates remain pending/not ready. No Rust/Python production suite, source-service integration, network-denied package installation, GUI, migration, GPU workload or native release was executed. No independent reviewer was run; independent review is a later explicit acceptance requirement. + +**Next slice:** Q1 after current checkout/consumer/retained-state preparation and exact source-work admission. Runtime R1 remains gated by AQ-HTTP. diff --git a/docs/plans/artifact-acquisition/issues.md b/docs/plans/artifact-acquisition/issues.md new file mode 100644 index 00000000..9c7469a7 --- /dev/null +++ b/docs/plans/artifact-acquisition/issues.md @@ -0,0 +1,22 @@ +# Acquisition issues and dispositions + +All repairs are planned, not implemented. Evidence references [the source audit](reports/codebase-audit.md), [the canonical contract](../../contracts/artifact-acquisition.md), and its linked primary sources. Severity is consequence within this scope, not an asserted exploited vulnerability. + +| ID | Severity / issue | Owner / disposition | Deciding evidence and revisit condition | +| --- | --- | --- | --- | +| AQ-I01 | High: model-specific destination authority cannot become the generic artifact root | Acquisition/core: fix in Q1 | AC01/AC04/AC09; re-plan if safe neutral capability extraction cannot preserve model custody | +| AQ-I02 | High: retained model/HF records and recovery transitions need explicit migration | Core/recovery: fix in Q1 | AC04–AC06; actual source/deployment inventory before mutation | +| AQ-I03 | High: cancellation/restart must retain worker and byte-use ownership | Acquisition: fix in Q1 | AC05/AC06; no dropped-future or timeout-as-cleanup proof | +| AQ-I04 | High: acquisition/import/install completion and duplicate transfer owners | Acquisition plus consumers: consolidate in Q1 | AC03/AC05/AC08/AC18 | +| AQ-I05 | High: cross-plan duplicate authority or prerequisite cycle | Integrator: fixed in plan design; implementation evidence pending | Gate graph and Q1/Q2 old-consumer evidence; review after any milestone dependency change | +| AQ-I06 | High: generic retrieval can be mistaken for package compatibility or executable trust | Runtime/package/security: Q2, with Q1 trust contract | AC07/AC11/AC12; keep origin and byte digest separate | +| AQ-I07 | Medium: new generic module could accidentally require inference or overclaim feature isolation | Core/composition: Q1/Q3/Q4 | AC09/AC15/AC17; no unsupported minimal-build claim | +| AQ-I08 | Medium: source roadmap can expand prerequisite beyond useful consumer result | Integrator: Q1 contract first; Q3 required S3 scope; future features deferred below | AC18 and gate-status record | +| AQ-D01 | Deferred: native Xet reconstruction | Acquisition/source owner | Revisit when a required HF source cannot use the supported existing file path or Xet transfer benefits are an admitted goal; evaluate maintained Rust implementation, no homemade protocol | +| AQ-D02 | Deferred: peers and concurrent multi-source failover | Acquisition/distribution owner | Revisit with actual node/authorization contract and content-equivalence proof; no cross-origin ETag comparison | +| AQ-D03 | Deferred: chunk CAS, reflink optimization and coalescing | Acquisition/storage owner | Revisit with measured duplication/network/storage need and explicit retention consumers; ordinary files and safe release are required now | +| AQ-D04 | Scoped retained dependency: package-resolver metadata and managed-Python-provider traffic | Package/runtime owner | Q2 inventories and accepts exact scope; migrate only through supported integration when it changes a meaningful transfer claim | +| AQ-D05 | Deferred: universal remote write/search/browsing or dynamic executable source plugins | Source owner | Revisit only for an explicit product/trust/API requirement; direct HTTP/S3 object acquisition is in scope now | +| AQ-E01 | Pending: required-real AWS/non-AWS/MinIO/native/desktop evidence | Assigned source/distribution integrator | AC14/AC17; missing environment blocks those claims, not fictional acceptance | + +Pending cleanup replay and unrelated whole-runtime remediation are owned by the existing Rust/library plan. Preserve current refusal while migrating the selected transfer family. Do not mark those unrelated work items accepted from this plan's link/schema checks. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md new file mode 100644 index 00000000..aada92ed --- /dev/null +++ b/docs/plans/artifact-acquisition/plan.md @@ -0,0 +1,145 @@ +# Plan: source-neutral artifact acquisition + +**Plan status:** `Planned` — implementation direction selected for this coordinated planning package; no production implementation is authorized or claimed by this delivery. +**Objective acceptance status:** `pending`. +**Current phase:** contract and source-grounded implementation preparation. +**Exactly one next slice:** **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** +**Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. +**Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. +**Operation for a later implementation session:** `start` this exact plan; refresh repository state and the prerequisite dispositions below first. +**Baselines:** Pumas `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`; Coding-Standards `39d55dc330d44ecf940364ceada9d2527f7c7ea0`; delivered runtime plan revision 3 is the input, revision 4 is the companion output. + +## Objective and scope + +Provide one reusable acquisition owner that obtains authorized, specifically identified artifacts from HTTP/Hugging Face and S3-compatible object stores, preserves resumability and recovery guarantees, verifies the required content evidence, and supplies ordinary local files with an owned lifetime. Model import and executable/package installation consume that result without implementing their own byte-transfer lifecycle. + +The initial production consumers are the existing Hugging Face model workflow, the existing llama.cpp archive installer, and the existing Torch package-installation path. They establish the interface before the later runtime-installation and arbitrary-model-adapter changes depend on it. + +The complete acquisition plan includes HTTP, the current Hugging Face resolution path, exact package-file acquisition, and S3-compatible object retrieval. It is more than a URL download utility. Full acceptance requires the real source/consumer, restart, desktop, and affected platform evidence named below. The first accepted gate can unblock runtime work before the complete acquisition roadmap is accepted. + +Source intent: `docs/breif/s3-model-fetch.md` and `docs/breif/intent-discovery-distribution.md`. They remain explanatory intent; this plan owns current execution sequencing. Existing model intent APIs remain the normal model-facing interface. Runtime archives and wheels do not become models to reuse them. + +### Included boundaries + +Source resolution and immutable file manifests; source readers; range/stream transfer; governed retries and capacity; source authorization refresh; attempt persistence and restart; contained destination effects; integrity evidence; ordinary-file handoff and retention; model/native/package consumer integrations; relevant RPC/UI projections and source-configuration workflows; supported retained-state evolution. + +### Preserved constraints + +Pumas core remains independently usable. Acquisition starts without Python, Torch, model-adapter imports, inference plugins, or a populated model index. A storage root and acquisition metadata may be needed; a model database is not. Reuse existing HTTP, hashing, capability-filesystem, task-custody and atomic-publication mechanisms where their actual invariants fit. + +Preserve model selection/revision, partial-model visibility, awaited importer completion, current root/destination exclusion and explicit refusal of unaccepted Pending cleanup replay. Preserve native build selection, wheel/source trust, staged installation, process ownership and existing generation lifetimes at their separate owners. + +Source resolution never grants execution permission. Acquisition readiness never means imported, installed, loaded or runnable. Credentials and signed URLs never define content identity. Normal files remain the consumer boundary without a mandatory second full copy of model weights. + +### Explicit exclusions and continuation + +Xet-native reconstruction, peer discovery, source racing, cross-origin partial-byte failover, chunk CAS, deduplicating concurrent transfers, a virtual filesystem, remote/multi-tenant service deployment, write access to remote stores, a universal package solver, arbitrary scripts, and new model execution tasks are excluded from the initial implementation. They are explicitly deferred with owner and triggers in [issues](issues.md), not represented as stub production capabilities. Existing HF retrieval continues through its supported file path; a source requiring an unimplemented reconstruction protocol returns a truthful unsupported/unavailable result, not fabricated complete coverage. + +Source-neutrality is achieved by a real shared contract and independently replaceable readers, not by promising every storage provider. S3-compatible endpoints are configuration variants of the same integration where their tested protocol permits it. + +## Binding decisions and owners + +### AD1 — one domain boundary for acquired files + +The proposed canonical interface is [Artifact acquisition contract](../../contracts/artifact-acquisition.md). Acquisition owns that contract; the companion runtime plan references it. The contract separates an immutable selected file set from refreshable retrieval authorization, durable acquisition ownership from in-memory task generations, and verified files from consumer publication. + +Place the source-neutral implementation in `pumas-core::acquisition` by default. App-manager already depends on core. The acquisition module has no dependency on app-manager, a model importer, Torch, or the desktop. The existing composition root creates and drains one acquisition owner for its scope; embedding can construct that owner directly with supported storage/network capabilities. No additional process, global singleton, scheduler, or crate is justified by current facts. + +### AD2 — independent owners, not a universal job engine + +| Concern | Canonical owner | Result promised | +| --- | --- | --- | +| Model/release/package selection | Existing model or runtime/package integration | Exact selected source revision/build/dependency artifacts | +| Source resolution | HF, HTTP or S3 resolver implementation | Complete artifact manifest and resolvable source references | +| Protocol access | Source reader and established HTTP/S3 libraries | Correct conditional full/range bytes or typed source failure | +| Transfer, custody and byte verification | Shared acquisition service | Verified ordinary files under a live/durable handoff contract | +| Model validation/import/index | Existing library consumer | Model publication, only after its own finalization | +| Archive/package installation | App-manager/package consumer | Validated installed unit, only after its own publication | +| Adapter loading and inference | Companion runtime/adapter plan | Compatible bound execution, not an acquisition responsibility | + +The contract's source reader is a bounded I/O capability; it does not carry model classification, install hooks, ABI selection, inference logic, or a duplicate retry/store manager. Packaging authentication and protocol signing remain with maintained tooling, not a home-grown signing engine. + +### AD3 — bootstrap-safe and incrementally extracted + +Q1 reuses and extracts the existing HF lifecycle rather than renaming the entire HF client into a generic manager. Source-specific resolution, model-destination authorization and importer settlement remain on the model side. Generalized attempt custody, persistence and byte effects become one acquisition owner. + +Current model-specific fields and root UUIDs cannot become mandatory runtime-artifact fields. Durable state has one authority per fact: acquisition state in the acquisition owner; model/installation publication at their consumers. A legacy adapter can read the retained supported format at an explicit migration boundary, but it cannot run a parallel transfer writer for the same attempt. + +Q1's small native-installer bridge changes only selected acquisition calls and their progress/handoff integration. It uses the currently supported tag-based installer until runtime R1; it does not implement installation-ID migration early. That proves a real consumer without depending on the downstream refactor. + +### AD4 — normal files and evidence-scoped reuse + +Use consumer-approved capability-backed staging roots, with transfer mutation grants and sealed read handoffs. Models may stage on their destination filesystem so publication can preserve ordinary paths without always duplicating the full artifact. Native archives/wheels can use owned staging or verified retained input cache. Shared raw inputs remain immutable; mutable install outputs cannot be writable hardlinks to them. + +A borrowed path is valid only while its associated use custody is retained. Child processes reading archives or wheels retain that custody through actual exit/cleanup. Restart uses durable consumer demand and verified file/workspace identity, not resurrection of a serialized OS handle or trust in a displayed path. No expiry timer alone authorizes deletion. + +Q1 does not require shared transfer coalescing, content-addressed filenames, or an LRU service. It does require safe release/reclamation of its owned staging and explicit retained-input policy. Identical complete bytes may be reused only with matching content evidence and the receiving consumer's authorization. Size/filename equality alone is insufficient. + +### AD5 — HTTP and object stores must preserve representation identity + +HTTP whole-file and range access implement the specific contract in the canonical document. Resume is conditional on stable source/representation evidence, and final verification satisfies the consumer's required evidence. A server returning a full body to a range request cannot have that body appended at an old offset. Source refresh cannot silently choose a new object or mix revisions. + +S3 resolves endpoint/bucket/key plus an immutable version or sufficiently strong content evidence. ETags are conditional validators, not universally cryptographic digests. An S3 prefix listing is not an atomic multi-file snapshot: either use an explicit manifest or pin every listed object and report incomplete/racing enumeration rather than asserting a coherent package. Remote object keys and local logical paths have separate validation contracts. + +Reuse current reqwest-based HTTP facilities. Evaluate the maintained Rust `object_store` S3 implementation as the first candidate for Q3's needed endpoint/credential/version/range semantics; its API is not the public Pumas contract. The bounded dependency decision and rejection conditions are in [design review](reports/architecture-review.md). Exact version, feature set, license compatibility and supported target evidence must be recorded before adding a dependency. A failed candidate requires a named alternative decision, not custom S3 protocol code by convenience. + +### AD6 — scoped authorization and bounded operational policy + +The authorized source configuration specifies endpoints, trust, permitted redirects and credential references. Keep credentials request-scoped and out of durable artifact identity, progress and public diagnostics. Support explicit private/local object-store endpoints; do not impose a blanket private-network ban that makes legitimate configured stores unusable. Conversely, arbitrary model metadata cannot grant access to local services or cloud metadata endpoints. + +The acquisition owner controls overall attempts, cancellation, backoff and resumption. Integrate or disable nested SDK retries so budgets do not multiply invisibly. Every task, file descriptor, network request, buffer, progress queue and blocking hash/write job has governed capacity. Reuse measured current settings where semantically compatible; declare units and saturation outcomes. Progress can coalesce, but terminal/control information cannot disappear. Time budgets may interrupt authorized I/O, not manufacture proof that file effects stopped. + +### AD7 — package semantics stay outside the fetcher + +Q2 consumes the exact accepted Torch/wheel artifact closure from the existing package owner. The resolver can use supported metadata/candidate access; the final acquisition-to-installation leg uses verified local inputs and is tested with network denied. Preserve original resolution provenance separately from local installation paths. Do not treat a pip report as a lockfile, rewrite a private pip downloader, or assert that all pip/managed-Python network traffic is shared. + +Runtime R2 adds arbitrary registered adapters later using that same accepted file-set handoff. Q2 does not depend on R2's adapter catalog. Bootstrap/provider-managed downloads receive a written migrate/retain-with-scope disposition; any retained mechanism has a distinct standard-tool responsibility, not a second Pumas payload downloader. + +### AD8 — own evolution before mutation + +The existing download schema, recovery admissions, root markers, hidden history and cleanup custody have retained consumers. Q1 must reconcile the active Rust recovery owner and enumerate supported source states before migration. Preserve legacy IDs through explicit mapping where needed for existing UI/consumer operations. Unsupported/corrupt or custody-unresolved states remain visible and non-authorizing. + +Reopen after each interrupted publication boundary to source, destination or an explicit uncertainty state. Do not mark completion, delete partials, auto-retry imports/installations, or replay Pending cleanup from guessed facts. New readers reject unknown future formats. Older writers must be retired or isolated; a new format number cannot constrain an old binary. Rollback requires evidence about both stores and subsequent authored changes, not just a backup file. + +## Milestones and release gates + +Each row is one coherent semantic unit including producer and actual consumers. [Write sets](reports/write-sets.md) are part of the row's allowed paths; each implementation admission binds the exact changed files and tests before editing. + +| Milestone | Goal | Dependencies | Gate / evidence | State | +| --- | --- | --- | --- | --- | +| **Q1** | Shared HTTP lifecycle, neutral persistence/handoff, and real HF/native consumer cutover; existing UI outcomes preserved. | Source/retained-state preparation and active recovery-owner handoff. No runtime milestone. | `AQ-HTTP`: AC01–AC10, AC15, AC16, AC18; real HF model import and native archive extraction plus corresponding UI/contract/cancellation/reopen evidence. | Planned | +| **Q2** | Exact wheel-file-set acquisition and local-only consumption in the existing Torch installer. | AQ-HTTP. | `AQ-PACKAGES`: AC11, AC12 and Q1 regression evidence affected by this composition; network-denied installation and actual package identity. | Planned | +| **Q3** | S3-compatible acquisition using the same lifecycle, direct explicit source workflow, tested credentials/version semantics, and a real model import through the existing model-facing operations. | AQ-HTTP. Q2 is the default next serial integration; Q3 can be delegated after shared files stabilize. | `AQ-S3`: AC13, AC14 and source-neutrality regressions; native AWS S3, one non-AWS compatible service, local MinIO, and S3-to-model-library evidence. | Planned | +| **Q4** | Complete affected public/installed/native qualification, migration documentation and removal of superseded authority. | Q1–Q3 implemented. | `AQ-COMPLETE`: all AC01–AC18 satisfied and all four milestones Accepted; packaged/independent-consumer/native-platform evidence in AC17. | Planned | + +[Dependency gates](reports/dependency-gates.md) is the single gate-status and consumer-handoff record. Every gate is currently **not ready**. A gate opens only after its claims pass for identified material source, supported API and stated target scope; availability is not inferred from a commit label or partial implementation. Material incompatible changes require targeted re-verification of affected gates. + +Runtime **R1 requires AQ-HTTP**. Runtime **R2 requires AQ-PACKAGES** as well as R1. Runtime use of S3 requires AQ-S3 on the relevant target. S3, Xet, peers or chunk CAS are not secretly prerequisites of HTTP runtime management. The runtime plan does not implement or accept these gates itself. Both plans share one serial integration owner for overlapping files. + +### Q1 starting boundary + +First refresh the selected checkout and current standards, preserve unrelated work, and reconcile the active recovery plan. State the supported retained-state population and exact seam between transfer completion and model/native finalization. Add regression evidence before/with changes. Implement a usable end-to-end HF/native path, not only a new trait or DTO. Do not open another generic engine while keeping old HF and native transfer owners active for the same migrated population. + +Bounded investigations belong inside their milestones: verify the current destination grant can support a neutral workspace, identify a real native artifact's accepted integrity evidence, and prove restart handoff with existing store readers. Stop each investigation when the chosen interface can be implemented safely; update the issue record only if its result changes ownership, migration or evidence. + +## Acceptance and verification + +[Acceptance matrix](reports/acceptance-matrix.md) owns criteria, evidence kind, environment, execution mode and named procedures. Every production claim remains pending. Static checks and disposable fixtures support, but do not replace, real transfer, package, desktop, native and deployment evidence. Source review and this package's link checks do not certify code compliance. + +Composed-design review is **applicable**. [Architecture review](reports/architecture-review.md) answers all eight probes for this design and its runtime consumer, plus the authority-scope and dependency questions. The required simplicity result is reduced caller knowledge: a new source changes a source integration, not runtime installers or model adapters; a new installer consumes existing verified artifacts without learning HTTP/S3 recovery. + +Independent read-only review covers durable migration, credential/destination authority, cancellation/retention and final producer/consumer composition. Reviewers never edit; narrow verification reuses the reviewer. Shared contracts, durable formats, generators, lockfiles and current plan state have one writer. Disjoint implementation roles and escalation rules are in the write-set report. No fixed commit count or exact-parent topology is prescribed. + +## Blockers, re-plan triggers and limits + +No code is implemented in this delivery. The active recovery ownership and real retained-state population must be reconciled before their mutation; those are bounded admission conditions, not a requirement to finish unrelated remediation. Pending cleanup remains refused. Native/GPU/GUI/network credentials unavailable to an execution session block only the claims requiring them; all final production acceptance still waits for its required evidence. + +Re-plan when a new consumer changes authority, the proposed state store cannot preserve supported recovery, a file lease cannot survive required worker cleanup, a source cannot establish required identity, an S3 dependency cannot provide the selected API/target, or downstream semantics leak into acquisition. Adding a file within an already admitted owner only amends its concrete write set. + +Authorized planning does not authorize deleting user files, publishing releases, migrating a live root, changing external repositories, or rewriting history. Prototype and real-source tests use disposable roots and approved test resources. No new production timeout/failure guarantee is inferred from test harness deadlines. + +## Linked records and terminal documentation + +[Ledger](execution-ledger.md) · [Issues](issues.md) · [Audit](reports/codebase-audit.md) · [Architecture](reports/architecture-review.md) · [Acceptance](reports/acceptance-matrix.md) · [Write sets](reports/write-sets.md) · [Dependency gates](reports/dependency-gates.md) · [Standards/source review](reports/standards-and-sources.md) · [Companion runtime plan](../runtime-installations-and-model-adapters/plan.md). + +On adoption, link this plan from the source brief and add the bounded ownership disposition to the active Rust/library recovery and upstream-runtime plans. Do not create another master execution plan. The shared contract remains marked proposed until implemented; update it in the same slices. At acceptance, move durable decisions to current owner documentation/ADR as required, and follow Pumas's documented terminal-plan lifecycle rather than retain duplicate active instructions. Git and the delivered older packages preserve history. diff --git a/docs/plans/artifact-acquisition/reports/acceptance-matrix.md b/docs/plans/artifact-acquisition/reports/acceptance-matrix.md new file mode 100644 index 00000000..603b2ecd --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/acceptance-matrix.md @@ -0,0 +1,60 @@ +# Acquisition acceptance matrix + +**Production status: all AC01–AC18 pending.** No code, production tests, source service, GUI, migration or installed artifact was executed in this planning delivery. A required unavailable environment changes its claim to blocked, not satisfied. All claim owners are roles for the assigned integrator to resolve. + +| ID | Observable claim / deciding procedure | Evidence kind | Environment | Mode | Milestone / owner | Status | +| --- | --- | --- | --- | --- | --- | --- | +| AC01 | **Immutable manifest and file identity.** Reject malformed versions, duplicate/colliding logical paths, contradictory evidence and incomplete sets. Preserve source key bytes, same-revision selection, zero-byte files and unknown sizes without fake percentages. | focused + contract | not-applicable | automated | Q1 / Acquisition/core | pending | +| AC02 | **Conditional HTTP and byte verification.** Controlled server exercises correct 200/206, ignored Range, bad Content-Range, 304/416, changed validator, truncated/extra body and compressed representation. Assert exact final bytes or refusal, never mixed-source/revision publication. | contract + integration | simulated HTTP server; real streaming/files | automated | Q1 / Acquisition | pending | +| AC03 | **Two real existing consumers.** Use actual HF resolution/download/import and current native archive installer through shared acquisition. Assert importer settlement and native consumer publication separately; no model row for executable archives. | system + integration | required-real HF and approved llama.cpp archive; supported host | either | Q1 / Acquisition + model/native consumer | pending | +| AC04 | **Retained recovery and migration.** Exercise supported current/legacy store fixtures, hidden admissions, revocation, uncertain publication and Pending cleanup refusal. Interrupt and reopen through production readers; preserve authored/model state and require old-writer retirement/isolation before actual mutation. | contract + system | representative retained-state replicas and real filesystem; actual deployment facts for rollout | either | Q1 / Core/recovery | pending | +| AC05 | **Consumer commit and handoff crash windows.** Interrupt before/after FilesReady, domain commit and settlement. Reopen, reacquire authority, preserve exact generation and demand, avoid repeated side effects, retain uncertain inputs, and never remove adopted output. Inject importer and extraction failures. | integration + system | representative filesystem/process/store adapters | automated | Q1 / Acquisition + consumers | pending | +| AC06 | **Cancellation, pause, cleanup and use custody.** Cancel queued/active/verifying/consuming work; verify no writes after Paused, stale generation cannot act, blocking/child work retains files through cleanup, release is idempotent and eviction preserves all live/durable demands. Coalescing is not required. | integration + system | controlled real files/workers/children | automated | Q1 / Acquisition | pending | +| AC07 | **Authorization and provenance.** Test unauthorized sources/cache use, local-service access, redirect credential scope, expired locators, identity-preserving refresh, unsafe paths, wrong digest and logs containing seeded secrets. Explicit private endpoint works without authorizing arbitrary internal hosts. | contract + integration | controlled HTTP/credential/filesystem boundaries | automated | Q1 / Security/acquisition | pending | +| AC08 | **Progress and actual user control.** Existing desktop HF/native controls start, pause/resume/cancel as supported; progress separates bytes and finalization, recovers missed events and stale requests, and never shows installed/model-ready at byte completion. | user-workflow + contract | representative Electron/backend plus controlled delayed sources | either | Q1 / Desktop + consumers | pending | +| AC09 | **Bootstrap and owner independence.** Construct neutral acquisition without Python/Torch/adapter imports and without opening a model DB. Fetch native input, then execute owner shutdown twice; observe complete or truthful incomplete drainage, not detached work. | integration + system | representative no-inference/empty-root backend | automated | Q1 / Core/composition | pending | +| AC10 | **Resource and file-representation behavior.** Verify configured queue/stream/buffer/hash admission bounds and overload behavior under concurrent transfers; measure peak RAM, disk/writes and network bytes on representative large artifact. Ordinary-file same-filesystem path does not unconditionally retain a second full model copy; copying fallback is explicit. | integration + system | representative filesystem and workload; simulated saturation | automated | Q1 / Acquisition | pending | +| AC11 | **Exact local package installation.** Existing Torch integration acquires a retained exact wheel closure, installs with network denied, rejects alternate same-name/version bytes and absent dependencies, verifies installed distributions/interpreter/build, and preserves source provenance. | contract + system | required-real supported Python/tooling and approved Torch wheel closure | automated | Q2 / Package/runtime | pending | +| AC12 | **Package and bootstrap boundary accounting.** Record resolver metadata/candidate and managed-Python provider traffic separately. Prove final payload installation has no hidden remote URL or re-resolution; input leases last through package child cleanup. No fake claim that every resolver byte used shared transfer. | integration + system | representative real package subprocess with egress observation | automated | Q2 / Package/runtime | pending | +| AC13 | **S3 semantics through shared owner.** Test versioned object/range reads, multipart-style ETags, credential rotation, changed unversioned objects, prefix pagination interruption, logical-path mapping, addressing modes and optional SDK retry budgets through the same store/lifecycle. | contract + integration | controlled S3-compatible endpoint with fault injection | automated | Q3 / S3/acquisition | pending | +| AC14 | **Real object-store and user workflow.** Fetch a pinned multi-file manifest on AWS S3, one non-AWS S3 service and local MinIO; verify normal files and producer/consumer results. Import an S3-sourced model through the supported model workflow, then observe its published library record through GetModel or EnsureModel. Use native source configuration/inspection API and representative desktop flow without separate provider-specific downloader. | system + user-workflow | required-real authorized AWS, non-AWS endpoint, MinIO and representative desktop | either | Q3 / S3 + desktop | pending | +| AC15 | **Existing static and feature contracts.** Run affected core/app-manager/RPC tests, Rust fmt/check/clippy and no-default variants, Torch and TS gates when changed. Do not promote empty current core feature flags into a new minimal-build claim. | supporting static + integration | representative pinned toolchains | automated | Q1/owning slice / Integrator | pending | +| AC16 | **Producer/consumer and error contract.** Canonical DTOs regenerate from owner, RPC/IPC/preload/renderer reject wrong version/type/path/status, preserve errors and progress meaning, and support named current public clients or explicit versioned cutover. | contract + integration | representative Rust/Node and actual affected interfaces | automated | Q1/owning slice / Contract/desktop | pending | +| AC17 | **Installed/native/deployment qualification.** Outside checkout, use actual packaged backend/UI to acquire model/native and S3 inputs; execute affected filesystem/cancellation/reopen mechanisms on each declared supported OS. Close independent client and old-writer dispositions; retain per-target proof and known exclusions. | release-artifact + system + user-workflow | required-real packaged/native targets and deployment facts | either | Q4 / Distribution/integrator | pending | +| AC18 | **Composed simplicity and single ownership.** Trace new source, native build, adapter artifact and recovery repair against locality matrix. Verify one authoritative transfer writer per migrated attempt, distinct consumer commits, deletion of selected duplicate byte loops and no downstream-to-acquisition dependency. | architecture review (separate from test success) | not-applicable | manual | Q1/each material composition / Independent reviewer | pending | + + +## Real procedures and proof boundaries + +**Q1 model/native path.** Use disposable model and installation roots. Pin a small actual HF artifact/commit and one native release archive through current supported selection. Start through the real public model/native operations, inspect intermediate progress, interrupt/resume one transfer, and wait through model importer and installer finalization. Assert exact expected input bytes, model record only for the model, correct extracted executable role, and preserved unrelated state. Native startup/stop can use the existing path where available, but new installation binding remains runtime R1's claim. Repeat the real UI operation under recorded Electron conditions. Test setup does not need the new adapter registry or runtime installation schema. + +**Q2 package path.** Produce a version-checked accepted resolution with the supported managed interpreter/tooling. Let shared acquisition fetch the approved wheel set. Close/deny network access to the installation process using an actual enforced mechanism and retain its evidence. Install from generated exact local inputs, verify package and native-build identity, then run the scoped CPU import/tensor/startup checks owned by the current installer. A successful package import is not model inference. Repeat with one corrupted/substituted wheel, one missing closure member and cancellation during package child work. Resolver network access before that leg is separately recorded, not counted as an offline failure or hidden. + +**Q3 S3 path.** Use test-owned credentials and a fixture object set with recorded versions/digests. Validate a real AWS endpoint, a named independent S3-compatible endpoint, and local MinIO. Record actual supported features; an emulator proves its own behavior, not AWS or the other service. Fetch the same artifact description without changing consumer code. Through the supported model-facing operation, acquire one S3-backed model, await importer completion, and confirm its exact published library record through GetModel or EnsureModel; a byte-ready transfer alone does not pass. Exercise credential refresh, range resume and object-change refusal. The source settings and progress workflow must be observable through the actual API/UI. Failed listing pages cannot publish a complete package. + +**Migration/restart.** Create disposable replicas of supported existing metadata and partial files. Inject interruption at each admission, byte-ready, consumer-commit, ownership-transfer and settlement boundary. Reopen through the production reader and source/consumer composition. Check content, state and effect authority rather than only error text. Prove old-writer retirement/isolation for the actual deployment before any real retained root is migrated. Explicitly keep unaccepted Pending replay unavailable. + +**Native and resource evidence.** State selected OS/filesystem, features, artifact size, parallelism and budgets. Tests cover actual mechanisms on Windows/macOS when those promises change. A Linux process or cross-compile cannot close the native execution claim. Runtime gates may open for the qualified target scope, not for every platform. Performance observations establish resource behavior only for that workload; no unsupported throughput improvement is required or asserted. + +## Planned test placement and supporting commands + +The write-set report assigns the new integration targets `rust/crates/pumas-core/tests/artifact_acquisition.rs` and `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`. They do not exist as executed tests in this package. Once implemented, the owner runs their actual named cases, records commands/results and broadens to affected aggregate checks: + +```sh +cargo test --manifest-path rust/Cargo.toml -p pumas-library --test artifact_acquisition +cargo test --manifest-path rust/Cargo.toml -p pumas-app-manager --test artifact_acquisition_install +./scripts/rust/check.sh +npm run -w frontend lint +npm run -w frontend check:types +npm run -w frontend test:run +npm run -w electron test +npm run -w electron validate +python3 -m ruff check torch-server +python3 -m unittest discover -s torch-server/tests +``` + +Use repository-defined current exporter/freshness and feature/native commands after inspecting their actual definitions. Do not invent a passed CI result or copy a fixture snapshot as independent consumer proof. Package-network denial and real service tests need their named actual environment, not a monkeypatched network function alone. + +## Evidence record + +For each claim record candidate/material identity, source/fixture and authoritative oracle, command or operator procedure, environment, outcome, scope limits and evidence link. Keep failed/skipped/unavailable results separate. Link acceptance from the gate record only after the claimed observable result exists. Avoid adding an omnibus production verifier; focused tests and existing tools are preferred when they prove the obligation. diff --git a/docs/plans/artifact-acquisition/reports/architecture-review.md b/docs/plans/artifact-acquisition/reports/architecture-review.md new file mode 100644 index 00000000..edaaee23 --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/architecture-review.md @@ -0,0 +1,89 @@ +# Composed-design review: acquisition and downstream runtime management + +**Applicability:** applicable. The design changes source/consumer boundaries, durable ownership, shared transport and runtime composition. This is a reviewed design proposal, not implemented acceptance or an independent external review. + +## Authority-scope declaration + +The shared contract contains stable values describing selected bytes and their use lifetime. It references source authorization, model selection, package closure and consumer operation identities without taking over their policy. It cannot become an umbrella manifest for model modality, CUDA selection, UI layout, package solving, installation lifecycle and transport scheduling. + +Acquisition owns transfer/recovery and byte evidence. Source readers own protocol-specific access. Model and runtime consumers own publication. Their states and versions evolve separately; they are linked by exact request identity and explicit settlement. A common storage format or package distribution is not grounds to merge their authorities. + +## 1. Independent concerns and their dimensions + +| Concern | What / who | How / when | Where / why | +| --- | --- | --- | --- | +| Semantic selection | Model or runtime/package owner selects exact required inputs | Resolver/domain policy before acquisition | Existing core or app-manager integration; avoid implicit artifact substitution | +| Source description/access | Source integration identifies a pinned object and scoped access | Metadata, conditional reads, credential refresh during resolution/transfer | HF/HTTP/S3 boundary; source-specific protocol knowledge stays local | +| Acquisition | One owner transfers and verifies selected files | Governed async/stream/blocking work, durable attempt and cleanup | Neutral core module; eliminate per-consumer transfer implementations | +| Storage/use | Capability-backed workspace and explicit consumer demand | Stage, seal, read, transfer ownership or release | Model filesystem or artifact input storage; preserve ordinary paths and restart custody | +| Consumer publication | Model importer or runtime installer | Its existing validation and durable commit after bytes are ready | Its existing owner; byte completion cannot promise installed/usable state | +| Projection | RPC/UI and public/embedding callers | Decode, observe, control and reconcile snapshots | Delivery boundaries; no second source of successful state | + +## 2. Required versus accidental interleaving + +Required: exact file/revision, permitted source, destination custody, attempt generation, validation evidence and consumer retention must stay related throughout a transfer and handoff. These facts constrain the same operation. + +Accidental coupling removed: model root UUID/repo ID in every transfer; HF importer inside generic completion; URLs as identities; image adapter names in source selection; package install code inside network transport; native runtime download loops; a single completed boolean spanning acquisition/import/install/run. + +A durable demand has a different lifetime from a network request and from a runtime process. Time limits govern the specific operation, not the truth of cleanup or readiness. + +## 3. Caller and composition-root knowledge + +Callers choose exact content and a documented evidence policy, receive verified normal files, and settle custody after their own operation. They do not reconstruct HTTP ranges, retry loops, S3 signing, partial recovery or file-integrity checks. They still own the meaningful obligations of selection/trust and consumption; the abstraction cannot hide those by assuming execution permission. + +The root supplies existing runtime/network configuration, approved storage capabilities, source integrations and shutdown ownership. Core remains independent of app-manager and inference imports. A direct neutral acquirer is constructible without a model database. Managed-Python downloads can therefore bootstrap without a cycle, while their supported package-provider protocol stays separately owned. + +## 4. Representative changes and forced owners + +| Change | Legitimate affected owner(s) | Should not change | +| --- | --- | --- | +| HTTP pause/resume correction | Acquisition HTTP reader/lifecycle and relevant cross-consumer tests | Separate HF/native algorithms or adapters | +| Add an S3-compatible endpoint | Source configuration and actual compatibility evidence | Installer state, model-adapter registry, per-cloud download code | +| Add a new protocol source | Source reader/resolver and composition registration; contract only for genuinely new semantics | Model/index schema or runtime build logic | +| Add a runtime build or adapter wheel | Domain selection/package metadata and installer/adapter evidence | HTTP/S3 lifecycle | +| Change model importer | Model publication and handoff/settlement tests | Source protocol implementation | +| Change acquisition storage format | Acquisition store, migration and affected public projections | Installed-runtime ID meaning or model selection | +| Add Xet or a chunk cache later | Acquisition reconstruction/retention implementation and named evidence | Ordinary-file consumer API, unless a newly required guarantee is explicitly adopted | + +The locality condition is semantic, not a promise of one-file changes. Tests and packaging may legitimately span owners when their actual boundary changes. + +## 5. Stable interfaces versus leaked detail + +Keep validated artifact descriptions and file-use capability stable. Do not leak `reqwest::Response`, `object_store` types, pip report parsing, HF URL templates, raw auth strings, journal layouts or mutable cache paths into app-manager consumers. Source-scoped opaque version evidence can be retained without asking callers to interpret it. + +Do not force all existing HTTP use into this owner: search metadata, health probes and generation are not artifact transfer merely because they use HTTP. Reuse a network stack without equating distinct lifetimes. + +## 6. Independent verification, failure and replacement + +Source failure can leave a resumable attempt without corrupting published models/installations. Consumer failure can leave a verified input available under retained demand, without making it runnable. A registry update cannot alter a selected acquisition. A read-only model-library consumer need not acquire execution infrastructure. + +The source reader is replaceable behind conditional-read/evidence semantics. The storage implementation is replaceable only through its actual migration contract. Public model/install facades retain their completion meanings. Q1's two existing consumers establish this separation before runtime refactoring begins. + +## 7. Deletion test and retained machinery + +| Mechanism | What happens if removed | Admission decision | +| --- | --- | --- | +| Source-neutral acquisition owner | Byte lifecycle reappears separately in HF, native and package consumers | Necessary shared responsibility | +| Minimal artifact specification | Identity, verification and file-set completeness become unstructured caller assumptions | Necessary contract; no universal workflow manifest | +| Workspace/read custody | Cancellation/eviction/restart can invalidate files still in use | Necessary; extend existing capability/task mechanisms | +| Durable demand/attempt settlement | Crashes between transfer and import/install cannot be reconciled safely | Necessary state, owned once | +| Consumer publication owner | Acquired bytes get conflated with installed/imported success | Must remain separate | +| Package solver and S3 protocol dependency | Difficult standardized semantics need bespoke implementations | Reuse maintained libraries/tooling | +| Extra daemon, global CAS, VFS, coalescing scheduler | Q1–Q3 still deliver their requested outcomes | Not admitted now | +| Mirrored transfer journals or permanent per-source byte loops | No necessary capability is lost after the selected consumer cutover | Remove/disposition at each owning slice | + +## 8. Cumulative complexity and scope + +The intended retained structure is one deep acquisition service, a narrow source-access boundary with real HTTP/S3 implementations, an owned store/workspace implementation, and existing model/native/package consumers. Files can be split by lifecycle/authority; there is no prescribed layer count or class framework. + +The shared contract closes the prerequisite without pulling runtime execution into acquisition. Q1 uses the existing native path; Q2 uses the existing package path. No new requirement forces runtime R1/R2 to exist before their prerequisites can be accepted. Store migration and credential/representation proof are inherent complexity; contain them rather than reduce them to unchecked booleans. + +## Bounded library and mechanism decisions + +**HTTP:** existing core uses reqwest/Tokio and hashing facilities. Retain those capabilities; isolate the specific file-read semantics and keep provenance/integrity/custody with acquisition. Review legacy helper calls before removing a public helper, and retain it only as a supported delegating facade when its existing promise fits. + +**S3:** `object_store` is the first candidate, not an installed/qualified dependency claim. Its documented conditional/range/version operations and S3 endpoint/credential builder make it a plausible source adapter. Before Q3 expands around it, build the smallest reader against the pinned Cargo toolchain and observe: explicit endpoint/addressing, supplied/refreshable credentials, versions/If-Match, precise range/error propagation, cancellation, retry control and optional-feature isolation. Record the exact crate version, feature set, licenses, provenance and supported OS. Stop when those facts decide suitability. If a necessary guarantee cannot be represented, evaluate a direct maintained S3 SDK against the same requirement; no automatic fallback or in-house signer. References: [GetOptions](https://docs.rs/object_store/latest/object_store/struct.GetOptions.html), [S3 builder](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html). + +**Persistence:** the current versioned JSON store already owns attempt/admission/release and uncertain publication. Generalize its proven mechanism and explicitly version records; use one ledger authority for migrated acquisitions. A new database/service is not selected merely because the output is generic. If real contention/transaction/volume requirements invalidate that mechanism, record the deciding evidence and re-plan at the persistence owner before changing the store. + +**Package tooling:** use the current supported report/resolution mechanism and a real local-input install probe. Test a valid wheel set plus an alternate same-version wheel and a missing dependency with network denied. That determines the exact handoff before independent adapter registration depends on it; it does not require a universal lockfile or private pip API. diff --git a/docs/plans/artifact-acquisition/reports/codebase-audit.md b/docs/plans/artifact-acquisition/reports/codebase-audit.md new file mode 100644 index 00000000..7aa71fac --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/codebase-audit.md @@ -0,0 +1,51 @@ +# Source audit supporting the paired layer design + +**Date:** 2026-09-29. **Pumas:** `04e7f1568f00693c0ef26c77e0150e5e4dd112ea` (`work/torch-version-management`). **Standards:** `39d55dc330d44ecf940364ceada9d2527f7c7ea0`. No repository source, user environment, branch, installed artifact or production test was changed or executed. + +## Method and extent + +The attached revision-three plan and remote-acquisition review were read in full. Current GitHub branch and standards refs were checked. The Pumas feature branch is unchanged. Standards advanced one commit from `91ceb0fe`; the inspected comparison contains regression tests, their plan records and generated suite inputs, not changes to the previously read normative Core/Router/Planning/Architecture contracts. + +This round additionally inspected the actual destination/recovery types, download-store records and task lifecycle, plus the core manifest. Prior source findings at the unchanged Pumas commit are carried forward with their scope. This is a bounded invariant-family audit; it does not claim every repository file, deployment, native target or external caller was inspected. Search results are navigation, never proof of complete consumer absence. + +## Findings and resulting design decisions + +| ID | Source finding / review result | Material consequence | Plan disposition | +| --- | --- | --- | --- | +| AQ-I01 | `DownloadRecoveryDestination`, `DownloadDestinationRoot` and recovery tickets carry model-relative identity; the root machinery validates `.pumas-library-id.json`. | Directly exporting that type as a generic wheel/archive destination would preserve model-specific authority. | Extract capability-backed workspace mechanics without making model ID/UUID mandatory; leave model authorization at its consumer. Q1, AC01/AC04/AC09. | +| AQ-I02 | `PersistedDownload` contains `repo_id`, `DownloadRequest`, optional HF evidence and model-root destination identities; its versioned store also owns exact-attempt admissions, releases and uncertainty. | A nullable-field patch is not enough; neither discarding the store nor duplicating it is justified. | Generalize the owned record at an explicit migration boundary, preserve retained custody/tombstone/refusal semantics, and give migrated attempts one writer. Q1, AC04–AC06. | +| AQ-I03 | `hf/lifecycle.rs` registers task roles/generations and destination queues; pointer identities prevent stale tasks acting on successors. | A new `download()` future plus Drop cleanup would lose required supervision. | Reuse that lifecycle depth and isolate neutral execution identity from source/model-specific inputs. Q1, AC05/AC06. | +| AQ-I04 | Prior pinned inspection shows HF owns model importer completion as well as byte work; native installer and `network/download.rs` contain other byte loops. | Shared acquisition cannot equate byte success with imported/installed completion or merely rename the HF client. | One transfer owner; separate domain commits; real HF/native bridge in Q1. AC03/AC05/AC08. | +| AQ-I05 | The runtime r3 plan contains generic acquisition as S1a, while runtime consumers are needed to prove it. | Splitting plans carelessly could create two authorities or a prerequisite cycle. | Acquisition Q1 uses existing native behavior, Q2 existing package behavior; runtime R1/R2 wait on accepted gates. | +| AQ-I06 | Runtime/package artifact selection includes build/origin/hash policies and package-tool resolution. | A generic S3/HTTP read does not prove executable approval or Python compatibility. | Preserve package/native policies; Q2 validates exact acquired local inputs with denied network during installation. | +| AQ-I07 | Current core directly uses reqwest, Tokio, sha2, capability filesystem and atomic store mechanisms; current no-default feature flags are not a full minimal-core implementation. | There is a reuse path without a new service/crate, but no basis for claiming all old dependencies disappear. | Neutral core module, optional source dependency, measured feature/target claims only. Q1/Q3/Q4. | +| AQ-I08 | The source brief's scope includes S3 and future Xet/peers/CAS, but source manifests appear after transport phases in its suggested sequence. | Different consumers need file identity and handoff before concrete transport expansion. | Establish the minimal manifest and consumer contract in Q1; Q3 adds S3; future reconstruction/peers/CAS remain explicit deferrals. | + +### Exact source population inspected this round + +- [rust/crates/pumas-core/src/model_library/download_recovery.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/model_library/download_recovery.rs) — lines 1–290: model/library markers, recovery values and held destination/root capability. +- [rust/crates/pumas-core/src/model_library/download_store.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/model_library/download_store.rs) — lines 1–235: schema 5, model/HF snapshot fields, exact admissions and uncertain publication. +- [rust/crates/pumas-core/src/model_library/hf/lifecycle.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs) — lines 1–235: task generations, destination claims and queue/lifetime ownership. +- [rust/crates/pumas-core/Cargo.toml](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/Cargo.toml) — full returned manifest: dependency/feature placement. + +### Carried-forward pinned evidence + +The preceding reviews read [the generic-fetch brief](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/breif/s3-model-fetch.md), [the model intent brief](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/breif/intent-discovery-distribution.md), [HF owner](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/model_library/hf/mod.rs), [HTTP helper](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/network/download.rs), [native installer](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-app-manager/src/version_manager/installer.rs), [package resolver/installer](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/torch-server/resolve_runtime.py), and [active recovery plan](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/plans/current-standards-remediation-2026-09-03/rust-library-and-rpc/plan.md). These establish existing paths and limitations, not passing evidence for new code. + +The existing source brief is not an already implemented acquisition service. In particular, accepted recovery checkpoints are not permission to replay still-unaccepted Pending cleanup. Existing namespace and directory spelling are preserved. + +## External mechanism facts used to constrain the plan + +HTTP partial response assembly requires representation/range evidence; an ignored range is not an append operation. S3 version selection, range behavior and ETag semantics require a source-specific adapter rather than guessed generic checksum fields. Official references are in the [shared contract](../../../contracts/artifact-acquisition.md); they are source semantics, not Pumas conformance evidence. + +The maintained `object_store` API is a plausible S3 implementation candidate, not a dependency installed or approved by this task. Exact target/features/license and operational checks remain Q3 admission. Supported pip reports describe resolutions but are not accepted as install input themselves; Q2 must prove its selected local-input procedure. + +## Informed layer boundary + +The shared state-machine and workspace invariants belong below both model import and runtime install. Model/release/package selection stays above acquisition; source access varies below it. A lower-level reader transfers protocol bytes but does not own another acquisition lifecycle. Verified-file custody connects the layers without collapsing their completion or rollback authority. + +A new source then changes source integration and its evidence. A new model adapter changes adapter metadata/code and its execution evidence. A new runtime build changes native/package selection. None should require a new transfer state machine. That is the design claim to examine during implementation, separately from whether tests are green. + +## Remaining bounded evidence needs + +Before Q1 source writes: current local/dirty state, exact direct producer/consumer/external API population, supported persisted source states and old-writer disposition, destination-grant extraction and actual native integrity policy. Before Q2: actual package-local-input translation and network denial. Before Q3: maintained S3 implementation/target decision and real endpoint credentials. Missing facts prevent only their corresponding unsafe action or acceptance claim; they do not authorize weaker fallback or a new parallel implementation. diff --git a/docs/plans/artifact-acquisition/reports/dependency-gates.md b/docs/plans/artifact-acquisition/reports/dependency-gates.md new file mode 100644 index 00000000..5f5c4135 --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/dependency-gates.md @@ -0,0 +1,59 @@ +# Acquisition prerequisites and runtime handoffs + +**Owner:** acquisition integration, with one serial cross-plan integrator. This is the single status record for acquisition-provided gates. The runtime plan references these rows and does not independently declare them ready. +**All gates:** not ready. No production evidence exists for the proposed implementation. + +| Gate | Provider milestone / claims | Required consumer observation | Unblocks | Current status | +| --- | --- | --- | --- | --- | +| AQ-HTTP | Q1 / AC01–AC10, AC15, AC16, AC18 | Existing HF model acquisition reaches awaited model import; existing llama.cpp installer consumes the same neutral verified-file handoff and reaches its own validated extraction/publication. Cancellation/restart/UI evidence included. | Runtime R1 on the qualified targets. | Not ready | +| AQ-PACKAGES | Q2 / AC11, AC12 plus affected AQ-HTTP regressions | Existing Torch package integration consumes an approved exact wheel set locally with network denied during installation; no new adapter registry needed. | Runtime R2 after R1. | Not ready | +| AQ-S3 | Q3 / AC13, AC14 plus source-independent regressions | The same manifest/transfer/handoff contract works with version/credential/range conditions on AWS S3, one non-AWS compatible service and local MinIO; an S3-sourced model completes import and is observed through GetModel or EnsureModel. | Runtime or model S3 acquisition on qualified endpoint/target combinations. | Not ready | +| AQ-COMPLETE | Q4 / AC01–AC18 and all milestones accepted | Actual installed/public/native consumers and source-migration/deletion dispositions satisfy the complete acquisition scope. | Acquisition plan acceptance; not a hidden prerequisite for HTTP-only runtime work. | Not ready | + +## One-way sequencing + +```text +Acquisition Q1 -- AQ-HTTP --------> Runtime R1 + | | + +--> Q2 -- AQ-PACKAGES ----> Runtime R2 --> R3 --> R4 --> R5 + | + +--> Q3 -- AQ-S3 ----------> S3-enabled consumer operation + | + Q1 + Q2 + Q3 --> Q4 --> AQ-COMPLETE +``` + +Default serial order is Q1, Q2, then the integrator selects the ready runtime R1 or acquisition Q3 work from product priorities and disjoint writes. Each plan still has exactly one next slice. Parallel development is allowed only under its declared ownership; integration remains serial for shared contracts/state/generator files. + +Q1 uses the existing native installer. Q2 uses the existing package installer. Neither depends on the new RuntimeInstallationId or registered-adapter implementation. This removes the circular dependency that would result if prerequisite evidence required runtime R1/R2 first. A narrow native/package bridge belongs to acquisition's write set until its acceptance; later runtime refactoring changes its consumer while preserving the contract. + +## Gate scope and evidence + +Before marking a gate ready, record: + +- reviewed material source/candidate identity and implemented contract revision; +- exact claims and evidence links, including actual producer and consumer; +- OS/architecture, source/endpoint and dependency context to which it applies; +- known unsupported/unavailable variants and preserved preexisting behavior; +- public/persisted consumer dispositions; and +- integrator/reviewer outcome. + +A Linux result is not Windows/macOS evidence. Runtime gates are evaluated for the target being integrated. Final cross-platform/release promises remain with Q4 and runtime R5. A source-only build or mocked success cannot open a required-real gate. + +A status update or documentation-only change does not invalidate reviewed code. A change to content semantics, authorization, custody, persistence, package handoff or wire compatibility triggers a targeted review and re-run of the affected claims. Gate records retain the last accepted scope without authorizing an incompatible new candidate. + +## Single-writer ownership and cutover + +Acquisition owns the canonical shared contract, transfer lifecycle and gate records. Runtime owns installation identities, model-adapter registration and bound execution. Shared file writes in `pumas-core`, app-manager installer, Torch package integration, RPC/export and renderer projections are reserved to one integrator while a predecessor slice modifies them. + +A worker may propose a contract change but cannot independently change the shared contract, both plan states and generated outputs. The integrator collects findings, reviews the changed composition, amends both plans and the affected gate status, then assigns disjoint implementation work. This is ordinary serial integration; a separate stale-proposal protocol is needed only if outstanding conflicting plan proposals actually coexist. + +## Replaced revision-three sequencing + +| Previous item | Current owner / meaning | +| --- | --- | +| Runtime S1a, generic acquisition | Superseded by acquisition Q1. No acquisition milestone remains in runtime. | +| Runtime S1b, installation identity | Runtime R1, gated by AQ-HTTP. | +| Runtime S2, packages plus adapters | Exact package acquisition/handoff foundation is Q2; independent registration is runtime R2, gated by AQ-PACKAGES. | +| Runtime S3/S4/S5 | Runtime R3/R4/R5; outcome scope preserved. | +| Runtime A29/A31/A32/A33/A34 shared-layer claims | Acquisition AC claims own the shared-layer proof; runtime retains explicit consumer obligations and references, not copied gate authority. | +| Runtime A30 exact installed closure | Runtime keeps the registered-adapter consumption claim; Q2 proves the prerequisite with the current package integration. These are different consumer observations. | diff --git a/docs/plans/artifact-acquisition/reports/standards-and-sources.md b/docs/plans/artifact-acquisition/reports/standards-and-sources.md new file mode 100644 index 00000000..8c57dd27 --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/standards-and-sources.md @@ -0,0 +1,57 @@ +# Standards applicability, sources and verification limits + +## Baselines + +Pumas feature-branch head observed: `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`. Coding-Standards head observed: `39d55dc330d44ecf940364ceada9d2527f7c7ea0`. Source reads used the connected GitHub tools. The delivered r3 plan was read through Files and its exact mounted ZIP was inspected/copied for this documentation revision. No source archive or production runtime was required for this task. + +The standards comparison from `91ceb0fe` to `39d55dc330d44ecf940364ceada9d2527f7c7ea0` was inspected. The previously read normative Core, Router, Planning, Architecture, Code Design, Contracts/Evolution, Implementation, Verification, Proportionality, Dependencies and Security material remains the governing context. This round additionally read Persistence, Concurrency and Rust Async. These are manual applicability/source observations, not a claim that an executable standards-router or whole-repository compliance suite was run. + +## Applicable obligations and their representation + +| Actual task fact | Applicable authority | Where the plan addresses it | +| --- | --- | --- | +| Material sequencing and two dependent plans | Planning, Implementation, Proportionality | Each plan's lifecycle, one next slice, Q/R dependencies, gate ownership, bounded unknowns and re-plan rules | +| New source/consumer/state boundaries | Architecture and Code Design | Both eight-probe composed-design reviews, authority matrix and deletion/change-locality tests | +| Persisted attempts and cross-process/independent consumers | Contracts/Evolution, Persistence and relevant IPC profiles at implementation | Shared contract, supported-source migration, old-writer disposition, gate evidence and generated-consumer cutover | +| Concurrent transfer, cancellation and cleanup | Concurrency, Rust Async and Rust profile closure at implementation | Single supervised owner, exact generations, capacity, leases, true shutdown and failure results | +| S3/Python/network protocol semantics | Dependencies and Security | Maintained library/tooling decisions, exact source/version/integrity and authorized provisioning; no homemade solver/signer | +| User interaction and generated TypeScript | Frontend, TypeScript/Async, Generated Contract and relevant language-binding profiles at implementation | Real UI path in each owning slice, canonical export, malformed/stale response tests and scope-preserving errors | +| Supported targets and filesystem identity | Cross-Platform, Rust Cross-Platform and platform verification at implementation | Actual native evidence, ordinary files, safe root identity and per-target gates | +| Source/public/installed result claims | Verification and documentation/release obligations where selected | AC matrix with kind/environment/mode/owner; package/readiness limits; durable contract stays proposed until implemented | + +The implementation integrator routes the concrete exact write set through the current Router and retrieves its required closure before editing, including conditionally selected diagnostics, protocols/schemas, Rust API/dependency/tooling, documentation, licensing and platform/GUI evidence. The current deliverable records the task facts; it does not claim an automated routing certificate. Routine extra files within an established owner amend that write set instead of starting a broad new investigation. + +## Normative links + +- [CORE-STANDARDS.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/CORE-STANDARDS.md) +- [STANDARDS-ROUTER.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/STANDARDS-ROUTER.md) +- [workflows/planning.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/workflows/planning.md) +- [workflows/implementation.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/workflows/implementation.md) +- [workflows/verification.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/workflows/verification.md) +- [workflows/development-proportionality.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/workflows/development-proportionality.md) +- [topics/architecture.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/architecture.md) +- [topics/code-design.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/code-design.md) +- [topics/contracts.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/contracts.md) +- [topics/contracts/evolution.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/contracts/evolution.md) +- [topics/dependencies.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/dependencies.md) +- [topics/security.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/security.md) +- [profiles/boundaries/persistence.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/profiles/boundaries/persistence.md) +- [topics/concurrency.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/topics/concurrency.md) +- [profiles/languages/rust/async.md](https://github.com/MrScripty/Coding-Standards/blob/39d55dc330d44ecf940364ceada9d2527f7c7ea0/profiles/languages/rust/async.md) + +## Code and intent authority + +[Acquisition source audit](codebase-audit.md) records exact new ranges and pinned earlier evidence. The [runtime source audit](../../runtime-installations-and-model-adapters/reports/codebase-audit.md) preserves the original runtime/adapter findings. Repository intent comes from [Generic Model Fetching Backend](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/breif/s3-model-fetch.md) and [Intent/discovery brief](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/breif/intent-discovery-distribution.md). Existing project ownership and gate commands come from [Contributing](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/CONTRIBUTING.md), [Architecture](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/ARCHITECTURE.md) and [Development](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/DEVELOPMENT.md). + +## Primary external references checked for this layer design + +- [RFC 9110](https://www.rfc-editor.org/rfc/rfc9110.html): representation and conditional/range response semantics, not a local implementation proof. +- [AWS S3 GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) and [Object](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html): versions, ranges and ETag limits. +- [object_store GetOptions](https://docs.rs/object_store/latest/object_store/struct.GetOptions.html) and [S3 builder](https://docs.rs/object_store/latest/object_store/aws/struct.AmazonS3Builder.html): a candidate's documented interface, not package approval, a version pin or target qualification. +- [pip installation report](https://pip.pypa.io/en/stable/reference/installation-report/) and [pip install](https://pip.pypa.io/en/stable/cli/pip_install/): resolution reporting versus actual local install input. + +These primary sources were inspected on September 29, 2026. Pin implementation dependencies to versions actually resolved/tested with Pumas's toolchain; do not infer qualification from current online docs. No third-party source, font, model weights or dependency binary is redistributed in this planning package. + +## Delivery verification + +The root `validation.json` records checks on this documentation package only: required files/fields, internal links/anchors, distinct plan paths, milestone/gate graph, transferred claim dispositions and ZIP/manifest integrity. It is not a Pumas build/test, standards-engine run, independent design review or a source/GUI/native acceptance result. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md new file mode 100644 index 00000000..7a515fad --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -0,0 +1,52 @@ +# Acquisition implementation write sets and coordination + +These are the proposed exact paths/closed path families. No production source is changed by this delivery. Before a slice edits files, its integrator records the exact members and actual tests in its ledger. New files below are intentional proposed paths, not claims of existing code. Same-owner amendments are recorded; new authority/consumer boundaries trigger re-plan. + +## Q1: one HTTP acquisition owner with real consumers + +**New canonical module paths, used only when the concern warrants the split:** +`rust/crates/pumas-core/src/acquisition/{mod.rs,types.rs,service.rs,store.rs,workspace.rs,http.rs}`. Source-reader abstraction and current HF adaptation can remain in these owners or `acquisition/sources/{mod.rs,http.rs}` if that is the clearer implementation; choose one actual layout at admission, not both. The semantic split is independent source access, durable transfer owner and capability workspace, not file-count reduction. + +**Existing owners allowed to change:** +- `rust/crates/pumas-core/src/lib.rs`, `network/{mod.rs,download.rs}`, `model_library/hf/{mod.rs,download.rs,lifecycle.rs,types.rs,metadata.rs}`; +- `rust/crates/pumas-core/src/model_library/{download_store.rs,download_recovery.rs,partial_download.rs}` only for the extracted authority and explicit supported migration; +- directly affected core `api/hf.rs`, `api/state.rs` and current model-importer/intent completion call sites, selected from the actual producer/consumer trace before editing; +- `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,ollama.rs,progress.rs}` for the acquisition bridge and transfer progress, not installed-unit identity migration; +- current core atomic JSON/capability-filesystem modules only where the same selected invariant requires a targeted change, never as an unrelated filesystem rewrite. + +**Tests:** proposed `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`, plus existing co-located HF lifecycle/recovery/installer regression tests. Fixtures must reach the owner under test with independent expected byte/effect outcomes. + +**Serial adjacent writes:** `rust/crates/pumas-rpc/src/contract.rs`, `contract/export.rs`, affected HF/version/status handlers, `electron/src/{preload.ts,rpc-method-registry.ts,ipc-validation.ts}`, actual corresponding frontend download/install/source views and generated DTOs. Enumerate outputs from the actual exporter rather than guess or edit generated files manually. Reuse the existing model/native UI; this is not a dashboard redesign. + +**Docs:** the canonical shared contract, acquisition plan records, bounded handoffs in the existing HF/Rust remediation and upstream-runtime plans, and current architecture/development documentation where behavior has landed. Preserve the `docs/breif` path spelling; do not rename unrelated source-intent files. + +**Forbidden:** runtime installation-ID/profile migration, new model-adapter registry, package dependency reinterpretation, arbitrary source/plugin execution, consumer data deletion, claims of completed Pending replay, and concurrent second writers to the same transfer state. + +## Q2: exact package-file handoff + +Allowed: Q1 acquisition types/service only for demonstrated missing file-set/lease semantics; `torch-server/resolve_runtime.py`, retained preview/lock consumers in `rust/crates/pumas-app-manager/src/version_manager/{torch_preview.rs,installer/torch.rs}`, corresponding existing package/integrity/progress tests, and `artifact_acquisition_install.rs`. Keep package resolution/install semantics with those consumers. The exact managed-Python/provider files are first traced for a migrate-versus-retain traffic disposition; no speculative private integration is authorized. + +No arbitrary adapter registry or new host/task API is needed for this prerequisite. The existing wheel path proves local exact consumption. Runtime R2 subsequently owns the new adapter-package caller and its acceptance. + +## Q3: S3-compatible source + +Proposed `rust/crates/pumas-core/src/acquisition/s3.rs` (or the already selected source directory), source configuration/credential projection at the existing authority, manifest resolution and focused S3 cases in the acquisition test target. Dependency admission can change `rust/Cargo.toml`, `rust/Cargo.lock`, `rust/crates/pumas-core/Cargo.toml` and licensing/ownership inventory through one integrator. Source URI/configuration operations and the smallest useful frontend source workflow change their canonical RPC/generated/preload/renderer projections together. + +No broad list/search capability, cloud-specific UI product, remote writes, custom signer, new model modality or provider-specific installer is included. AWS/non-AWS/MinIO environment setup is test-owned, not user account mutation without authorization. + +## Q4: qualification and cutover + +Affected installed/public/native consumers, source installation/build packaging and exact generated outputs. Change CI/release scripts only where necessary to observe a named acceptance claim. Final docs include the active source-of-truth guide and consumer/migration limits. Remove a public legacy helper only with its actual supported consumer/version disposition; otherwise retain a delegating facade without a second lifecycle owner. No release publication is authorized. + +## Shared roles and integration + +| Role | Primary write authority | Shared/forbidden | Required handoff | +| --- | --- | --- | --- | +| Acquisition worker | Admitted neutral service/HTTP internals and focused tests | Shared DTO/store migration requires integrator; no app-manager identity refactor | Invariants, exact files, cases/results, lifetime and remaining gaps | +| Consumer worker | Current HF/native/package bridge after contract fixed | No independent transfer state or source policy | Actual producer/consumer observation and completion semantics | +| S3 worker | S3 reader/config adapter after gate contract fixed | No global retries/store/manifest redesign or shared lockfile edits | Version/range/auth/endpoint evidence and dependency decision | +| Desktop worker | Admitted views/interaction tests against generated contract | No backend compatibility or completion policy; generated outputs integrator-owned | Real workflow, async freshness, keyboard/focus and error observations | +| Integrator | Both plan states, shared contract, persistence changes, lockfiles, exports, shared fixtures | Preserves unrelated work and history | Gate status, exact staged diff, integration evidence and commit disposition | +| Reviewer | Read-only candidate/context/evidence | Never edits | Architecture, migration, trust and lifecycle findings; narrow repair verification | + +Default serial order prevents Q1/Q2 installer changes racing runtime R1/R2. Record disjoint primary/allowed-adjacent sets when delegating. Reuse existing branches/worktree conventions; the presence of two plans alone does not require new worktrees or a custom multiagent coordination framework. diff --git a/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md b/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md new file mode 100644 index 00000000..89ca79a6 --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md @@ -0,0 +1,13 @@ +# Runtime execution ledger + +## 2026-09-29 — revision 4, separate acquisition prerequisite + +The user requested a dedicated acquisition implementation plan alongside this runtime plan. Revision 3 is the input. Its source intent and runtime/adapter decisions are preserved, but generic acquisition S1a is superseded by the separate acquisition Q1. S1b becomes R1, gated by AQ-HTTP. Exact package-file acquisition is acquisition Q2; registered-adapter R2 additionally requires AQ-PACKAGES. R3–R5 retain the existing admission, independent Z-Image extension and qualification objectives. + +Created one canonical proposed acquisition contract and linked both plans to it. Acquisition's existing HF/native/package consumers prove prerequisites without waiting for new installation identity or adapter registration. Shared source and generated files have one integrator, and acquisition alone owns the AQ gate status. + +Pumas branch baseline is unchanged at `04e7f156`; current standards ref is `39d55dc`. Production acceptance remains pending. This delivery changes only planning artifacts in the working container. No repository write, implementation test, actual runtime/model operation, live migration, public release or independent external review occurred. + +The earlier entry-point mechanism probe is historical limited evidence in the previous delivery; it is not re-run, re-packaged as new evidence or used to mark runtime claims satisfied. Earlier delivered packages preserve revision history; this package avoids copies of obsolete active plans. + +**Exactly one next runtime slice:** R1 after AQ-HTTP is ready for its actual target. The coordinated program starts with acquisition Q1. Plan-only preparation may continue without opening the runtime source gate. diff --git a/docs/plans/runtime-installations-and-model-adapters/issues.md b/docs/plans/runtime-installations-and-model-adapters/issues.md new file mode 100644 index 00000000..d3d3c124 --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/issues.md @@ -0,0 +1,20 @@ +# Runtime issues and dispositions — revision 4 + +All code fixes remain planned. [Source audit](reports/codebase-audit.md) and the earlier delivered r3 package provide the baseline evidence; [acceptance](reports/acceptance-matrix.md) owns the current runtime claims. + +| Family | Severity / issue | Owner and disposition | Verification / revisit | +| --- | --- | --- | --- | +| E01/E02/E11 | High: concrete installation versus tag/global process ambiguity | R1 runtime/profile owner | A01/A02/A09/A20/A27 | +| E03/E06/E07 | High: missing adapter-specific admission and multiple live consumer paths | R3 managed execution/host/desktop | A03/A07/A08/A12/A18 | +| E04/E05 | High: requirement identity and additive semantics versus complete-environment policy | R3 core dependency owner | A04/A05/A19/A22; preserve authored old meaning until versioned | +| E08 | High: old-writer profile loss during new schema migration | R1/R5 core/integrator | A10/A19; actual retirement/isolation before retained-state mutation | +| E09/E10 | Medium: fixed bundle inventory and generated/handwritten UI semantics | R2 and each affected producer/consumer owner | A06/A15/A16/A21 | +| E13–E16 | High: native/Python common management and open revision-safe registration | R1/R2 app-manager/host | A20/A21/A23/A24/A27 | +| E17–E19 | High: task assumptions, control responsiveness and executable code authorization | R2/R3 host/security | A22/A25/A26/A28 | +| E23–E28 | High: acquisition boundary and duplicate/layer authority | Acquisition plan owns shared repair; runtime consumes gates | AC claims plus runtime A08/A09/A17/A25/A28/A30 | +| RT-P01 | Blocking dependency: AQ-HTTP not ready | Acquisition Q1; runtime R1 source integration waits | Target-scoped gate record, not an assumed helper API | +| RT-P02 | Blocking dependency for registration install: AQ-PACKAGES not ready | Acquisition Q2; runtime R2 waits after R1 | Existing and then registered-package local-install evidence | +| RT-D01 | Deferred: marketplace, arbitrary task host, native dylib ABI, orchestration | Respective runtime/host owner | Revisit only for explicit product/trust/protocol requirement, not an ordinary adapter addition | +| RT-E01 | Pending required-real image/native/package/UI/old-deployment evidence | Assigned runtime/host/distribution owners | Close only with named A claims and actual environment | + +Existing Nunchaku and FLUX.2 qualification remains exact-tuple historical evidence. Native management, a standard loader and standalone Z-Image need new evidence through the new contract. Temporary facades have a real supported-consumer owner and cutover trigger; preserve no alias merely because it existed. diff --git a/docs/plans/runtime-installations-and-model-adapters/plan.md b/docs/plans/runtime-installations-and-model-adapters/plan.md new file mode 100644 index 00000000..d8395d8a --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/plan.md @@ -0,0 +1,194 @@ +# Plan: runtime installations and registered model adapters + +**Revision:** 4 — coordinated with the separate prerequisite acquisition plan; supersedes revision 3 at the same canonical runtime plan path. + +**Status:** Planned, proposed for owner adoption; production implementation has not started. + +**Objective acceptance:** pending; real runtime, model, migration, native-platform, and desktop acceptance has not been performed. + +**Current phase:** runtime design ready for prerequisite-led implementation; AQ-HTTP is not ready. + +**Exactly one next slice:** **R1 — shared installation identity and bound launch for Torch and llama.cpp; source integration is gated by AQ-HTTP.** + +**Canonical repository path on adoption:** `docs/plans/runtime-installations-and-model-adapters/plan.md`. + +**Integration owner:** Pumas runtime integration, sharing one serial integrator with the acquisition plan for overlapping contracts/files. +**Prerequisite owner:** [Artifact acquisition](../artifact-acquisition/plan.md); [gate status](../artifact-acquisition/reports/dependency-gates.md). R1 may begin only after its target-scoped AQ-HTTP gate is ready. Read-only preparation can proceed independently. + +**Baselines:** Pumas `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`; Coding-Standards `39d55dc330d44ecf940364ceada9d2527f7c7ea0`. + +## 1. Objective + +Reuse the generic artifact-acquisition direction in `docs/breif/s3-model-fetch.md` for obtaining verified bytes. Make app-manager the coherent application-facing manager of installed external runtimes/executables, with native llama.cpp and Python/Torch as concrete implementations. Allow an open population of model adapters to be registered independently of a Pumas build. A model may use a standard loader, that loader plus supplemental requirements, or approved custom loading/execution code. + +The externally meaningful chain is: + +**Selected model artifact + task → selected adapter revision → effective requirements and approved component/code inputs → compatible installed runtime → bound profile/process → adapter preflight → verified loaded-model receipt → supported operation.** + +The manager may install and launch an executable without that fact asserting model support. An inference provider separately implements the execution protocol. Model adapters may share that provider and runtime; registering another model adapter does not create another runtime provider or require another global enum variant. + +A normal adapter addition for an existing supported host/task must not require a new Pumas build, a new RPC/gateway branch, a hardcoded UI option, or a new installation-state variant. A genuinely new execution host or public task protocol can require a runtime integration; adapter registration is not a promise to interpret arbitrary protocols. + +## 2. Scope and exclusions + +### Included + +- Consumer integration with the separately owned [acquisition contract](../../contracts/artifact-acquisition.md). This plan does not implement a second acquisition service, source protocol, transfer store or gate authority. +- Common runtime installation identity, inspection, launch-target selection, use custody, profile binding, and removal semantics for native executables and Python environments. +- Torch and llama.cpp exercised as real consumers of that common contract. Migrate affected Ollama entry points/representations in the shared family; preserve externally managed and in-process distinctions. +- Data-only model requirement specializations and versioned registered adapter implementation packages. +- Registration, discovery, explicit selection, authorized installation, lazy loading, revision replacement, disablement, unregister/removal, and failure isolation at their owning boundaries. +- Existing generic Torch text loading, Nunchaku, FLUX.2, and standalone Z-Image represented within the same adapter host contract, with only task operations that are actually implemented and qualified advertised. +- Dependency interpretation, exact model/component references, real slot UI/serving/gateway consumers, independent deployed packages, generated DTOs, and safe retained-state migration. + +### Preserved constraints + +Pumas remains a reusable model library with optional runtime conveniences. Pantograph orchestration, scheduling, and workload placement are outside this change. Core/library-only operation must not import or provision inference packages to inspect metadata. + +Keep upstream Torch discovery independent of qualification pins, managed Python provisioning, explicit install/bind/select/default/start/stop, typed unsupported versus inconclusive results, numeric image dimensions, and duration-unbounded admitted generation. Existing startup/loading/probe/install budgets keep their separate owners and meanings. + +Keep staged publication/recovery, immutable installation evidence, exact process generations, listener/child/device custody, and compensation. Reuse those mechanisms; do not add parallel mutable installation catalogs, process owners, or model-dependency authorities. + +### Excluded + +Full S3/Xet/peer/chunk-CAS implementation is not a prerequisite of this runtime plan; those mechanisms remain with the acquisition workstream. A plugin marketplace, automatic downloading/importing code from model cards, arbitrary shell install hooks, a universal package manager, a native Rust dynamic-library ABI, transparent hot replacement of live Python modules, multi-tenant isolation, remote scheduling, and universal support for every model or operation are excluded. Trusted external model-adapter packages and normal registration are **included**, unlike revision 1. + +## 3. Domain decisions and ownership + +### D1. Distinguish runtime installations, execution providers, adapters, and requirements + +A **runtime installation** is an addressable installed unit. A native unit contains its selected executable/library artifacts and build identity; a Python unit includes the interpreter/ABI and resolved distribution closure. No native installation is forced to invent Python, wheel, or sidecar fields. + +An **execution provider/runtime driver** knows executable roles, launch arguments, environment setup, supported protocol/task behavior, and readiness evidence. Provider-specific launch preparation belongs behind app-manager's management interface. Existing neutral OS/process custody may remain in core as a reused mechanism; there is exactly one operational owner of a child/session. + +A **model adapter** turns approved model inputs and configuration into a usable model session for a supported host/task. Its definition may select an existing loader or reference independently supplied implementation code. Its implementation owns model-specific construction, preprocessing, execution invocation, result conversion, and release hooks as needed by its supported operations. It does not install packages, choose global defaults, own subprocess adoption, or define the public gateway protocol. + +**Requirements** describe what an adapter/model needs. Installing them creates or verifies a suitable environment; it does not manufacture missing loader behavior. The same adapter may serve many models, and a model may have several possible adapters. Supplements alone do not require a new executable adapter. + +### D2. Use one common installation/binding contract with specialized internals + +Use a validated opaque `RuntimeInstallationId` rather than a Torch release tag as the common installed-unit address. Keep release/build/platform/interpreter/package identities as typed installation facts. Native and Python installation implementations retain release/build and dependency selection, install-time verification, installed-unit publication and migration. Source-independent byte transfer, resume, transfer persistence and content-integrity checks belong to the shared acquisition owner. Source resolvers retain source-specific location and authorization semantics. + +A profile stores its requested installation binding when managed. A process receipt records the actual installation, executable/role, relevant host bundle identity, generation and effective device/configuration context. The native receipt must be grounded in the manager's actual launched executable and custody; llama.cpp is not required to echo a Pumas environment ID or speak a Python sidecar handshake. + +Common operations are inspect/list, resolve installation choices, explicitly install, bind, prepare/launch, observe/stop, and remove an unreferenced installation. Use the existing version manager as a migration facade or refactor it in place; do not let old tag state and a new catalog independently mutate the same installations. + +Global selection is a preference for future explicitly accepted bindings, not evidence about running processes. An explicit profile is evaluated against its actual binding. An unbound/ambiguous legacy profile reports binding-required. Removal operates on exact installation references and live use, not app-wide PID guesses. + +Torch and llama.cpp are the immediate concrete locality test. General executable launch uses a resolved executable and argument vector, not a binary name derived from the app ID or a universal `serve` argument. Executable roles distinguish service launch from an auxiliary command when an actual integration needs that distinction; an executable does not inherently have an HTTP endpoint. This plan initially qualifies the existing native/Python service paths, rather than advertising arbitrary command or protocol support without an implementation. External service connections do not imply permission to install, adopt, or stop their processes. Embedded ONNX retains its in-process lifecycle. + +### D3. Register an open population of adapters, without global eager imports + +Introduce one logical model-adapter catalog with versioned definitions and explicit registration operations. It may combine bundled definitions and installed operator registrations, but those are origin/projection distinctions, not competing authorities. Reuse catalog/persistence utilities where their semantics fit; the current mixed UI plugin config is not automatically the canonical execution contract. + +Each registration references a stable namespaced adapter ID, an exact revision/content identity, a supported adapter-host API, tasks and applicability facts, required component roles, supported option contract, and either a standard loader with declarative options or a pinned implementation package/entry point. Package metadata owns that package's dependency declaration; adapter metadata references or specializes it without creating an independently editable copy of the same requirements. + +Read registration metadata without importing implementation code or optional inference dependencies. Keep a verified metadata reference available before environment provisioning so Pumas can explain/install missing dependencies. Verify that metadata against the selected package bytes again during install; discovery is not authorization or proof of readiness. + +For Python implementation packages, prefer the standard distribution entry-point mechanism and public `importlib.metadata` APIs over a custom import/packaging framework. The exact entry-point group and package name are owned contract choices to establish at implementation, not assumed from this review. Explicitly authorize and bind the distribution/version/digest/entry point; do not scan ambient global Python and load every advertised entry point. + +No fixed supported-adapter list or compile-time adapter enum controls registration. Bound individual payloads and resource usage and paginate enumeration; arbitrary extensibility is not unbounded memory allocation. Known unsupported host API/task/requirement variants have explicit diagnostics and are not silently interpreted as generic text loaders. + +### D4. Define registration and revision lifetime + +The catalog distinguishes definition validity/enabled state from installation compatibility, per-process availability, and per-model loaded state. A registered adapter with missing dependencies stays inspectable. A package's presence or import success does not publish a runnable model. + +Same ID/revision with identical content can be an idempotent registration. Different content claiming the same immutable identity is a conflict. Different revisions may coexist. Choose revisions explicitly or through a documented compatible selection policy; never let filesystem order, hash-map insertion, or newest-version guessing select the winner. + +Updates publish a new immutable definition/package reference. Loaded sessions pin their exact definition, package/host binding, installation and generation. A disable operation blocks new admissions; existing sessions retain their resources until their documented terminal lifecycle. Explicit stop/revoke is a different operator action. Unregister removes future selection authority; physical package/installation removal waits for retained references and use custody. + +A catalog refresh is atomic at its owner and produces explicit diagnostics for invalid candidates. Preserve a last known snapshot only as an explicitly labeled observation; it cannot authorize new work when current authority is unknown. Do not silently omit broken entries or continue serving with stale revocation state. Prevent factory/client caches from retaining an obsolete registration as current execution authority. + +Python process generations retain a stable package/module set. A new adapter package or incompatible dependency resolution uses a new/recreated environment and process, or a documented restart; no in-place `pip install` or hot module replacement in a live managed environment. Multiple compatible adapters can share one environment and process subject to the host's actual lifecycle contract. + +### D5. Compose requirements at the existing owners + +Effective requirements are **runtime-host requirements + selected adapter requirements + applicable authored model supplements**. Their resolved software artifacts are handed to shared acquisition; a model-specific `GetModel` request is not used to disguise a wheel or executable as a model. Apply platform/Python markers, native runtime constraints, source/hash constraints, and code/component inputs according to their owning semantics. Reuse actual package resolvers; do not implement Python package semantics in Rust. + +A standard model may have no supplements. A model requiring an extra package can still use a standard loader. A custom construction/execution requirement uses an approved adapter implementation. Native llama.cpp requirements are build/features/device/artifact requirements, not a fabricated Python dependency environment. + +The current dependency `env_id` is a profile/context identity, not an installed-runtime ID. Distinguish additive supplements from legacy full-environment profiles before composition. Preserve authored pins/provenance and unresolved states; unequal hashes alone do not establish additive conflict. New additive interpretation assigns torchvision/torchaudio or native-extension requirements to actual consumers, not automatically to every model modality. + +Conflicting requirements return their sources and possible explicit installation actions. A successful resolution does not waive model/task compatibility. An explicit install may group constraints transactionally but cannot silently change user pins, registries, selected adapters, or unrelated running environments. + +### D6. Apply trust to executable packages, not just filenames + +Approval is an explicit operator decision or configured trust policy, not a Pumas-maintained whitelist of adapter IDs. A compatible new adapter can be authorized and registered without maintainer changes or a Pumas release. + +Support trusted externally supplied adapter code without making model metadata permission to execute it. Registration and dependency inspection are read-only with respect to execution. Code installation/import requires the accepted operator or automation authority, exact source identity, integrity evidence, and supported host contract. + +Replace unconditional `trust_remote_code=True` in the affected managed loading family with an explicit approved-code decision. A custom adapter can use approved pinned code through the supported host; an unknown model cannot silently enter a remote-code path. Native extensions and package build steps are executable trust decisions too. Preserve the supported binary-wheel baseline; source builds or setup scripts need their own explicit procedure rather than happening inside discovery. + +Python environments and subprocesses isolate dependency/lifetime state, not hostile-code permissions. This plan makes no sandbox claim. Host API objects provide only the documented resources; approved plugins still execute with the deployed process permissions. Model assets and code are separate references with distinct authority and retention. + +### D7. Unify admission and task execution without hiding unsupported tasks + +The app-manager managed execution service composes library facts, selected adapter, requirements, installation, profile/process custody, host preflight, load correlation, and publication. Core supplies neutral contracts and existing model/profile/serving owners; core does not depend on app-manager. Preserve native embedding and local IPC by injecting/delegating through the optional integration, not by moving all functionality into RPC-only code. + +Move model-specific selection out of RPC repository checks, Python ad-hoc dispatch, probe lists and gateway allowlists. The provider advertises the tasks/operations its host actually implements. The adapter describes its task support. Public availability is the intersection with current installation compatibility and a verified loaded session; arbitrary strings in a plugin cannot create an unsupported public endpoint. + +Existing text code assumes a Transformers tokenizer/model and synchronous generation. Encapsulate that as the standard text implementation, not the universal custom-adapter contract. Before exposing it through managed public admission, qualify cancellation, executor responsiveness and shared generation lifetime. A new audio/video/task operation is explicitly unsupported until its real host/gateway contract is implemented; no claim that an adapter declaration alone adds one. + +Load receipts correlate exact artifact/components, adapter revision and implementation package, runtime installation/fingerprint, host bundle, process generation, operation, effective device, slot/session and supported task result. Retain producer/consumer validation; an echoed identifier alone is not launch evidence. Listing, generation, unload, cancellation and compensation use the same binding and cannot target a successor at a reused endpoint. + +### D8. Preserve independent compatibility and migration obligations + +Runtime releases, installation manifests, model metadata, adapter definitions/packages, adapter-host API, private sidecar protocol, and public gateway contracts have independent change reasons and versions. Adding an adapter within an existing host API does not require a sidecar wire bump or Pumas release. Changing required binding fields does require an explicit private-protocol transition; protocol 3 is the current baseline, not a permanent hardcoded proposal for the successor. + +Coordinate changed internal source/DTO/generated/UI consumers in their owning slice. Give public libraries, bindings and independently installed clients explicit supported-version or breaking-release dispositions. Inspect and migrate actual supported retained states without deleting user installations or moving virtual environments as though they were portable directories. + +Older profile writers currently accept future schemas. New reader checks cannot fix those deployed binaries. Migration therefore requires retiring old writers or isolating the new authoritative storage and managed resource ownership. Keep backups and interruption/reopen evidence, but never overwrite newer authored state merely because a backup exists. Physical removal is a separately authorized operation. + +### D9. Consume the prerequisite contract; keep downstream authority + +[Artifact acquisition](../../contracts/artifact-acquisition.md) is the single proposed handoff authority. Its [plan](../artifact-acquisition/plan.md) owns HTTP/HF and S3 source handling, transfer/control/recovery, evidence-scoped ordinary-file handoff, and the prerequisite package-file integration. The runtime plan owns selection/trust, installed-unit publication, registered adapters and bound execution. + +Acquire exact approved artifacts through that interface without fake model records or source-specific downloader code. Keep the input lease until extraction/package children and cleanup finish; commit installed state only after installer verification. Model assets remain owned by the library. An installed runtime already satisfying its binding can be inspected/launched offline without querying remote sources merely to re-establish file acquisition. + +Runtime R1 requires AQ-HTTP for the integrated target. R2 requires R1 and AQ-PACKAGES; it verifies the new registered-adapter consumer even though acquisition Q2 already proved the existing package consumer. Any S3 path additionally requires AQ-S3 for its endpoint/target scope. Native Xet, peers and chunk CAS remain acquisition continuation work, not hidden runtime prerequisites. + +The existing HF/native/package bridges used by Q1/Q2 are prerequisite evidence and do not depend on R1/R2. Once the acquisition integrator accepts those shared files, runtime may refactor their consumer under the same contract. Exact status, scope, evidence and invalidation are owned in [dependency gates](../artifact-acquisition/reports/dependency-gates.md), not copied here. + +## 4. Milestones, write sets and prerequisite gates + +[Write sets](reports/write-sets.md) are part of these boundaries. Both plans use current exact source/consumer facts, preserve unrelated work and assign shared schemas, lockfiles, migrations and generated outputs to one writer. + +| Slice | Coherent outcome | Dependencies / gate | State | +| --- | --- | --- | --- | +| **R1** | Shared installation identity, explicit profile binding and actual executable-bound launch for Torch and llama.cpp, including affected existing native consumers and UI addressing. | AQ-HTTP; runtime A01/A02/A09/A10/A12/A16/A17/A19/A20/A27 evidence at the relevant scope. | Planned; gated | +| **R2** | Register/list/select versioned adapter definitions; acquire through the accepted file-set contract, then install locally and lazily load an independent adapter. | R1 + AQ-PACKAGES; post-build registration, exact local package consumption and catalog lifecycle/trust evidence. S3 use additionally requires AQ-S3. | Planned | +| **R3** | One managed model-admission/task interface with existing native/Torch implementations migrated. | R2; requirements, exact receipts, slot/gateway/launch/unload consistency, real standard text/images and owned cancellation. | Planned | +| **R4** | Standalone Z-Image as a separately registered extension with no Nunchaku dependency. | R3; frozen Pumas host, exact real model and requested-size output, cancellation/reuse. | Planned | +| **R5** | Final installed/public/deployment qualification and removal of superseded execution authority. | R1–R4 implemented and each used acquisition gate ready for the promised scope; all runtime claims satisfied. | Planned | + +R1 uses native and Python consumers immediately. R2 proves actual independently installed registration, not only DTOs. R4 is a genuine extension-locality test. Required producer/generated/UI changes land in the same slice as their owner, not in a final cleanup. Every gate is tied to its exact qualified scope; no all-source/all-platform acceptance is inferred. + +Read-only investigation and design preparation may proceed before AQ-HTTP, but this plan does not start runtime implementation by building another downloader. Acquisition Q1 is the coordinated program's next implementation slice. Acquisition Q3 can proceed alongside later runtime work once writes are disjoint; runtime's HTTP/package integration does not wait for Xet/CAS or for unrelated acquisition release claims. + +## 5. Acceptance and review + +The [acceptance matrix](reports/acceptance-matrix.md) preserves A01–A28 and A30 as runtime-owned claims and records explicit acquisition/dependent-consumer dispositions for A29/A31–A34. Every implementation claim remains unsatisfied. The historical Python entry-point probe belongs to the earlier delivered package and is not production qualification; no probe is rerun or promoted here. + +Composed-design review is **applicable**. All eight probes and authority scopes are answered in [architecture review](reports/architecture-review.md). Review the final artifact by its caller knowledge and change locality, not by module counts or green test totals. An adapter extension must not force edits across app-manager installation state, core provider enums, RPC, gateway and renderer. + +Independent review covers migration/old writers, executable trust, session/cancellation/removal lifecycle, adapter revision/host API evolution, and final composition. Reviewers are read-only and reused for narrow repairs. Shared contracts, generated artifacts, manifests, lockfiles and plan state have one integration writer. Delegate implementation only with disjoint primary writes and explicit adjacent/shared-file rules. + +## 6. Bounded preparation, blockers and re-plan triggers + +Before R1, read the acquisition gate record and confirm AQ-HTTP evidence applies to the actual target/candidate. Refresh local branch/dirty state, shared active plans, public launch consumers, native artifact layout/integrity, supported legacy state and old-writer retirement/isolation. The acquisition contract owns transfer custody; runtime must not reimplement it. Missing migration/deployment facts block their unsafe effects, not bounded read-only preparation. + +Before R2, establish the exact package/metadata entry-point convention and host API, supported declaration/options validation vocabulary and namespace ownership. Resolve third-party packages from a reviewed source; no guessed version pins appear in this plan. Before R4, verify the selected standard Z-Image dependency/API/model-layout tuple through the smallest real observation that can change the implementation. + +Re-plan for a new trust boundary or task host, incompatible independent deployment, migration data risk, duplicated authority, unbounded/ownerless work, or propagation of adapter semantics back into shared runtime/UI code. Adding another ordinary adapter or another file within an existing owner is not itself a re-plan trigger. A new native dynamic-plugin ABI or universal install language would require a separate justified decision. + +Acceptance transitions through Implemented/Verifying/Accepted only as named evidence permits. Record unavailable native/GPU/GUI/deployment legs accurately. Source review is not certification that the current or future entire codebase is standards compliant. + +## 7. Adoption, authorization and links + +Adopt this revision at `docs/plans/runtime-installations-and-model-adapters/plan.md` alongside `docs/plans/artifact-acquisition/plan.md`. Acquisition Q1 replaces former runtime S1a; R1 replaces S1b; R2–R5 replace S2–S5. The [cross-plan record](../artifact-acquisition/reports/dependency-gates.md) owns that disposition and readiness. Do not adopt the old Torch-only plan as another active authority. + +Reconcile active upstream-Torch, cross-platform, diffusion and standards-remediation owners by superseding only affected current decisions. Preserve their historic exact-tuple evidence and unsupported/unaccepted limits. Do not delete or rewrite old plans merely to erase history; update current owner documents when their behavior lands. + +A future implementation session explicitly selects this plan and `start` after its gate is ready. This planning delivery does not mutate the repository, authorize live environment migration/deletion, publish packages/releases, or rewrite shared history. Keep coherent commits and independent read-only review; no fixed commit topology is prescribed. + +[Ledger](execution-ledger.md) · [Issues](issues.md) · [Source audit](reports/codebase-audit.md) · [Adapter contract](reports/adapter-contract.md) · [Architecture](reports/architecture-review.md) · [Acceptance](reports/acceptance-matrix.md) · [Write sets](reports/write-sets.md) · [Standards/source inventory](../artifact-acquisition/reports/standards-and-sources.md) · [Shared acquisition contract](../../contracts/artifact-acquisition.md). diff --git a/docs/plans/runtime-installations-and-model-adapters/reports/acceptance-matrix.md b/docs/plans/runtime-installations-and-model-adapters/reports/acceptance-matrix.md new file mode 100644 index 00000000..9904508b --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/reports/acceptance-matrix.md @@ -0,0 +1,88 @@ +# Runtime acceptance matrix — revision 4 + +All runtime-owned claims below remain **pending**. Acquisition gate evidence is tracked only in the companion plan; it does not replace the new runtime consumer observations. No production acceptance is executed in this delivery. + +Owners below are roles to assign: runtime integrator (R), core/dependency owner (C), adapter/host owner (H), desktop/distribution owner (D), independent reviewer (V). Each row names a criterion, evidence kind, environment and execution mode. More specific methods below are part of the row. + +| ID | Observable criterion | Evidence kind | Required environment / mode | Owner; slice | Status | +| --- | --- | --- | --- | --- | --- | +| A01 | Two different configurations of one Torch release coexist, reopen, bind and remove independently; failed/cancelled sibling installation preserves the other. | Integration + system | Representative managed host; automated | R; R1 | pending | +| A02 | A preferred installation never substitutes for the actual B-bound profile; include same-release variants and stale/racing selection. | Contract + integration + system | Controlled boundary plus real launch; automated | R; R1/R3 | pending | +| A03 | Missing packages, symbols, native requirements or device support reject selected adapter admission before model load; unrelated compatible adapters remain eligible. | Contract + integration | Controlled failures plus representative import environment; automated | H; R2/R3 | pending | +| A04 | No model supplements is valid; declared unresolved/invalid/inapplicable-context and failed queries do not become empty requirements. | Focused + contract | No material platform dependency; automated | C; R3 | pending | +| A05 | Compatible additive declarations compose; real conflicting pins/sources/hashes/Python/markers retain provenance; old complete-environment semantics are not silently weakened. | Contract + integration | Established package parser/resolver; automated | C; R3 | pending | +| A06 | Changed interpreter, distributions, adapter code or host bundle invalidates relevant evidence even with unchanged Torch version; hardware facts do not become package identity. | Integration + contract | Representative installation; automated | R/H; R1–R3 | pending | +| A07 | Loaded receipt correlates actual model/components, adapter revision/package, installation, bundle, process generation, operation, effective device and slot; mismatched ready responses cannot publish. | Contract + integration | Controlled host/native integration responses; automated | R/H; R3 | pending | +| A08 | Replacement/cancellation/disconnect/failed cleanup cannot publish or mutate a successor; required installation/adapter/device custody lasts through actual cleanup. | System + integration | Controlled real children; representative host plus simulated device work where scoped; automated | R/H; R3 | pending | +| A09 | Removal obeys live and persisted installation/adapter references; unrelated profiles continue; bind/start/remove races have one owning transition. | Integration + system | Representative filesystem/process host; automated | R; R1/R2 | pending | +| A10 | Supported legacy state imports idempotently or under verified one-shot conditions; interruptions reopen safely; authored data retained; old writers retired or isolated. | Contract + system | Representative retained-state replicas plus actual deployment facts; either | C/R; R1/R5 | pending | +| A11 | Binding-bearing private protocol accepts its actual supported host and rejects retained incompatible sidecars without fabricated defaults; independent adapter packages use compatible host API. | Contract + system | Installed host plus protocol fixtures; automated | H; R2/R3 | pending | +| A12 | Actual slot UI, serve, explicit launch/trial, listing, generation and unload use the same managed binding; direct/external control cannot mint owned readiness. | Integration + user-workflow | Representative desktop/backend; either | D/R; R1/R3 | pending | +| A13 | Standard Z-Image real load and requested-size generation work with no Nunchaku distribution or attempted import; cancellation and later reuse succeed. | System + user-workflow | Required-real supported model/assets/GPU and resolved environment; either | H; R4 | pending | +| A14 | Existing Nunchaku and FLUX.2 exact-tuple evidence is refreshed through the new contract; their own unsupported cases and cleanup remain correct. | System + user-workflow | Required-real qualified assets/hardware; either | H; R3/R5 | pending | +| A15 | Installed packaged Pumas outside checkout discovers and runs a separately installed adapter; actual UI bind/load/operation and resulting artifact are observed. | Release-artifact + user-workflow | Required-real packaged target and relevant model hardware; either | D/H; R4/R5 | pending | +| A16 | Changed DTOs regenerate from canonical source; real producer/preload/renderer preserve identity, typed failures and stale-invocation handling; no literal adapter menu list. | Contract + integration | Representative Rust/Node toolchain; automated | D; owning slice | pending | +| A17 | Library-only/no-default remains coherent without adapter imports/provisioning; changed native support is verified per supported OS without implying all-task support. | Integration + system | Required toolchains and required-real target OS for changed lifecycle; automated | R/D; R1/R5 | pending | +| A18 | Representative native build change, model supplement, independent adapter addition, host API change, revision update and UI change match the architecture locality matrix. | Architecture review | Source/installed change paths; manual | V; each material boundary/final | pending | +| A19 | Changed public/embedding/local-IPC/binding/independent clients have explicit version/cutover dispositions and matching evidence; public factory absence is not assumed from search. | Contract + system | Actual consumer/deployment facts; either | R/C; R1/R5 | pending | +| A20 | Native llama.cpp uses shared install/bind/observe/stop/remove contracts; R1 proves actual executable startup/stop, R3 proves model load and supported output. | System | Required-real native bundle/model and representative supported host; either | R; R1/R3 | pending | +| A21 | Freeze Pumas build, install/register an independently authored adapter package for an existing task, load and execute it without editing/rebuilding Pumas or adding core enum/UI cases. | System + release-artifact | Required-real built Pumas and separate package; either, with recorded workflow | H/D; R2 registration, R3 execution, R4 real adapter | pending | +| A22 | A standard model uses an existing adapter; extra requirements using the same behavior need no new implementation; genuinely custom behavior uses a registered implementation and does not fall back to generic loading. | Contract + system | Controlled models plus representative supported real path; automated | C/H; R3 | pending | +| A23 | Catalog discovery has no adapter imports or implicit installs; duplicate/conflicting/malformed records are visible; list pagination remains complete with a catalog larger than one page. | Contract + integration | Controlled package/catalog corpus; automated | R/H; R2 | pending | +| A24 | Exact registration is idempotent; conflicting same-ID/revision content fails; v1/v2 coexist as selected; disabling prevents new admissions while pinned sessions retain their version until explicit cleanup. | Contract + system | Controlled persistence/process revisions; automated | R/H; R2/R3 | pending | +| A25 | Unapproved model/plugin code cannot import/build/install through discovery or model load; approved immutable code takes the declared path; changed package bytes or wrong origin fails before use. | Security/contract + system | Controlled packages, sentinel side effects and real trust boundary; automated | H/R; R2/R3 | pending | +| A26 | A registered task implementation not shaped as a Transformers model/tokenizer executes its supported contract; unsupported task/host versions reject without creating gateway capability; text work does not block cancellation/control. | Contract + system | Controlled custom implementation plus actual host process; automated | H; R3 | pending | +| A27 | Native launch selects the correct executable role and library/build context, preserves argv boundaries, rejects invalid paths, and distinguishes owned/external/in-process operations; no Python placeholders or universal serve argument. | Contract + system | Representative native files/processes and supported OS; automated | R; R1 | pending | +| A28 | Compatible adapters share an installation; incompatible dependency closures use separate selected installations; adding/updating/removing one never hot-mutates shared live packages or loses references. | Integration + system | Real isolated environments and controlled conflicts; automated | R/H; R2/R3 | pending | +| A30 | Retained exact Torch/adapter artifact closure is acquired and installed from local inputs with network denied during installation. Reject wrong hash/build/wheel and direct-URL redownload; preserve original provenance and verify installed closure. | Contract + system | Real supported Python/package tool and approved wheel set; automated | R/H; R2 | pending | + +## Required procedures and proof boundaries + +### Native and Python management (A01/A02/A09/A20/A27) + +Use an isolated launcher root. Through actual management APIs create or explicitly import verified native and Python installations, inspect their IDs and family-specific evidence, bind profiles, launch/observe/stop and reopen. Exercise distinct variants of a release where available; fixtures do not substitute for unavailable real variant evidence. A native profile receipt must point to the exact launched executable and build. It must not be satisfied by an arbitrary stub declaring its family. R1 may qualify native lifecycle before R3's model/task integration, but final acceptance needs both. + +### Registered package after build (A21/A23/A24/A25/A26) + +Build Pumas once and record the artifact identity. Prepare a separate versioned adapter distribution and metadata; keep it out of Pumas source/build inputs. Register metadata without imports, inspect needed dependencies, authorize exact package provisioning into a compatible new installation, select and bind, load through the real host and call a supported operation via the managed/public path. Record the package/definition/runtime/session identities. Change the adapter revision and verify new admissions versus existing sessions. Duplicate a registration identity and inject a broken import. Test unsupported host/task, unapproved code and changed bytes with side-effect sentinels. No `include_str!`, hardcoded enum, UI literal or source edit is allowed to make the independent adapter reachable. + +R2 closes only actual registry/host discovery/probe obligations; R3 closes actual operation behavior. Standard Z-Image in R4 repeats the test with a real model, rather than treating a toy adapter as model qualification. + +### Supplements and execution (A04/A05/A22/A28) + +Use three cases: existing generic loader with no custom supplements; same loader with an actual additional requirement; a selected custom implementation with behavior not supplied by generic Transformers. Resolve with the selected package tool and preserve markers, sources, hashes and Python constraints. A historical complete-environment profile must retain its selected interpretation or be explicitly migrated. Name/dependency presence cannot invent runtime support. Test reuse of a compatible environment and explicit separation of incompatible ones. + +### Standard and existing image models (A13/A14) + +Record exact artifact/component refs, adapter package revision, resolved dependencies, interpreter/Torch/build, host identity, hardware and request. Standard Z-Image must run without Nunchaku installed or imported. Validate actual output bytes/dimensions, including 1280×720 when that declared adapter contract supports it; exercise cancellation and subsequent reuse. Nunchaku and FLUX.2 retain separately recorded qualified tuples. An import probe, successful load or simulated image result is not the generation/user-workflow claim. + +### Migration, compatibility and desktop (A10/A11/A12/A15/A19) + +Enumerate actual retained records and deployed consumers before migration. Test disposable replicas with interruption at authoritative transitions. Retire old writers or isolate new storage and managed resource ownership; do not rely on new-reader guards to constrain old code. Preserve authored selections or give an explicit binding-required result. Old sidecars/clients receive their supported compatibility outcome. Run installed desktop selection/registration/binding/loading outside the checkout; startup/jsdom/serialization alone does not prove the workflow. + +### Cost and scope + +Keep evidence at its owner and exercise negative cases at the boundary that can cause the forbidden effect. Use existing test frameworks and package tooling. No new general registry verifier, fingerprint service or property framework is justified merely to inflate evidence counts. Expensive hashing occurs at owned integrity transitions, not each generation. A frozen-build test supplies unique extension evidence; a mocked registry cannot replace it. + +## Supporting checks + +Use the repository's pinned toolchain and documented Rust/Node/Python commands in the [shared supporting commands](../../artifact-acquisition/reports/acceptance-matrix.md#planned-test-placement-and-supporting-commands). Select actual focused tests for each owning slice, then affected aggregate/static/feature/target checks. Confirm contract-export outputs and command from current source before running it. No Pumas command from that guide was executed in this review. + +Record candidate, command/procedure, inputs, environment, result, limitations and evidence location for every executed claim. Required unavailable hardware, native OS, GUI or deployed-consumer evidence remains unsatisfied. Passing lint/typechecks is not full behavior acceptance, and test success is not itself evidence of architectural simplicity. + + +## Acquisition claims and retained runtime obligations + +Shared-layer proof is owned by the [acquisition acceptance matrix](../../artifact-acquisition/reports/acceptance-matrix.md). Gate status is owned by its [dependency record](../../artifact-acquisition/reports/dependency-gates.md). The old claim IDs below are explicitly dispositioned rather than deleted without a trace. + +| Prior runtime claim | Acquisition-owned proof | Runtime obligation that remains | +| --- | --- | --- | +| A29 shared HF/native transfer | AC03, AQ-HTTP | R1 consumes the accepted handoff; no fake model record or replacement downloader. | +| A31 acquisition/handoff crash and cancellation | AC04–AC06 | A08/A09/A10 exercise the new installer/adapter/process consumer and retain input custody through its cleanup. | +| A32 source identity, credentials and trust | AC01/AC02/AC07/AC13 | A25 enforces executable/adapter authorization; source support cannot grant code execution. | +| A33 input leases and immutable cache | AC05/AC06/AC10 | A08/A09/A28 exercise runtime/adapter consumers without live mutation or premature release. | +| A34 bootstrap/offline/recovery | AC04/AC09/AC12/AC15 | A17 plus R1/R2 integration prove installed runtime operation remains possible without unnecessary remote acquisition. | + +A30 remains runtime-owned because registered adapter installation is a new consumer. AQ-PACKAGES proves the same contract first with the existing package installer, without waiting for that registry. Runtime gate evidence includes exact installed closure, private protocol and task semantics in addition to shared byte evidence. + +For R1/R2 admission record the required acquisition gate's exact source/contract/target evidence. Re-run only affected shared claims when this consumer changes their material meaning. Do not require full S3/Xet/CAS completion for an HTTP/package operation, and do not advertise S3 consumption before AQ-S3 is ready. diff --git a/docs/plans/runtime-installations-and-model-adapters/reports/adapter-contract.md b/docs/plans/runtime-installations-and-model-adapters/reports/adapter-contract.md new file mode 100644 index 00000000..0860260e --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/reports/adapter-contract.md @@ -0,0 +1,94 @@ +# Adapter definition, implementation and lifecycle + +This is the proposed semantic contract, not a finalized wire schema or implemented API. Public names/signatures must be integrated with the current contract owner and generated consumers in R2. The division of responsibilities is binding for this revision; spelling and private module layout are not. + +## What an adapter means + +An adapter supplies the missing behavior between a model artifact and an execution host. Dependencies describe the software/native/artifact conditions that behavior needs. Environment management satisfies those requirements. Neither a package list nor an import alone describes tokenization, component assembly, specialized model construction, generation invocation or result conversion. + +Three supported model cases: + +| Case | Representation | New executable adapter required? | +| --- | --- | --- | +| Standard supported artifact and task | Existing standard loader selected from package facts | No | +| Standard loader works after additional dependencies/options | Existing loader plus authored supplements and validated options | No | +| Specialized loader or execution behavior required | Registered adapter implementation with its package and requirements | Yes, when no existing adapter implements that behavior | + +A model can reference a preferred adapter revision without embedding the implementation or copying its dependencies into model metadata. An adapter can apply to a family of models. A single compatible installation can support multiple adapters. Incompatible dependency closures normally require separate immutable installations and processes; registration itself does not require one environment per adapter. + +## Small definition shape + +The owning definition must establish these meanings: + +- Identity: stable namespaced adapter ID, immutable revision/content identity, origin and supported adapter-host API range/version. +- Applicability: supported task kinds, runtime family/capabilities, package architecture/format/quantization evidence and explicit artifact/component roles. Repository names may supply provenance, not replace compatibility. +- Construction: either an existing loader ID plus validated options or a pinned distribution/entry point with integrity/provenance. Requirements for a distribution come from its package metadata; adapter-native/model-role constraints remain separately owned. +- Configuration: versioned accepted options and a supported validation representation. The generic UI can expose basic schema-defined options; specialized optional controls do not redefine compatibility. +- Operational behavior: task interfaces implemented, resource/device restrictions, probe entry point where needed, and the documented cleanup/cancellation behavior of the host interface. + +This is not a universal manifest for UI layout, installation catalog state, hardware inventory, profile defaults, mutable status or all Pumas policy. Those independent authorities remain referenced, not absorbed. + +## Python package mechanism + +Prefer conventional Python distribution metadata/entry points for executable registrations. PyPA defines installed entry-point groups/names/object references and leaves collision behavior to the consuming application. Python's `importlib.metadata` exposes metadata separately from `EntryPoint.load()`. See the prior-reviewed [PyPA entry-point specification](https://packaging.python.org/en/latest/specifications/entry-points/) and [Python metadata API](https://docs.python.org/3/library/importlib.metadata.html); the [common source inventory](../../artifact-acquisition/reports/standards-and-sources.md) records the review limits. + +Pumas must still own the semantics that packaging does not supply: authorization, schema/host version checks, adapter ID collision policy, immutable revision selection, environment scope, admission, lifetime and readiness. + +Inspect a package definition/metadata from its reviewed artifact without importing it. Retain the verified definition reference so requirements can be shown before installation. At installation, verify definition identity and implementation entry point against the exact artifact installed. At runtime, discover only within the admitted installation and only load its selected approved entry point. Do not resolve an entry point by name from an ambient or unrelated environment. + +Externally registered Python adapter code is not imported into the core-library or Electron/renderer processes. Python code is loaded in the selected owned Python host process. A native runtime with no Python adapter uses its registered runtime integration and supported native loading protocol instead of launching a Python sidecar for symmetry. + +## Proposed operations and outcomes + +Use intent-level operations whose names align with existing API conventions: + +| Operation | Required behavior | +| --- | --- | +| Register definition | Validate schema, IDs, immutable revision and references; record explicit authority; no model load or implicit package installation. Return registered/already-registered/conflict/invalid/unsupported/unavailable distinctly. | +| Inspect/list definitions | Metadata-only and paginated; include enabled/disabled/invalid/unavailable entries and origin. Availability is a separate installation/process observation. | +| Resolve execution | For exact model/task/profile and optional adapter preference, return the selected definition and missing requirements or compatible installation choices. No hidden installation, binding change or fallback to a different model. | +| Install requirements | Explicit approved resolution/publication; preserve per-requirement provenance and terminal outcomes. Do not mutate a busy environment. | +| Bind and launch | Persist chosen installation/profile relationship; launch or validate exact current process binding with retained use custody. | +| Load/execute/unload | Use admitted adapter revision, targets/options, generation and slot/session. Return task-typed results; preserve cleanup obligations even if caller disconnects. | +| Disable/unregister/update | Affect future admissions as documented; retain references for current sessions. New revision is separate content. Physical deletion waits for references/custody. | + +Do not expose a new public planning-session protocol just to coordinate internal calls. A service can carry an immutable admitted value within one operation. + +## Selection without accidental winners + +Explicit compatible adapter/revision selection wins only after validation. A declared model preference may narrow candidates. Without an explicit choice, select only when an owned deterministic policy establishes a unique suitable candidate; otherwise return ambiguity and the concrete choices. Do not run arbitrary candidate code to discover an adapter during ordinary library listing. Optional trusted probes are a separate execution operation in the selected host. + +One adapter reporting unsupported is not permission to swap models, enable remote code, relax pins or start a different provider. Extra protocol capabilities are additive where the contract permits; an unknown operation/host API is explicitly unsupported. + +## Lifecycle and invalidation + +Keep facts separate rather than one `available` boolean: + +1. Definition registered and enabled at a catalog revision. +2. Requirements resolved/compatible for a selected installation. +3. Implementation installed and host/API compatible. +4. Current process has passed relevant adapter preflight. +5. Exact selected model has loaded in a pinned session. +6. A specific task request completed and its result passed the destination contract. + +Model metadata/component changes invalidate the model decision. Adapter revision changes invalidate new admissions referencing its old selection policy but do not rewrite active sessions. Environment mutation/recreation invalidates its prior fingerprint evidence. Driver/device observations have their own scope; a driver change is not a new package identity. Bundle/API changes follow their own compatibility rules. + +A live generation pins its actual implementation and resources. Disable closes new admission. Ordinary unregister does not uninstall packages under a running process. Immediate revocation requires explicit stop authority and observed terminal cleanup; it cannot claim cleanup merely because a timer expired. Normal failures in a plugin are contained as adapter/session failures; native crashes affect their process and are not described as isolated merely because imports were lazy. Process separation for crash containment is an explicit policy, not a sandbox claim. + +## Security and customization + +Trust approval belongs to the operator or configured policy, not a centrally compiled list of permitted adapters. Supporting an open adapter population must not require a Pumas maintainer to approve each adapter ID or release new source. + +Custom Python code is supported when it is explicitly approved and bound to a pinned package or supported immutable local artifact. A directory/script editable in place must be snapshotted/verified under a declared development mechanism before managed production admission; live mutable source cannot inherit immutable-package guarantees. + +Existing loaders' blanket `trust_remote_code=True` must not become the new default. Code trust applies to model-provided files, adapter packages, compiled extensions and install/build steps. Dependency solving and metadata listing must not execute unapproved setup/import code. + +## Required extension test + +Build Pumas and its host once, record that candidate, then add a separately distributed compatible adapter definition/package that was not enumerated in source. Register it, inspect missing dependencies without import, explicitly install into an isolated managed environment, bind/start, load a suitable model and execute a supported operation through the public consumer. Verify no Pumas source, enum, gateway allowlist, sidecar dispatch list, frontend type union or core bundle file list was changed to add that adapter. + +Repeat with two definition revisions, conflicting packages requiring separate environments, duplicate IDs, missing dependencies, disabled/removed definitions, host API mismatch and one failing import. Standard Z-Image in R4 is a real independently shipped extension test, not only a synthetic fixture. + +## Acquisition prerequisite + +Adapter metadata and implementation artifacts use the [shared acquisition contract](../../../contracts/artifact-acquisition.md). Source resolvers and model-execution adapters are different extension points. The acquisition plan owns byte access, verification and input lifetime; runtime owns requirement/code approval, package installation, catalog revisions and execution. R2 requires AQ-PACKAGES and R1; S3 retrieval additionally requires AQ-S3. A missing source gate is not permission to add an adapter-specific downloader. diff --git a/docs/plans/runtime-installations-and-model-adapters/reports/architecture-review.md b/docs/plans/runtime-installations-and-model-adapters/reports/architecture-review.md new file mode 100644 index 00000000..1f0f3ab8 --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/reports/architecture-review.md @@ -0,0 +1,63 @@ +# Composed-design review: runtime management as an acquisition consumer + +**Applicable.** This replaces the revision-three composition review for the current runtime plan. It is a source-grounded design review, not external independent review or implementation acceptance. The companion [acquisition review](../../artifact-acquisition/reports/architecture-review.md) owns the shared lower-layer design. + +## Authority scope + +Core owns model facts, authored supplements, exact model/component references, profile state, stable binding values and existing custody/publication invariants. Acquisition owns selected bytes and their handoff. App-manager owns installed external runtime configuration, native/Python build selection, installation and its coherent lifecycle interface. Runtime integration owns supported protocol/operation mechanics. Model adapters own model-specific construction/execution and requirements; their catalog owns immutable registration/revision authority. Public transports and UI project those decisions. No one manifest/catalog absorbs all these authorities. + +## 1. Independent concerns and dimensions + +Acquisition obtains selected files for an authorized consumer through an owned transfer into approved storage. Installation consumes them using native/package semantics, validates and publishes an installation. A profile selects that installed unit; a process receipt records the actual launched unit/executable/generation. Registered adapter code constructs and executes the selected model through a supported host/task. These concerns have independent who/what/how/when/where/why dimensions: their selection authority, implementation, change cadence and terminal results differ even when one user action composes them. + +## 2. Necessary and accidental interleaving + +A model load necessarily binds model/component revisions, adapter definition/package, installation content, host code/protocol, profile/device settings, process generation and operation. Its required acquisition demand is retained until installation consumes the files, but network source location is not runtime identity. + +Remove tag-as-installation, global-preference-as-process-identity, dependency-profile-hash-as-installed-unit, adapter-as-provider, literal adapter menus, repository-ID loading branches, imported code during discovery, and one success boolean covering acquired/installed/loaded. Keep material lifecycle obligations visible instead of hiding them in a service name. + +## 3. Caller and composition knowledge + +A caller supplies model/task/profile or an explicit install/registration action, receives a typed result and controls its documented lifetime. It should not derive runtime paths, choose source URLs, implement pip/HTTP recovery, infer adapter IDs from repo names, inspect PID text as ownership or manually reconstruct publication safety. + +The existing optional composition builds app-manager from core/acquisition/contracts and private clients, not the reverse. No new daemon/global runtime/process registry is introduced. The generic library and metadata inspection remain usable without inference provisioning. + +## 4. Change locality + +| Representative change | Owners that change | Owners that should remain generic | +| --- | --- | --- | +| Add an ordinary model adapter | Adapter package/definition, registration and real qualification | Acquisition, provider enum, installer state, RPC/gateway model branches, hardcoded UI options | +| Add a runtime build | Runtime/native/package selection and probes | Acquisition lifecycle or model taxonomy | +| Add S3 retrieval | Acquisition/source integration | Adapter loader, runtime installed-ID meaning | +| Change model supplements | Core authored dependency interpretation and affected admission | Generic source transport or global modality package list | +| Change host/task protocol | Host/protocol authority and actual producer/consumers | Unrelated source reader or acquisition store | +| Change input retention/recovery | Acquisition owner and affected installer consumption tests | A second runtime download engine | +| Change device/profile settings | Profile/driver/process binding | Immutable remote content identity | + +## 5. Stable values and leaked implementation detail + +Use RuntimeInstallationId, exact model/artifact references, adapter revision and constructed launch/load receipts. Consume the shared file lease rather than reqwest/S3/pip transport types. Native installations have no fabricated Python fields or forced Pumas-sidecar handshake; actual launched executable custody supplies native identity. Keep approved-code decisions explicit without creating a universal install-script language. + +## 6. Independent evolution and failure + +A registered adapter may be missing dependencies without disappearing from the catalog. Acquisition can succeed while installation fails. A new package revision does not hot-replace a running module set. Changing a global default does not rewrite bound profiles. Unsupported tasks remain unsupported despite metadata claims. Process replacement/cleanup cannot act on a successor. Installed runtime operation does not require a new remote download when its binding is already satisfied. + +Independent package, host API, source schema, persisted profile and public gateway contracts receive their own compatibility decisions. Their common repository or release does not make them one versioned artifact. + +## 7. Deletion test + +| Mechanism | Deletion result | Decision | +| --- | --- | --- | +| Common installed-unit identity/binding | Same-release variants and actual launched identity become ambiguous | Retain | +| Deep managed execution service | Every UI/RPC/gateway caller rebuilds adapter/requirements/process admission | Retain within app-manager | +| Open registered adapter definitions and packages | Every new model implementation requires a Pumas build/list change | Required by user scope | +| Existing process/device custody | Cancellation and replacement safety disappear | Extend, never duplicate | +| Shared acquisition consumer interface | Runtime-specific download code returns | Consume prerequisite, not a new runtime module owner | +| Generic orchestration engine, marketplace, dylib ABI | Requested current outcomes still work | Exclude | +| Duplicate catalogs/compatibility aliases without a supported consumer | No accepted capability lost | Remove at the owning cutover | + +## 8. Cumulative complexity and plan composition + +Retain the necessary installation, adapter, dependency, execution and custody responsibilities but remove repeated interpretation. Acquisition Q1/Q2 supplies a proven handoff first using old consumers; runtime R1/R2 supplies richer new consumers afterward. The dependency is one-way and stage-specific. Neither plan owns the other's acceptance state. + +Review the actual implemented representative changes, not file count or test totals. Re-plan if ordinary adapters still force edits across transport, GUI and global provider code, or if a second mutable installation/download/process owner is introduced. Final code review includes executable trust, retained-state migration, lifecycle and registered-package extension against a frozen host. diff --git a/docs/plans/runtime-installations-and-model-adapters/reports/codebase-audit.md b/docs/plans/runtime-installations-and-model-adapters/reports/codebase-audit.md new file mode 100644 index 00000000..caed2f2e --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/reports/codebase-audit.md @@ -0,0 +1,30 @@ +# Runtime source audit — coordinated revision 4 + +Baseline Pumas `04e7f1568f00693c0ef26c77e0150e5e4dd112ea` is unchanged. This report carries forward the preceding pinned code review; this round's additional inspection concentrated on the shared acquisition boundary. No new whole-repository or executed runtime qualification is claimed. Source-derived scenarios remain unexecuted unless evidence is explicitly recorded in a later ledger. + +| Finding family | Evidence-backed concern | Primary pinned source | Current owner / acceptance | +| --- | --- | --- | --- | +| E01/E02/E11 | Tag-only installed identity and verify-A/reuse-B path; global/PID gate does not represent per-installation use | [rust/crates/pumas-app-manager/src/version_manager/mod.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-app-manager/src/version_manager/mod.rs) | R1, A01/A02/A09 | +| E03/E06/E07 | Generic image capability, repo-specific selection and actual raw-slot UI alternate path | [rust/crates/pumas-rpc/src/handlers/serving_torch.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-rpc/src/handlers/serving_torch.rs) | R3, A03/A07/A12 | +| E04/E05 | Dependency profile/context env_id, hash-based conflicts and modality pins need explicit additive interpretation | [rust/crates/pumas-core/src/model_library/dependencies.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/model_library/dependencies.rs) | R3, A04/A05/A22 | +| E08 | Future profile schema accepted into an old reader/writer; a new schema number cannot protect against that binary | [rust/crates/pumas-core/src/runtime_profiles/route_config.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/runtime_profiles/route_config.rs) | R1/R5, A10/A19 | +| E09/E10 | Fixed sidecar file inventory and partly handwritten tag-specific UI require producer/distribution cutover | [torch-server/probe_runtime.py](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/torch-server/probe_runtime.py) | R2/owning consumer slices, A06/A15/A16 | +| E13/E14 | App-manager already installs native and Python runtimes; generic process factory and version launcher have different assumptions | [rust/crates/pumas-app-manager/src/process/factory.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-app-manager/src/process/factory.rs) | R1, A20/A27 | +| E15/E16 | Provider integration enum is not an open model-adapter registry; current plugin loader overwrites duplicate IDs and reload semantics differ from cached managers | [rust/crates/pumas-core/src/plugins/loader.rs](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/rust/crates/pumas-core/src/plugins/loader.rs) | R2, A21/A23/A24 | +| E17/E18 | Provider advertises image-only Torch while text code assumes Transformers and directly executes synchronous generation | [torch-server/openai_api.py](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/torch-server/openai_api.py) | R3, A22/A26 | +| E19 | Affected loaders unconditionally allow remote model code | [torch-server/loaders/safetensors_loader.py](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/torch-server/loaders/safetensors_loader.py) | R2/R3, A25 | +| E23–E28 | Generic acquisition brief exists; HF/native/package paths need a shared byte handoff without erasing their consumer policies | [docs/breif/s3-model-fetch.md](https://github.com/MrScripty/Pumas-Library/blob/04e7f1568f00693c0ef26c77e0150e5e4dd112ea/docs/breif/s3-model-fetch.md) | Acquisition Q1/Q2/Q3 and runtime dependent-consumer tests | + +## Preserved strengths and boundaries + +Earlier reads established useful staged installation/publication recovery, explicit Torch in-place dependency-repair rejection, owned process generations/listener custody, ready-slot checks and compensating unloads, device custody through cancelled loads, and a valid empty-supplement result. Extend those guarantees; do not duplicate them to satisfy generic naming. + +The new acquisition plan now owns the transfer-family refactor and its existing-consumer proof. The runtime plan keeps installed identity, registered code, effective requirements, supported tasks, exact process/load receipts and actual GUI behavior. A working acquisition layer does not close those runtime claims. + +## Current additional review + +[The acquisition audit](../../artifact-acquisition/reports/codebase-audit.md) identifies the actual model-bound destination and persisted record shapes that prevent treating the existing HF client as a finished generic layer. The shared contract and one-way prerequisite gates replace the old runtime S1a sequencing. New adapter registration continues to be open within the supported host/task contract and independent of Pumas builds; the two plans must not regress that requirement into a bundled allowlist. + +## Limits and adoption + +Refresh current local/dirty state, public/binding callers, generated output population, native artifact layout, actual legacy stores and old-writer deployments before their cutover. Search alone does not establish that public factory methods are unused. No user installation is an authorized test fixture. The source audit is not a certificate that every source file currently complies with the standards. diff --git a/docs/plans/runtime-installations-and-model-adapters/reports/write-sets.md b/docs/plans/runtime-installations-and-model-adapters/reports/write-sets.md new file mode 100644 index 00000000..d0284d86 --- /dev/null +++ b/docs/plans/runtime-installations-and-model-adapters/reports/write-sets.md @@ -0,0 +1,69 @@ +# Proposed write sets and integration boundaries + +This is a source-grounded boundary inventory, not authorization to edit the user's repository. Exact new module names below are proposed. Before each slice, inspect current local/branch state, direct callers and generated outputs. A same-owner file discovered within the stated semantic boundary is a documented amendment; a new public/trust/ownership boundary triggers a decision review. + +## Prerequisite boundary and shared-file custody + +Acquisition's [write-set report](../../artifact-acquisition/reports/write-sets.md) owns Q1/Q2 HTTP/HF/native/package bridges and shared transfer/state changes. Runtime source integration starts only after the relevant [gate](../../artifact-acquisition/reports/dependency-gates.md) is ready. It does not introduce `acquisition/` source changes without an acquisition-owned contract amendment. The integrator serializes overlapping app-manager installer, Torch package integration, public contract/export and UI writes. + +## R1 — installation identity and bound executable launch + +Existing primary files/families: +- `rust/crates/pumas-app-manager/src/lib.rs` +- `rust/crates/pumas-app-manager/src/version_manager/{mod.rs,state.rs,installer.rs,launcher.rs}` +- `rust/crates/pumas-app-manager/src/version_manager/installer/torch.rs` and its currently linked publication/recovery tests +- `rust/crates/pumas-app-manager/src/process/{mod.rs,factory.rs,traits.rs}` — reconcile public legacy surfaces, not replace custody blindly +- `rust/crates/pumas-core/src/models/runtime_profile.rs` +- `rust/crates/pumas-core/src/runtime_profiles.rs` and `runtime_profiles/{route_config.rs,launch_specs.rs,launch_strategy.rs,process_owner.rs}` +- `rust/crates/pumas-core/src/api/{runtime_profiles.rs,state_runtime_profiles.rs}` +- Directly affected installed-version metadata/path helpers, located through their current callers before migration +- `rust/crates/pumas-rpc/src/handlers/runtime_profiles.rs`, affected `handlers/versions/` operations and launch/stop handlers + +Proposed additions, only when their responsibility cannot be kept coherently in the existing owner: app-manager `runtime_installations/` for common identity/lifecycle facade and native/Python detail; a neutral core installed-runtime binding type where consumed by public/persisted profiles. No second mutable catalog and no new crate by default. + +Integration-owned adjacent writes: RPC command/schema export, generated consumers, profile/install selection frontend, Electron preload/registry, focused fixtures and active-plan/ADR handoffs. The full output list must come from the exporter. Update actual controls in R1, not only the backend. Native launch prep moves/delegates with its core/embedding/local-IPC callers in the same accepted boundary. + +R1 proves real Torch and llama.cpp lifecycle; AQ-HTTP supplies the acquisition boundary. Affected Ollama shared paths must either adopt the same neutral contract or have an explicit supported facade/cutover disposition; no broken third consumer. ONNX/external negative cases remain in the core profile tests. + +## R2 — metadata catalog and independent adapter packages (requires AQ-PACKAGES) + +Include `torch-server/resolve_runtime.py`, retained-resolution consumers in app-manager `version_manager/{torch_preview.rs,installer/torch.rs}`, local wheel staging/input projection, relevant package integrity tests and acquisition progress consumers. Q2 already provides the existing package-file handoff. R2 adds the independently registered adapter consumer rather than redesigning it. Package solving remains with tooling. Preserve original source provenance while making the installation leg consume exact verified local files. Audit managed-Python provider network/bootstrap hooks separately; do not replace unsupported private APIs or assert that all provider traffic is migrated. + +Primary owners: optional app-manager catalog/registration/inspection service; Python adapter host discovery/API modules; installation integration for selected implementation distributions. Proposed `model_adapters/` modules are semantic ownership, not a required directory count. + +Direct existing files to reconcile: `torch-server/{model_manager.py,probe_runtime.py,resolve_runtime.py,control_api.py,serve.py}`, existing runtime embedding/integrity inputs, `rust/crates/pumas-app-manager/src/torch_client.rs`, `core/plugins/{schema.rs,loader.rs}` only for genuinely shared/replaced authority, and affected RPC/GUI projection. + +A separate adapter SDK/host contract can remain a small Python package/module; package its declaration/API independently only if the actual external package consumer requires that distribution. Establish name, API version, metadata/entry-point ownership and test boundary before publishing. Do not invent a native Rust dylib ABI, per-model provider variant or universal executable install language. + +Add controlled adapter fixtures and a real independent package used outside Pumas build inputs. The historical stdlib probe is not production fixture/SDK source to copy wholesale. + +## R3 — shared model admission and supported task behavior + +Primary existing population: +- `rust/crates/pumas-core/src/model_library/{dependencies.rs,dependency_pins.rs}` and direct core API/DTO/projection consumers selected by the semantic change +- Existing `models/artifact_load_target.rs`, package-fact selection and approved load-target APIs (extend only when an actual missing contract is demonstrated) +- `rust/crates/pumas-core/src/providers/mod.rs` for host/task capability meaning, not model-ID enumeration +- `rust/crates/pumas-rpc/src/handlers/{serving.rs,serving_torch.rs,torch.rs,runtime_profiles.rs,openai_gateway.rs,openai_gateway_images.rs}` and affected native serving handlers discovered from provider dispatch +- Core serving publication/observation and profile ownership APIs needed to carry exact binding values +- `torch-server/{model_manager.py,control_api.py,openai_api.py,image_api.py,diffusion.py,flux2.py}` and `loaders/{__init__.py,safetensors_loader.py,dllm_loader.py,sherry_loader.py}` +- Actual `frontend/src/components/app-panels/sections/TorchModelSlotsSection.tsx` and related active-slot/profile views + +Proposed managed-execution module belongs in app-manager; transport consumes it. Move current adapter assumptions into the adapter host implementations and eliminate duplicated policy from RPC/gateway/probe/UI. Keep approved custom-code authorization on the reachable managed family. Ordinary registered adapters must not require new changes to those shared handlers after this slice. + +## R4 — standard Z-Image extension + +Primary writes are the separately packaged adapter definition/implementation, its declared dependencies, model/task fixtures and real acceptance procedure. The frozen Pumas build/host must already be sufficient. Any necessary shared-host change is evidence of a missing host contract and must be resolved in the owning earlier boundary, not hidden as an ordinary plugin addition. + +Do not embed the external adapter's source into Rust to make the test pass. Tests may reference the independent package as an input; Pumas product source must not require it to build. Preserve the original qualified image implementations and exact output contracts. + +## R5 — release/deployment and terminal source cleanup + +Only affected public/binding consumers, generated output closure, packaging/embedding inventory, native QA workflows/procedures and current owner docs. Public factory/dead-path removal requires the actual supported consumer/version disposition. No release/tag publication, user-environment deletion, repository history rewrite or external repository write is authorized by this document. + +## Shared artifacts and concurrent work + +One integrator owns core types, wire/host API definitions, manifests, lockfiles, generated outputs, shared fixtures, active plan and migration decisions. Workers receive a fixed contract and non-overlapping primary writes. Environment/native internals, adapter-host internals and UI may be delegated only after their shared contracts are stable; list permitted adjacent changes and required evidence. Reviewers are read-only. Integrate serially and recheck the current candidate; do not let workers independently redesign a shared DTO or update generated files by hand. + +## Before breaking or destructive changes + +Read actual call sites of public app-manager/core launch APIs, current metadata helpers/exporters, local dirty changes, supported retained legacy records and independently deployed clients. Inspect exact native archive role/library layout. Missing consumer/state facts block their corresponding cutover, not unrelated reversible implementation. A global source search showing no local caller does not authorize breaking a public API by itself. From ef74e84a85d7eefd082a87559935e8ae7eb23815 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 14:12:54 -0700 Subject: [PATCH 02/20] docs(acquisition): admit Q1 and record starting evidence --- docs/README.md | 4 +- docs/contracts/artifact-acquisition.md | 2 +- .../artifact-acquisition/execution-ledger.md | 9 ++++ docs/plans/artifact-acquisition/issues.md | 4 +- docs/plans/artifact-acquisition/plan.md | 19 ++++--- .../reports/coding-standards-mcp-usability.md | 23 ++++++++ .../reports/dependency-gates.md | 2 +- .../reports/q1-starting-state.md | 53 +++++++++++++++++++ .../execution-ledger.md | 4 ++ .../plan.md | 10 ++-- 10 files changed, 114 insertions(+), 16 deletions(-) create mode 100644 docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md create mode 100644 docs/plans/artifact-acquisition/reports/q1-starting-state.md diff --git a/docs/README.md b/docs/README.md index c0b888a3..cba2d3ac 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,8 +30,8 @@ and remediation inputs, not current operating instructions. ## Active and Planned Work -- [Artifact acquisition](plans/artifact-acquisition/plan.md) — proposed plan for reusable HTTP, package, and S3 acquisition; implementation gates are not ready -- [Runtime installations and model adapters](plans/runtime-installations-and-model-adapters/plan.md) — proposed plan for installation identity, bound profiles, and model-specific adapters; implementation is gated by acquisition evidence +- [Artifact acquisition](plans/artifact-acquisition/plan.md) — Q1 shared HTTP acquisition is active; AQ-HTTP is not ready +- [Runtime installations and model adapters](plans/runtime-installations-and-model-adapters/plan.md) — runtime implementation remains gated by acquisition evidence - [2026-09-03 current-standards remediation program](plans/current-standards-remediation-2026-09-03/plan.md) - [Local intent API and transport-independent domain language](plans/local-intent-api-2026-09-12/plan.md) — complete within the recorded local scope; native resolution, acquisition, durable declarations and existing IPC/RPC projections accepted; nodes/fleets/new networking deferred until after the next release diff --git a/docs/contracts/artifact-acquisition.md b/docs/contracts/artifact-acquisition.md index f575b1af..7027ed98 100644 --- a/docs/contracts/artifact-acquisition.md +++ b/docs/contracts/artifact-acquisition.md @@ -1,6 +1,6 @@ # Artifact acquisition contract -**Status:** Proposed for the paired acquisition/runtime plans; not an implemented public API. +**Status:** Q1 implementation contract; its public API and acceptance remain pending. **Canonical owner:** Pumas acquisition integration. **Implementation authority:** [Acquisition plan](../plans/artifact-acquisition/plan.md). **Consumer:** [Runtime installation and model-adapter plan](../plans/runtime-installations-and-model-adapters/plan.md), plus the existing model-library and native/package integrations. diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index d4510567..664416fe 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -11,3 +11,12 @@ Created this acquisition plan, a proposed shared contract, gate record, source a **Evidence:** source/docs inspection and mechanical delivery checks only. All AC production claims and AQ gates remain pending/not ready. No Rust/Python production suite, source-service integration, network-denied package installation, GUI, migration, GPU workload or native release was executed. No independent reviewer was run; independent review is a later explicit acceptance requirement. **Next slice:** Q1 after current checkout/consumer/retained-state preparation and exact source-work admission. Runtime R1 remains gated by AQ-HTTP. + +## 2026-09-29 — Q1 started from accepted current main + +- Resolved `a8359512` to `a8359512a580aa25fb2f9c9e4cd7e0dd64fd970d`. It was not an ancestor of current accepted `main` `e37bbf4b964a0e2aadf25f80ab71edd8fa6b3eb3`; the common ancestor is `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`. Preserved the original planning commit through the plan-only integration merge `f4dd7ff9` on `work/acquisition-q1-http`. No accepted source was reset or replaced. +- Current GitHub inventory: no open PRs; PRs #4, #5 and #6 are the latest merged runtime/Torch changes. On the exact base SHA, Build run `36624219733` and Scorecard run `36624219480` both completed successfully. These are base-only CI results. See the [starting-state report](reports/q1-starting-state.md) for links and source evidence. +- Source preparation traced the normal model workflow, durable download owner, native runtime installer, and existing generic download API. The acquisition module, actual shared transfer owner and both production cutovers do not exist yet. The source inventory and independent architecture review are preliminary only; no Q1 gate claim is satisfied. +- Existing local-TCP restart/import tests and historical native installation evidence retain their original scope. No real HF service, current llama.cpp archive, desktop workflow, live migration, or Q1 production test has been executed in this slice. +- The store reader currently supports schema 5 and explicit schema 4 upgrade; deployed record population and old-writer retirement are unknown. No live retained state was opened or modified. Pending/unresolved cleanup replay remains refused under its existing recovery owner. +- The sole implementation slice remains Q1. Q2, Q3 and runtime R1 remain dependency-gated; no acquisition gate is ready. diff --git a/docs/plans/artifact-acquisition/issues.md b/docs/plans/artifact-acquisition/issues.md index 9c7469a7..0d3ad0bd 100644 --- a/docs/plans/artifact-acquisition/issues.md +++ b/docs/plans/artifact-acquisition/issues.md @@ -1,6 +1,6 @@ # Acquisition issues and dispositions -All repairs are planned, not implemented. Evidence references [the source audit](reports/codebase-audit.md), [the canonical contract](../../contracts/artifact-acquisition.md), and its linked primary sources. Severity is consequence within this scope, not an asserted exploited vulnerability. +Q1 is active; production repairs and acceptance claims remain pending until evidence is recorded. Evidence references [the source audit](reports/codebase-audit.md), [the canonical contract](../../contracts/artifact-acquisition.md), and its linked primary sources. Severity is consequence within this scope, not an asserted exploited vulnerability. | ID | Severity / issue | Owner / disposition | Deciding evidence and revisit condition | | --- | --- | --- | --- | @@ -20,3 +20,5 @@ All repairs are planned, not implemented. Evidence references [the source audit] | AQ-E01 | Pending: required-real AWS/non-AWS/MinIO/native/desktop evidence | Assigned source/distribution integrator | AC14/AC17; missing environment blocks those claims, not fictional acceptance | Pending cleanup replay and unrelated whole-runtime remediation are owned by the existing Rust/library plan. Preserve current refusal while migrating the selected transfer family. Do not mark those unrelated work items accepted from this plan's link/schema checks. + +Q1 source preparation confirmed a public `DownloadManager` with no in-repository production caller, but did not establish its external compatibility population; removal or a compatibility disposition remains open. The supported download-store reader currently accepts schema 5 and upgrades schema 4, while deployment inventory and older-writer isolation are unavailable. Q1 must preserve both facts and keep live-root mutation blocked until that deployment evidence exists. The native llama.cpp cache currently uses size/filename admission and direct-final-directory extraction; these are admitted Q1 repair findings under AQ-I04/AQ-I07. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index aada92ed..8917578e 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -1,13 +1,14 @@ # Plan: source-neutral artifact acquisition -**Plan status:** `Planned` — implementation direction selected for this coordinated planning package; no production implementation is authorized or claimed by this delivery. +**Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** contract and source-grounded implementation preparation. -**Exactly one next slice:** **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** +**Current phase:** Q1 source, retained-state, and authority reconciliation; production implementation and required real acceptance are pending. +**Exactly one next slice:** Continue **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. -**Operation for a later implementation session:** `start` this exact plan; refresh repository state and the prerequisite dispositions below first. -**Baselines:** Pumas `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`; Coding-Standards `39d55dc330d44ecf940364ceada9d2527f7c7ea0`; delivered runtime plan revision 3 is the input, revision 4 is the companion output. +**Operation:** `start` this exact plan on `work/acquisition-q1-http`. +**Planning baseline:** Pumas `a8359512a580aa25fb2f9c9e4cd7e0dd64fd970d`, preserved and integrated by `f4dd7ff9`. +**Implementation base:** accepted Pumas `main` `e37bbf4b964a0e2aadf25f80ab71edd8fa6b3eb3`; Coding-Standards `39d55dc330d44ecf940364ceada9d2527f7c7ea0`; delivered runtime plan revision 3 is the input, revision 4 is the companion output. ## Objective and scope @@ -107,7 +108,7 @@ Each row is one coherent semantic unit including producer and actual consumers. | Milestone | Goal | Dependencies | Gate / evidence | State | | --- | --- | --- | --- | --- | -| **Q1** | Shared HTTP lifecycle, neutral persistence/handoff, and real HF/native consumer cutover; existing UI outcomes preserved. | Source/retained-state preparation and active recovery-owner handoff. No runtime milestone. | `AQ-HTTP`: AC01–AC10, AC15, AC16, AC18; real HF model import and native archive extraction plus corresponding UI/contract/cancellation/reopen evidence. | Planned | +| **Q1** | Shared HTTP lifecycle, neutral persistence/handoff, and real HF/native consumer cutover; existing UI outcomes preserved. | Source/retained-state preparation and active recovery-owner handoff. No runtime milestone. | `AQ-HTTP`: AC01–AC10, AC15, AC16, AC18; real HF model import and native archive extraction plus corresponding UI/contract/cancellation/reopen evidence. | In progress; gate not ready | | **Q2** | Exact wheel-file-set acquisition and local-only consumption in the existing Torch installer. | AQ-HTTP. | `AQ-PACKAGES`: AC11, AC12 and Q1 regression evidence affected by this composition; network-denied installation and actual package identity. | Planned | | **Q3** | S3-compatible acquisition using the same lifecycle, direct explicit source workflow, tested credentials/version semantics, and a real model import through the existing model-facing operations. | AQ-HTTP. Q2 is the default next serial integration; Q3 can be delegated after shared files stabilize. | `AQ-S3`: AC13, AC14 and source-neutrality regressions; native AWS S3, one non-AWS compatible service, local MinIO, and S3-to-model-library evidence. | Planned | | **Q4** | Complete affected public/installed/native qualification, migration documentation and removal of superseded authority. | Q1–Q3 implemented. | `AQ-COMPLETE`: all AC01–AC18 satisfied and all four milestones Accepted; packaged/independent-consumer/native-platform evidence in AC17. | Planned | @@ -122,6 +123,10 @@ First refresh the selected checkout and current standards, preserve unrelated wo Bounded investigations belong inside their milestones: verify the current destination grant can support a neutral workspace, identify a real native artifact's accepted integrity evidence, and prove restart handoff with existing store readers. Stop each investigation when the chosen interface can be implemented safely; update the issue record only if its result changes ownership, migration or evidence. +### Q1 current-checkout reconciliation — 2026-09-29 + +The exact refs, PR/CI state, pre-existing worktrees and untracked work, source-owner trace, supported store formats, review findings and evidence limits are recorded in the [Q1 starting-state report](reports/q1-starting-state.md). That report is read-only source/repository evidence and does not satisfy acceptance claims. The actual deployed retained-state population and older-writer retirement/isolation remain unknown; no live root was read or changed. Q1 may proceed with source work and disposable fixtures, while live-root mutation remains blocked. The original `a8359512…` plan commit is preserved and integrated into the task branch; runtime R1 remains gated by AQ-HTTP. + ## Acceptance and verification [Acceptance matrix](reports/acceptance-matrix.md) owns criteria, evidence kind, environment, execution mode and named procedures. Every production claim remains pending. Static checks and disposable fixtures support, but do not replace, real transfer, package, desktop, native and deployment evidence. Source review and this package's link checks do not certify code compliance. @@ -140,6 +145,6 @@ Authorized planning does not authorize deleting user files, publishing releases, ## Linked records and terminal documentation -[Ledger](execution-ledger.md) · [Issues](issues.md) · [Audit](reports/codebase-audit.md) · [Architecture](reports/architecture-review.md) · [Acceptance](reports/acceptance-matrix.md) · [Write sets](reports/write-sets.md) · [Dependency gates](reports/dependency-gates.md) · [Standards/source review](reports/standards-and-sources.md) · [Companion runtime plan](../runtime-installations-and-model-adapters/plan.md). +[Ledger](execution-ledger.md) · [Issues](issues.md) · [Audit](reports/codebase-audit.md) · [Architecture](reports/architecture-review.md) · [Acceptance](reports/acceptance-matrix.md) · [Write sets](reports/write-sets.md) · [Dependency gates](reports/dependency-gates.md) · [Standards/source review](reports/standards-and-sources.md) · [Coding-Standards MCP usability](reports/coding-standards-mcp-usability.md) · [Companion runtime plan](../runtime-installations-and-model-adapters/plan.md). On adoption, link this plan from the source brief and add the bounded ownership disposition to the active Rust/library recovery and upstream-runtime plans. Do not create another master execution plan. The shared contract remains marked proposed until implemented; update it in the same slices. At acceptance, move durable decisions to current owner documentation/ADR as required, and follow Pumas's documented terminal-plan lifecycle rather than retain duplicate active instructions. Git and the delivered older packages preserve history. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md new file mode 100644 index 00000000..1657dc06 --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -0,0 +1,23 @@ +# Coding-Standards MCP usability feedback + +This report records agent experience using the Coding-Standards MCP during Acquisition Q1 preparation. It is separate from acquisition acceptance: MCP routing and usability do not establish product-code compliance, and product tests do not establish that the MCP workflow was usable. + +## Primary integrator + +- **Useful calls:** `routing_facts` exposed the valid fact categories and values; an initial route with minimal facts made the router's information model clear. Compact routes with bounded content returned an explicit snapshot, selected standards, and unresolved-fact count. `read_many` let the integrator inspect the commit, planning, implementation, verification, architecture, contract, and documentation obligations. Rerouting after composition seams were clarified confirmed applicability for the actual Q1 boundary. +- **Confusing or redundant steps:** Tool discovery descriptions were verbose. The first broad route requested 32 policy contents and produced output large enough to truncate. A broad `read_many` similarly returned tens of thousands of tokens. A later route displayed policies already read, so applicability and policy reading were not easy to distinguish. +- **Missing context:** The MCP did not know the repository's current accepted plan status, retained-state population, public consumers, active recovery owner, or actual CI/source evidence. Those facts had to be gathered from the repository, GitHub, and the source tree. The route cannot certify that the source or tests satisfy the selected rules. +- **Smallest sufficient workflow:** Read routing facts once; route the concrete slice without policy contents; read Core plus only the selected standards and workflows relevant to its actual changed boundary in small pages; inspect source and run the required evidence; reroute only when ownership or scope changes; then use the final review operation against the implemented candidate. +- **Recommendations:** Keep route output compact by default and separate applicability reasons from policy bodies. Make content pages token-budgeted, expose previously read policy IDs, and put unresolved facts first. Clarify in the result that a complete route means applicability was resolved, not that implementation compliance was verified. A same-snapshot “changed design” review summary could call out new or removed obligations without replaying all policy text. + +## Independent architecture reviewer + +- **Useful calls:** `routing_facts`, a route including content, and the pinned snapshot made it possible to identify and read Architecture, Rust Async, Persistence, Contract Evolution, Core, and Planning. The final route selected 24 standards with zero unresolved categories; this was a complete applicability result, not a compliance certificate. +- **Confusing or redundant steps:** Initial tool discovery was verbose. The broad route returned roughly 68,000 tokens and truncated, repeated relationship rationales, and later policy reads repeated content already shown inline. +- **Missing context:** The MCP could not reveal the repository's persisted-state population, source/consumer ownership, accepted plan prerequisites, or shutdown composition. The reviewer had to inspect source and plan documents outside the MCP to assess those obligations. +- **Smallest sufficient workflow:** Route concrete facts once, omit inline policy content, read only the relevant policies in small pages, inspect source/evidence independently, and reroute when the design materially changes. +- **Recommendations:** Add an output-size budget, avoid repeating policy prose and relationship explanations, show which policies have already been read, and provide a same-facts design-change summary. Keep route completeness distinct from source/evidence compliance. + +## Participation + +The primary integrator and the independent architecture reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. diff --git a/docs/plans/artifact-acquisition/reports/dependency-gates.md b/docs/plans/artifact-acquisition/reports/dependency-gates.md index 5f5c4135..45f71b9a 100644 --- a/docs/plans/artifact-acquisition/reports/dependency-gates.md +++ b/docs/plans/artifact-acquisition/reports/dependency-gates.md @@ -5,7 +5,7 @@ | Gate | Provider milestone / claims | Required consumer observation | Unblocks | Current status | | --- | --- | --- | --- | --- | -| AQ-HTTP | Q1 / AC01–AC10, AC15, AC16, AC18 | Existing HF model acquisition reaches awaited model import; existing llama.cpp installer consumes the same neutral verified-file handoff and reaches its own validated extraction/publication. Cancellation/restart/UI evidence included. | Runtime R1 on the qualified targets. | Not ready | +| AQ-HTTP | Q1 / AC01–AC10, AC15, AC16, AC18 | Existing HF model acquisition reaches awaited model import; existing llama.cpp installer consumes the same neutral verified-file handoff and reaches its own validated extraction/publication. Cancellation/restart/UI evidence included. | Runtime R1 on the qualified targets. | Q1 in progress; not ready | | AQ-PACKAGES | Q2 / AC11, AC12 plus affected AQ-HTTP regressions | Existing Torch package integration consumes an approved exact wheel set locally with network denied during installation; no new adapter registry needed. | Runtime R2 after R1. | Not ready | | AQ-S3 | Q3 / AC13, AC14 plus source-independent regressions | The same manifest/transfer/handoff contract works with version/credential/range conditions on AWS S3, one non-AWS compatible service and local MinIO; an S3-sourced model completes import and is observed through GetModel or EnsureModel. | Runtime or model S3 acquisition on qualified endpoint/target combinations. | Not ready | | AQ-COMPLETE | Q4 / AC01–AC18 and all milestones accepted | Actual installed/public/native consumers and source-migration/deletion dispositions satisfy the complete acquisition scope. | Acquisition plan acceptance; not a hidden prerequisite for HTTP-only runtime work. | Not ready | diff --git a/docs/plans/artifact-acquisition/reports/q1-starting-state.md b/docs/plans/artifact-acquisition/reports/q1-starting-state.md new file mode 100644 index 00000000..2be57f41 --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/q1-starting-state.md @@ -0,0 +1,53 @@ +# Q1 starting state and source inventory + +**Captured:** 2026-09-29. **Evidence type:** read-only repository, GitHub PR/CI, and source review. **Acceptance effect:** none; all Q1 claims remain pending. + +## Repository and history + +- Repository: `/media/jeremy/OrangeCream/Linux Software/repos/owned/ai-systems/Pumas-Library`. +- Starting `HEAD`, `origin/main`, and local `main`: `e37bbf4b964a0e2aadf25f80ab71edd8fa6b3eb3`. `HEAD` was equal to `origin/main` before task work. +- Requested plan commit: `a8359512a580aa25fb2f9c9e4cd7e0dd64fd970d`. It is a child of `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`, which is the merge base with current `main`; it is not an ancestor of current `main`. The commit contains the coordinated plans and shared contract. It is preserved as an ancestor of the Q1 task branch through merge commit `f4dd7ff9`. +- Initial local branches: `main`; the only advertised remote heads were `main` and `ci/windows-native`. Existing archive refs and unrelated detached worktrees were left untouched. The task created no extra worktree. +- Initial worktree state contained the unrelated untracked [`docs/breif/future.md`](../../../breif/future.md). It remains unmodified and unstaged. `pumas-coordinated-plans.zip` was not present in this checkout. No stash, reset, clean, or deletion was performed. +- The primary branch is `work/acquisition-q1-http`, created from the exact accepted base above. The first commit `f4dd7ff9` integrates only the original plan/contract documentation and the docs index; it preserves the original `a8359512` commit. + +## Current PR and CI state + +The read-only GitHub query found no open pull requests. The latest relevant merged PRs are: + +- [PR #6 — Torch installation flow, package validation, and progress](https://github.com/MrScripty/Pumas-Library/pull/6), merged 2026-09-29. +- [PR #5 — Managed Torch runtime foundation and cross-platform installation](https://github.com/MrScripty/Pumas-Library/pull/5), merged 2026-09-29. +- [PR #4 — Torch image adapter integration and dimension fixes](https://github.com/MrScripty/Pumas-Library/pull/4), merged 2026-09-29. + +On starting SHA `e37bbf4b964a0e2aadf25f80ab71edd8fa6b3eb3`, [Build run 36624219733](https://github.com/MrScripty/Pumas-Library/actions/runs/36624219733) and [Scorecard run 36624219480](https://github.com/MrScripty/Pumas-Library/actions/runs/36624219480) both completed successfully. They qualify only the starting base, not this Q1 candidate. + +## Current acquisition owners and paths + +| Concern | Current source owner | Q1 boundary | +| --- | --- | --- | +| HF repository/revision/file selection and model destination | `rust/crates/pumas-core/src/api/hf.rs`; `rust/crates/pumas-core/src/model_library/hf/{metadata,types,download}.rs` | Preserve model selection and destination authorization at the model consumer. | +| HF worker generation, transfer state and finalization | `rust/crates/pumas-core/src/model_library/hf/lifecycle.rs` and `hf/download.rs`; `import_completed_download` awaits the importer | Move shared transfer custody coherently; keep model import/publication distinct. | +| Durable current HF attempt state | `rust/crates/pumas-core/src/model_library/download_store.rs`; recovery/root grants in `model_library/download_recovery.rs` | This is HF-specific today; it cannot silently become the neutral runtime store. | +| Normal intent acquisition | `rust/crates/pumas-core/src/intent/acquisition.rs` through prepared/owned HF operations | Keep this normal model-facing path connected to the canonical acquisition owner. | +| llama.cpp release/build selection and installation | `rust/crates/pumas-app-manager/src/version_manager/installer.rs` | Keep selection, extraction, install verification and installed metadata with app-manager; replace its independent byte-transfer lifecycle. | +| Existing generic HTTP helper | `rust/crates/pumas-core/src/network/download.rs`, exported from `network` | Public API; no internal production caller was found. External compatibility is unknown, so its removal or delegation is unresolved. | + +The HF transfer loop builds revision-scoped URLs, sends Range requests, writes guarded partial files, observes cancellation/pause, validates selected content, promotes bytes and then waits for importer settlement. A focused existing local TCP fixture, `pinned_aux_and_payload_retry_resume_after_reopen_and_import_exact_revision`, exercises truncated transfer, failed/successful range resume, reopen, byte identity and import. It is not a live Hugging Face service run. + +The llama.cpp path calls `download_archive` from `installer.rs`, keeps an installer-owned retry/progress/cancel loop, and caches under `launcher-data/cache/downloads`. `is_cached_download_valid` currently accepts matching file size; `GitHubAsset` has name, size, URL and content type but no digest. `do_llama_cpp_install` removes the existing tag directory and extracts directly into the final directory before `finalize_llama_cpp_installation` writes metadata. Existing tests cover platform asset selection and extraction/wrapper behavior, not a full live archive acquisition/reopen/publication sequence. Historical real llama.cpp and model-serving observations remain scoped to their recorded older candidates. + +## Retained-state and recovery facts + +Current code reads download-store schema 5 and has a specific schema 4 to 5 migration. The durable document contains download snapshots, recovery revocations, lifecycle cleanup quarantines, admission attempts, queue admissions, and released queue-admission proofs. Snapshots include model repository, filenames, model destination, status, request, revision and optional Hugging Face evidence; destination authority is separately reconstructed from the model root. + +The current standards-remediation plan accepts selected incremental recovery and local-intent operations within their recorded Linux evidence. It explicitly leaves broader C3 recovery and hard-process-crash claims open and refuses unresolved admission/quarantine cleanup replay. The active local-intent plan records M1–M4 complete within its own bounded scope; those results do not accept the proposed shared acquisition owner. + +Source-supported formats are visible, but no actual deployed retained root or older running writer inventory was read. Therefore the deployed record population, old-reader/writer retirement, and safe rollout/rollback facts are **unavailable**. Do not modify live roots or claim migration safety until an owner records those facts. Disposable v4/v5 fixtures and current-reader reopen tests are independent and may proceed. + +## Plan and authority dispositions + +- Acquisition Q1 is the only active implementation slice. AQ-HTTP, AQ-PACKAGES, AQ-S3 and AQ-COMPLETE remain not ready. +- The Torch package resolver/install path remains with the current Torch plans until acquisition Q2; Q1 does not change package resolution or attempt a network-denied install. +- Runtime R1 remains unstarted until AQ-HTTP passes at its target scope and the prerequisite is merged into current `main`. Runtime R2 remains gated by R1 and AQ-PACKAGES. +- No task-created worktree exists. All pre-existing `.muse` and `passeur_cache` worktrees, archived branches, and user-owned files remain untouched. +- Q1 real HF service/import, current llama.cpp archive/install, desktop control, capacity, deployed migration, and old-writer evidence have not run. No production code or live data changed in the preparation recorded here. diff --git a/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md b/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md index 89ca79a6..ae2b751a 100644 --- a/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md +++ b/docs/plans/runtime-installations-and-model-adapters/execution-ledger.md @@ -11,3 +11,7 @@ Pumas branch baseline is unchanged at `04e7f156`; current standards ref is `39d5 The earlier entry-point mechanism probe is historical limited evidence in the previous delivery; it is not re-run, re-packaged as new evidence or used to mark runtime claims satisfied. Earlier delivered packages preserve revision history; this package avoids copies of obsolete active plans. **Exactly one next runtime slice:** R1 after AQ-HTTP is ready for its actual target. The coordinated program starts with acquisition Q1. Plan-only preparation may continue without opening the runtime source gate. + +## 2026-09-29 — Q1 implementation started on current accepted main + +The original companion plan commit `a8359512a580aa25fb2f9c9e4cd7e0dd64fd970d` is preserved on the acquisition Q1 branch based on current accepted `main` `e37bbf4b964a0e2aadf25f80ab71edd8fa6b3eb3`. The acquisition plan is active; AQ-HTTP remains not ready. R1 has not started because its prerequisite has not been accepted or merged. Current Build and Scorecard success on the base commit do not qualify a runtime or acquisition candidate. No runtime source, profile, process-binding, adapter, dependency or model-admission behavior changed in this slice. diff --git a/docs/plans/runtime-installations-and-model-adapters/plan.md b/docs/plans/runtime-installations-and-model-adapters/plan.md index d8395d8a..6d9c640e 100644 --- a/docs/plans/runtime-installations-and-model-adapters/plan.md +++ b/docs/plans/runtime-installations-and-model-adapters/plan.md @@ -2,21 +2,23 @@ **Revision:** 4 — coordinated with the separate prerequisite acquisition plan; supersedes revision 3 at the same canonical runtime plan path. -**Status:** Planned, proposed for owner adoption; production implementation has not started. +**Status:** Planned and adopted; runtime implementation has not started because R1 is gated by AQ-HTTP. **Objective acceptance:** pending; real runtime, model, migration, native-platform, and desktop acceptance has not been performed. -**Current phase:** runtime design ready for prerequisite-led implementation; AQ-HTTP is not ready. +**Current phase:** waiting for Acquisition Q1 to satisfy and merge AQ-HTTP; read-only preparation may continue. -**Exactly one next slice:** **R1 — shared installation identity and bound launch for Torch and llama.cpp; source integration is gated by AQ-HTTP.** +**Exactly one next runtime slice:** **R1 — shared installation identity and bound launch for Torch and llama.cpp; begin only after AQ-HTTP is accepted and merged for the target in scope.** -**Canonical repository path on adoption:** `docs/plans/runtime-installations-and-model-adapters/plan.md`. +**Canonical repository path:** `docs/plans/runtime-installations-and-model-adapters/plan.md`. **Integration owner:** Pumas runtime integration, sharing one serial integrator with the acquisition plan for overlapping contracts/files. **Prerequisite owner:** [Artifact acquisition](../artifact-acquisition/plan.md); [gate status](../artifact-acquisition/reports/dependency-gates.md). R1 may begin only after its target-scoped AQ-HTTP gate is ready. Read-only preparation can proceed independently. **Baselines:** Pumas `04e7f1568f00693c0ef26c77e0150e5e4dd112ea`; Coding-Standards `39d55dc330d44ecf940364ceada9d2527f7c7ea0`. +**Current accepted integration base:** Pumas `e37bbf4b964a0e2aadf25f80ab71edd8fa6b3eb3`. Acquisition Q1 is active on its task branch; AQ-HTTP remains not ready. Runtime source changes must start from updated accepted `main` after that gate is merged. + ## 1. Objective Reuse the generic artifact-acquisition direction in `docs/breif/s3-model-fetch.md` for obtaining verified bytes. Make app-manager the coherent application-facing manager of installed external runtimes/executables, with native llama.cpp and Python/Torch as concrete implementations. Allow an open population of model adapters to be registered independently of a Pumas build. A model may use a standard loader, that loader plus supplemental requirements, or approved custom loading/execution code. From 50105338fd9cb67e53a54720f466b5bd3408a3c9 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 14:19:10 -0700 Subject: [PATCH 03/20] docs(acquisition): define durable handoff transitions --- docs/contracts/artifact-acquisition.md | 13 +++++++++++++ docs/plans/artifact-acquisition/execution-ledger.md | 1 + 2 files changed, 14 insertions(+) diff --git a/docs/contracts/artifact-acquisition.md b/docs/contracts/artifact-acquisition.md index 7027ed98..8870fc36 100644 --- a/docs/contracts/artifact-acquisition.md +++ b/docs/contracts/artifact-acquisition.md @@ -107,6 +107,19 @@ Acquisition FilesReady and model/runtime publication are distinct commits. Consu Consumer settlement is idempotent for its exact generation. Old acknowledgements cannot release a successor. Durable unfinished consumption retains custody across restart even though in-memory RAII handles no longer exist. If confirmation is unavailable, surface recovery-required and bounded retained state, not deletion by age. +### Durable handoff transition ownership + +| Transition | Durable writer and exact identity | Filesystem authority and cancellation | Restart and reclamation | +| --- | --- | --- | --- | +| Admission → transfer | Acquisition records the selected manifest, demand, and attempt generation before starting network work. | The acquisition workspace grant is scoped to this generation; the caller cannot revive a serialized display path. | An admitted record without a live worker is recovered only after reopening its grant and revalidating the source identity. No input is reclaimed while the demand is retained. | +| Transfer → paused, waiting, or verifying | Acquisition records the observed transition; saved byte counts are progress only. | Pause is published only after the generation's write-capable worker and nested file effects have drained. Cancellation prevents successor writes and retains cleanup custody until they drain. | Reopen validates the actual partial bytes and matching source validator before a range request; otherwise it restarts that file from zero. | +| Verification → FilesReady | Acquisition durably flushes and verifies the complete selected set before it publishes FilesReady and a sealed read/use lease. | The live lease refers to the held workspace authority and exact attempt generation. A path string alone does not keep inputs alive. | Reopen re-establishes required byte evidence before issuing a new lease. No consumer may observe a set as complete before FilesReady is durable. | +| FilesReady → consumer using | Acquisition records the consumer demand and operation generation before handing out a lease; the model or runtime consumer remains authoritative for its own operation state. | The consumer holds the lease through importer/extractor work and its cleanup. Cancel is routed through that consumer and does not release acquisition custody early. | An unfinished durable demand retains the files after process restart even though the in-memory lease is gone. | +| Consumer using → committed | The consumer writes its own publication record; acquisition records the exact-generation settlement acknowledgement separately. | The consumer's publication authority owns installed/imported output. Acquisition may reclaim only its own workspace after it observes that output identity and the consumer's worker cleanup. | If consumer commit precedes acquisition acknowledgement, reconcile against the consumer's exact durable generation; do not repeat import/install or remove uncertain inputs. | +| Consumer committed/failed → settled | Acquisition settles only the acknowledged demand and attempt generation; the consumer does not rewrite acquisition state. | Adopted output is recorded before workspace release and is never deleted by acquisition cleanup. Failure retains uncertain input until required cleanup has a positive result. | Old acknowledgements cannot release a successor. Unknown cleanup or publication visibility remains recovery-required; elapsed time is not a reclamation proof. | + +The composition root owns shutdown ordering: stop new admission, drain each consumer and its cleanup while its leases remain valid, then drain acquisition workers and persist their truthful resumable or terminal disposition. A consumer-specific shutdown must not close the shared owner while another registered consumer can still use it. + ## 10. Failure, retries, progress and shutdown Differentiate malformed request, unsupported source/representation, authorization-required/denied, source-changed, integrity mismatch, not found, network inconclusive/transient, storage full, busy/capacity, cancellation, consumer failure, and visibility/durability uncertainty. Project through existing public error contracts with bounded non-sensitive diagnostics; the list does not mandate a new cross-project universal error enum. diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 664416fe..09da4c59 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -19,4 +19,5 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - Source preparation traced the normal model workflow, durable download owner, native runtime installer, and existing generic download API. The acquisition module, actual shared transfer owner and both production cutovers do not exist yet. The source inventory and independent architecture review are preliminary only; no Q1 gate claim is satisfied. - Existing local-TCP restart/import tests and historical native installation evidence retain their original scope. No real HF service, current llama.cpp archive, desktop workflow, live migration, or Q1 production test has been executed in this slice. - The store reader currently supports schema 5 and explicit schema 4 upgrade; deployed record population and old-writer retirement are unknown. No live retained state was opened or modified. Pending/unresolved cleanup replay remains refused under its existing recovery owner. +- Before the first Q1 source edit, the staged write set is limited to the validated manifest value module: `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs}` and `rust/crates/pumas-core/src/lib.rs`, with co-located manifest tests. Planned evidence: `cargo test -p pumas-library acquisition::manifest` plus formatting and clippy for the touched Rust crate. This establishes validated identities only; it does not satisfy AQ-HTTP, transfer lifecycle, persistence, consumer integration, or gate acceptance. The complete Q1 write set remains the one in [write sets](reports/write-sets.md); the exact next vertical consumer cutover will be recorded before those edits. - The sole implementation slice remains Q1. Q2, Q3 and runtime R1 remain dependency-gated; no acquisition gate is ready. From 8b96cab5cbb7ff5104d6e33692538d3680b32fc8 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 16:46:31 -0700 Subject: [PATCH 04/20] feat(acquisition): route HF through shared HTTP protocol --- .../artifact-acquisition/execution-ledger.md | 11 + docs/plans/artifact-acquisition/issues.md | 1 + docs/plans/artifact-acquisition/plan.md | 2 +- .../reports/coding-standards-mcp-usability.md | 4 + .../reports/write-sets.md | 8 +- .../crates/pumas-core/src/acquisition/http.rs | 665 ++++++++++ .../pumas-core/src/acquisition/manifest.rs | 851 +++++++++++++ rust/crates/pumas-core/src/acquisition/mod.rs | 18 + rust/crates/pumas-core/src/lib.rs | 1 + .../src/model_library/download_recovery.rs | 103 +- .../model_library/hf/acquisition_source.rs | 173 +++ .../src/model_library/hf/download.rs | 1120 ++++++++++++----- .../pumas-core/src/model_library/hf/mod.rs | 1 + .../pumas-core/src/model_library/hf/types.rs | 28 +- .../pumas-core/src/model_library/mod.rs | 1 - .../src/model_library/partial_download.rs | 137 -- rust/crates/pumas-core/src/tests.rs | 6 +- 17 files changed, 2660 insertions(+), 470 deletions(-) create mode 100644 rust/crates/pumas-core/src/acquisition/http.rs create mode 100644 rust/crates/pumas-core/src/acquisition/manifest.rs create mode 100644 rust/crates/pumas-core/src/acquisition/mod.rs create mode 100644 rust/crates/pumas-core/src/model_library/hf/acquisition_source.rs delete mode 100644 rust/crates/pumas-core/src/model_library/partial_download.rs diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 09da4c59..8b0f6c7a 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -21,3 +21,14 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - The store reader currently supports schema 5 and explicit schema 4 upgrade; deployed record population and old-writer retirement are unknown. No live retained state was opened or modified. Pending/unresolved cleanup replay remains refused under its existing recovery owner. - Before the first Q1 source edit, the staged write set is limited to the validated manifest value module: `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs}` and `rust/crates/pumas-core/src/lib.rs`, with co-located manifest tests. Planned evidence: `cargo test -p pumas-library acquisition::manifest` plus formatting and clippy for the touched Rust crate. This establishes validated identities only; it does not satisfy AQ-HTTP, transfer lifecycle, persistence, consumer integration, or gate acceptance. The complete Q1 write set remains the one in [write sets](reports/write-sets.md); the exact next vertical consumer cutover will be recorded before those edits. - The sole implementation slice remains Q1. Q2, Q3 and runtime R1 remain dependency-gated; no acquisition gate is ready. + +## 2026-09-29 — Q1 manifest/HTTP protocol and HF consumer slice + +- Implemented the versioned source-neutral artifact manifest and HTTP response/body-streaming protocol. The ordinary Hugging Face download path now builds the manifest from its selected revision and file evidence and consumes the shared HTTP protocol. The existing Hugging Face owner still owns durable admission, the task generation, cancellation/pause/retry, progress publication, destination capability, persistence, file finalization, and import settlement. This is a real protocol/consumer cutover in one existing workflow, not the shared durable acquisition lifecycle required by AQ-HTTP. +- Exact source write set: `rust/crates/pumas-core/src/{lib.rs,tests.rs,acquisition/{mod.rs,manifest.rs,http.rs},model_library/{mod.rs,download_recovery.rs,hf/{mod.rs,download.rs,types.rs,acquisition_source.rs},partial_download.rs}`. The old aggregate-size-only `partial_download` finalizer was removed after search found no remaining production consumer; no independent downloader replaced it. No dependency, durable schema, RPC, generated contract, frontend, or native installer was changed. The supported reader remains schema 5 with its existing schema 4 upgrade; no live retained state was opened or modified. +- Restore now hydrates aggregate size/hash only for an unambiguous single-file selection. Hashless pinned restored files do not finalize/import. If an immutable pinned source must be compared to a pre-existing hashless final, the first supervised attempt discards any unproved `.part` prefix and requests a complete representation; mismatch preserves the final and staged response. The pair comparison uses held capability-relative handles, checks identity and metadata, and fills fixed-size chunks to handle short reads. It explicitly does not claim isolation from an uncooperative local writer that modifies a file in place and restores metadata. AQ-I09 tracks that remaining boundary for the completed handoff review. +- Independent read-only architecture review found and then rechecked two high-severity bypasses: stale `.part` bytes could have been resumed to falsely match the final, and pinned restore could accept a hashless final. Both are repaired with the exact regression `existing_hashless_file_uses_fresh_source_and_is_not_imported_after_reopen`; the reviewer confirmed source-level closure. The reviewer did not run that test; the integrator did. +- **Actual candidate tests/checks:** `cargo test --manifest-path rust/Cargo.toml -p pumas-library model_library::hf:: -- --test-threads=1` passed 244/244 HF unit tests using controlled local HTTP servers and disposable filesystems. The new `xx` final / `xx` stale prefix / `ABCD` selected source case asserts no Range header, preservation on mismatch, and non-import after reopen. The complete `cargo test --manifest-path rust/Cargo.toml -p pumas-library -- --test-threads=1` then passed 1464 unit tests (6 ignored), the crate's integration suites (including the 36-test API suite), and doctests; the importer-recovery fixture now supplies its payload SHA-256 so it reaches the importer-failure behavior with verified input. These local-server tests are synthetic source fixtures, not live Hugging Face service evidence. `cargo clippy --manifest-path rust/Cargo.toml -p pumas-library --all-targets -- -D warnings`, `cargo check --manifest-path rust/Cargo.toml -p pumas-library --no-default-features`, `cargo fmt --manifest-path rust/Cargo.toml --all`, and `git diff --check` passed; final clippy/diff checks are repeated at the commit boundary. +- **Still unexecuted / not claimed:** live HF source acquisition, current llama.cpp archive through the shared handoff, desktop HF/native controls, shared-owner cancellation/shutdown/reopen, real deployed-state migration/old-writer isolation, current-candidate hosted CI, and all Q1 system/manual/platform acceptance. Local loopback permission was required for the controlled HTTP tests. The first sandbox-only full run had 23 listener `PermissionDenied` failures; the authorized loopback rerun passed. Base CI remains historical/base-only and GitHub currently returns no combined status or PR-triggered workflow run for accepted `main`. +- **Standards MCP:** final route against `snapshot:v1:c181902d-6cad-40ae-80ef-83f091932539` selected 37 standards with zero unresolved fact categories. Targeted `read_many` calls read 13 applicable policies covering persistence/evolution, async lifecycle, Rust filesystem security, contracts, architecture, code design, API, cross-platform, verification, planning, and proportionality. Findings were checked against source and actual candidate tests; the route itself is not compliance evidence. The MCP authoring `review` operation does not review application source. The independent reviewer reused earlier routed obligations and made no new MCP calls in the narrow repair review. See the separate [MCP usability report](reports/coding-standards-mcp-usability.md). +- Plan state remains Q1 in progress; AQ-HTTP, AQ-PACKAGES, AQ-S3 and AQ-COMPLETE remain not ready. Q2/Q3/runtime R1 are not started on an unaccepted prerequisite. The next implementation work continues Q1 with the durable owner and existing llama.cpp consumer; no runtime R1 branch is authorized yet. diff --git a/docs/plans/artifact-acquisition/issues.md b/docs/plans/artifact-acquisition/issues.md index 0d3ad0bd..f797eff9 100644 --- a/docs/plans/artifact-acquisition/issues.md +++ b/docs/plans/artifact-acquisition/issues.md @@ -12,6 +12,7 @@ Q1 is active; production repairs and acceptance claims remain pending until evid | AQ-I06 | High: generic retrieval can be mistaken for package compatibility or executable trust | Runtime/package/security: Q2, with Q1 trust contract | AC07/AC11/AC12; keep origin and byte digest separate | | AQ-I07 | Medium: new generic module could accidentally require inference or overclaim feature isolation | Core/composition: Q1/Q3/Q4 | AC09/AC15/AC17; no unsupported minimal-build claim | | AQ-I08 | Medium: source roadmap can expand prerequisite beyond useful consumer result | Integrator: Q1 contract first; Q3 required S3 scope; future features deferred below | AC18 and gate-status record | +| AQ-I09 | Medium: hashless source comparison is not isolation from an uncooperative local writer that can modify a file in place and restore metadata before model import | Acquisition/import handoff: Q1 claim is explicitly limited; resolve through a same-owner handle/lifetime handoff or document and verify the supported local-writer threat model before any broader claim | AC05–AC07; revisit at shared durable handoff and final composed review | | AQ-D01 | Deferred: native Xet reconstruction | Acquisition/source owner | Revisit when a required HF source cannot use the supported existing file path or Xet transfer benefits are an admitted goal; evaluate maintained Rust implementation, no homemade protocol | | AQ-D02 | Deferred: peers and concurrent multi-source failover | Acquisition/distribution owner | Revisit with actual node/authorization contract and content-equivalence proof; no cross-origin ETag comparison | | AQ-D03 | Deferred: chunk CAS, reflink optimization and coalescing | Acquisition/storage owner | Revisit with measured duplication/network/storage need and explicit retention consumers; ordinary files and safe release are required now | diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index 8917578e..fbb19f0c 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,7 +2,7 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** Q1 source, retained-state, and authority reconciliation; production implementation and required real acceptance are pending. +**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The shared durable handoff owner, llama.cpp consumer, retained-store evolution, desktop path, and required real-source qualification remain pending. **Exactly one next slice:** Continue **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index 1657dc06..1ba3b532 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -6,9 +6,11 @@ This report records agent experience using the Coding-Standards MCP during Acqui - **Useful calls:** `routing_facts` exposed the valid fact categories and values; an initial route with minimal facts made the router's information model clear. Compact routes with bounded content returned an explicit snapshot, selected standards, and unresolved-fact count. `read_many` let the integrator inspect the commit, planning, implementation, verification, architecture, contract, and documentation obligations. Rerouting after composition seams were clarified confirmed applicability for the actual Q1 boundary. - **Confusing or redundant steps:** Tool discovery descriptions were verbose. The first broad route requested 32 policy contents and produced output large enough to truncate. A broad `read_many` similarly returned tens of thousands of tokens. A later route displayed policies already read, so applicability and policy reading were not easy to distinguish. +- **Additional final-route experience:** The summary carried the snapshot UUID without its required `snapshot:v1:` prefix; the MCP rejected that exact handle with a clear pattern error, after which the full opaque handle worked. A complete route still required explicit empty values for the framework and workflow-profile categories. Supplying `content.limit: 32` returned all 32 policy bodies at once (about 85,000 output tokens and truncated); a targeted `read_many` of 11 exact selected policies was more useful for checking this candidate. - **Missing context:** The MCP did not know the repository's current accepted plan status, retained-state population, public consumers, active recovery owner, or actual CI/source evidence. Those facts had to be gathered from the repository, GitHub, and the source tree. The route cannot certify that the source or tests satisfy the selected rules. - **Smallest sufficient workflow:** Read routing facts once; route the concrete slice without policy contents; read Core plus only the selected standards and workflows relevant to its actual changed boundary in small pages; inspect source and run the required evidence; reroute only when ownership or scope changes; then use the final review operation against the implemented candidate. - **Recommendations:** Keep route output compact by default and separate applicability reasons from policy bodies. Make content pages token-budgeted, expose previously read policy IDs, and put unresolved facts first. Clarify in the result that a complete route means applicability was resolved, not that implementation compliance was verified. A same-snapshot “changed design” review summary could call out new or removed obligations without replaying all policy text. +- **Smallest sufficient workflow for this slice:** Preserve the complete opaque snapshot handle; call `routing_facts` only when the current fact vocabulary is not already known; route the changed boundary without inline policy text; read the exact selected obligations in a small `read_many`; inspect source and test evidence outside the MCP; reroute the final implementation. Empty framework/profile answers are needed to close applicability when those categories are not relevant. ## Independent architecture reviewer @@ -21,3 +23,5 @@ This report records agent experience using the Coding-Standards MCP during Acqui ## Participation The primary integrator and the independent architecture reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. + +The architecture reviewer’s narrow follow-up inspected the repaired current source without making a new MCP call and reused the prior routed obligations; its findings confirm the two identified integrity paths are closed at source level, not that Q1 is accepted. The MCP `review` operation is an authoring workflow for changes to the standards corpus; it does not review application source. Application compliance was checked by final routing, standards reads, source/test inspection, and the independent code review. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index 7a515fad..78846b48 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -1,15 +1,15 @@ # Acquisition implementation write sets and coordination -These are the proposed exact paths/closed path families. No production source is changed by this delivery. Before a slice edits files, its integrator records the exact members and actual tests in its ledger. New files below are intentional proposed paths, not claims of existing code. Same-owner amendments are recorded; new authority/consumer boundaries trigger re-plan. +These are the admitted exact paths/closed path families. The actual source files selected for the current Q1 slice are listed in the execution ledger; remaining paths below are authorized Q1 boundaries, not claims that those implementations exist. Before a slice edits files, its integrator records the exact members and actual tests in its ledger. New authority/consumer boundaries trigger re-plan. ## Q1: one HTTP acquisition owner with real consumers -**New canonical module paths, used only when the concern warrants the split:** -`rust/crates/pumas-core/src/acquisition/{mod.rs,types.rs,service.rs,store.rs,workspace.rs,http.rs}`. Source-reader abstraction and current HF adaptation can remain in these owners or `acquisition/sources/{mod.rs,http.rs}` if that is the clearer implementation; choose one actual layout at admission, not both. The semantic split is independent source access, durable transfer owner and capability workspace, not file-count reduction. +**Canonical module paths:** `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs,http.rs}` currently hold validated source-neutral selections and the HTTP representation/body-streaming protocol. The remaining Q1 design still needs one durable transfer owner and capability workspace; place those with the canonical module rather than creating another downloader. Source-reader adaptation may remain in that module or a focused `acquisition/sources/` child if the actual design supports it. **Existing owners allowed to change:** - `rust/crates/pumas-core/src/lib.rs`, `network/{mod.rs,download.rs}`, `model_library/hf/{mod.rs,download.rs,lifecycle.rs,types.rs,metadata.rs}`; -- `rust/crates/pumas-core/src/model_library/{download_store.rs,download_recovery.rs,partial_download.rs}` only for the extracted authority and explicit supported migration; +- `rust/crates/pumas-core/src/model_library/{download_store.rs,download_recovery.rs}` only for the extracted authority and explicit supported migration; +- `rust/crates/pumas-core/src/tests.rs` when a builder/reopen fixture depends on the selected completion-evidence invariant; - directly affected core `api/hf.rs`, `api/state.rs` and current model-importer/intent completion call sites, selected from the actual producer/consumer trace before editing; - `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,ollama.rs,progress.rs}` for the acquisition bridge and transfer progress, not installed-unit identity migration; - current core atomic JSON/capability-filesystem modules only where the same selected invariant requires a targeted change, never as an unrelated filesystem rewrite. diff --git a/rust/crates/pumas-core/src/acquisition/http.rs b/rust/crates/pumas-core/src/acquisition/http.rs new file mode 100644 index 00000000..c3fb049f --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/http.rs @@ -0,0 +1,665 @@ +//! HTTP representation admission shared by artifact consumers. + +use super::{ArtifactManifest, ManifestValidationError}; +use crate::error::{PumasError, Result}; +use futures::StreamExt; +use reqwest::header::{ACCEPT_ENCODING, AUTHORIZATION, CONTENT_ENCODING, CONTENT_RANGE, RANGE}; +use reqwest::{Response, StatusCode}; + +/// A checked response to one whole-file or suffix-range artifact request. +/// Body transfer remains owned by the caller's supervised acquisition attempt. +#[derive(Debug)] +pub(crate) struct HttpArtifactResponse { + pub(crate) body: Response, + pub(crate) resumed: bool, + pub(crate) total_size: Option, +} + +/// Consumer-owned file effect used while the source-neutral HTTP component +/// streams one selected representation. +#[async_trait::async_trait] +pub(crate) trait HttpArtifactSink: Send { + async fn write_all(&mut self, bytes: &[u8]) -> Result<()>; + async fn flush(&mut self) -> Result<()>; +} + +/// Existing supervised operation supplies cancellation and progress projection +/// without transferring its lifecycle ownership to the protocol adapter. +#[async_trait::async_trait] +pub(crate) trait HttpAttemptHost: Send { + async fn pause_requested(&self); + fn pause_requested_now(&self) -> bool; + fn cancel_requested(&self) -> bool; + async fn record_progress(&mut self, downloaded_for_file: u64) -> Result<()>; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum HttpBodyOutcome { + Complete { downloaded: u64 }, + Paused, + Cancelled, +} + +/// Open the selected representation without putting access material into the +/// manifest. A server which ignores Range returns a full response with +/// `resumed == false`, requiring the caller to replace its partial file. +pub(crate) async fn open_http_artifact( + client: &reqwest::Client, + url: &str, + manifest: &ArtifactManifest, + file_index: usize, + resume_from: u64, + authorization: Option<&str>, +) -> Result { + let file = manifest + .files() + .get(file_index) + .ok_or_else(|| invalid_response("HTTP request selected an unknown manifest file"))?; + if resume_from > 0 && !manifest.permits_resume(file_index) { + return Err(resume_identity_required()); + } + let mut request = client.get(url).header(ACCEPT_ENCODING, "identity"); + if let Some(authorization) = authorization { + request = request.header(AUTHORIZATION, authorization); + } + if resume_from > 0 { + request = request.header(RANGE, format!("bytes={resume_from}-")); + } + let response = request.send().await.map_err(|error| PumasError::Network { + message: "HTTP artifact request failed".into(), + cause: Some(error.without_url().to_string()), + })?; + + validate_identity_encoding(&response)?; + let status = response.status(); + match (resume_from, status) { + (offset, StatusCode::PARTIAL_CONTENT) if offset > 0 => { + let (start, end, total) = parse_content_range(&response)?; + if start != offset + || end < start + || end.checked_add(1) != Some(total) + || file + .expected_size() + .is_some_and(|expected| expected != total) + { + return Err(invalid_response( + "HTTP range response does not match the selected artifact representation", + )); + } + let range_length = end - start + 1; + if response + .content_length() + .is_some_and(|content_length| content_length != range_length) + { + return Err(invalid_response( + "HTTP range response length contradicts Content-Range", + )); + } + Ok(HttpArtifactResponse { + body: response, + resumed: true, + total_size: Some(total), + }) + } + (_, StatusCode::OK) => { + // A full response to Range is safe only when the consumer opens the + // partial file from zero and replaces its prior contents. + let content_length = response.content_length(); + if file + .expected_size() + .zip(content_length) + .is_some_and(|(expected, observed)| expected != observed) + { + return Err(invalid_response( + "HTTP full response length contradicts the selected artifact", + )); + } + Ok(HttpArtifactResponse { + body: response, + resumed: false, + total_size: file.expected_size().or(content_length), + }) + } + (_, StatusCode::PARTIAL_CONTENT) => Err(invalid_response( + "HTTP source returned a partial representation without a range request", + )), + (_, status) if status == StatusCode::TOO_MANY_REQUESTS => Err(PumasError::RateLimited { + service: "artifact source".into(), + retry_after_secs: response + .headers() + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse().ok()), + }), + (_, status) if status == StatusCode::REQUEST_TIMEOUT || status.is_server_error() => { + Err(PumasError::Network { + message: format!("HTTP artifact source returned {status}"), + cause: None, + }) + } + (_, status) => Err(PumasError::DownloadFailed { + url: "artifact source".into(), + message: format!("HTTP {status}"), + }), + } +} + +/// Drain an admitted HTTP response through the consumer's held write +/// capability. This owns stream byte accounting and completion checks; the +/// consumer host retains scheduling, retry, cancellation settlement and file +/// publication authority. +pub(crate) async fn stream_http_artifact( + response: HttpArtifactResponse, + requested_resume_from: u64, + sink: &mut dyn HttpArtifactSink, + host: &mut dyn HttpAttemptHost, +) -> Result { + let total = response.total_size; + let mut downloaded = if response.resumed { + requested_resume_from + } else { + 0 + }; + let mut body = response.body.bytes_stream(); + loop { + let next = tokio::select! { + biased; + _ = host.pause_requested() => { + sink.flush().await?; + return Ok(HttpBodyOutcome::Paused); + } + next = body.next() => next, + }; + let Some(chunk) = next else { + break; + }; + if host.pause_requested_now() { + sink.flush().await?; + return Ok(HttpBodyOutcome::Paused); + } + if host.cancel_requested() { + return Ok(HttpBodyOutcome::Cancelled); + } + let chunk = chunk.map_err(|error| PumasError::Network { + message: "HTTP artifact stream failed".into(), + cause: Some(error.without_url().to_string()), + })?; + let next_downloaded = downloaded + .checked_add(chunk.len() as u64) + .ok_or_else(|| invalid_response("HTTP artifact byte count overflowed"))?; + if total.is_some_and(|total| next_downloaded > total) { + return Err(invalid_response( + "HTTP artifact body exceeded its selected representation length", + )); + } + sink.write_all(&chunk).await?; + downloaded = next_downloaded; + host.record_progress(downloaded).await?; + } + sink.flush().await?; + if total.is_some_and(|total| downloaded != total) { + return Err(PumasError::Network { + message: format!( + "Incomplete HTTP artifact body: received {downloaded} of {} bytes", + total.unwrap_or_default() + ), + cause: None, + }); + } + Ok(HttpBodyOutcome::Complete { downloaded }) +} + +fn validate_identity_encoding(response: &Response) -> Result<()> { + if let Some(value) = response.headers().get(CONTENT_ENCODING) { + let value = value.to_str().map_err(|_| { + invalid_response("HTTP artifact response used an unreadable content encoding") + })?; + if !value.eq_ignore_ascii_case("identity") { + return Err(invalid_response( + "HTTP artifact response used an encoded representation", + )); + } + } + Ok(()) +} + +fn parse_content_range(response: &Response) -> Result<(u64, u64, u64)> { + let value = response + .headers() + .get(CONTENT_RANGE) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| invalid_response("HTTP range response omitted Content-Range"))?; + let value = value + .strip_prefix("bytes ") + .ok_or_else(|| invalid_response("HTTP range response used an unsupported range unit"))?; + let (range, total) = value + .split_once('/') + .ok_or_else(|| invalid_response("HTTP Content-Range has no complete length"))?; + let (start, end) = range + .split_once('-') + .ok_or_else(|| invalid_response("HTTP Content-Range has no byte interval"))?; + let start = parse_range_integer(start)?; + let end = parse_range_integer(end)?; + let total = parse_range_integer(total)?; + if total == 0 || end >= total { + return Err(invalid_response( + "HTTP Content-Range has an invalid complete length", + )); + } + Ok((start, end, total)) +} + +fn parse_range_integer(value: &str) -> Result { + if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(invalid_response( + "HTTP Content-Range contains an invalid integer", + )); + } + value + .parse() + .map_err(|_| invalid_response("HTTP Content-Range integer is out of range")) +} + +fn invalid_response(message: &'static str) -> PumasError { + PumasError::Validation { + field: "artifact.http.response".into(), + message: message.into(), + } +} + +fn resume_identity_required() -> PumasError { + PumasError::Validation { + field: "artifact.http.resume".into(), + message: ManifestValidationError::ResumeIdentityRequired.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::acquisition::{ + ArtifactFile, ArtifactRevisionEvidence, ArtifactSourceIdentity, + FileVerificationRequirement, RevisionStrength, + }; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + fn selected_file(size: u64) -> ArtifactFile { + ArtifactFile::new( + "weights.bin", + "weights.bin", + Some(size), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap() + } + + fn weak_file(size: u64) -> ArtifactFile { + ArtifactFile::new( + "archive.tar", + "archive.tar", + Some(size), + None, + FileVerificationRequirement::CompleteRepresentation, + ) + .unwrap() + } + + fn manifest(file: ArtifactFile, strength: RevisionStrength) -> ArtifactManifest { + let revision = + ArtifactRevisionEvidence::new("test.revision", "revision-1", strength).unwrap(); + let source = ArtifactSourceIdentity::new("test", "source", revision).unwrap(); + ArtifactManifest::new(source, vec![file]).unwrap() + } + + async fn serve_once(response: String) -> (reqwest::Url, tokio::task::JoinHandle) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + loop { + let read = stream.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + stream.write_all(response.as_bytes()).await.unwrap(); + let _ = stream.shutdown().await; + String::from_utf8(request).unwrap() + }); + ( + reqwest::Url::parse(&format!("http://{address}/artifact")).unwrap(), + server, + ) + } + + fn response(status: &str, headers: &str, body: &str) -> String { + format!( + "HTTP/1.1 {status}\r\n{headers}Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) + } + + #[derive(Default)] + struct TestSink { + bytes: Vec, + flushed: bool, + } + + #[async_trait::async_trait] + impl HttpArtifactSink for TestSink { + async fn write_all(&mut self, bytes: &[u8]) -> Result<()> { + self.bytes.extend_from_slice(bytes); + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + self.flushed = true; + Ok(()) + } + } + + struct TestHost { + pause: bool, + cancel: bool, + progress: Vec, + } + + #[async_trait::async_trait] + impl HttpAttemptHost for TestHost { + async fn pause_requested(&self) { + if !self.pause { + std::future::pending().await + } + } + + fn cancel_requested(&self) -> bool { + self.cancel + } + + fn pause_requested_now(&self) -> bool { + self.pause + } + + async fn record_progress(&mut self, downloaded_for_file: u64) -> Result<()> { + self.progress.push(downloaded_for_file); + Ok(()) + } + } + + #[tokio::test] + async fn range_resume_requires_exact_content_range_and_pinned_total() { + let (url, server) = serve_once(response( + "206 Partial Content", + "Content-Range: bytes 3-5/6\r\n", + "def", + )) + .await; + let client = reqwest::Client::new(); + let reply = open_http_artifact( + &client, + url.as_str(), + &manifest(selected_file(6), RevisionStrength::Immutable), + 0, + 3, + None, + ) + .await + .unwrap(); + assert!(reply.resumed); + assert_eq!(reply.total_size, Some(6)); + assert_eq!(reply.body.bytes().await.unwrap().as_ref(), b"def"); + let request = server.await.unwrap().to_ascii_lowercase(); + assert!(request.contains("range: bytes=3-")); + assert!(request.contains("accept-encoding: identity")); + } + + #[tokio::test] + async fn ignored_range_returns_full_body_for_replacement_from_zero() { + let (url, server) = serve_once(response("200 OK", "", "abcdef")).await; + let reply = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(selected_file(6), RevisionStrength::Immutable), + 0, + 3, + None, + ) + .await + .unwrap(); + assert!(!reply.resumed); + assert_eq!(reply.body.bytes().await.unwrap().as_ref(), b"abcdef"); + assert!(server + .await + .unwrap() + .to_ascii_lowercase() + .contains("range: bytes=3-")); + } + + #[tokio::test] + async fn malformed_or_changed_range_responses_are_rejected() { + for headers in [ + "Content-Range: bytes 2-5/6\r\n", + "Content-Range: bytes 3-5/7\r\n", + "Content-Range: bytes 3-4/6\r\n", + "", + ] { + let (url, server) = serve_once(response("206 Partial Content", headers, "def")).await; + assert!(open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(selected_file(6), RevisionStrength::Immutable), + 0, + 3, + None, + ) + .await + .is_err()); + let _ = server.await.unwrap(); + } + } + + #[tokio::test] + async fn a_partial_response_is_not_a_complete_file_response() { + let (url, server) = serve_once(response( + "206 Partial Content", + "Content-Range: bytes 0-2/6\r\n", + "abc", + )) + .await; + let error = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(6), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap_err(); + assert!(matches!(error, PumasError::Validation { .. })); + let _ = server.await.unwrap(); + } + + #[tokio::test] + async fn rejects_non_identity_content_encoding() { + let (url, server) = + serve_once(response("200 OK", "Content-Encoding: gzip\r\n", "bytes")).await; + let error = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(5), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap_err(); + assert!( + matches!(error, PumasError::Validation { field, .. } if field == "artifact.http.response") + ); + let _ = server.await.unwrap(); + } + + #[tokio::test] + async fn http_failure_does_not_disclose_ephemeral_access_url() { + let (mut url, server) = serve_once(response("404 Not Found", "", "missing")).await; + url.query_pairs_mut() + .append_pair("X-Amz-Credential", "seeded-secret-value"); + let error = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(7), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap_err(); + assert!(!format!("{error:?} {error}").contains("seeded-secret-value")); + let request = server.await.unwrap(); + assert!(request.contains("X-Amz-Credential=seeded-secret-value")); + } + + #[tokio::test] + async fn body_transfer_checks_actual_length_and_flushes_before_completion() { + let (url, server) = serve_once(response("200 OK", "", "abcdef")).await; + let opened = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(6), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap(); + let mut sink = TestSink::default(); + let mut host = TestHost { + pause: false, + cancel: false, + progress: Vec::new(), + }; + assert_eq!( + stream_http_artifact(opened, 0, &mut sink, &mut host) + .await + .unwrap(), + HttpBodyOutcome::Complete { downloaded: 6 } + ); + assert_eq!(sink.bytes, b"abcdef"); + assert!(sink.flushed); + assert_eq!(host.progress.last(), Some(&6)); + let _ = server.await.unwrap(); + + let (url, server) = serve_once( + "HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\nabc".into(), + ) + .await; + let opened = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(6), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap(); + let mut sink = TestSink::default(); + let mut host = TestHost { + pause: false, + cancel: false, + progress: Vec::new(), + }; + assert!(matches!( + stream_http_artifact(opened, 0, &mut sink, &mut host).await, + Err(PumasError::Network { .. }) + )); + assert_eq!(sink.bytes, b"abc"); + let _ = server.await.unwrap(); + } + + #[tokio::test] + async fn body_transfer_flushes_pause_and_respects_cancellation() { + let (url, server) = serve_once(response("200 OK", "", "complete")).await; + let opened = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(8), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap(); + let mut sink = TestSink::default(); + let mut host = TestHost { + pause: true, + cancel: false, + progress: Vec::new(), + }; + assert_eq!( + stream_http_artifact(opened, 0, &mut sink, &mut host) + .await + .unwrap(), + HttpBodyOutcome::Paused + ); + assert!(sink.bytes.is_empty()); + assert!(sink.flushed); + let _ = server.await.unwrap(); + + let (url, server) = serve_once(response("200 OK", "", "complete")).await; + let opened = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(8), RevisionStrength::Weak), + 0, + 0, + None, + ) + .await + .unwrap(); + let mut sink = TestSink::default(); + let mut host = TestHost { + pause: false, + cancel: true, + progress: Vec::new(), + }; + assert_eq!( + stream_http_artifact(opened, 0, &mut sink, &mut host) + .await + .unwrap(), + HttpBodyOutcome::Cancelled + ); + assert!(sink.bytes.is_empty()); + let _ = server.await.unwrap(); + } + + #[tokio::test] + async fn weak_source_without_a_digest_cannot_resume_a_partial() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let url = reqwest::Url::parse(&format!("http://{address}/artifact")).unwrap(); + let error = open_http_artifact( + &reqwest::Client::new(), + url.as_str(), + &manifest(weak_file(6), RevisionStrength::Weak), + 0, + 3, + None, + ) + .await + .unwrap_err(); + assert!( + matches!(error, PumasError::Validation { field, .. } if field == "artifact.http.resume") + ); + drop(listener); + } +} diff --git a/rust/crates/pumas-core/src/acquisition/manifest.rs b/rust/crates/pumas-core/src/acquisition/manifest.rs new file mode 100644 index 00000000..1d0420fc --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/manifest.rs @@ -0,0 +1,851 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use std::collections::{HashMap, HashSet}; +use thiserror::Error; + +/// Current serialized artifact-manifest version. +pub const CURRENT_MANIFEST_VERSION: u16 = 1; + +/// Whether a source revision is immutable or only identifies one complete read. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RevisionStrength { + Immutable, + Weak, +} + +/// A resolver's claim about the selected revision; construction validates its +/// shape, not its authority or mapping to the bytes that a reader returns. +/// +/// The resolver supplies an authority such as `huggingface.commit` or +/// `s3.version_id`. Acquisition verifies that evidence with the corresponding +/// source adapter; the value is not itself permission to access or execute bytes. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactRevisionEvidence { + authority: String, + value: String, + strength: RevisionStrength, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ArtifactRevisionEvidenceWire { + authority: String, + value: String, + strength: RevisionStrength, +} + +impl ArtifactRevisionEvidence { + pub fn new( + authority: impl Into, + value: impl Into, + strength: RevisionStrength, + ) -> Result { + let authority = authority.into(); + let value = value.into(); + validate_evidence_authority(&authority)?; + validate_stable_text(&value)?; + Ok(Self { + authority, + value, + strength, + }) + } + + pub fn authority(&self) -> &str { + &self.authority + } + + pub fn value(&self) -> &str { + &self.value + } + + pub fn strength(&self) -> RevisionStrength { + self.strength + } +} + +impl<'de> Deserialize<'de> for ArtifactRevisionEvidence { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let wire = ArtifactRevisionEvidenceWire::deserialize(deserializer)?; + Self::new(wire.authority, wire.value, wire.strength).map_err(serde::de::Error::custom) + } +} + +/// Stable, non-secret identity of a selected source revision. +/// +/// Retrieval URLs, credentials, and signed query strings belong to ephemeral +/// source access and are intentionally absent from this value. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactSourceIdentity { + provider: String, + source_id: String, + revision: ArtifactRevisionEvidence, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ArtifactSourceIdentityWire { + provider: String, + source_id: String, + revision: ArtifactRevisionEvidence, +} + +impl ArtifactSourceIdentity { + pub fn new( + provider: impl Into, + source_id: impl Into, + revision: ArtifactRevisionEvidence, + ) -> Result { + let provider = provider.into(); + let source_id = source_id.into(); + if !valid_provider(&provider) { + return Err(ManifestValidationError::InvalidSourceIdentity); + } + validate_identity_text(&source_id)?; + Ok(Self { + provider, + source_id, + revision, + }) + } + + pub fn provider(&self) -> &str { + &self.provider + } + + pub fn source_id(&self) -> &str { + &self.source_id + } + + pub fn revision(&self) -> &ArtifactRevisionEvidence { + &self.revision + } +} + +impl<'de> Deserialize<'de> for ArtifactSourceIdentity { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let wire = ArtifactSourceIdentityWire::deserialize(deserializer)?; + Self::new(wire.provider, wire.source_id, wire.revision).map_err(serde::de::Error::custom) + } +} + +/// Provenance for an expected SHA-256 supplied during source resolution. +/// Admission must still decide whether this authority is trusted for its use. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct Sha256Evidence { + authority: String, + value: String, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Sha256EvidenceWire { + authority: String, + value: String, +} + +impl Sha256Evidence { + pub fn new( + authority: impl Into, + value: impl Into, + ) -> Result { + let authority = authority.into(); + validate_evidence_authority(&authority)?; + let value = canonical_sha256(&value.into())?; + Ok(Self { authority, value }) + } + + pub fn authority(&self) -> &str { + &self.authority + } + + pub fn value(&self) -> &str { + &self.value + } +} + +impl<'de> Deserialize<'de> for Sha256Evidence { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let wire = Sha256EvidenceWire::deserialize(deserializer)?; + Self::new(wire.authority, wire.value).map_err(serde::de::Error::custom) + } +} + +/// Minimum byte evidence the acquisition implementation must establish. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FileVerificationRequirement { + Sha256, + SizeAndImmutableRevision, + CompleteRepresentation, +} + +/// One selected logical file, opaque source key, and required verification +/// policy. The policy is a declared requirement, not byte verification. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactFile { + logical_path: String, + source_key: String, + expected_size: Option, + expected_sha256: Option, + verification: FileVerificationRequirement, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ArtifactFileWire { + logical_path: String, + source_key: String, + expected_size: Option, + expected_sha256: Option, + verification: FileVerificationRequirement, +} + +impl ArtifactFile { + pub fn new( + logical_path: impl Into, + source_key: impl Into, + expected_size: Option, + expected_sha256: Option, + verification: FileVerificationRequirement, + ) -> Result { + let logical_path = logical_path.into(); + let source_key = source_key.into(); + validate_logical_path(&logical_path)?; + validate_source_key(&source_key)?; + match (verification, expected_sha256.as_ref(), expected_size) { + (FileVerificationRequirement::Sha256, None, _) => { + return Err(ManifestValidationError::Sha256Required) + } + (FileVerificationRequirement::Sha256, Some(_), _) => {} + (_, Some(_), _) => return Err(ManifestValidationError::UnexpectedSha256), + (FileVerificationRequirement::SizeAndImmutableRevision, None, None) => { + return Err(ManifestValidationError::ExpectedSizeRequired) + } + _ => {} + } + Ok(Self { + logical_path, + source_key, + expected_size, + expected_sha256, + verification, + }) + } + + pub fn logical_path(&self) -> &str { + &self.logical_path + } + + /// The protocol key is preserved exactly; it is never converted to a path + /// or used to retain a retrieval URL. Source resolution owns that distinction. + pub fn source_key(&self) -> &str { + &self.source_key + } + + pub fn expected_size(&self) -> Option { + self.expected_size + } + + pub fn expected_sha256(&self) -> Option<&Sha256Evidence> { + self.expected_sha256.as_ref() + } + + pub fn verification(&self) -> FileVerificationRequirement { + self.verification + } +} + +impl<'de> Deserialize<'de> for ArtifactFile { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let wire = ArtifactFileWire::deserialize(deserializer)?; + Self::new( + wire.logical_path, + wire.source_key, + wire.expected_size, + wire.expected_sha256, + wire.verification, + ) + .map_err(serde::de::Error::custom) + } +} + +/// A structurally validated selection of source objects and local logical +/// names. Construction does not authenticate the resolver's revision or digest +/// claims; the acquisition admission boundary must establish that mapping. +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct ArtifactManifest { + schema_version: u16, + source: ArtifactSourceIdentity, + files: Vec, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ArtifactManifestWire { + schema_version: u16, + source: ArtifactSourceIdentity, + files: Vec, +} + +impl ArtifactManifest { + pub fn new( + source: ArtifactSourceIdentity, + files: Vec, + ) -> Result { + Self::with_version(CURRENT_MANIFEST_VERSION, source, files) + } + + fn with_version( + schema_version: u16, + source: ArtifactSourceIdentity, + files: Vec, + ) -> Result { + if schema_version != CURRENT_MANIFEST_VERSION { + return Err(ManifestValidationError::UnsupportedVersion(schema_version)); + } + if files.is_empty() { + return Err(ManifestValidationError::EmptyFileSet); + } + if source.revision.strength == RevisionStrength::Weak + && files.iter().any(|file| { + file.verification == FileVerificationRequirement::SizeAndImmutableRevision + }) + { + return Err(ManifestValidationError::ImmutableRevisionRequired); + } + + let mut exact_paths = HashSet::with_capacity(files.len()); + let mut portable_paths = HashSet::with_capacity(files.len()); + let mut source_evidence = HashMap::with_capacity(files.len()); + let mut total = Some(0_u64); + for file in &files { + if !exact_paths.insert(file.logical_path.as_str()) { + return Err(ManifestValidationError::DuplicateLogicalPath); + } + if !portable_paths.insert(case_insensitive_path_key(&file.logical_path)) { + return Err(ManifestValidationError::CollidingLogicalPath); + } + let evidence = source_evidence + .entry(file.source_key.as_str()) + .or_insert((None, None)); + if evidence + .0 + .is_some_and(|known| file.expected_size.is_some_and(|expected| known != expected)) + || evidence.1.is_some_and(|known| { + file.expected_sha256 + .as_ref() + .is_some_and(|expected| known != expected.value.as_str()) + }) + { + return Err(ManifestValidationError::ConflictingSourceEvidence); + } + if evidence.0.is_none() { + evidence.0 = file.expected_size; + } + if evidence.1.is_none() { + evidence.1 = file + .expected_sha256 + .as_ref() + .map(|digest| digest.value.as_str()); + } + total = total.and_then(|sum| file.expected_size.and_then(|size| sum.checked_add(size))); + } + + for path in &portable_paths { + let mut prefix = String::new(); + let components: Vec<_> = path.split('/').collect(); + for component in components.iter().take(components.len().saturating_sub(1)) { + if !prefix.is_empty() { + prefix.push('/'); + } + prefix.push_str(component); + if portable_paths.contains(&prefix) { + return Err(ManifestValidationError::CollidingLogicalPath); + } + } + } + + if files.iter().all(|file| file.expected_size.is_some()) && total.is_none() { + return Err(ManifestValidationError::SizeOverflow); + } + + Ok(Self { + schema_version, + source, + files, + }) + } + + pub fn schema_version(&self) -> u16 { + self.schema_version + } + + pub fn source(&self) -> &ArtifactSourceIdentity { + &self.source + } + + pub fn files(&self) -> &[ArtifactFile] { + &self.files + } + + /// Whether the selected file has evidence that makes a retained partial + /// safe to combine with a later response. An immutable source revision + /// pins the representation; otherwise a whole-file digest can reject any + /// mixed or changed representation before publication. + pub fn permits_resume(&self, file_index: usize) -> bool { + self.files.get(file_index).is_some_and(|file| { + self.source.revision.strength == RevisionStrength::Immutable + || file.expected_sha256.is_some() + }) + } + + /// Whether a final or complete partial file has selected integrity + /// evidence that must be checked before it can be reused or published. + pub fn requires_file_verification(&self, file_index: usize) -> bool { + self.files.get(file_index).is_some_and(|file| { + self.source.revision.strength == RevisionStrength::Immutable + || file.expected_sha256.is_some() + }) + } + + /// Returns `None` when any selected file has an unknown size. + pub fn total_bytes(&self) -> Option { + self.files + .iter() + .try_fold(0_u64, |sum, file| sum.checked_add(file.expected_size?)) + } +} + +impl<'de> Deserialize<'de> for ArtifactManifest { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let wire = ArtifactManifestWire::deserialize(deserializer)?; + Self::with_version(wire.schema_version, wire.source, wire.files) + .map_err(serde::de::Error::custom) + } +} + +#[derive(Clone, Debug, Eq, Error, PartialEq)] +pub enum ManifestValidationError { + #[error("artifact source identity is invalid")] + InvalidSourceIdentity, + #[error( + "artifact source identity field is empty, contains URL material, or contains controls" + )] + InvalidSourceText, + #[error("artifact source revision or digest authority is invalid")] + InvalidEvidenceAuthority, + #[error("artifact revision evidence contains an absolute retrieval URL")] + InvalidRevisionEvidence, + #[error("size-and-revision verification requires an immutable source revision")] + ImmutableRevisionRequired, + #[error("SHA-256 verification requires an expected digest")] + Sha256Required, + #[error("expected SHA-256 is inconsistent with the selected verification policy")] + UnexpectedSha256, + #[error("size-and-revision verification requires an expected size")] + ExpectedSizeRequired, + #[error("artifact file set is empty")] + EmptyFileSet, + #[error("artifact manifest version {0} is unsupported")] + UnsupportedVersion(u16), + #[error("artifact logical path is invalid")] + InvalidLogicalPath, + #[error("artifact source key is empty or contains control characters")] + InvalidSourceKey, + #[error("artifact SHA-256 must contain exactly 64 hexadecimal characters")] + InvalidSha256, + #[error("artifact manifest contains the same logical path more than once")] + DuplicateLogicalPath, + #[error("artifact manifest contains file/directory paths that collide under case-insensitive comparison")] + CollidingLogicalPath, + #[error("artifact source object has contradictory size or digest evidence")] + ConflictingSourceEvidence, + #[error("artifact manifest size total overflows u64")] + SizeOverflow, + #[error( + "artifact partial cannot resume without immutable revision or expected digest evidence" + )] + ResumeIdentityRequired, +} + +fn valid_provider(value: &str) -> bool { + !value.is_empty() + && value.len() <= 64 + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) + && value.as_bytes()[0].is_ascii_lowercase() +} + +fn validate_evidence_authority(value: &str) -> Result<(), ManifestValidationError> { + if value.is_empty() + || value.len() > 128 + || !value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || b"._-".contains(&byte) + }) + || !value.as_bytes()[0].is_ascii_lowercase() + { + return Err(ManifestValidationError::InvalidEvidenceAuthority); + } + Ok(()) +} + +fn validate_stable_text(value: &str) -> Result<(), ManifestValidationError> { + if value.is_empty() || value.len() > 16 * 1024 || value.chars().any(char::is_control) { + return Err(ManifestValidationError::InvalidRevisionEvidence); + } + if value.contains("://") { + return Err(ManifestValidationError::InvalidRevisionEvidence); + } + Ok(()) +} + +fn validate_identity_text(value: &str) -> Result<(), ManifestValidationError> { + if value.is_empty() + || value.len() > 16 * 1024 + || value.chars().any(char::is_control) + || value.contains("://") + || value.contains(['?', '#']) + { + return Err(ManifestValidationError::InvalidSourceText); + } + Ok(()) +} + +fn validate_source_key(value: &str) -> Result<(), ManifestValidationError> { + if value.is_empty() || value.len() > 16 * 1024 || value.chars().any(char::is_control) { + return Err(ManifestValidationError::InvalidSourceKey); + } + Ok(()) +} + +fn validate_logical_path(value: &str) -> Result<(), ManifestValidationError> { + // This checks source-independent traversal and common portability hazards. + // Materialization still checks the actual target filesystem's name and + // normalization collisions before creating any destination entry. + if value.is_empty() + || value.len() > 4096 + || value.starts_with('/') + || value.contains(['\\', ':']) + || value.chars().any(char::is_control) + { + return Err(ManifestValidationError::InvalidLogicalPath); + } + + for component in value.split('/') { + if component.is_empty() + || component == "." + || component == ".." + || component.ends_with(['.', ' ']) + { + return Err(ManifestValidationError::InvalidLogicalPath); + } + let stem = component + .split('.') + .next() + .unwrap_or_default() + .to_ascii_uppercase(); + if matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL") + || (stem.len() == 4 + && (stem.starts_with("COM") || stem.starts_with("LPT")) + && matches!(stem.as_bytes()[3], b'1'..=b'9')) + { + return Err(ManifestValidationError::InvalidLogicalPath); + } + } + Ok(()) +} + +fn case_insensitive_path_key(value: &str) -> String { + value + .split('/') + .map(|component| component.to_lowercase()) + .collect::>() + .join("/") +} + +fn canonical_sha256(value: &str) -> Result { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(ManifestValidationError::InvalidSha256); + } + Ok(value.to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn revision(strength: RevisionStrength) -> ArtifactRevisionEvidence { + ArtifactRevisionEvidence::new("huggingface.commit", "commit-123", strength).unwrap() + } + + fn source() -> ArtifactSourceIdentity { + ArtifactSourceIdentity::new( + "huggingface", + "org/repository", + revision(RevisionStrength::Immutable), + ) + .unwrap() + } + + fn complete_file(path: &str, source_key: &str, size: Option) -> ArtifactFile { + ArtifactFile::new( + path, + source_key, + size, + None, + FileVerificationRequirement::CompleteRepresentation, + ) + .unwrap() + } + + #[test] + fn manifest_preserves_source_keys_and_reports_unknown_totals_truthfully() { + let manifest = ArtifactManifest::new( + source(), + vec![ + complete_file( + "model/weights.safetensors", + "model/weights.safetensors", + Some(0), + ), + complete_file("model/config.json", "model/config.json?version=1", None), + ], + ) + .unwrap(); + + assert_eq!(manifest.schema_version(), CURRENT_MANIFEST_VERSION); + assert_eq!( + manifest.files()[1].source_key(), + "model/config.json?version=1" + ); + assert_eq!(manifest.total_bytes(), None); + } + + #[test] + fn manifest_rejects_empty_duplicate_and_case_or_prefix_collisions() { + assert_eq!( + ArtifactManifest::new(source(), Vec::new()).unwrap_err(), + ManifestValidationError::EmptyFileSet + ); + let duplicate = complete_file("weights.bin", "one", Some(1)); + assert_eq!( + ArtifactManifest::new(source(), vec![duplicate.clone(), duplicate]).unwrap_err(), + ManifestValidationError::DuplicateLogicalPath + ); + for paths in [ + ["Model/Weights.bin", "model/weights.BIN"], + ["a", "a/b"], + ["a/b", "a"], + ["A", "a/b"], + ] { + let files = paths + .into_iter() + .map(|path| complete_file(path, path, Some(1))) + .collect(); + assert_eq!( + ArtifactManifest::new(source(), files).unwrap_err(), + ManifestValidationError::CollidingLogicalPath + ); + } + } + + #[test] + fn manifest_rejects_contradictory_evidence_for_one_source_object() { + let first = ArtifactFile::new( + "weights-a.bin", + "objects/weights", + Some(4), + Some(Sha256Evidence::new("publisher.sha256", "a".repeat(64)).unwrap()), + FileVerificationRequirement::Sha256, + ) + .unwrap(); + let conflicting_digest = ArtifactFile::new( + "weights-b.bin", + "objects/weights", + Some(4), + Some(Sha256Evidence::new("publisher.sha256", "b".repeat(64)).unwrap()), + FileVerificationRequirement::Sha256, + ) + .unwrap(); + assert_eq!( + ArtifactManifest::new(source(), vec![first.clone(), conflicting_digest]).unwrap_err(), + ManifestValidationError::ConflictingSourceEvidence + ); + + let conflicting_size = complete_file("weights-b.bin", "objects/weights", Some(5)); + assert_eq!( + ArtifactManifest::new(source(), vec![first, conflicting_size]).unwrap_err(), + ManifestValidationError::ConflictingSourceEvidence + ); + } + + #[test] + fn manifest_rejects_unsafe_paths_but_preserves_opaque_protocol_keys() { + for path in [ + "../weights.bin", + "/weights.bin", + "a\\b", + "NUL.txt", + "folder/../x", + ] { + assert!( + ArtifactFile::new( + path, + "object", + Some(1), + None, + FileVerificationRequirement::CompleteRepresentation, + ) + .is_err(), + "{path}" + ); + } + let opaque_key = "object://bucket/key?version=1"; + let file = complete_file("weights.bin", opaque_key, Some(1)); + assert_eq!(file.source_key(), opaque_key); + } + + #[test] + fn sha256_policy_records_digest_authority_and_zero_size() { + let digest = Sha256Evidence::new("huggingface.lfs.sha256", "A".repeat(64)).unwrap(); + let file = ArtifactFile::new( + "empty", + "empty-file", + Some(0), + Some(digest), + FileVerificationRequirement::Sha256, + ) + .unwrap(); + let manifest = ArtifactManifest::new(source(), vec![file]).unwrap(); + assert_eq!(manifest.total_bytes(), Some(0)); + assert_eq!( + manifest.files()[0].expected_sha256().unwrap().value(), + "a".repeat(64) + ); + assert_eq!( + manifest.files()[0].expected_sha256().unwrap().authority(), + "huggingface.lfs.sha256" + ); + } + + #[test] + fn verification_policy_requires_its_declared_evidence() { + assert_eq!( + ArtifactFile::new( + "weights.bin", + "weights.bin", + Some(1), + None, + FileVerificationRequirement::Sha256, + ) + .unwrap_err(), + ManifestValidationError::Sha256Required + ); + let weak_source = ArtifactSourceIdentity::new( + "huggingface", + "org/repository", + revision(RevisionStrength::Weak), + ) + .unwrap(); + let file = ArtifactFile::new( + "weights.bin", + "weights.bin", + Some(1), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap(); + assert_eq!( + ArtifactManifest::new(weak_source.clone(), vec![file]).unwrap_err(), + ManifestValidationError::ImmutableRevisionRequired + ); + assert!(ArtifactManifest::new( + weak_source, + vec![complete_file("weights.bin", "weights.bin", None)] + ) + .is_ok()); + let weak_source = ArtifactSourceIdentity::new( + "huggingface", + "org/repository", + revision(RevisionStrength::Weak), + ) + .unwrap(); + let hashed_file = ArtifactFile::new( + "weights.bin", + "weights.bin", + Some(1), + Some(Sha256Evidence::new("publisher.sha256", "a".repeat(64)).unwrap()), + FileVerificationRequirement::Sha256, + ) + .unwrap(); + assert!(ArtifactManifest::new(weak_source, vec![hashed_file]).is_ok()); + } + + #[test] + fn deserialization_rejects_unknown_versions_fields_and_unvalidated_values() { + let source = serde_json::to_value(source()).unwrap(); + let value = serde_json::json!({ + "schema_version": CURRENT_MANIFEST_VERSION + 1, + "source": source, + "files": [{"logical_path":"x","source_key":"x","expected_size":1,"expected_sha256":null,"verification":"complete_representation"}] + }); + assert!(serde_json::from_value::(value).is_err()); + + let malformed = serde_json::json!({ + "schema_version": CURRENT_MANIFEST_VERSION, + "source": {"provider":"huggingface","source_id":"org/repo","revision":{"authority":"huggingface.commit","value":"commit","strength":"immutable"}}, + "files": [{"logical_path":"../x","source_key":"x","expected_size":1,"expected_sha256":null,"verification":"complete_representation"}] + }); + assert!(serde_json::from_value::(malformed).is_err()); + + let unknown_field = serde_json::json!({ + "schema_version": CURRENT_MANIFEST_VERSION, + "source": {"provider":"huggingface","source_id":"org/repo","revision":{"authority":"huggingface.commit","value":"commit","strength":"immutable"}}, + "files": [{"logical_path":"x","source_key":"x","expected_size":1,"expected_sha256":null,"verification":"complete_representation","url":"https://example.invalid/x"}] + }); + assert!(serde_json::from_value::(unknown_field).is_err()); + } + + #[test] + fn source_evidence_and_manifest_totals_reject_unsafe_values() { + assert_eq!( + ArtifactRevisionEvidence::new( + "huggingface.commit", + "https://host/object?token=secret", + RevisionStrength::Immutable, + ) + .unwrap_err(), + ManifestValidationError::InvalidRevisionEvidence + ); + let files = vec![ + complete_file("first", "first", Some(u64::MAX)), + complete_file("second", "second", Some(1)), + ]; + assert_eq!( + ArtifactManifest::new(source(), files).unwrap_err(), + ManifestValidationError::SizeOverflow + ); + } +} diff --git a/rust/crates/pumas-core/src/acquisition/mod.rs b/rust/crates/pumas-core/src/acquisition/mod.rs new file mode 100644 index 00000000..2510b32a --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/mod.rs @@ -0,0 +1,18 @@ +//! Source-neutral descriptions of selected artifact bytes. +//! +//! This module owns validated selection data and the shared HTTP response and +//! body-streaming protocol. Durable custody, lifecycle admission, publication, +//! and consumer settlement still belong to their current production owners. + +mod http; +mod manifest; + +pub(crate) use http::{ + open_http_artifact, stream_http_artifact, HttpArtifactSink, HttpAttemptHost, HttpBodyOutcome, +}; + +pub use manifest::{ + ArtifactFile, ArtifactManifest, ArtifactRevisionEvidence, ArtifactSourceIdentity, + FileVerificationRequirement, ManifestValidationError, RevisionStrength, Sha256Evidence, + CURRENT_MANIFEST_VERSION, +}; diff --git a/rust/crates/pumas-core/src/lib.rs b/rust/crates/pumas-core/src/lib.rs index 4c051b44..909561ec 100644 --- a/rust/crates/pumas-core/src/lib.rs +++ b/rust/crates/pumas-core/src/lib.rs @@ -30,6 +30,7 @@ #[cfg(feature = "uniffi")] uniffi::setup_scaffolding!(); +pub mod acquisition; pub mod cache; pub mod cancel; pub mod config; diff --git a/rust/crates/pumas-core/src/model_library/download_recovery.rs b/rust/crates/pumas-core/src/model_library/download_recovery.rs index 92d666fb..e8f1d5f7 100644 --- a/rust/crates/pumas-core/src/model_library/download_recovery.rs +++ b/rust/crates/pumas-core/src/model_library/download_recovery.rs @@ -203,24 +203,6 @@ pub(crate) struct DownloadRecoveryDestination { #[cfg(test)] type CleanupParentSync = dyn Fn(&Dir) -> io::Result<()> + Send + Sync; -impl super::partial_download::PartialDownloadFiles for DownloadRecoveryDestination { - fn file_len(&self, filename: &str) -> Result> { - Ok(self.file_len(filename)?) - } - fn part_len(&self, filename: &str) -> Result> { - Ok(self.part_len(filename)?) - } - fn rename_part_to_file(&self, filename: &str) -> Result<()> { - Ok(self.rename_part_to_file(filename)?) - } - fn remove_part(&self, filename: &str) -> Result<()> { - Ok(self.remove_part(filename)?) - } - fn remove_marker(&self) -> Result<()> { - Ok(self.remove_marker()?) - } -} - struct CreationAnchor { directory: Dir, relative: PathBuf, @@ -1158,6 +1140,71 @@ impl DownloadRecoveryDestination { Ok(()) } + /// Compare an immutable-source response staged in `.part` with an + /// existing final file without replacing either file. Identity and + /// metadata checks detect replacement and ordinary concurrent changes; + /// this is not isolation from an uncooperative writer that can modify a + /// file in place while restoring its metadata. + pub(crate) fn download_part_matches_final(&self, filename: &str) -> Result { + let (parent, name) = self.file_parent(filename, false)?; + let part_name = format!( + "{name}{}", + crate::config::NetworkConfig::DOWNLOAD_TEMP_SUFFIX + ); + let mut final_options = OpenOptions::new(); + final_options.read(true); + nofollow_options(&mut final_options); + let mut part_options = OpenOptions::new(); + part_options.read(true); + nofollow_options(&mut part_options); + let mut final_file = parent.open_with(&name, &final_options)?.into_std(); + let mut part_file = parent.open_with(&part_name, &part_options)?.into_std(); + let final_before = Metadata::from_file(&final_file)?; + let part_before = Metadata::from_file(&part_file)?; + if !final_before.is_file() || !part_before.is_file() { + return Err(invalid_capability_path().into()); + } + let final_identity = filesystem_identity(&final_before).ok_or_else(|| { + invalid_download_integrity("Platform cannot bind the existing file during comparison") + })?; + let part_identity = filesystem_identity(&part_before).ok_or_else(|| { + invalid_download_integrity("Platform cannot bind the staged file during comparison") + })?; + + let comparison = if final_before.len() == part_before.len() { + compare_file_contents(&mut final_file, &mut part_file, final_before.len()) + } else { + Ok(false) + }; + + let final_after = Metadata::from_file(&final_file)?; + let part_after = Metadata::from_file(&part_file)?; + let current_final = parent.symlink_metadata(&name)?; + let current_part = parent.symlink_metadata(&part_name)?; + let unchanged = current_final.is_file() + && current_part.is_file() + && filesystem_identity(&final_before) == Some(final_identity) + && filesystem_identity(&final_after) == Some(final_identity) + && filesystem_identity(¤t_final) == Some(final_identity) + && filesystem_identity(&part_before) == Some(part_identity) + && filesystem_identity(&part_after) == Some(part_identity) + && filesystem_identity(¤t_part) == Some(part_identity) + && final_before.len() == final_after.len() + && final_before.len() == current_final.len() + && part_before.len() == part_after.len() + && part_before.len() == current_part.len() + && final_before.modified()? == final_after.modified()? + && final_before.modified()? == current_final.modified()? + && part_before.modified()? == part_after.modified()? + && part_before.modified()? == current_part.modified()?; + if !unchanged { + return Err(invalid_download_integrity( + "Downloaded files changed during source comparison", + )); + } + Ok(comparison?) + } + pub(crate) fn open_part(&self, file: &str, append: bool) -> io::Result { let (parent, name) = self.file_parent(file, true)?; let name = format!( @@ -1275,6 +1322,26 @@ impl DownloadRecoveryDestination { } } +fn compare_file_contents( + left: &mut std::fs::File, + right: &mut std::fs::File, + length: u64, +) -> io::Result { + let mut left_buffer = [0_u8; 64 * 1024]; + let mut right_buffer = [0_u8; 64 * 1024]; + let mut remaining = length; + while remaining > 0 { + let chunk_length = remaining.min(left_buffer.len() as u64) as usize; + left.read_exact(&mut left_buffer[..chunk_length])?; + right.read_exact(&mut right_buffer[..chunk_length])?; + if left_buffer[..chunk_length] != right_buffer[..chunk_length] { + return Ok(false); + } + remaining -= chunk_length as u64; + } + Ok(true) +} + fn directory_identity(directory: &Dir) -> io::Result { filesystem_identity(&directory.dir_metadata()?).ok_or_else(invalid_capability_path) } diff --git a/rust/crates/pumas-core/src/model_library/hf/acquisition_source.rs b/rust/crates/pumas-core/src/model_library/hf/acquisition_source.rs new file mode 100644 index 00000000..d8b101b4 --- /dev/null +++ b/rust/crates/pumas-core/src/model_library/hf/acquisition_source.rs @@ -0,0 +1,173 @@ +//! Hugging Face selection and access adapter for shared acquisition types. + +use crate::acquisition::{ + ArtifactFile, ArtifactManifest, ArtifactRevisionEvidence, ArtifactSourceIdentity, + FileVerificationRequirement, RevisionStrength, Sha256Evidence, +}; +use crate::error::PumasError; +use crate::model_library::artifact_identity::DownloadRevision; +use crate::model_library::hf::types::FileToDownload; + +/// Map the already selected HF revision and file set into the source-neutral +/// contract consumed by acquisition. This records resolver claims; it does +/// not authenticate them or grant permission to execute the downloaded bytes. +pub(crate) fn manifest_for_download( + repo_id: &str, + revision: &DownloadRevision, + files: &[FileToDownload], +) -> crate::Result { + let strength = if revision.as_persisted().is_some() { + RevisionStrength::Immutable + } else { + RevisionStrength::Weak + }; + let revision_evidence = + ArtifactRevisionEvidence::new("huggingface.commit", revision.as_str(), strength) + .map_err(manifest_error)?; + let source = ArtifactSourceIdentity::new("huggingface", repo_id, revision_evidence) + .map_err(manifest_error)?; + + let files = files + .iter() + .map(|file| { + let expected_sha256 = file + .sha256 + .as_deref() + .map(|value| Sha256Evidence::new("huggingface.lfs.sha256", value)) + .transpose() + .map_err(manifest_error)?; + let verification = if expected_sha256.is_some() { + FileVerificationRequirement::Sha256 + } else if file.size.is_some() && strength == RevisionStrength::Immutable { + FileVerificationRequirement::SizeAndImmutableRevision + } else { + FileVerificationRequirement::CompleteRepresentation + }; + ArtifactFile::new( + &file.filename, + &file.filename, + file.size, + expected_sha256, + verification, + ) + .map_err(manifest_error) + }) + .collect::>>()?; + + ArtifactManifest::new(source, files).map_err(manifest_error) +} + +/// Build an ephemeral retrieval URL from a stable HF source key. Each path +/// segment is encoded by `Url`; the URL is access material and is never placed +/// in the artifact manifest or durable identity. +pub(crate) fn retrieval_url( + base: &str, + repo_id: &str, + revision: &DownloadRevision, + file: &ArtifactFile, +) -> crate::Result { + let mut url = reqwest::Url::parse(base).map_err(|_| PumasError::Config { + message: "Hugging Face download base URL is invalid".into(), + })?; + { + let mut path = url.path_segments_mut().map_err(|_| PumasError::Config { + message: "Hugging Face download base URL cannot contain path segments".into(), + })?; + path.pop_if_empty(); + path.extend(repo_id.split('/')) + .push("resolve") + .push(revision.as_str()) + .extend(file.source_key().split('/')); + } + Ok(url) +} + +fn manifest_error(error: impl std::fmt::Display) -> PumasError { + PumasError::Validation { + field: "artifact.manifest".into(), + message: error.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn manifest_preserves_pinned_lfs_evidence_and_encodes_only_retrieval_url() { + let revision = + DownloadRevision::from_commit("0123456789abcdef0123456789abcdef01234567").unwrap(); + let file = FileToDownload { + filename: "weights/model file.safetensors".into(), + size: Some(5), + sha256: Some("a".repeat(64)), + }; + let manifest = manifest_for_download("org/repo", &revision, &[file]).unwrap(); + assert_eq!(manifest.source().provider(), "huggingface"); + assert_eq!(manifest.source().source_id(), "org/repo"); + assert_eq!(manifest.source().revision().value(), revision.as_str()); + assert_eq!( + manifest.files()[0].source_key(), + "weights/model file.safetensors" + ); + assert_eq!( + manifest.files()[0].expected_sha256().unwrap().authority(), + "huggingface.lfs.sha256" + ); + let url = retrieval_url( + "https://huggingface.co", + "org/repo", + &revision, + &manifest.files()[0], + ) + .unwrap(); + assert_eq!( + url.as_str(), + "https://huggingface.co/org/repo/resolve/0123456789abcdef0123456789abcdef01234567/weights/model%20file.safetensors" + ); + } + + #[test] + fn weak_legacy_revision_does_not_claim_resume_identity() { + let manifest = manifest_for_download( + "org/repo", + &DownloadRevision::legacy_main(), + &[FileToDownload { + filename: "config.json".into(), + size: None, + sha256: None, + }], + ) + .unwrap(); + assert_eq!( + manifest.source().revision().strength(), + RevisionStrength::Weak + ); + assert_eq!( + manifest.files()[0].verification(), + FileVerificationRequirement::CompleteRepresentation + ); + assert!(!manifest.permits_resume(0)); + assert!(!manifest.requires_file_verification(0)); + } + + #[test] + fn legacy_digest_remains_a_required_verification_and_resume_identity() { + let manifest = manifest_for_download( + "org/repo", + &DownloadRevision::legacy_main(), + &[FileToDownload { + filename: "weights.bin".into(), + size: Some(4), + sha256: Some("a".repeat(64)), + }], + ) + .unwrap(); + assert_eq!( + manifest.files()[0].verification(), + FileVerificationRequirement::Sha256 + ); + assert!(manifest.permits_resume(0)); + assert!(manifest.requires_file_verification(0)); + } +} diff --git a/rust/crates/pumas-core/src/model_library/hf/download.rs b/rust/crates/pumas-core/src/model_library/hf/download.rs index 4dcf6da1..75e36936 100644 --- a/rust/crates/pumas-core/src/model_library/hf/download.rs +++ b/rust/crates/pumas-core/src/model_library/hf/download.rs @@ -20,9 +20,6 @@ use crate::model_library::download_store::{ DownloadPersistence, LifecycleCleanupDisposition, LifecycleQuarantine, LifecycleQuarantineDomain, PersistedDownload, PersistedDownloadInventory, }; -use crate::model_library::partial_download::{ - finalize_download_artifact_with_files, infer_expected_sizes_with_files, -}; use crate::model_library::sharding; use crate::model_library::types::{DownloadRequest, DownloadStatus, ModelDownloadProgress}; use crate::model_library::SelectedArtifactIdentity; @@ -68,6 +65,38 @@ struct PendingDownloadPublication { completed: tokio::sync::oneshot::Sender<()>, } +fn existing_artifact_requires_source_comparison( + file: &crate::acquisition::ArtifactFile, + observed_size: u64, + immutable_revision: bool, +) -> Result { + if file + .expected_size() + .is_some_and(|expected| expected != observed_size) + { + return Err(PumasError::Validation { + field: "download.integrity".into(), + message: format!( + "Existing selected file {} does not match its expected size; preserving it for reconciliation", + file.source_key() + ), + }); + } + if file.expected_sha256().is_some() { + return Ok(false); + } + if immutable_revision { + return Ok(true); + } + Err(PumasError::Validation { + field: "download.integrity".into(), + message: format!( + "Existing selected file {} has no whole-file digest or custody receipt and its source revision is mutable; preserving it for reconciliation", + file.source_key() + ), + }) +} + #[derive(Default)] struct DownloadPublicationState { queue: VecDeque, @@ -824,6 +853,13 @@ impl PreparedDownloadTask { } async fn finalize_pinned_restored_files(&self, context: &TaskContext) -> Result { + // A restored file with no admitted whole-file digest cannot be tied to + // the selected source from its pathname, size, or immutable revision + // alone. Leave it under its existing recovery custody for an ordinary + // worker to compare against a fresh complete source representation. + if self.files.iter().any(|file| file.sha256.is_none()) { + return Ok(false); + } for file in &self.files { if self .destination @@ -863,13 +899,63 @@ impl PreparedDownloadTask { Ok(true) } + async fn finalize_digest_verified_restored_files(&self, context: &TaskContext) -> Result { + for file in &self.files { + // Legacy snapshots can lack enough identity to prove a completed + // file. Keep them resumable/reconcilable instead of promoting or + // importing data based on aggregate byte counts alone. + if file.sha256.is_none() { + return Ok(false); + } + + if self + .destination + .file_len(context, &file.filename) + .await? + .is_some() + { + self.destination.verify_file(context, file, false).await?; + if self + .destination + .part_len(context, &file.filename) + .await? + .is_some() + { + self.destination + .remove_part(context, &file.filename) + .await?; + } + continue; + } + + let Some(part_size) = self.destination.part_len(context, &file.filename).await? else { + return Ok(false); + }; + if file.size.is_some_and(|expected| expected != part_size) { + return Ok(false); + } + if let Err(error) = self.destination.verify_file(context, file, true).await { + if file.size.is_none() && matches!(error, PumasError::HashMismatch { .. }) { + // With no size, a hash mismatch can simply mean that the + // retained part had not reached the end of the object. + return Ok(false); + } + return Err(error); + } + self.destination + .rename_part_to_file(context, &file.filename) + .await?; + } + Ok(true) + } + async fn finalize_restored( &self, context: &TaskContext, initial_status: DownloadStatus, ) -> std::result::Result<(), RestoredFinalizationError> { let mut destination_guard = Some(self.destination_lock.clone().lock_owned().await); - let (attempt, total_bytes) = { + let attempt = { let mut states = self.downloads.write().await; let state = current_worker_state( &mut states, @@ -886,7 +972,7 @@ impl PreparedDownloadTask { .attempt_id .clone(); state.status = DownloadStatus::Downloading; - (attempt, state.total_bytes) + attempt }; let provenance_destination = self.destination.capability().clone(); let provenance_revision = self.revision.clone(); @@ -909,31 +995,8 @@ impl PreparedDownloadTask { let complete = if self.revision.as_persisted().is_some() { self.finalize_pinned_restored_files(context).await? } else { - context - .run_fallible_blocking_named("finalize restored download files", { - let destination = self.destination.capability().clone(); - let filenames = self - .files - .iter() - .map(|file| file.filename.clone()) - .collect::>(); - move || -> Result { - let sizes = - infer_expected_sizes_with_files(&destination, &filenames, total_bytes)?; - Ok( - finalize_download_artifact_with_files( - &destination, - &filenames, - &sizes, - )? - .complete, - ) - } - }) - .await - .map_err(|error| { - PumasError::Other(format!("Restore finalization owner failed: {error}")) - })?? + self.finalize_digest_verified_restored_files(context) + .await? }; if !matches!(context.drain_blocking().await, Ok(0)) { return Err( @@ -958,6 +1021,7 @@ impl PreparedDownloadTask { state.files_completed = state.files.len(); } self.verify_pinned_final_files(context).await?; + self.destination.remove_marker(context).await?; let info = self .downloads .read() @@ -1480,6 +1544,25 @@ impl DownloadDestination { } } + async fn part_matches_final(&self, task_context: &TaskContext, filename: &str) -> Result { + match self { + Self::Recovery(destination) | Self::Managed(destination) => { + let destination = destination.clone(); + let filename = filename.to_string(); + task_context + .run_blocking_named("compare staged and existing artifact files", move || { + destination.download_part_matches_final(&filename) + }) + .await + .map_err(|error| { + PumasError::Other(format!( + "download source comparison task failed: {error}" + )) + })? + } + } + } + async fn remove_part(&self, task_context: &TaskContext, filename: &str) -> Result<()> { let operation = if self.is_recovery() { "remove partial download file" @@ -1700,6 +1783,94 @@ impl DownloadFile { } } +struct HuggingFaceHttpSink<'a> { + file: DownloadFile, + task_context: &'a TaskContext, +} + +#[async_trait::async_trait] +impl crate::acquisition::HttpArtifactSink for HuggingFaceHttpSink<'_> { + async fn write_all(&mut self, bytes: &[u8]) -> Result<()> { + self.file.write_all(self.task_context, bytes).await + } + + async fn flush(&mut self) -> Result<()> { + self.file.flush(self.task_context).await + } +} + +struct HuggingFaceHttpAttemptHost<'a> { + downloads: &'a Arc>>, + publications: &'a Arc, + destination_lock: &'a Arc>, + destination_guard: &'a mut Option>, + download_id: &'a str, + destination: &'a DownloadDestination, + cancel_flag: &'a AtomicBool, + pause_flag: &'a AtomicBool, + task_context: &'a TaskContext, + bytes_offset: u64, + started_at: Instant, + last_publish: Instant, +} + +#[async_trait::async_trait] +impl crate::acquisition::HttpAttemptHost for HuggingFaceHttpAttemptHost<'_> { + async fn pause_requested(&self) { + self.task_context.pause_requested(self.pause_flag).await; + } + + fn pause_requested_now(&self) -> bool { + self.pause_flag.load(Ordering::Acquire) + } + + fn cancel_requested(&self) -> bool { + #[cfg(test)] + self.task_context.observe_cancellation_check(); + self.cancel_flag.load(Ordering::Relaxed) + } + + async fn record_progress(&mut self, downloaded_for_file: u64) -> Result<()> { + let elapsed = self.started_at.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + downloaded_for_file as f64 / elapsed + } else { + 0.0 + }; + let overall_downloaded = self.bytes_offset + downloaded_for_file; + let mut downloads = self.downloads.write().await; + let state = current_worker_state( + &mut downloads, + self.download_id, + self.task_context, + &[DownloadStatus::Downloading], + )?; + state.downloaded_bytes = overall_downloaded; + state.speed = speed; + state.progress = state + .total_bytes + .map(|total| overall_downloaded as f32 / total as f32) + .unwrap_or(0.0); + drop(downloads); + + if self.last_publish.elapsed() >= DOWNLOAD_PROGRESS_PUBLISH_INTERVAL { + publish_worker_snapshot_and_revalidate( + self.publications, + self.downloads, + self.download_id, + self.task_context, + self.destination, + self.destination_lock, + self.destination_guard, + &[DownloadStatus::Downloading], + ) + .await?; + self.last_publish = Instant::now(); + } + Ok(()) + } +} + async fn recovery_filesystem_operation( task_context: &TaskContext, operation: &'static str, @@ -3895,6 +4066,9 @@ impl HuggingFaceClient { use crate::config::NetworkConfig; use crate::network::RetryConfig; + let artifact_manifest = + super::acquisition_source::manifest_for_download(repo_id, revision, files)?; + #[cfg(test)] task_context.observe_worker_projection("worker-entry"); let mut destination_guard = Some(destination_lock.clone().lock_owned().await); @@ -4088,69 +4262,86 @@ impl HuggingFaceClient { for (file_idx, file_info) in files.iter().enumerate() { let filename = &file_info.filename; - + let artifact_file = + artifact_manifest + .files() + .get(file_idx) + .ok_or_else(|| PumasError::Config { + message: "Resolved Hugging Face artifact manifest lost a selected file" + .into(), + })?; + let verify_selected_file = artifact_manifest.requires_file_verification(file_idx); // Ensure parent directory exists (needed for subdirectory files // like transformer/model.safetensors in diffusion repos) destination.prepare_file(&task_context, filename).await?; - // Skip files that already exist (completed from previous run) + let mut compare_existing_source = false; + // Digest-backed local files can be verified directly. Without a + // digest, only an immutable source can prove an existing file by + // fetching the selected bytes and comparing them under custody. if let Some(existing_size) = destination.file_len(&task_context, filename).await? { - if revision.as_persisted().is_some() { + compare_existing_source = existing_artifact_requires_source_comparison( + artifact_file, + existing_size, + artifact_manifest.source().revision().strength() + == crate::acquisition::RevisionStrength::Immutable, + )?; + if !compare_existing_source { destination .verify_file(&task_context, file_info, false) .await?; - } - if destination - .part_len(&task_context, filename) - .await? - .is_some() - { - if let Err(error) = destination.remove_part(&task_context, filename).await { - warn!( - "Failed to remove stale partial file for {}/{}: {}", - repo_id, filename, error - ); + if destination + .part_len(&task_context, filename) + .await? + .is_some() + { + if let Err(error) = destination.remove_part(&task_context, filename).await { + warn!( + "Failed to remove stale partial file for {}/{}: {}", + repo_id, filename, error + ); + } } - } - bytes_offset += existing_size; - info!( - "Skipping already-downloaded file {}/{} ({} bytes)", - repo_id, filename, existing_size - ); + bytes_offset += existing_size; + info!( + "Skipping already-downloaded file {}/{} ({} bytes)", + repo_id, filename, existing_size + ); - // Update state - #[cfg(test)] - task_context.observe_worker_projection("before-existing-file-projection"); - { - let mut downloads = downloads.write().await; - let state = current_worker_state( - &mut downloads, + // Update state + #[cfg(test)] + task_context.observe_worker_projection("before-existing-file-projection"); + { + let mut downloads = downloads.write().await; + let state = current_worker_state( + &mut downloads, + download_id, + &task_context, + &[DownloadStatus::Downloading], + )?; + state.files_completed = file_idx + 1; + state.downloaded_bytes = bytes_offset; + if let Some(total) = state.total_bytes { + state.progress = bytes_offset as f32 / total as f32; + } + } + #[cfg(test)] + if file_idx + 1 == files.len() { + task_context.observe_worker_projection("terminal-cleanup-committed"); + } + publish_worker_snapshot_and_revalidate( + &download_publications, + &downloads, download_id, &task_context, + destination, + &destination_lock, + &mut destination_guard, &[DownloadStatus::Downloading], - )?; - state.files_completed = file_idx + 1; - state.downloaded_bytes = bytes_offset; - if let Some(total) = state.total_bytes { - state.progress = bytes_offset as f32 / total as f32; - } - } - #[cfg(test)] - if file_idx + 1 == files.len() { - task_context.observe_worker_projection("terminal-cleanup-committed"); + ) + .await?; + continue; } - publish_worker_snapshot_and_revalidate( - &download_publications, - &downloads, - download_id, - &task_context, - destination, - &destination_lock, - &mut destination_guard, - &[DownloadStatus::Downloading], - ) - .await?; - continue; } // Fire aux-complete callback at the boundary between auxiliary and weight files. @@ -4281,13 +4472,13 @@ impl HuggingFaceClient { let download_base = HF_HUB_BASE; #[cfg(test)] let download_base = download_base_url.as_deref().unwrap_or(download_base); - let url = format!( - "{}/{}/resolve/{}/{}", + let url = super::acquisition_source::retrieval_url( download_base, repo_id, - revision.as_str(), - filename - ); + revision, + artifact_file, + )? + .to_string(); let mut last_error: Option = None; @@ -4338,18 +4529,33 @@ impl HuggingFaceClient { } // Determine resume offset from existing .part file - let resume_from_byte = destination + let mut resume_from_byte = destination .part_len(&task_context, filename) .await? .unwrap_or(0); - if destination - .finalize_complete_part_file( - &task_context, - file_info, - revision.as_persisted().is_some(), - ) - .await? + if compare_existing_source && attempt == 1 && resume_from_byte > 0 { + // A retained partial has no durable evidence tying its + // prefix to this source. Start this comparison from a + // fresh complete representation; retries in this same + // supervised worker may resume bytes it just received. + destination.remove_part(&task_context, filename).await?; + resume_from_byte = 0; + } + + if resume_from_byte > 0 && !artifact_manifest.permits_resume(file_idx) { + info!( + "Discarding partial file {}/{} because its selected source has no resume identity", + repo_id, filename + ); + destination.remove_part(&task_context, filename).await?; + resume_from_byte = 0; + } + + if !compare_existing_source + && destination + .finalize_complete_part_file(&task_context, file_info, verify_selected_file) + .await? { file_completed = true; break; @@ -4397,7 +4603,8 @@ impl HuggingFaceClient { &url, destination, filename, - file_info.size, + &artifact_manifest, + file_idx, resume_from_byte, bytes_offset, &cancel_flag, @@ -4424,19 +4631,34 @@ impl HuggingFaceClient { ) .await; } - if revision.as_persisted().is_some() { + if verify_selected_file { destination .verify_file(&task_context, file_info, true) .await?; } - // Rename .part to final path atomically - destination - .rename_part_to_file(&task_context, filename) - .await - .map_err(|e| PumasError::DownloadFailed { - url: url.clone(), - message: format!("Failed to rename temp file: {}", e), - })?; + if compare_existing_source { + if !destination + .part_matches_final(&task_context, filename) + .await? + { + return Err(PumasError::Validation { + field: "download.integrity".into(), + message: format!( + "Existing selected file {filename} differs from the immutable source artifact; preserving both files for reconciliation" + ), + }); + } + destination.remove_part(&task_context, filename).await?; + } else { + // Rename .part to final path atomically. + destination + .rename_part_to_file(&task_context, filename) + .await + .map_err(|e| PumasError::DownloadFailed { + url: "Hugging Face artifact".into(), + message: format!("Failed to rename temp file: {}", e), + })?; + } file_completed = true; break; @@ -4580,8 +4802,8 @@ impl HuggingFaceClient { ); } - if revision.as_persisted().is_some() { - for file in files { + for (file_idx, file) in files.iter().enumerate() { + if artifact_manifest.requires_file_verification(file_idx) { destination.verify_file(&task_context, file, false).await?; } } @@ -4710,7 +4932,8 @@ impl HuggingFaceClient { /// Execute a single download attempt, optionally resuming from a byte offset. /// - /// `file_size_expected` is the expected size of this individual file. + /// `artifact_manifest` is the validated source-neutral selection for this + /// download, and `file_index` identifies the selected file for resume policy. /// `bytes_offset` is bytes already downloaded from previous files in a multi-file download. /// Overall progress is calculated as `(bytes_offset + file_downloaded) / overall_total`. #[allow(clippy::too_many_arguments)] @@ -4724,7 +4947,8 @@ impl HuggingFaceClient { url: &str, destination: &DownloadDestination, filename: &str, - file_size_expected: Option, + artifact_manifest: &crate::acquisition::ArtifactManifest, + file_index: usize, resume_from_byte: u64, bytes_offset: u64, cancel_flag: &Arc, @@ -4733,90 +4957,65 @@ impl HuggingFaceClient { auth_header: Option<&str>, task_context: &TaskContext, ) -> Result<()> { - use futures::StreamExt; - - let mut request = client.get(url); - if let Some(auth) = auth_header { - request = request.header("Authorization", auth); - } if resume_from_byte > 0 { - request = request.header("Range", format!("bytes={}-", resume_from_byte)); info!("Resuming download from byte {}", resume_from_byte); } let response = tokio::select! { biased; _ = task_context.pause_requested(pause_flag) => return Err(PumasError::DownloadPaused), - response = request.send() => response, - } - .map_err(|e| PumasError::Network { - message: format!("Download request failed: {}", e), - cause: Some(e.to_string()), - })?; - - let status = response.status(); - - // Check for non-success responses (but 206 Partial Content is expected for resume) - if !status.is_success() && status != reqwest::StatusCode::PARTIAL_CONTENT { - return Err(PumasError::DownloadFailed { - url: url.to_string(), - message: format!("HTTP {}", status), - }); - } + response = crate::acquisition::open_http_artifact( + client, + url, + artifact_manifest, + file_index, + resume_from_byte, + auth_header, + ) => response, + }?; - // Determine if we're actually resuming - let is_resuming = resume_from_byte > 0 && status == reqwest::StatusCode::PARTIAL_CONTENT; + let is_resuming = response.resumed; if resume_from_byte > 0 && !is_resuming { - warn!("Server does not support Range requests, restarting from zero"); + warn!("Server ignored Range; replacing the partial file from byte zero"); } - // Per-file total for completeness verification - let file_total = if is_resuming { - file_size_expected - } else { - response.content_length().or(file_size_expected) - }; - // Open file: append for resume, create for fresh start - let mut file = destination + let file = destination .open_part(task_context, filename, is_resuming) .await?; - - let mut downloaded: u64 = if is_resuming { resume_from_byte } else { 0 }; - let mut stream = response.bytes_stream(); - let start_time = std::time::Instant::now(); - let mut last_publish = Instant::now(); - - loop { - let chunk = tokio::select! { - biased; - _ = task_context.pause_requested(pause_flag) => { - file.flush(task_context).await?; - return Err(PumasError::DownloadPaused); - } - chunk = stream.next() => chunk, - }; - let Some(chunk) = chunk else { - break; + let (sink, outcome) = { + let mut sink = HuggingFaceHttpSink { file, task_context }; + let mut host = HuggingFaceHttpAttemptHost { + downloads, + publications: download_publications, + destination_lock, + destination_guard, + download_id, + destination, + cancel_flag, + pause_flag, + task_context, + bytes_offset, + started_at: Instant::now(), + last_publish: Instant::now(), }; - #[cfg(test)] - task_context.observe_cancellation_check(); - if cancel_flag.load(Ordering::Relaxed) { - drop(file); - let _ = destination.remove_part(task_context, filename).await; - // Terminal cancellation belongs to the generation-replacing - // finalizer, which observes this worker and its nested work. - return Err(PumasError::DownloadCancelled); - } - - if pause_flag.load(Ordering::Relaxed) { - file.flush(task_context).await?; - drop(file); - // Preserve .part file for resume + let outcome = crate::acquisition::stream_http_artifact( + response, + resume_from_byte, + &mut sink, + &mut host, + ) + .await?; + (sink, outcome) + }; + drop(sink); + match outcome { + crate::acquisition::HttpBodyOutcome::Complete { downloaded: _ } => Ok(()), + crate::acquisition::HttpBodyOutcome::Paused => { #[cfg(test)] task_context.observe_worker_projection("pause-during-stream"); - return Self::settle_worker_pause( + Self::settle_worker_pause( downloads, download_publications, download_id, @@ -4824,73 +5023,15 @@ impl HuggingFaceClient { persistence, destination_guard, ) - .await; - } - - let chunk = chunk.map_err(|e| PumasError::Network { - message: format!("Download stream error: {}", e), - cause: Some(e.to_string()), - })?; - - file.write_all(task_context, &chunk).await?; - downloaded += chunk.len() as u64; - - // Update overall progress (bytes_offset accounts for completed files) - let elapsed = start_time.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - downloaded as f64 / elapsed - } else { - 0.0 - }; - - let overall_downloaded = bytes_offset + downloaded; - - let mut download_states = downloads.write().await; - let state = current_worker_state( - &mut download_states, - download_id, - task_context, - &[DownloadStatus::Downloading], - )?; - state.downloaded_bytes = overall_downloaded; - state.speed = speed; - state.progress = if let Some(total) = state.total_bytes { - overall_downloaded as f32 / total as f32 - } else { - 0.0 - }; - drop(download_states); - - if last_publish.elapsed() >= DOWNLOAD_PROGRESS_PUBLISH_INTERVAL { - publish_worker_snapshot_and_revalidate( - download_publications, - downloads, - download_id, - task_context, - destination, - destination_lock, - destination_guard, - &[DownloadStatus::Downloading], - ) - .await?; - last_publish = Instant::now(); + .await } - } - - file.flush(task_context).await?; - drop(file); - - // Verify this file's download completeness - if let Some(total) = file_total { - if downloaded != total { - return Err(PumasError::Network { - message: format!("Incomplete download: got {} of {} bytes", downloaded, total), - cause: None, - }); + crate::acquisition::HttpBodyOutcome::Cancelled => { + let _ = destination.remove_part(task_context, filename).await; + // Terminal cancellation belongs to the generation-replacing + // finalizer, which observes this worker and its nested work. + Err(PumasError::DownloadCancelled) } } - - Ok(()) } async fn persist_status_update_owned( @@ -6277,6 +6418,12 @@ impl HuggingFaceClient { })?, ) }; + if revision.as_persisted().is_none() && files.iter().any(|file| file.sha256.is_none()) { + return Err(PumasError::Validation { + field: "download.resume".into(), + message: "Cannot resume a mutable source selection without a digest for every file; start a fresh selection to reacquire the current source".into(), + }); + } let cancel_flag = Arc::new(AtomicBool::new(false)); let pause_flag = Arc::new(AtomicBool::new(false)); let mut prepared_download = self @@ -6652,7 +6799,7 @@ mod tests { vec![LfsFileInfo { filename: "second.gguf".into(), size: 8, - sha256: "a".repeat(64), + sha256: "eebbf6457e46a7f63acdf9b97390f790ba443d60cfa44b607da7e5c40aa1cc1d".into(), }], Vec::new(), ); @@ -6765,7 +6912,7 @@ mod tests { vec![LfsFileInfo { filename: "second.gguf".into(), size: 8, - sha256: "a".repeat(64), + sha256: "eebbf6457e46a7f63acdf9b97390f790ba443d60cfa44b607da7e5c40aa1cc1d".into(), }], Vec::new(), ); @@ -7132,7 +7279,7 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".into(), size: 8, - sha256: "a".repeat(64), + sha256: "9c56cc51b374c3ba189210d5b6d4bf57790d351c96c47c02190ecf1e430635ab".into(), }], Vec::new(), ); @@ -7229,7 +7376,7 @@ mod tests { if paused.is_ok() && matches!(stall, StalledResponse::ImmediateResume) { // Do not drain the paused generation first: public Paused is the // promise that callers may immediately request a successor. - assert_resumed_partial_completes(&mut client, &id, &destination).await; + assert_resumed_partial_completes(&mut client, &id, &destination, true).await; release_sender.send(()).unwrap(); assert!(server.await.unwrap()); return; @@ -7277,7 +7424,7 @@ mod tests { Some(DownloadStatus::Paused) ); if stall_body { - assert_resumed_partial_completes(&mut restarted, &id, &destination).await; + assert_resumed_partial_completes(&mut restarted, &id, &destination, true).await; } } @@ -7285,6 +7432,7 @@ mod tests { client: &mut HuggingFaceClient, id: &str, destination: &Path, + expect_range: bool, ) { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -7296,11 +7444,14 @@ mod tests { assert!(headers.len() < 4096); headers.push(socket.read_u8().await.unwrap()); } - assert!(String::from_utf8(headers) - .unwrap() - .to_ascii_lowercase() - .contains("range: bytes=5-")); - socket.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 3\r\nContent-Range: bytes 5-7/8\r\nConnection: close\r\n\r\nfgh").await.unwrap(); + let headers = String::from_utf8(headers).unwrap().to_ascii_lowercase(); + if expect_range { + assert!(headers.contains("range: bytes=5-")); + socket.write_all(b"HTTP/1.1 206 Partial Content\r\nContent-Length: 3\r\nContent-Range: bytes 5-7/8\r\nConnection: close\r\n\r\nfgh").await.unwrap(); + } else { + assert!(!headers.contains("range:")); + socket.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\nabcdefgh").await.unwrap(); + } }); assert!(client.resume_download(id).await.unwrap()); tokio::time::timeout(Duration::from_secs(3), async { @@ -7342,7 +7493,8 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".into(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211" + .into(), }], Vec::new(), ); @@ -7498,7 +7650,8 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".into(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211" + .into(), }], Vec::new(), ); @@ -7772,6 +7925,7 @@ mod tests { }; use crate::ModelRecord; use serde_json::json; + use sha2::Digest; use std::sync::atomic::AtomicUsize; use tempfile::TempDir; @@ -7967,63 +8121,332 @@ mod tests { } } - #[test] - fn progress_library_identity_comes_only_from_the_bound_destination() { + fn set_test_file_digest(state: &mut DownloadState, filename: &str, bytes: &[u8]) { + let digest = hex::encode(sha2::Sha256::digest(bytes)); + let file = state + .files + .iter_mut() + .find(|file| file.filename == filename) + .expect("digest fixture must identify one selected file"); + file.sha256 = Some(digest.clone()); + if state.files.len() == 1 && state.filename == filename { + state.known_sha256 = Some(digest); + } + } + + #[test] + fn progress_library_identity_comes_only_from_the_bound_destination() { + let temp = TempDir::new().unwrap(); + let verified = + verified_recovery(temp.path(), "different-publisher/model", &["weights.gguf"]); + let mut state = recovery_test_state( + &verified, + "identity-download", + DownloadStatus::Paused, + false, + ); + assert_eq!( + progress_from_state(&state).library_model_id.as_deref(), + Some("llm/acme/model") + ); + state.make_managed_for_test(); + assert_eq!( + progress_from_state(&state).library_model_id.as_deref(), + Some("llm/acme/model") + ); + let snapshot = persisted_recovery_test_state(&state); + let restored = DownloadState::from_persisted( + &snapshot, + 2, + state.destination.clone().unwrap(), + DownloadRevision::legacy_main(), + ); + assert_eq!( + progress_from_state(&restored).library_model_id.as_deref(), + Some("llm/acme/model") + ); + state.destination = None; + assert!( + progress_from_state(&state).library_model_id.is_none(), + "ambient path and repository labels cannot invent association" + ); + } + + fn persisted_recovery_test_state(state: &DownloadState) -> PersistedDownload { + PersistedDownload { + download_id: state.download_id.clone(), + repo_id: state.repo_id.clone(), + filename: state.filename.clone(), + filenames: state + .files + .iter() + .map(|file| file.filename.clone()) + .collect(), + dest_dir: state.dest_dir.clone(), + total_bytes: state.total_bytes, + status: state.status, + download_request: state.download_request.clone().unwrap(), + revision: state.revision.as_persisted().map(str::to_owned), + created_at: "2026-09-03T00:00:00Z".to_string(), + known_sha256: state.known_sha256.clone(), + huggingface_evidence: None, + } + } + + #[test] + fn restored_primary_integrity_is_hydrated_only_for_an_unambiguous_single_file() { + let temp = TempDir::new().unwrap(); + let single = verified_recovery(temp.path(), "acme/single", &["weights.gguf"]); + let state = recovery_test_state(&single, "single-integrity", DownloadStatus::Paused, false); + let mut snapshot = persisted_recovery_test_state(&state); + snapshot.known_sha256 = Some("a".repeat(64)); + let restored = DownloadState::from_persisted( + &snapshot, + 2, + state.destination.clone().unwrap(), + DownloadRevision::legacy_main(), + ); + assert_eq!(restored.files[0].size, Some(4)); + let primary_sha256 = "a".repeat(64); + assert_eq!( + restored.files[0].sha256.as_deref(), + Some(primary_sha256.as_str()) + ); + + let multi = verified_recovery(temp.path(), "acme/multi", &["first.gguf", "second.gguf"]); + let state = recovery_test_state(&multi, "multi-integrity", DownloadStatus::Paused, false); + let mut snapshot = persisted_recovery_test_state(&state); + snapshot.known_sha256 = Some("b".repeat(64)); + let restored = DownloadState::from_persisted( + &snapshot, + 2, + state.destination.clone().unwrap(), + DownloadRevision::legacy_main(), + ); + assert!(restored.files.iter().all(|file| file.size.is_none())); + assert!(restored.files.iter().all(|file| file.sha256.is_none())); + } + + #[test] + fn size_and_immutable_revision_do_not_authorize_reuse_without_a_local_digest() { + let revision = + DownloadRevision::from_commit("0123456789abcdef0123456789abcdef01234567").unwrap(); + let manifest = super::super::acquisition_source::manifest_for_download( + "acme/model", + &revision, + &[FileToDownload { + filename: "weights.gguf".into(), + size: Some(4), + sha256: None, + }], + ) + .unwrap(); + + assert!( + existing_artifact_requires_source_comparison(&manifest.files()[0], 4, true).unwrap() + ); + assert!( + existing_artifact_requires_source_comparison(&manifest.files()[0], 5, true) + .unwrap_err() + .to_string() + .contains("expected size") + ); + assert!( + existing_artifact_requires_source_comparison(&manifest.files()[0], 4, false) + .unwrap_err() + .to_string() + .contains("source revision is mutable") + ); + } + + #[tokio::test] + async fn existing_hashless_file_uses_fresh_source_and_is_not_imported_after_reopen() { + use tokio::io::AsyncWriteExt; + let temp = TempDir::new().unwrap(); - let verified = - verified_recovery(temp.path(), "different-publisher/model", &["weights.gguf"]); - let mut state = recovery_test_state( - &verified, - "identity-download", - DownloadStatus::Paused, - false, + let library = Arc::new( + crate::model_library::ModelLibrary::new(temp.path().join("library")) + .await + .unwrap(), + ); + let destination = library.build_model_path("vision", "acme", "model"); + let mut client = configured_download_client(temp.path().join("cache")).unwrap(); + client + .configure_download_destination_root(library.library_root()) + .unwrap(); + client.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( + library.clone(), + ))); + let revision = + DownloadRevision::from_commit("0123456789abcdef0123456789abcdef01234567").unwrap(); + let revision_value = revision.as_str().to_owned(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let request = read_pinned_test_request(&mut socket).await; + assert!(request.starts_with(&format!( + "GET /acme/model/resolve/{revision_value}/model.onnx HTTP/1.1" + ))); + assert!( + !request.to_ascii_lowercase().contains("\r\nrange:"), + "a retained prefix cannot authorize a ranged source comparison" + ); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\nConnection: close\r\n\r\nABCD") + .await + .unwrap(); + }); + client.set_test_download_base_url(base_url); + *client.auth_token.write().await = None; + cache_pinned_repo_tree( + &client, + "acme/model", + &revision, + Vec::new(), + vec!["model.onnx".into()], ); + std::fs::create_dir_all(&destination).unwrap(); + let existing = b"xxCD"; + std::fs::write(destination.join("model.onnx"), existing).unwrap(); + // This prefix would combine with a ranged suffix from ABCD to make + // the stale final appear equal despite belonging to different bytes. + std::fs::write(destination.join("model.onnx.part"), b"xx").unwrap(); + let mut request = recovery_test_request("acme/model", &["model.onnx".into()]); + request.filenames = None; + request.filename = Some("model.onnx".into()); + request.model_type = Some("vision".into()); + request.pipeline_tag = Some("image-classification".into()); + let download_id = client + .start_download_at_revision(&request, &destination, None, revision) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + loop { + client.observe_finished_download_tasks().await; + let settled = + client + .downloads + .read() + .await + .get(&download_id) + .is_some_and(|state| { + state.status == DownloadStatus::Error && !state.task_registered + }); + if settled { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("a differing immutable source file must fail before import"); + server.await.unwrap(); + + let states = client.downloads.read().await; + let state = states.get(&download_id).unwrap(); + assert!(state + .error + .as_deref() + .unwrap() + .contains("differs from the immutable source artifact")); assert_eq!( - progress_from_state(&state).library_model_id.as_deref(), - Some("llm/acme/model") + std::fs::read(destination.join("model.onnx")).unwrap(), + existing ); - state.make_managed_for_test(); assert_eq!( - progress_from_state(&state).library_model_id.as_deref(), - Some("llm/acme/model") + std::fs::read(destination.join("model.onnx.part")).unwrap(), + b"ABCD" ); - let snapshot = persisted_recovery_test_state(&state); - let restored = DownloadState::from_persisted( - &snapshot, - 2, - state.destination.clone().unwrap(), - DownloadRevision::legacy_main(), + assert!(library.load_metadata(&destination).unwrap().is_none()); + drop(states); + drop(client); + + // A failed live comparison must remain failed after restart. The + // pinned revision and byte count cannot promote the hashless final. + let mut reopened = configured_download_client(temp.path().join("cache")).unwrap(); + reopened + .configure_download_destination_root(library.library_root()) + .unwrap(); + reopened.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( + library.clone(), + ))); + assert!(reopened + .restore_persisted_downloads() + .await + .unwrap() + .is_empty()); + assert_eq!( + reopened.get_download_status(&download_id).await, + Some(DownloadStatus::Error) ); assert_eq!( - progress_from_state(&restored).library_model_id.as_deref(), - Some("llm/acme/model") + std::fs::read(destination.join("model.onnx")).unwrap(), + existing ); - state.destination = None; - assert!( - progress_from_state(&state).library_model_id.is_none(), - "ambient path and repository labels cannot invent association" + assert_eq!( + std::fs::read(destination.join("model.onnx.part")).unwrap(), + b"ABCD" ); + assert!(destination.join(".pumas_download").exists()); + assert!(library.load_metadata(&destination).unwrap().is_none()); } - fn persisted_recovery_test_state(state: &DownloadState) -> PersistedDownload { - PersistedDownload { - download_id: state.download_id.clone(), - repo_id: state.repo_id.clone(), - filename: state.filename.clone(), - filenames: state - .files - .iter() - .map(|file| file.filename.clone()) - .collect(), - dest_dir: state.dest_dir.clone(), - total_bytes: state.total_bytes, - status: state.status, - download_request: state.download_request.clone().unwrap(), - revision: state.revision.as_persisted().map(str::to_owned), - created_at: "2026-09-03T00:00:00Z".to_string(), - known_sha256: None, - huggingface_evidence: None, - } + #[tokio::test] + async fn existing_same_size_file_with_wrong_digest_is_preserved_and_rejected() { + let temp = TempDir::new().unwrap(); + let client = configured_download_client(temp.path().join("cache")).unwrap(); + let revision = + DownloadRevision::from_commit("0123456789abcdef0123456789abcdef01234567").unwrap(); + cache_pinned_repo_tree( + &client, + "acme/model", + &revision, + vec![LfsFileInfo { + filename: "weights.gguf".into(), + size: 4, + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".into(), + }], + Vec::new(), + ); + let destination = temp.path().join("library/model"); + std::fs::create_dir_all(&destination).unwrap(); + let existing = b"evil"; + std::fs::write(destination.join("weights.gguf"), existing).unwrap(); + let request = recovery_test_request("acme/model", &["weights.gguf".into()]); + + let download_id = client + .start_download_at_revision(&request, &destination, None, revision) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(3), async { + loop { + client.observe_finished_download_tasks().await; + let settled = + client + .downloads + .read() + .await + .get(&download_id) + .is_some_and(|state| { + state.status == DownloadStatus::Error && !state.task_registered + }); + if settled { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("digest mismatch must fail before the existing file is reused"); + + let states = client.downloads.read().await; + let state = states.get(&download_id).unwrap(); + assert!(state.error.as_deref().unwrap().contains("Hash mismatch")); + assert_eq!( + std::fs::read(destination.join("weights.gguf")).unwrap(), + existing + ); } fn install_promotable_recovery_transition( @@ -8971,6 +9394,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "ambient-resume-missing-row"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Paused, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); persist_state_fixture(&persistence, &mut state); client @@ -9457,7 +9881,7 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".into(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".into(), }], Vec::new(), ); @@ -9927,6 +10351,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "resume-stale-transition-check"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Paused, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); persist_state_fixture(&persistence, &mut state); let mut downloads_guard = client.downloads.write().await; @@ -10402,7 +10827,7 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".to_string(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".into(), }], vec!["config.json".to_string()], ); @@ -10490,7 +10915,7 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".to_string(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".into(), }], Vec::new(), ); @@ -10592,7 +11017,7 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".to_string(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".into(), }], Vec::new(), ); @@ -10874,6 +11299,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "ambient-completion-order"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Queued, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); let cancel_flag = state.cancel_flag.clone(); let pause_flag = state.pause_flag.clone(); @@ -11017,7 +11443,7 @@ mod tests { vec![LfsFileInfo { filename: "model.onnx".into(), size: 4, - sha256: "a".repeat(64), + sha256: "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7".into(), }], Vec::new(), ); @@ -11290,7 +11716,7 @@ mod tests { vec![LfsFileInfo { filename: "successor.onnx".into(), size: 4, - sha256: "b".repeat(64), + sha256: "c6c1c9a9c8543f1e4cd980064cf1625eeb61a90703b2464fff039f21682508b3".into(), }], Vec::new(), ); @@ -11492,6 +11918,7 @@ mod tests { sha256: None, }, ]; + set_test_file_digest(&mut state, "config.json", b"{}"); state.filename = "config.json".to_string(); state.total_bytes = Some(4); let cancel_flag = state.cancel_flag.clone(); @@ -11603,6 +12030,7 @@ mod tests { sha256: None, }, ]; + set_test_file_digest(&mut state, "config.json", b"{}"); state.filename = "config.json".to_string(); state.total_bytes = Some(4); let cancel_flag = state.cancel_flag.clone(); @@ -11725,6 +12153,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "completion-callback-panic"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Queued, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); let cancel_flag = state.cancel_flag.clone(); let pause_flag = state.pause_flag.clone(); @@ -11806,6 +12235,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "ambient-completion-failure"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Queued, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); let cancel_flag = state.cancel_flag.clone(); let pause_flag = state.pause_flag.clone(); @@ -12114,6 +12544,9 @@ mod tests { .unwrap(); let download_id = "ordinary-resume"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Paused, false); + let done_sha256 = "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211"; + state.files[0].sha256 = Some(done_sha256.into()); + state.known_sha256 = Some(done_sha256.into()); state.make_managed_for_test(); persist_state_fixture(&persistence, &mut state); client @@ -12183,6 +12616,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "cancelled-ordinary-resume"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Paused, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); persist_state_fixture(&persistence, &mut state); client @@ -12248,6 +12682,7 @@ mod tests { let download_id = format!("cancelled-{mode}-resume-after-commit"); let mut state = recovery_test_state(&verified, &download_id, DownloadStatus::Paused, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); if !recovery { state.make_managed_for_test(); persist_state_fixture(&persistence, &mut state); @@ -12701,7 +13136,7 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".to_string(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".into(), }], Vec::new(), ); @@ -12823,7 +13258,8 @@ mod tests { vec![LfsFileInfo { filename: "weights.gguf".to_string(), size: 4, - sha256: "a".repeat(64), + sha256: "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211" + .into(), }], Vec::new(), ); @@ -16853,6 +17289,7 @@ mod tests { follower.filename = "follower.gguf".into(); follower.filenames = vec![follower.filename.clone()]; follower.download_request = recovery_test_request(&follower.repo_id, &follower.filenames); + follower.known_sha256 = Some(hex::encode(sha2::Sha256::digest(b"complete"))); admit_snapshot_at_root(&store, &head, root); admit_snapshot_at_root(&store, &follower, root); std::fs::write(destination.join("head.gguf.part"), b"old").unwrap(); @@ -17691,7 +18128,7 @@ mod tests { } #[tokio::test] - async fn test_restore_auto_finalizes_byte_complete_persisted_download() { + async fn restore_does_not_finalize_byte_complete_download_without_digest() { let tmp = TempDir::new().unwrap(); let dest_dir = tmp.path().join("library").join("llm/test/ready-model"); std::fs::create_dir_all(&dest_dir).unwrap(); @@ -17740,6 +18177,69 @@ mod tests { .unwrap(); let completed = client.restore_persisted_downloads().await.unwrap(); + assert!(completed.is_empty()); + assert_eq!(client.list_downloads().await.len(), 1); + assert_eq!(persistence.load_all().len(), 1); + assert!(!dest_dir.join("model.gguf").exists()); + assert_eq!( + std::fs::read(dest_dir.join("model.gguf.part")).unwrap(), + b"done" + ); + assert!(dest_dir.join(".pumas_download").exists()); + } + + #[tokio::test] + async fn restore_finalizes_single_file_only_after_digest_verification() { + let tmp = TempDir::new().unwrap(); + let dest_dir = tmp.path().join("library").join("llm/test/ready-model"); + std::fs::create_dir_all(&dest_dir).unwrap(); + std::fs::write(dest_dir.join("model.gguf.part"), b"done").unwrap(); + std::fs::write(dest_dir.join(".pumas_download"), b"{}").unwrap(); + + let persistence = Arc::new(DownloadPersistence::new(tmp.path())); + admit_snapshot_at_root( + &persistence, + &PersistedDownload { + download_id: "verified-ready-download".to_string(), + repo_id: "owner/model".to_string(), + filename: "model.gguf".to_string(), + filenames: vec!["model.gguf".to_string()], + dest_dir: dest_dir.clone(), + total_bytes: Some(4), + status: DownloadStatus::Error, + revision: None, + download_request: DownloadRequest { + repo_id: "owner/model".to_string(), + family: "test".to_string(), + official_name: "Ready Model".to_string(), + model_type: Some("llm".to_string()), + quant: None, + filename: Some("model.gguf".to_string()), + filenames: None, + pipeline_tag: Some("text-generation".to_string()), + bundle_format: None, + pipeline_class: None, + release_date: None, + download_url: None, + model_card_json: None, + license_status: None, + }, + created_at: chrono::Utc::now().to_rfc3339(), + known_sha256: Some( + "a4c3ed04a95a3da14a9d235c83d868bed7c0f45cf7f3faa751ee8f50598d2211".to_string(), + ), + huggingface_evidence: None, + }, + tmp.path(), + ); + + let mut client = HuggingFaceClient::new(tmp.path()).unwrap(); + client.set_persistence(persistence.clone()); + client + .configure_download_destination_root(tmp.path()) + .unwrap(); + let completed = client.restore_persisted_downloads().await.unwrap(); + assert!(client.list_downloads().await.is_empty()); assert_eq!(completed.len(), 1); assert!(persistence.load_all().is_empty()); @@ -17756,6 +18256,7 @@ mod tests { let download_id = "restore-completion-handoff"; let mut original = recovery_test_state(&verified, download_id, DownloadStatus::Paused, false); + set_test_file_digest(&mut original, "original.gguf", b"done"); original.make_managed_for_test(); let persistence = Arc::new(DownloadPersistence::new(temp.path())); persist_state_fixture(&persistence, &mut original); @@ -18200,6 +18701,7 @@ mod tests { let verified = verified_recovery(&library_root, "acme/model", &["weights.gguf"]); let download_id = "guard-free-worker-publication"; let mut state = recovery_test_state(&verified, download_id, DownloadStatus::Queued, false); + set_test_file_digest(&mut state, "weights.gguf", b"done"); state.make_managed_for_test(); let files = state.files.clone(); let cancel_flag = state.cancel_flag.clone(); @@ -18535,6 +19037,16 @@ mod tests { .await .unwrap(); + let (mut reopened_config, _) = listener.accept().await.unwrap(); + let reopened_config_request = read_pinned_test_request(&mut reopened_config).await; + assert!(reopened_config_request.starts_with(&format!( + "GET /acme/model/resolve/{COMMIT}/config.json HTTP/1.1" + ))); + reopened_config + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{}") + .await + .unwrap(); + let (mut resumed, _) = listener.accept().await.unwrap(); let resumed_request = read_pinned_test_request(&mut resumed).await; assert!(resumed_request.starts_with(&format!( @@ -18997,7 +19509,9 @@ mod tests { let error = reopened.restore_persisted_downloads().await.unwrap_err(); assert!( - error.to_string().contains("contradicts durable admission"), + error + .to_string() + .contains("Pinned repository evidence changed for admitted file weights.gguf"), "{error}" ); assert_eq!( diff --git a/rust/crates/pumas-core/src/model_library/hf/mod.rs b/rust/crates/pumas-core/src/model_library/hf/mod.rs index 299b3e55..ae31bada 100644 --- a/rust/crates/pumas-core/src/model_library/hf/mod.rs +++ b/rust/crates/pumas-core/src/model_library/hf/mod.rs @@ -14,6 +14,7 @@ //! - [`download`] - Download management with pause/resume/cancel //! - [`auth`] - Authentication token management +mod acquisition_source; mod auth; mod bundles; mod download; diff --git a/rust/crates/pumas-core/src/model_library/hf/types.rs b/rust/crates/pumas-core/src/model_library/hf/types.rs index 74d3ff03..3046a863 100644 --- a/rust/crates/pumas-core/src/model_library/hf/types.rs +++ b/rust/crates/pumas-core/src/model_library/hf/types.rs @@ -267,14 +267,32 @@ impl DownloadState { other => other, }; - // Current persisted snapshots always contain an explicit file set. + // The durable snapshot stores aggregate size and the primary file's + // digest, not per-file metadata. Those values identify one file only + // when the persisted selection contains exactly that primary file. + // Never attach aggregate or primary evidence to an arbitrary member + // of a multi-file selection. + let single_file_evidence = (entry.filenames.len() == 1 + && entry + .filenames + .first() + .is_some_and(|filename| filename == &entry.filename)) + .then_some(( + &entry.filename, + entry.total_bytes, + entry.known_sha256.as_deref(), + )); let files: Vec = entry .filenames .iter() - .map(|f| FileToDownload { - filename: f.clone(), - size: None, // Not persisted per-file; verified on disk - sha256: None, + .map(|filename| { + let evidence = + single_file_evidence.filter(|(primary, _, _)| *primary == filename.as_str()); + FileToDownload { + filename: filename.clone(), + size: evidence.and_then(|(_, size, _)| size), + sha256: evidence.and_then(|(_, _, sha256)| sha256.map(str::to_owned)), + } }) .collect(); diff --git a/rust/crates/pumas-core/src/model_library/mod.rs b/rust/crates/pumas-core/src/model_library/mod.rs index 38109e8d..b1c4663a 100644 --- a/rust/crates/pumas-core/src/model_library/mod.rs +++ b/rust/crates/pumas-core/src/model_library/mod.rs @@ -44,7 +44,6 @@ mod model_type_resolver; mod mutation_authority; mod naming; mod package_facts; -mod partial_download; mod read_only; pub mod sharding; mod task_signature; diff --git a/rust/crates/pumas-core/src/model_library/partial_download.rs b/rust/crates/pumas-core/src/model_library/partial_download.rs deleted file mode 100644 index a0f43e18..00000000 --- a/rust/crates/pumas-core/src/model_library/partial_download.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Filesystem recovery for downloads interrupted after their final byte arrived. - -use crate::error::Result; -use std::collections::{HashMap, HashSet}; -use std::path::Path; - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub(crate) struct PartialArtifactFinalization { - pub complete: bool, - pub promoted_files: usize, -} - -/// Filesystem operations used by the shared partial-artifact policy. The -/// adapter owns path authority and cleanup failure semantics. -pub(crate) trait PartialDownloadFiles { - fn file_len(&self, filename: &str) -> Result>; - fn part_len(&self, filename: &str) -> Result>; - fn rename_part_to_file(&self, filename: &str) -> Result<()>; - fn remove_part(&self, filename: &str) -> Result<()>; - fn remove_marker(&self) -> Result<()>; -} - -fn is_size_accounted_payload(relative_path: &str) -> bool { - Path::new(relative_path) - .extension() - .and_then(|extension| extension.to_str()) - .is_some_and(|extension| { - matches!( - extension.to_ascii_lowercase().as_str(), - "gguf" | "safetensors" | "bin" | "pt" | "pth" | "ckpt" | "onnx" | "npz" - ) - }) -} - -/// Infer one unfinished file's expected size from the selected artifact total. -/// Multi-file inference is restricted to payload formats whose bytes are included -/// in Hugging Face's persisted LFS total; auxiliary files require remote sizes. -pub(crate) fn infer_expected_sizes_with_files( - files: &impl PartialDownloadFiles, - expected_files: &[String], - total_size: Option, -) -> Result> { - let Some(total_size) = total_size.filter(|size| *size > 0) else { - return Ok(HashMap::new()); - }; - if expected_files.len() == 1 { - return Ok(HashMap::from([(expected_files[0].clone(), total_size)])); - } - if expected_files.is_empty() - || !expected_files - .iter() - .all(|filename| is_size_accounted_payload(filename)) - { - return Ok(HashMap::new()); - } - - let mut completed_size = 0_u64; - let mut unfinished = Vec::new(); - let mut seen = HashSet::new(); - for relative_path in expected_files { - if !seen.insert(relative_path) { - continue; - } - if let Some(size) = files.file_len(relative_path)? { - completed_size = completed_size.saturating_add(size); - } else if files.part_len(relative_path)?.is_some() { - unfinished.push(relative_path.clone()); - } else { - return Ok(HashMap::new()); - } - } - - if unfinished.len() != 1 { - return Ok(HashMap::new()); - } - let Some(expected_size) = total_size.checked_sub(completed_size) else { - return Ok(HashMap::new()); - }; - if expected_size == 0 { - return Ok(HashMap::new()); - } - - Ok(HashMap::from([(unfinished.remove(0), expected_size)])) -} - -/// Promote every exact-size `.part` file once the selected artifact -/// is locally complete. Unknown-size partials are left untouched. -pub(crate) fn finalize_download_artifact_with_files( - files: &impl PartialDownloadFiles, - expected_files: &[String], - expected_sizes: &HashMap, -) -> Result { - if expected_files.is_empty() { - return Ok(PartialArtifactFinalization::default()); - } - - let mut seen = HashSet::new(); - let mut promotions = Vec::new(); - let mut stale_parts = Vec::new(); - - for relative_path in expected_files { - if !seen.insert(relative_path) { - continue; - } - if files.file_len(relative_path)?.is_some() { - if files.part_len(relative_path)?.is_some() { - stale_parts.push(relative_path); - } - continue; - } - - let Some(part_size) = files.part_len(relative_path)? else { - return Ok(PartialArtifactFinalization::default()); - }; - let Some(expected_size) = expected_sizes.get(relative_path) else { - return Ok(PartialArtifactFinalization::default()); - }; - if part_size != *expected_size { - return Ok(PartialArtifactFinalization::default()); - } - promotions.push(relative_path); - } - - for filename in &promotions { - files.rename_part_to_file(filename)?; - } - for filename in &stale_parts { - files.remove_part(filename)?; - } - - files.remove_marker()?; - - Ok(PartialArtifactFinalization { - complete: true, - promoted_files: promotions.len(), - }) -} diff --git a/rust/crates/pumas-core/src/tests.rs b/rust/crates/pumas-core/src/tests.rs index 387f9ef4..9141bd0f 100644 --- a/rust/crates/pumas-core/src/tests.rs +++ b/rust/crates/pumas-core/src/tests.rs @@ -288,6 +288,10 @@ async fn builder_retains_failed_download_import_and_retries_before_completion() let destination = library_root.join("vision/idea-research/grounding-dino-base"); std::fs::create_dir_all(&destination).unwrap(); let payload = b"not-a-real-model"; + let known_sha256 = { + use sha2::Digest; + Some(hex::encode(sha2::Sha256::digest(payload))) + }; std::fs::write(destination.join("detector.onnx.part"), payload).unwrap(); std::fs::write(destination.join(".pumas_download"), b"{}").unwrap(); // Structurally valid provenance passes destination admission, while an @@ -322,7 +326,7 @@ async fn builder_retains_failed_download_import_and_retries_before_completion() license_status: Some("apache-2.0".into()), }, created_at: chrono::Utc::now().to_rfc3339(), - known_sha256: None, + known_sha256, huggingface_evidence: None, }; let mut client = HuggingFaceClient::new(temp.path().join("fixture-cache")).unwrap(); From 98c19910faeebbb75056ddd604b99554305b14a8 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 17:48:24 -0700 Subject: [PATCH 05/20] feat(acquisition): resolve verified GitHub release assets Resolve an exact release tag through the existing GitHub metadata owner and map publisher asset identity and SHA-256 evidence into the source-neutral manifest. Keep the established release DTO and cache schema unchanged, and keep the signed retrieval URL out of the manifest. Record Q1 scope and evidence without advancing AQ-HTTP. --- .../artifact-acquisition/execution-ledger.md | 25 ++ docs/plans/artifact-acquisition/plan.md | 2 +- .../reports/coding-standards-mcp-usability.md | 36 +- .../reports/write-sets.md | 9 +- .../src/acquisition/github_release.rs | 183 ++++++++++ rust/crates/pumas-core/src/acquisition/mod.rs | 5 + rust/crates/pumas-core/src/network/github.rs | 326 +++++++++++++++--- 7 files changed, 536 insertions(+), 50 deletions(-) create mode 100644 rust/crates/pumas-core/src/acquisition/github_release.rs diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 8b0f6c7a..72483747 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -32,3 +32,28 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - **Still unexecuted / not claimed:** live HF source acquisition, current llama.cpp archive through the shared handoff, desktop HF/native controls, shared-owner cancellation/shutdown/reopen, real deployed-state migration/old-writer isolation, current-candidate hosted CI, and all Q1 system/manual/platform acceptance. Local loopback permission was required for the controlled HTTP tests. The first sandbox-only full run had 23 listener `PermissionDenied` failures; the authorized loopback rerun passed. Base CI remains historical/base-only and GitHub currently returns no combined status or PR-triggered workflow run for accepted `main`. - **Standards MCP:** final route against `snapshot:v1:c181902d-6cad-40ae-80ef-83f091932539` selected 37 standards with zero unresolved fact categories. Targeted `read_many` calls read 13 applicable policies covering persistence/evolution, async lifecycle, Rust filesystem security, contracts, architecture, code design, API, cross-platform, verification, planning, and proportionality. Findings were checked against source and actual candidate tests; the route itself is not compliance evidence. The MCP authoring `review` operation does not review application source. The independent reviewer reused earlier routed obligations and made no new MCP calls in the narrow repair review. See the separate [MCP usability report](reports/coding-standards-mcp-usability.md). - Plan state remains Q1 in progress; AQ-HTTP, AQ-PACKAGES, AQ-S3 and AQ-COMPLETE remain not ready. Q2/Q3/runtime R1 are not started on an unaccepted prerequisite. The next implementation work continues Q1 with the durable owner and existing llama.cpp consumer; no runtime R1 branch is authorized yet. + +## 2026-09-29 — Q1 native source identity and digest slice admitted + +- Independent read-only review of the Q1 candidate found the native llama.cpp installer still accepts cached archives by filename and size, deletes an existing version directory before extraction, extracts directly into the published directory, and lacks a server-shutdown drain for its discarded install task. These are Q1 consumer/custody defects, not evidence that the shared owner exists. The reviewer recommended first recording source integrity and then repairing native staging/task custody without changing R1 identity. +- The initial write-set review found `GitHubAsset` is a public Rust struct re-exported by `pumas_library::network`; adding public fields would change downstream struct-literal compatibility with no external-consumer disposition. The implementation therefore preserves that DTO and its cache schema. The exact root source write set is `rust/crates/pumas-core/src/{acquisition/{mod.rs,github_release.rs},network/github.rs}` and focused source/manifest tests plus these plan records. Verified selection requires a positive live GitHub asset ID and a publisher `sha256:` digest; the retrieval URL remains separate. Legacy release caches stay readable but do not authorize verified selection. No `DownloadPersistence` format, live root, app-manager installer, or generated/RPC DTO is modified in this root source slice. +- GitHub's current REST release-asset representation includes an optional asset `digest` field; the official REST documentation shows `sha256:` and the upstream `ggml-org/llama.cpp` release API currently returns it for installable archives. This is source-metadata evidence only; it does not prove a Pumas acquisition, install, or supported-platform workflow. See [GitHub REST release asset documentation](https://docs.github.com/en/rest/releases/assets). +- In parallel, worker `/root/q1_native_custody_sol` is admitted from exact candidate `8b96cab5cbb7ff5104d6e33692538d3680b32fc8` with isolated primary writes in `pumas-app-manager/version_manager/{mod.rs,installer.rs,state.rs}` and `pumas-rpc/src/server.rs`, plus co-located tests. `state.rs` was added to the write set after independent review showed its public metadata-mutator methods bypass a manager-only installation lock. The worker must not modify shared source identity, acquisition persistence, or plan/gate records. Root remains the integration owner. The proposed custody bridge still uses the existing native downloader and cannot qualify AQ-HTTP until it consumes the shared acquisition lifecycle. +- The standards router was re-run for both launcher and library application forms, Rust API/async/cross-platform/dependency/security profiles, persistence, concurrency, protocol/schema evolution, security, resilience, dependencies, licensing, performance, and untrusted execution. The complete route returned no unresolved questions or policy counts. Targeted normative reads included Rust Cargo dependencies, Rust cross-platform, Rust security, Resilience, Untrusted Execution, Dependencies, Licensing, and Launcher Application. The official GitHub REST documentation was consulted for source-field semantics. Product acceptance remains separately determined by source and tests. +- This admitted source slice does not advance any gate. Q1 remains in progress and AQ-HTTP is not ready; no R1/R2/runtime work starts from the unaccepted prerequisite. + +### GitHub source-resolution implementation evidence + +- The existing public `GitHubAsset` shape and persisted release-cache schema remain unchanged after source inspection found the type is publicly re-exported and external struct-literal compatibility has no checked disposition. `GitHubClient::resolve_release_asset` fetches the exact release tag from the configured GitHub API, rejects mismatched tags/missing or ambiguous names, and uses the acquisition adapter to require a positive API asset ID plus a valid publisher SHA-256. The resulting manifest contains repository/asset identity, logical name, size and digest evidence; the signed retrieval URL exists only on the non-serializable selection value and is omitted from `Debug`. +- Exact root source write set is `rust/crates/pumas-core/src/{acquisition/{mod.rs,github_release.rs},network/{github.rs}}`, plus these plan records. The additive method belongs to the existing GitHub metadata owner; it does not download or persist bytes, add a downloader, change a release RPC/cache DTO, or change the durable acquisition store. It is not yet wired into the native installer, so this does not satisfy the Q1 producer/consumer criterion. +- Standards routing was repeated after the public API boundary changed and before final source review. Snapshot `snapshot:v1:2838a231-9170-40b0-b058-6fc1c2b228b5` selected 34 units with no unresolved routing facts. Targeted reads included Core, persistence, contracts/evolution/schema/protocol, Rust API/async/security, resilience, dependencies, commit and verification obligations. The route selects applicable obligations; it is not source-compliance evidence. +- Synthetic local HTTP API test `network::github::tests::release_asset_resolution_uses_fresh_metadata_and_keeps_ephemeral_url_out_of_manifest` passed and asserted the exact-tag endpoint, selected ID, size, and separation of ephemeral URL from manifest. `network::github::tests::release_asset_resolution_encodes_tag_as_one_path_segment` passed. Acquisition selection tests passed (2/2) for identity/digest preservation, legacy/missing evidence rejection, malformed digest rejection, and URL exclusion. These use synthetic metadata/local service; no live GitHub-to-installer acquisition is claimed. +- `cargo fmt --manifest-path rust/Cargo.toml --package pumas-library -- --check`, `cargo clippy --manifest-path rust/Cargo.toml -p pumas-library --all-targets -- -D warnings`, and `cargo check --manifest-path rust/Cargo.toml -p pumas-library --no-default-features` passed at this worktree candidate. The targeted source tests above passed. The previous full library suite applies to source commit `8b96cab5…`, not this uncommitted source candidate. +- Current Q1 gate remains in progress/not ready. Live GitHub-to-native acquisition, integration with the shared durable owner, lifecycle handoff, cancellation/reopen/restart, extraction publication, desktop behavior, and all other AQ-HTTP acceptance remain pending. + +### Current PR and independent review status + +- A live `gh pr view 7` query returned PR [#7](https://github.com/MrScripty/Pumas-Library/pull/7) open and draft, targeting `main`, at remote head `8b96cab5cbb7ff5104d6e33692538d3680b32fc8`. For that exact older head, Build run `36647045127` completed the listed Rust quality, frontend/contracts, headless-no-inference and three Torch native checks successfully; nine conditional build/package/release jobs were skipped. These are actual hosted checks for that SHA only. The current uncommitted source candidate has no hosted CI result, and skipped jobs are not treated as passes. +- Independent read-only review of native custody proposal `work/q1-native-custody` found four open findings: Ollama network I/O can stall RPC shutdown; public `VersionState` mutators bypass the installer-only metadata lock; cancelled native download may leave queued Tokio file I/O unobserved before custody reclamation; and `TempDir::drop` suppresses deletion errors. The write set now includes `version_manager/state.rs` so the metadata invariant can be fixed at its owner. The worker is repairing these on its isolated branch; no proposal is integrated or accepted yet. +- Independent shared-owner design review confirms current Q1 still has no shared durable acquisition owner. It traces HF-specific state/task/store ownership and the native independent download loop, and identifies one-root-grant coupling plus `mutation_authority.rs` as part of migration/reopen correctness. It recommends one migrated store authority, consumer-held leases through import/extraction, and composed shutdown. No source, tests or retained state were changed by the reviewer. +- Root rerouted the current Q1 source/consumer slice through Coding-Standards snapshot `snapshot:v1:264b9c2f-aca9-4b00-a344-530f8036a005`: 40 selected units, zero unresolved routing facts. Targeted policy reads covered Core, Persistence, Concurrency, Contract Evolution, Architecture, Rust Async, Rust API and Rust Security; the commit workflow was read before the source-slice commit review. Routing was used for obligations only; source and test evidence remain independent. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index fbb19f0c..c71cc00c 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,7 +2,7 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The shared durable handoff owner, llama.cpp consumer, retained-store evolution, desktop path, and required real-source qualification remain pending. +**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The existing GitHub client now has an additive exact-tag resolver that maps live publisher asset identity/digest metadata to a verified manifest while leaving the public/cache DTO unchanged. A native custody proposal is under repair on an isolated worker branch after independent review found shutdown, metadata-coordination, pending-file-I/O, and cleanup-error gaps. The resolver is not yet consumed by the native installer, and shared durable handoff ownership, retained-store evolution, desktop path, and required real-source qualification remain pending. **Exactly one next slice:** Continue **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index 1ba3b532..d419a2ca 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -20,8 +20,42 @@ This report records agent experience using the Coding-Standards MCP during Acqui - **Smallest sufficient workflow:** Route concrete facts once, omit inline policy content, read only the relevant policies in small pages, inspect source/evidence independently, and reroute when the design materially changes. - **Recommendations:** Add an output-size budget, avoid repeating policy prose and relationship explanations, show which policies have already been read, and provide a same-facts design-change summary. Keep route completeness distinct from source/evidence compliance. +## Primary integrator — source API boundary follow-up + +- **Useful calls:** I refreshed `routing_facts`, routed the changed public API/async/network boundary, and got a complete route with 34 selected units and zero unresolved categories. Compact single-policy reads provided the specific compatibility and lifecycle rules. The route helped identify that public API compatibility and typed failure behavior mattered even though the source adapter was internal to the repository. +- **Confusing or redundant steps:** The route result still repeated long relationship rationales for each policy. A `read_many` call for nine compact policies returned enough combined policy text to exceed a practical review/output window; focused individual reads were easier to inspect, though several were still large. The route's completeness and the amount of normative text read remained separate facts that needed explicit tracking. +- **Missing context:** The MCP did not know that `GitHubAsset` is publicly re-exported, nor whether downstream crate consumers construct it directly. I had to inspect `network/mod.rs`, search the Rust tree, and inspect Cargo packaging before rejecting a breaking public-field addition. It also could not observe whether the resolver was consumed by the native installer or what current tests had run. +- **Smallest sufficient workflow:** Use current routing facts; route concrete code and contract facts without inline policy bodies; read Core and the exact contract-evolution, Rust API/async, persistence, security, resilience, and verification units needed for the slice in a bounded set; inspect source and run tests separately; re-route when the public boundary/design changes; then check the final source against the selected obligations. +- **Recommendations:** Add a strict token/byte budget to `read_many`; support section-scoped reads with a concise obligation summary and authoritative full-text links; retain a visible list of already-read policy IDs across route continuations; shorten repeated relationship rationales; and state that current consumer inventory and source evidence must be established outside the MCP. Preserve the distinction between a complete route and implementation compliance. + +## Native-custody contributor — GPT-6.1 Sol High + +- **Useful calls:** `routing_facts`, a complete `route`, and snapshot-bound `read_many` selected and returned the Core, Rust async, concurrency, persistence, contracts, and commit obligations. The first route selected 27 standards; recognizing the metadata-publication boundary required a second route, which selected 29 with no unresolved facts. These calls selected guidance; the contributor used code inspection and tests for implementation evidence. +- **Confusing or redundant steps:** Route content-pagination repeated long policy-selection rationale. Large policy results combined with source diffs exceeded a useful output window; smaller content-only reads worked better. Broad tool discovery included unrelated authoring operations. +- **Missing context:** The MCP did not know the admitted worktree, current plan/write set, actual publication semantics, source ownership, or test environment. The contributor had to inspect source to recognize the metadata boundary even though persistence files were outside its write set. +- **Where the contributor left MCP:** Git state, plan details, source ownership, filesystem behavior, Cargo checks, and fixture outputs all came from repository tools, not the MCP. +- **Smallest sufficient workflow:** One `routing_facts` call, one complete-facts route, then snapshot-bound reads of only the applicable policies; use repository source and tests to assess actual compliance. Standards authoring/revision/publication calls were unnecessary. +- **Recommendations:** Provide continuations that return policy content without repeating route rationales; make verbose relationship detail opt-in; separate implementation-guidance discovery from standards-authoring tools; and make the route surface a concise checklist for indirect boundaries such as durable metadata publication. Retain the explicit distinction between routed guidance and implementation review. + +## Native-custody independent reviewer — GPT-6.1 Sol High + +- **Useful calls:** `describe_input(route)` clarified the input contract; `routing_facts` exposed the valid fact names and values; a complete route plus snapshot-bound `read_many` selected 27 policies with no unresolved questions. Concurrency, Rust async, persistence, resilience, and verification policies supported the source review. +- **Confusing or redundant steps:** Tool discovery was verbose. The reviewer first searched for a server named Coding-Standards, which was not configured, and initially missed the `standards_engine` tool namespace in `ALL_TOOLS`. `describe_input(route)` added little beyond the exposed declaration. Reading all 27 policies at once produced excessive output and repeated policy-selection rationale. +- **Missing context:** The MCP did not bind the route to the branch, diff, baseline, plan gate, source population, or test evidence. The reviewer had to inspect Git, the plan, the worker diff, Tokio/tempfile behavior, and source to identify shutdown, metadata coordination, pending filesystem I/O, and cleanup-error obligations. +- **Where the reviewer left MCP:** Repository, dependency-source, and test inspection were required for every product finding. No tests were run during the read-only review. +- **Smallest sufficient workflow:** Discover `standards_engine` tools from the provided callable catalog; use `routing_facts`, one complete route with explicit empty categories, then read only focused applicable policies in a bounded page; evaluate the actual diff and evidence in repository tools. `describe_input` is optional when the declared shape is already clear. +- **Recommendations:** Make the tool namespace easier to discover by linking it from the configured MCP catalog; include accepted base and plan/write-set context when the caller supplies it, while clearly distinguishing context from policy authority; offer a focused policy-text continuation without relationship rationale; and provide a concise read-only code-review workflow separate from standards-corpus authoring. Do not imply that a complete route certifies implementation. + +## Shared-owner design reviewer — GPT-6.1 Sol High + +- **Useful calls:** The reviewer initially searched for a Coding-Standards server and did not find one; after I pointed it to the exposed `standards_engine` tool namespace, it called `routing_facts`, routed concrete planning/verification/persistence/Rust facts to a complete result with zero unresolved questions, and read Architecture, replay, Persistence, Contract Evolution, Code Design, and Rust Async. The pinned snapshot was `snapshot:v1:78677912-3ddc-4107-83a3-c7f83542de1d`. +- **Confusing or redundant steps:** Searching by the product name led to an incorrect `list_mcp_resources` call against a nonexistent server. Tool-callable discovery and MCP-server/resource discovery were not clearly distinguished. Correcting the search resolved the confusion. +- **Missing context:** MCP output identified its interface version and unreviewed exposure but did not provide the current Q1 source ownership, retained-store facts, or plan gate by itself; the reviewer is collecting those from repository files. +- **Smallest sufficient workflow:** Discover callable `standards_engine` methods from `ALL_TOOLS`, call `routing_facts`, complete a slice-specific route, then read the few selected policies required for architecture/persistence/concurrency. Repository inspection remains the source of implementation facts. +- **Recommendations:** Surface configured MCP tool namespaces in initial agent context, distinguish callable tools from MCP servers/resources, and make the bounded read-only workflow discoverable without exposing unrelated authoring operations first. + ## Participation -The primary integrator and the independent architecture reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. +The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, and shared-owner design reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. The architecture reviewer’s narrow follow-up inspected the repaired current source without making a new MCP call and reused the prior routed obligations; its findings confirm the two identified integrity paths are closed at source level, not that Q1 is accepted. The MCP `review` operation is an authoring workflow for changes to the standards corpus; it does not review application source. Application compliance was checked by final routing, standards reads, source/test inspection, and the independent code review. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index 78846b48..b26925ed 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -4,17 +4,18 @@ These are the admitted exact paths/closed path families. The actual source files ## Q1: one HTTP acquisition owner with real consumers -**Canonical module paths:** `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs,http.rs}` currently hold validated source-neutral selections and the HTTP representation/body-streaming protocol. The remaining Q1 design still needs one durable transfer owner and capability workspace; place those with the canonical module rather than creating another downloader. Source-reader adaptation may remain in that module or a focused `acquisition/sources/` child if the actual design supports it. +**Canonical module paths:** `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs,http.rs}` currently hold validated source-neutral selections and the HTTP representation/body-streaming protocol. The GitHub release adapter is `acquisition/github_release.rs`, with its fresh asset-metadata resolver in the existing `network/github.rs` owner; the public/cache release DTO remains unchanged. The adapter maps publisher asset identity/digest evidence into a verified manifest while keeping the retrieval URL ephemeral. The remaining Q1 design still needs one durable transfer owner and capability workspace; place those with the canonical module rather than creating another downloader. Source-reader adaptation may remain in that module or a focused `acquisition/sources/` child if the actual design supports it. **Existing owners allowed to change:** -- `rust/crates/pumas-core/src/lib.rs`, `network/{mod.rs,download.rs}`, `model_library/hf/{mod.rs,download.rs,lifecycle.rs,types.rs,metadata.rs}`; +- `rust/crates/pumas-core/src/lib.rs`, `network/{mod.rs,download.rs,github.rs}`, `models/github.rs`, `model_library/hf/{mod.rs,download.rs,lifecycle.rs,types.rs,metadata.rs}`; +- `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs,http.rs}` and focused source adapters under that module when the selected source evidence requires them; - `rust/crates/pumas-core/src/model_library/{download_store.rs,download_recovery.rs}` only for the extracted authority and explicit supported migration; - `rust/crates/pumas-core/src/tests.rs` when a builder/reopen fixture depends on the selected completion-evidence invariant; - directly affected core `api/hf.rs`, `api/state.rs` and current model-importer/intent completion call sites, selected from the actual producer/consumer trace before editing; -- `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,ollama.rs,progress.rs}` for the acquisition bridge and transfer progress, not installed-unit identity migration; +- `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,ollama.rs,progress.rs,state.rs}` for the acquisition bridge and transfer progress, not installed-unit identity migration; - current core atomic JSON/capability-filesystem modules only where the same selected invariant requires a targeted change, never as an unrelated filesystem rewrite. -**Tests:** proposed `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`, plus existing co-located HF lifecycle/recovery/installer regression tests. Fixtures must reach the owner under test with independent expected byte/effect outcomes. +**Tests:** proposed `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`, plus existing co-located GitHub metadata, HF lifecycle/recovery, and installer regression tests. Fixtures must reach the owner under test with independent expected byte/effect outcomes. **Serial adjacent writes:** `rust/crates/pumas-rpc/src/contract.rs`, `contract/export.rs`, affected HF/version/status handlers, `electron/src/{preload.ts,rpc-method-registry.ts,ipc-validation.ts}`, actual corresponding frontend download/install/source views and generated DTOs. Enumerate outputs from the actual exporter rather than guess or edit generated files manually. Reuse the existing model/native UI; this is not a dashboard redesign. diff --git a/rust/crates/pumas-core/src/acquisition/github_release.rs b/rust/crates/pumas-core/src/acquisition/github_release.rs new file mode 100644 index 00000000..5ee918c7 --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/github_release.rs @@ -0,0 +1,183 @@ +//! GitHub release assets mapped into the source-neutral acquisition contract. + +use thiserror::Error; + +use super::manifest::{ + ArtifactFile, ArtifactManifest, ArtifactRevisionEvidence, ArtifactSourceIdentity, + FileVerificationRequirement, ManifestValidationError, RevisionStrength, Sha256Evidence, +}; +use crate::PumasError; + +/// A publisher-identified GitHub asset selected for verified acquisition. +/// +/// The immutable manifest excludes the retrieval URL. Callers pass the URL to +/// the source reader separately so signed or refreshable authorization is not +/// retained as artifact identity. +pub struct GitHubReleaseAssetSelection { + manifest: ArtifactManifest, + download_url: String, +} + +/// Metadata available only from a live GitHub release response. It stays +/// separate from the established public/cache DTO, which has a distinct +/// compatibility contract. +pub(crate) struct GitHubReleaseAssetMetadata { + pub id: Option, + pub name: String, + pub size: u64, + pub download_url: String, + pub digest: Option, +} + +impl GitHubReleaseAssetSelection { + pub fn manifest(&self) -> &ArtifactManifest { + &self.manifest + } + + pub fn download_url(&self) -> &str { + &self.download_url + } +} + +/// Select one GitHub release asset only when publisher identity and digest are +/// available. Legacy release caches remain readable but cannot authorize this +/// verified selection. +pub(crate) fn select_github_release_asset( + repository: &str, + asset: &GitHubReleaseAssetMetadata, +) -> Result { + let asset_id = asset + .id + .filter(|asset_id| *asset_id > 0) + .ok_or(GitHubAssetSelectionError::AssetIdentityRequired)?; + let digest = asset + .digest + .as_deref() + .and_then(|value| value.strip_prefix("sha256:")) + .ok_or(GitHubAssetSelectionError::Sha256DigestRequired)?; + let digest = Sha256Evidence::new("github.release_asset.digest", digest)?; + let revision = ArtifactRevisionEvidence::new( + "github.release_asset.id", + asset_id.to_string(), + RevisionStrength::Weak, + )?; + let source = ArtifactSourceIdentity::new("github", repository, revision)?; + let file = ArtifactFile::new( + asset.name.clone(), + asset_id.to_string(), + Some(asset.size), + Some(digest), + FileVerificationRequirement::Sha256, + )?; + let manifest = ArtifactManifest::new(source, vec![file])?; + + Ok(GitHubReleaseAssetSelection { + manifest, + download_url: asset.download_url.clone(), + }) +} + +#[derive(Clone, Debug, Eq, Error, PartialEq)] +pub enum GitHubAssetSelectionError { + #[error("GitHub release asset has no stable API identity")] + AssetIdentityRequired, + #[error("GitHub release asset has no publisher SHA-256 digest")] + Sha256DigestRequired, + #[error(transparent)] + Manifest(#[from] ManifestValidationError), +} + +#[derive(Debug, Error)] +pub enum GitHubReleaseAssetResolutionError { + #[error("GitHub API request failed: {0}")] + Api(#[from] PumasError), + #[error("GitHub release tag was empty")] + EmptyTag, + #[error("GitHub repository must have the form owner/repository")] + InvalidRepository, + #[error("GitHub API base URL cannot accept path segments")] + InvalidApiBase, + #[error("GitHub returned release tag `{returned}` for requested tag `{requested}`")] + TagMismatch { requested: String, returned: String }, + #[error("GitHub release `{tag}` has no asset named `{asset_name}`")] + AssetNotFound { tag: String, asset_name: String }, + #[error("GitHub release `{tag}` contains multiple assets named `{asset_name}`")] + AmbiguousAsset { tag: String, asset_name: String }, + #[error(transparent)] + Selection(#[from] GitHubAssetSelectionError), +} + +#[cfg(test)] +mod tests { + use super::*; + + fn asset() -> GitHubReleaseAssetMetadata { + GitHubReleaseAssetMetadata { + id: Some(42), + name: "llama-server.tar.gz".into(), + size: 1234, + download_url: "https://github.com/ggml-org/llama.cpp/releases/download/v1/llama-server.tar.gz?token=short-lived".into(), + digest: Some(format!("sha256:{}", "A".repeat(64))), + } + } + + #[test] + fn selection_preserves_publisher_identity_and_digest_but_not_url_in_manifest() { + let selection = select_github_release_asset("ggml-org/llama.cpp", &asset()).unwrap(); + let manifest = selection.manifest(); + let file = &manifest.files()[0]; + + assert_eq!(manifest.source().provider(), "github"); + assert_eq!(manifest.source().source_id(), "ggml-org/llama.cpp"); + assert_eq!( + manifest.source().revision().authority(), + "github.release_asset.id" + ); + assert_eq!(manifest.source().revision().value(), "42"); + assert_eq!(file.source_key(), "42"); + assert_eq!(file.expected_size(), Some(1234)); + assert_eq!( + file.expected_sha256().unwrap().authority(), + "github.release_asset.digest" + ); + assert_eq!(file.expected_sha256().unwrap().value(), "a".repeat(64)); + assert_eq!(file.verification(), FileVerificationRequirement::Sha256); + assert!(selection.download_url().contains("short-lived")); + assert!(!serde_json::to_string(manifest) + .unwrap() + .contains("short-lived")); + } + + #[test] + fn legacy_or_unverified_release_asset_cannot_be_selected() { + let mut legacy = asset(); + legacy.id = None; + assert!(matches!( + select_github_release_asset("ggml-org/llama.cpp", &legacy), + Err(GitHubAssetSelectionError::AssetIdentityRequired) + )); + + let mut no_digest = asset(); + no_digest.digest = None; + assert!(matches!( + select_github_release_asset("ggml-org/llama.cpp", &no_digest), + Err(GitHubAssetSelectionError::Sha256DigestRequired) + )); + + let mut malformed = asset(); + malformed.digest = Some("sha512:abcd".into()); + assert!(matches!( + select_github_release_asset("ggml-org/llama.cpp", &malformed), + Err(GitHubAssetSelectionError::Sha256DigestRequired) + )); + + let mut malformed = asset(); + malformed.digest = Some("sha256:not-hex".into()); + assert!(matches!( + select_github_release_asset("ggml-org/llama.cpp", &malformed), + Err(GitHubAssetSelectionError::Manifest( + ManifestValidationError::InvalidSha256 + )) + )); + } +} diff --git a/rust/crates/pumas-core/src/acquisition/mod.rs b/rust/crates/pumas-core/src/acquisition/mod.rs index 2510b32a..bc4bfabd 100644 --- a/rust/crates/pumas-core/src/acquisition/mod.rs +++ b/rust/crates/pumas-core/src/acquisition/mod.rs @@ -4,6 +4,7 @@ //! body-streaming protocol. Durable custody, lifecycle admission, publication, //! and consumer settlement still belong to their current production owners. +mod github_release; mod http; mod manifest; @@ -11,6 +12,10 @@ pub(crate) use http::{ open_http_artifact, stream_http_artifact, HttpArtifactSink, HttpAttemptHost, HttpBodyOutcome, }; +pub(crate) use github_release::{select_github_release_asset, GitHubReleaseAssetMetadata}; +pub use github_release::{ + GitHubAssetSelectionError, GitHubReleaseAssetResolutionError, GitHubReleaseAssetSelection, +}; pub use manifest::{ ArtifactFile, ArtifactManifest, ArtifactRevisionEvidence, ArtifactSourceIdentity, FileVerificationRequirement, ManifestValidationError, RevisionStrength, Sha256Evidence, diff --git a/rust/crates/pumas-core/src/network/github.rs b/rust/crates/pumas-core/src/network/github.rs index 7dd939ef..5b41786e 100644 --- a/rust/crates/pumas-core/src/network/github.rs +++ b/rust/crates/pumas-core/src/network/github.rs @@ -7,6 +7,10 @@ //! - Rate limit handling use super::web_source::{CacheStrategy, WebSource, WebSourceId}; +use crate::acquisition::{ + select_github_release_asset, GitHubReleaseAssetMetadata, GitHubReleaseAssetResolutionError, + GitHubReleaseAssetSelection, +}; use crate::config::{AppId, NetworkConfig}; use crate::models::{CacheStatus, GitHubReleasesCache}; use crate::network::client::HttpClient; @@ -27,6 +31,7 @@ use tokio::fs; use tokio::sync::Notify; use tokio::sync::{watch, Mutex, RwLock}; use tracing::{debug, info, warn}; +use url::Url; // Re-export for convenience pub use crate::models::{GitHubAsset, GitHubRelease}; @@ -39,6 +44,93 @@ struct StoredReleasesCache { listing_complete: bool, } +#[derive(serde::Deserialize)] +struct GitHubReleaseAssetResponse { + tag_name: String, + assets: Vec, +} + +#[derive(serde::Deserialize)] +struct GitHubReleaseAssetApiRecord { + id: Option, + name: String, + size: u64, + #[serde(rename = "browser_download_url")] + download_url: String, + digest: Option, +} + +fn build_release_tag_url( + api_base: &str, + repository: &str, + tag: &str, +) -> std::result::Result { + if tag.is_empty() || tag == "." || tag == ".." { + return Err(GitHubReleaseAssetResolutionError::EmptyTag); + } + let repository_parts: Vec<_> = repository.split('/').collect(); + if repository_parts.len() != 2 + || repository_parts.iter().any(|part| { + part.is_empty() + || *part == "." + || *part == ".." + || !part + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"._-".contains(&byte)) + }) + { + return Err(GitHubReleaseAssetResolutionError::InvalidRepository); + } + + let mut url = + Url::parse(api_base).map_err(|_| GitHubReleaseAssetResolutionError::InvalidApiBase)?; + { + let mut segments = url + .path_segments_mut() + .map_err(|_| GitHubReleaseAssetResolutionError::InvalidApiBase)?; + segments.pop_if_empty(); + segments + .push("repos") + .push(repository_parts[0]) + .push(repository_parts[1]) + .push("releases") + .push("tags") + .push(tag); + } + Ok(url) +} + +fn github_rate_limit_error(response: &reqwest::Response) -> Option { + let status = response.status(); + if !matches!( + status, + StatusCode::FORBIDDEN | StatusCode::TOO_MANY_REQUESTS + ) { + return None; + } + let retry_after = response + .headers() + .get("Retry-After") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .or_else(|| { + response + .headers() + .get("X-RateLimit-Reset") + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .and_then(|reset| { + let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs(); + Some(reset.saturating_sub(now)) + }) + }); + warn!("GitHub rate limited ({status}), retry after: {retry_after:?} seconds"); + Some(PumasError::RateLimited { + service: "GitHub".to_string(), + retry_after_secs: retry_after, + }) +} + /// Cache for GitHub releases. pub struct ReleasesCache { /// In-memory cache with TTL. @@ -412,6 +504,7 @@ where /// GitHub API client. pub struct GitHubClient { http: Arc, + api_base: String, cache: ReleasesCache, /// Whether we're currently fetching releases. is_fetching: AtomicBool, @@ -443,23 +536,23 @@ impl LlamaCppReleaseVariant { impl GitHubClient { /// Create a new GitHub client. pub fn new(cache_dir: PathBuf) -> Result { - let http = HttpClient::new()?; - Ok(Self { - http: Arc::new(http), - cache: ReleasesCache::new(cache_dir, NetworkConfig::GITHUB_RELEASES_TTL), - is_fetching: AtomicBool::new(false), - fetch_lock: RwLock::new(()), - pending_fetches: Mutex::new(HashMap::new()), - #[cfg(test)] - follower_joined: Notify::new(), - }) + Self::with_config( + cache_dir, + NetworkConfig::GITHUB_RELEASES_TTL, + NetworkConfig::GITHUB_API_BASE.to_string(), + ) } /// Create a new GitHub client with custom TTL. pub fn with_ttl(cache_dir: PathBuf, ttl: Duration) -> Result { + Self::with_config(cache_dir, ttl, NetworkConfig::GITHUB_API_BASE.to_string()) + } + + fn with_config(cache_dir: PathBuf, ttl: Duration, api_base: String) -> Result { let http = HttpClient::new()?; Ok(Self { http: Arc::new(http), + api_base: api_base.trim_end_matches('/').to_string(), cache: ReleasesCache::new(cache_dir, ttl), is_fetching: AtomicBool::new(false), fetch_lock: RwLock::new(()), @@ -744,6 +837,101 @@ impl GitHubClient { Ok(releases.into_iter().find(|r| r.tag_name == tag)) } + /// Resolve one release asset from a fresh GitHub API representation and + /// require publisher identity and SHA-256 evidence before acquisition. + /// The established release/cache DTO intentionally remains unchanged. + pub async fn resolve_release_asset( + &self, + repo: &str, + tag: &str, + asset_name: &str, + ) -> std::result::Result { + let url = build_release_tag_url(&self.api_base, repo, tag)?; + let retry_config = RetryConfig::new() + .with_max_attempts(3) + .with_base_delay(Duration::from_secs(2)); + let http = self.http.clone(); + let url_string = url.to_string(); + + let (result, stats) = retry_async( + &retry_config, + || { + let http = http.clone(); + let url = url_string.clone(); + async move { + let headers = vec![( + "Accept".to_string(), + "application/vnd.github.v3+json".to_string(), + )]; + http.get_with_headers(&url, &headers).await + } + }, + |error| error.is_retryable(), + ) + .await; + + if stats.attempts > 1 { + debug!( + "GitHub release asset resolution succeeded after {} attempts", + stats.attempts + ); + } + + let response = result?; + let status = response.status(); + if let Some(error) = github_rate_limit_error(&response) { + return Err(GitHubReleaseAssetResolutionError::Api(error)); + } + if !status.is_success() { + return Err(GitHubReleaseAssetResolutionError::Api( + PumasError::GitHubApi { + message: format!("GitHub API returned {status}"), + status_code: Some(status.as_u16()), + }, + )); + } + let release: GitHubReleaseAssetResponse = + response.json().await.map_err(|error| PumasError::Json { + message: format!("Failed to parse GitHub release asset metadata: {error}"), + source: None, + })?; + if release.tag_name != tag { + return Err(GitHubReleaseAssetResolutionError::TagMismatch { + requested: tag.to_string(), + returned: release.tag_name, + }); + } + + let mut matching = release + .assets + .into_iter() + .filter(|asset| asset.name == asset_name); + let asset = + matching + .next() + .ok_or_else(|| GitHubReleaseAssetResolutionError::AssetNotFound { + tag: tag.to_string(), + asset_name: asset_name.to_string(), + })?; + if matching.next().is_some() { + return Err(GitHubReleaseAssetResolutionError::AmbiguousAsset { + tag: tag.to_string(), + asset_name: asset_name.to_string(), + }); + } + + Ok(select_github_release_asset( + repo, + &GitHubReleaseAssetMetadata { + id: asset.id, + name: asset.name, + size: asset.size, + download_url: asset.download_url, + digest: asset.digest, + }, + )?) + } + /// Get cache status for a repository. pub async fn get_cache_status(&self, repo: &str) -> CacheStatus { self.cache @@ -857,10 +1045,7 @@ impl GitHubClient { let per_page = NetworkConfig::GITHUB_RELEASES_PER_PAGE; let url = format!( "{}/repos/{}/releases?per_page={}&page={}", - NetworkConfig::GITHUB_API_BASE, - repo, - per_page, - page + self.api_base, repo, per_page, page ); let retry_config = RetryConfig::new() @@ -896,36 +1081,8 @@ impl GitHubClient { let response = result?; let status = response.status(); - - if status == StatusCode::FORBIDDEN || status == StatusCode::TOO_MANY_REQUESTS { - // Rate limited - extract retry information from headers - let retry_after = response - .headers() - .get("Retry-After") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - // Also check X-RateLimit-Reset as fallback - .or_else(|| { - response - .headers() - .get("X-RateLimit-Reset") - .and_then(|v| v.to_str().ok()) - .and_then(|s| s.parse::().ok()) - .and_then(|reset| { - let now = SystemTime::now().duration_since(UNIX_EPOCH).ok()?.as_secs(); - Some(reset.saturating_sub(now)) - }) - }); - - warn!( - "GitHub rate limited ({}), retry after: {:?} seconds", - status, retry_after - ); - - return Err(PumasError::RateLimited { - service: "GitHub".to_string(), - retry_after_secs: retry_after, - }); + if let Some(error) = github_rate_limit_error(&response) { + return Err(error); } if !status.is_success() { @@ -1288,6 +1445,87 @@ mod tests { (client, temp_dir) } + #[tokio::test] + async fn release_asset_resolution_uses_fresh_metadata_and_keeps_ephemeral_url_out_of_manifest() + { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let root = TempDir::new().unwrap(); + let client = GitHubClient::with_config( + root.path().to_path_buf(), + Duration::from_secs(60), + format!("http://{address}/api"), + ) + .unwrap(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + loop { + let read = stream.read(&mut chunk).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&chunk[..read]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + } + + let body = format!( + r#"{{"tag_name":"v1.2","assets":[{{"id":42,"name":"server.tar.gz","size":17,"browser_download_url":"https://github.com/org/repo/releases/download/v1.2/server.tar.gz?sig=ephemeral","digest":"sha256:{}"}}]}}"#, + "a".repeat(64) + ); + stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ) + .as_bytes(), + ) + .await + .unwrap(); + String::from_utf8(request).unwrap() + }); + + let selection = client + .resolve_release_asset("org/repo", "v1.2", "server.tar.gz") + .await + .unwrap(); + let request = server.await.unwrap(); + + assert!(request.starts_with("GET /api/repos/org/repo/releases/tags/v1.2 HTTP/1.1")); + assert_eq!(selection.manifest().source().source_id(), "org/repo"); + assert_eq!(selection.manifest().files()[0].source_key(), "42"); + assert_eq!(selection.manifest().files()[0].expected_size(), Some(17)); + assert!(selection.download_url().contains("sig=ephemeral")); + assert!(!serde_json::to_string(selection.manifest()) + .unwrap() + .contains("ephemeral")); + } + + #[test] + fn release_asset_resolution_encodes_tag_as_one_path_segment() { + let url = build_release_tag_url( + "https://api.github.com", + "ggml-org/llama.cpp", + "release/candidate", + ) + .unwrap(); + assert!(url + .as_str() + .ends_with("/repos/ggml-org/llama.cpp/releases/tags/release%2Fcandidate")); + assert!(matches!( + build_release_tag_url("https://api.github.com", "owner/repo/extra", "v1"), + Err(GitHubReleaseAssetResolutionError::InvalidRepository) + )); + } + #[test] fn test_releases_cache_disk() { let temp_dir = TempDir::new().unwrap(); From 096d180372d8c694948ea41329d9a82dc3fbd371 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 18:20:16 -0700 Subject: [PATCH 06/20] refactor(acquisition): support multiple destination grants --- .../artifact-acquisition/execution-ledger.md | 9 +++ docs/plans/artifact-acquisition/plan.md | 2 +- .../reports/coding-standards-mcp-usability.md | 45 ++++++++++++ .../reports/write-sets.md | 4 ++ .../src/model_library/download_recovery.rs | 11 +++ .../src/model_library/hf/lifecycle.rs | 71 ++++++++++++++++--- 6 files changed, 133 insertions(+), 9 deletions(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 72483747..88b6618b 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -57,3 +57,12 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - Independent read-only review of native custody proposal `work/q1-native-custody` found four open findings: Ollama network I/O can stall RPC shutdown; public `VersionState` mutators bypass the installer-only metadata lock; cancelled native download may leave queued Tokio file I/O unobserved before custody reclamation; and `TempDir::drop` suppresses deletion errors. The write set now includes `version_manager/state.rs` so the metadata invariant can be fixed at its owner. The worker is repairing these on its isolated branch; no proposal is integrated or accepted yet. - Independent shared-owner design review confirms current Q1 still has no shared durable acquisition owner. It traces HF-specific state/task/store ownership and the native independent download loop, and identifies one-root-grant coupling plus `mutation_authority.rs` as part of migration/reopen correctness. It recommends one migrated store authority, consumer-held leases through import/extraction, and composed shutdown. No source, tests or retained state were changed by the reviewer. - Root rerouted the current Q1 source/consumer slice through Coding-Standards snapshot `snapshot:v1:264b9c2f-aca9-4b00-a344-530f8036a005`: 40 selected units, zero unresolved routing facts. Targeted policy reads covered Core, Persistence, Concurrency, Contract Evolution, Architecture, Rust Async, Rust API and Rust Security; the commit workflow was read before the source-slice commit review. Routing was used for obligations only; source and test evidence remain independent. + +## 2026-09-29 — Q1 shared-owner extraction preparation + +- A read-only composed-design trace found the current HF task owner stores one weak physical-root grant and one in-progress acquisition slot. Reusing this owner for model and native roots would reject the second root. The root-owned preparatory write set is `rust/crates/pumas-core/src/model_library/{download_recovery.rs,hf/lifecycle.rs}` plus this plan record and the co-located lifecycle test. `DestinationRootIdentity` now keys grant bookkeeping by physical-root identity; held `RootExecutionGrant` capabilities remain the effect authority. Independently reopened handles for one root coalesce to one retained grant, while separate roots can be acquired concurrently. Independent review confirmed that the key does not become filesystem authority and found no blocking defect. It noted inactive per-root slots remain for the owner lifetime; configured-root use is currently bounded, and a dynamically multi-root shared owner must prune only inactive slots or bound root admission. This does not move the task owner out of HF or add transfer/persistence ownership. +- **Actual candidate evidence:** `cargo test --manifest-path rust/Cargo.toml -p pumas-library model_library::hf::lifecycle::tests:: -- --test-threads=1` passed **37/37** lifecycle tests, including `one_owner_acquires_distinct_physical_root_grants_concurrently` for same-root reopen coalescing and two-root custody. `cargo clippy --manifest-path rust/Cargo.toml -p pumas-library --all-targets -- -D warnings`, `cargo check --manifest-path rust/Cargo.toml -p pumas-library --no-default-features`, `cargo fmt --manifest-path rust/Cargo.toml --package pumas-library -- --check`, and `git diff --check` passed. These are local, disposable-filesystem tests and static checks, not real HF/runtime consumer or restart/reopen acceptance. +- **Coding-Standards MCP:** Fresh snapshot `snapshot:v1:ea1dc4cb-2d14-4a69-866c-ded1b097e278` routed the slice to 37 standards with no unresolved questions. Focused reads covered Persistence, Concurrency, Contract Evolution, Architecture, Rust Async, Rust API, Verification, Commit, Rust Cross-Platform, Rust Dependencies, Security, Untrusted Execution, Resilience, Protocols, Schemas, Replay, and Code Design. The first route included eight policy bodies and produced output large enough to truncate; subsequent focused policy reads supplied the applicable obligations. This workflow feedback is separately recorded in the [MCP usability report](reports/coding-standards-mcp-usability.md). +- This sub-slice is authored atop Q1 source `98c19910faeebbb75056ddd604b99554305b14a8`. `gh pr checks 7` could not reach GitHub (`error connecting to api.github.com`); a direct PR-page fetch also returned a cache miss. The last prior observation for the pushed `98c19910…` head was Build run `36652117684` in progress, so no pass is claimed for this candidate. The user-owned `docs/breif/future.md` remains untouched and untracked. +- The isolated native proposal remains unintegrated. Its GPT-6.1 Sol High reviewer confirmed the four initial custody findings were addressed, then found a P2: canceled metadata transaction/removal waiters left blocking workers outside the shutdown receipt. The contributor reports that diff `642c96dde04fb49a133c56081c814abacc06b08c5959b87239fe755de700157a` registers these tasks with `VersionState`, drains them through manager shutdown, and exposes an explicit direct-owner drain. Independent follow-up review confirmed the P2 closed for `VersionManager` and direct `VersionState` owners and found no new P0/P1/P2 in that scope. It also identified the public `OllamaVersionManager` wrapper as an adjacent drain owner; a read-only tree search found no in-repository production caller, but its public API owns the same state. The exact worker write set is therefore expanded to include `version_manager/ollama.rs` only for its owned mutation drain and a focused regression. The reviewer explicitly did not extend the guarantee to that wrapper yet. The worker's exact-candidate tests report 254 app-manager, 265 RPC, 17 integration and 2 intent-integration cases passing, plus targeted canceled-waiter and static checks. Its native downloader remains independent, and AQ-HTTP is not ready. +- Q1/AQ-HTTP remains in progress/not ready. The shared durable owner/store, actual HF import and native consumer cutover, source resolver consumption, lifecycle reopen across handoff, desktop path, real source and all AC01–AC10/AC15/AC16/AC18 claims remain pending. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index c71cc00c..5be59422 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,7 +2,7 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The existing GitHub client now has an additive exact-tag resolver that maps live publisher asset identity/digest metadata to a verified manifest while leaving the public/cache DTO unchanged. A native custody proposal is under repair on an isolated worker branch after independent review found shutdown, metadata-coordination, pending-file-I/O, and cleanup-error gaps. The resolver is not yet consumed by the native installer, and shared durable handoff ownership, retained-store evolution, desktop path, and required real-source qualification remain pending. +**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The existing GitHub client now has an additive exact-tag resolver that maps live publisher asset identity/digest metadata to a verified manifest while leaving the public/cache DTO unchanged. The HF lifecycle owner now keys held root grants by physical-root identity, allowing one owner to retain distinct model/runtime roots and coalesce independently reopened handles; this is extraction preparation, not the shared acquisition cutover. An isolated native-custody proposal now has a candidate for shutdown drainage, metadata coordination, pending-file-I/O settlement, and fallible stage cleanup. Independent review confirmed managed `VersionManager` and direct `VersionState` worker drainage, then found the public `OllamaVersionManager` wrapper also owns `VersionState`; its exact adjacent write set is being added to close that drain boundary. The resolver is not yet consumed by the native installer, and shared durable handoff ownership, retained-store evolution, desktop path, and required real-source qualification remain pending. **Exactly one next slice:** Continue **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index d419a2ca..73eaf723 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -54,6 +54,51 @@ This report records agent experience using the Coding-Standards MCP during Acqui - **Smallest sufficient workflow:** Discover callable `standards_engine` methods from `ALL_TOOLS`, call `routing_facts`, complete a slice-specific route, then read the few selected policies required for architecture/persistence/concurrency. Repository inspection remains the source of implementation facts. - **Recommendations:** Surface configured MCP tool namespaces in initial agent context, distinguish callable tools from MCP servers/resources, and make the bounded read-only workflow discoverable without exposing unrelated authoring operations first. +## Primary integrator — shared-owner extraction preparation + +- **Useful calls:** For the new multi-root custody seam, I refreshed `routing_facts`, then routed the launcher/library, persistence/IPC, Rust API/async/cross-platform/dependency/security, concurrency, contracts/evolution, resilience, replay, and verification facts. The route completed with no unresolved facts under snapshot `snapshot:v1:ea1dc4cb-2d14-4a69-866c-ded1b097e278`. Focused snapshot-bound reads of Persistence, Concurrency, Contract Evolution, Architecture, Rust Async, Rust API, Verification, Commit, Rust Cross-Platform, Rust Dependencies, Security, Untrusted Execution, Resilience, Protocols, Schemas, Replay, and Code Design identified the relevant ownership and evidence obligations. +- **Confusing or redundant steps:** The route requested eight policy bodies and produced roughly 63,000 output tokens before truncation, even though only the applicability result and snapshot were needed. `read_many` returns full policy content without a section selector, so I filtered the structured result by headings and relevant sections. Repeating route relationship metadata alongside policy text remained a major output cost. +- **Missing context:** The MCP did not know the existing owner retained only one physical-root grant, that independently opened handles can refer to one root, or that model and native consumers need separate physical roots. Repository inspection and the shared-owner source review established those facts. It also could not report the active PR check because GitHub access failed in the current environment. +- **Where I left MCP:** Root identity semantics, async grant acquisition, the actual two-root test, Rust verification, repository state, and current CI availability were established with source and repository tools. The standards route selected obligations only; the code/test evidence is separate. +- **Smallest sufficient workflow:** Reuse the current fact vocabulary when available; call `routing_facts` only to refresh it; route without inline policy bodies; read a small set of exact selected policies in bounded calls; inspect source and run tests outside the MCP; reroute the final candidate if the owner boundary changes. +- **Recommendations:** Make route content opt-in and strictly byte-budgeted; offer section-scoped policy reads; report which policies were already read; include repository-supplied owner and candidate context as non-authoritative context; and show a compact delta when a slice adds or removes an ownership boundary. + +## Native-custody contributor — revised lifecycle review + +- **Useful calls:** A new `routing_facts`/`route` cycle selected 28 standards with no unresolved facts under snapshot `snapshot:v1:afd3e35d-bc08-4dc8-b848-597626ae279c`. The contributor reread concurrency, Rust async, persistence, and resilience against the repaired native-custody proposal and then executed source-backed lifecycle fixtures and Cargo checks. +- **Confusing or redundant steps:** Route results again carried substantial edge/handle metadata. Batched full policy content exceeded a useful output window; smaller content-only reads were easier to apply. +- **Missing context:** The MCP still did not identify the exact `VersionState` lifecycle owner or how shutdown observed metadata workers; those were discovered in repository source and through independent review. +- **Where the contributor left MCP:** All filesystem, task-registration, test, and build evidence came from the isolated worktree. The revised route did not establish AQ-HTTP or certify the native implementation. +- **Smallest sufficient workflow:** Refresh facts, route the precise changed boundary, reread only changed obligations, then inspect and test the exact candidate. +- **Recommendations:** Add a compact route with no repeated relationship graph, preserve a visible list of already-read obligations, and present changed-boundary implications without repeating unchanged policy text. + +## Native-custody contributor — canceled-worker ownership follow-up + +- **Useful calls:** The contributor reused its previously routed snapshot and reread Concurrency and Rust Async after the independent P2. That narrowed the design requirement from “retain the lock” to “retain and observe worker completion through shutdown.” The actual registry implementation and canceled-waiter completion/failure/panic fixtures were verified in the worker worktree. +- **Confusing or redundant steps:** No new discovery or broad reroute was needed. Policy text still lacked source-specific task-owner and state-construction context. +- **Missing context:** The MCP did not identify the existing `VersionState`/`InstallationTasks` owners or manager shutdown order; source inspection established how to register and drain the new mutation workers. +- **Where the contributor left MCP:** The worker registry, manager drain, repeated shutdown result, and tests were all assessed from source and exact-candidate Cargo runs. The MCP supplied obligations only. +- **Smallest sufficient workflow:** Reuse the prior route, reread only concurrency/async obligations implicated by the finding, then implement and test the source lifecycle. +- **Recommendations:** Keep exact prior-read and snapshot context visible, offer section-level policy output, and provide concrete owner/cancellation prompts without pretending the MCP knows repository lifecycle wiring. + +## Native-custody independent reviewer — revised candidate follow-up + +- **Useful calls:** The reviewer reused the prior registered facts and snapshot, routed the revised metadata/task-lifecycle scope, and read only Concurrency, Persistence, and Rust Async. That focused read directly exposed the missing terminal observer for canceled metadata workers. +- **Confusing or redundant steps:** Route rationale still repeated relation/handle metadata. Focused output improved on the earlier all-policy batch, but exact diff/base/gate facts still had to be supplied and verified externally. +- **Missing context:** MCP did not bind the review to the candidate digest or show that the new workers were omitted from `shutdown_installations`; source and test inspection were necessary. +- **Where the reviewer left MCP:** The P2 finding, exact source locations, changed-diff digest, and review status came from read-only Git/source inspection. No tests were run by this review. +- **Smallest sufficient workflow:** Reuse retained routing facts, route the actual change, read the three relevant policies, inspect source/diff and existing evidence, then report findings independently. +- **Recommendations:** Offer a narrow source-review mode that accepts an exact candidate reference as context, links only relevant policies, and says plainly that tools neither inspected the diff nor approved standards compliance. Keep standards-corpus authoring operations out of the default review entry point. + +## Native-custody independent reviewer — shutdown ownership repair verification + +- **Useful calls:** The reviewer used a snapshot-bound `route` plus a focused `read_many` of Rust Async and Concurrency. These rules directly separated retaining a metadata lock from retaining a completion observer and shutdown drain. +- **Confusing or redundant steps:** No new fact discovery was needed; the route and two policy reads were sufficient. Source inspection was still needed to confirm the lock, result channel, supervisor receipt, and shutdown guard ordering. +- **Missing context:** The MCP did not reveal the public OllamaVersionManager wrapper or its lack of a drain method; repository search did. No in-repository production caller was found, but the public wrapper remains a supported ownership surface unless its API contract says otherwise. +- **Where the reviewer left MCP:** Exact diff identity, the closed P2 disposition, fixture structure, and remaining wrapper limit came from read-only source/Git inspection. No tests were run by the reviewer. +- **Smallest sufficient workflow:** Reuse retained route facts, route the focused shutdown change, read Rust Async and Concurrency, then inspect the exact diff and tests. +- **Recommendations:** Keep the minimal policy-only workflow easy to repeat and make source review context explicit. A concise owner inventory would help identify wrappers that must delegate to a newly introduced lifecycle owner. + ## Participation The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, and shared-owner design reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index b26925ed..d734d0a9 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -15,6 +15,10 @@ These are the admitted exact paths/closed path families. The actual source files - `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,ollama.rs,progress.rs,state.rs}` for the acquisition bridge and transfer progress, not installed-unit identity migration; - current core atomic JSON/capability-filesystem modules only where the same selected invariant requires a targeted change, never as an unrelated filesystem rewrite. +The current root-owned extraction-preparation sub-slice is limited to `rust/crates/pumas-core/src/model_library/{download_recovery.rs,hf/lifecycle.rs}` and the co-located lifecycle regression. It adds a physical-root equality key and lets the existing supervised lifecycle owner hold distinct root grants concurrently while coalescing independently reopened handles for the same root. It does not move the lifecycle owner, add a transfer/persistence authority, or advance AQ-HTTP. + +The isolated native-custody worker's original four-file set is expanded by one exact adjacent file after review found the public `OllamaVersionManager` retains the shared `VersionState` but exposes no drain for its mutation registry. Its only permitted `ollama.rs` work is to close/drain that owned state lifecycle and add a focused regression; no Ollama downloader, source, archive, publication, or server-owner redesign is admitted. The same routed Rust async/concurrency/public-API obligations apply. + **Tests:** proposed `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`, plus existing co-located GitHub metadata, HF lifecycle/recovery, and installer regression tests. Fixtures must reach the owner under test with independent expected byte/effect outcomes. **Serial adjacent writes:** `rust/crates/pumas-rpc/src/contract.rs`, `contract/export.rs`, affected HF/version/status handlers, `electron/src/{preload.ts,rpc-method-registry.ts,ipc-validation.ts}`, actual corresponding frontend download/install/source views and generated DTOs. Enumerate outputs from the actual exporter rather than guess or edit generated files manually. Reuse the existing model/native UI; this is not a dashboard redesign. diff --git a/rust/crates/pumas-core/src/model_library/download_recovery.rs b/rust/crates/pumas-core/src/model_library/download_recovery.rs index e8f1d5f7..f144eae2 100644 --- a/rust/crates/pumas-core/src/model_library/download_recovery.rs +++ b/rust/crates/pumas-core/src/model_library/download_recovery.rs @@ -245,6 +245,10 @@ pub(crate) struct DestinationIdentity { relative: String, } +/// Equality key for one configured physical root; it grants no filesystem access. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct DestinationRootIdentity(FilesystemIdentity); + /// One configured root opened by the composition owner after directory setup. #[derive(Clone)] pub(crate) struct DownloadDestinationRoot(Arc); @@ -287,6 +291,13 @@ impl RootExecutionGrant { } impl DownloadDestinationRoot { + /// Equality identity for the configured physical root. This is only a key + /// for shared in-process grant bookkeeping; the held capability remains + /// the authority for every filesystem effect. + pub(crate) fn grant_identity(&self) -> DestinationRootIdentity { + DestinationRootIdentity(self.0.root_identity) + } + pub(crate) fn same_physical_root(&self, other: &Self) -> bool { self.0.root_identity == other.0.root_identity } diff --git a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs index 553d51be..22d0f59e 100644 --- a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs +++ b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs @@ -19,7 +19,7 @@ use tokio::sync::{oneshot, Notify}; use tokio::task::JoinHandle; use crate::model_library::download_recovery::{ - DestinationIdentity, DownloadDestinationRoot, RootExecutionGrant, + DestinationIdentity, DestinationRootIdentity, DownloadDestinationRoot, RootExecutionGrant, }; type FallibleBlockingReceiver = @@ -763,8 +763,7 @@ struct OwnerState { // Admission, ownership transfers, and shutdown capture share this mutex. // Entries leave these populations only for another registered observer. closed: bool, - root_grant: Weak, - root_grant_acquiring: bool, + root_grants: HashMap, tasks: HashMap, prepared: HashMap, retired: Vec, @@ -773,6 +772,12 @@ struct OwnerState { shutdown_driver: Option>, } +#[derive(Default)] +struct RootGrantSlot { + grant: Weak, + acquiring: bool, +} + struct InvocationWaiter { owner: Arc, id: String, @@ -995,6 +1000,7 @@ impl DownloadTaskOwner { context: &TaskContext, root: DownloadDestinationRoot, ) -> crate::Result> { + let identity = root.grant_identity(); loop { let changed = self.root_grant_changed.notified(); tokio::pin!(changed); @@ -1004,12 +1010,13 @@ impl DownloadTaskOwner { if state.closed { return Err(crate::PumasError::DownloadLifecycleClosed); } - if let Some(grant) = state.root_grant.upgrade() { + let slot = state.root_grants.entry(identity).or_default(); + if let Some(grant) = slot.grant.upgrade() { GrantAcquisition::Reuse(grant) - } else if state.root_grant_acquiring { + } else if slot.acquiring { GrantAcquisition::Wait } else { - state.root_grant_acquiring = true; + slot.acquiring = true; GrantAcquisition::Open } }; @@ -1045,10 +1052,11 @@ impl DownloadTaskOwner { .and_then(|result| result); { let mut state = self.state.lock().expect("HF task owner lock poisoned"); + let slot = state.root_grants.entry(identity).or_default(); if let Ok(grant) = &result { - state.root_grant = Arc::downgrade(grant); + slot.grant = Arc::downgrade(grant); } - state.root_grant_acquiring = false; + slot.acquiring = false; } self.root_grant_changed.notify_waiters(); return result; @@ -3111,6 +3119,53 @@ mod tests { owner.shutdown(|| async { Ok(()) }).await.unwrap(); } + #[tokio::test] + async fn one_owner_acquires_distinct_physical_root_grants_concurrently() { + let model_root_dir = tempfile::TempDir::new().unwrap(); + let runtime_root_dir = tempfile::TempDir::new().unwrap(); + let model_root = DownloadDestinationRoot::open(model_root_dir.path()).unwrap(); + let reopened_model_root = DownloadDestinationRoot::open(model_root_dir.path()).unwrap(); + let runtime_root = DownloadDestinationRoot::open(runtime_root_dir.path()).unwrap(); + let owner = Arc::new(DownloadTaskOwner::new()); + owner + .run_invocation(move |context| async move { + let (model, reopened_model, runtime) = tokio::join!( + context.with_root_grant(model_root.clone()), + context.with_root_grant(reopened_model_root.clone()), + context.with_root_grant(runtime_root.clone()), + ); + let model = model?; + let reopened_model = reopened_model?; + let runtime = runtime?; + assert!(Arc::ptr_eq( + model.root_grant.as_ref().unwrap(), + reopened_model.root_grant.as_ref().unwrap() + )); + assert!(!Arc::ptr_eq( + model.root_grant.as_ref().unwrap(), + runtime.root_grant.as_ref().unwrap() + )); + assert!(matches!( + model_root.try_acquire_execution_grant(), + Err(crate::PumasError::DownloadRootBusy) + )); + assert!(matches!( + runtime_root.try_acquire_execution_grant(), + Err(crate::PumasError::DownloadRootBusy) + )); + drop(model); + drop(reopened_model); + drop(runtime); + context.drain_blocking().await.unwrap(); + model_root.try_acquire_execution_grant()?; + runtime_root.try_acquire_execution_grant()?; + Ok(()) + }) + .await + .unwrap(); + owner.shutdown(|| async { Ok(()) }).await.unwrap(); + } + #[tokio::test] async fn refused_root_grant_does_not_poison_shutdown() { let temp = tempfile::TempDir::new().unwrap(); From 560cec716211c64cdfc04bc737c8bda94266fcf2 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 18:23:40 -0700 Subject: [PATCH 07/20] fix(runtime): retain native installer effects through shutdown --- .../src/version_manager/installer.rs | 922 +++++++++++++----- .../src/version_manager/mod.rs | 803 +++++++++++++-- .../src/version_manager/ollama.rs | 451 ++++++++- .../src/version_manager/state.rs | 778 ++++++++++----- rust/crates/pumas-rpc/src/server.rs | 62 +- 5 files changed, 2445 insertions(+), 571 deletions(-) diff --git a/rust/crates/pumas-app-manager/src/version_manager/installer.rs b/rust/crates/pumas-app-manager/src/version_manager/installer.rs index 5a380360..1d10dfbb 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/installer.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/installer.rs @@ -57,6 +57,104 @@ async fn path_exists(path: &Path) -> Result { /// Coordinates Torch cancellation with the irreversible publication boundary. pub(crate) struct TorchInstallControl(AtomicU8); +/// Mutable output and input bytes belong to one native attempt. The permanent +/// lock file coordinates native installers and metadata mutations; it must +/// never be unlinked. +struct NativeInstallWorkspace { + // Implicit destruction must preserve uncertain work. Reclamation is an + // explicit fallible operation after all file/extraction effects settle. + directory: PathBuf, + _lock: NativeVersionsLock, +} + +#[derive(Clone)] +pub(crate) struct NativeVersionsLock(Arc); + +impl Drop for NativeVersionsLock { + fn drop(&mut self) { + // Explicit unlock also releases locks inherited by a concurrently + // spawning child before that child's close-on-exec takes effect. + if Arc::strong_count(&self.0) == 1 { + let _ = fs2::FileExt::unlock(&*self.0); + } + } +} + +impl NativeVersionsLock { + pub(crate) fn try_acquire(versions: &Path) -> Result { + std::fs::create_dir_all(versions) + .map_err(|error| PumasError::io_with_path(error, versions))?; + let lock_path = versions.join(".llama-install.lock"); + let lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&lock_path) + .map_err(|error| PumasError::io_with_path(error, &lock_path))?; + fs2::FileExt::try_lock_exclusive(&lock) + .map_err(|error| PumasError::io_with_path(error, &lock_path))?; + Ok(Self(Arc::new(lock))) + } + + pub(crate) async fn acquire(versions: PathBuf) -> Result { + tokio::task::spawn_blocking(move || Self::try_acquire(&versions)) + .await + .map_err(|error| { + PumasError::Other(format!("Native mutation admission failed: {error}")) + })? + } +} + +impl NativeInstallWorkspace { + fn create(versions: &Path) -> Result { + let lock = NativeVersionsLock::try_acquire(versions)?; + let directory = tempfile::Builder::new() + .prefix(".llama-install-") + .tempdir_in(versions) + .map_err(|error| PumasError::io_with_path(error, versions))? + .keep(); + Ok(Self { + directory, + _lock: lock, + }) + } + + fn path(&self) -> &Path { + &self.directory + } + + fn cleanup(self) -> Result<()> { + std::fs::remove_dir_all(&self.directory).map_err(|error| PumasError::Io { + message: format!("Native staging cleanup incomplete; retained workspace requires reconciliation: {error}"), + path: Some(self.directory.clone()), + source: Some(error), + }) + } +} + +fn settled_result(outcome: Result, settlement: Result<()>) -> Result { + match (outcome, settlement) { + (result, Ok(())) => result, + (Ok(_), Err(error)) => Err(error), + (Err(error), Err(settlement)) => Err(PumasError::InstallationFailed { + message: format!("{error}; settlement incomplete: {settlement}"), + }), + } +} + +async fn settle_archive_file(mut file: fs::File, path: &Path, outcome: Result) -> Result { + let settlement = file.flush().await.map_err(|error| PumasError::Io { + message: format!("Failed to flush archive: {error}"), + path: Some(path.to_path_buf()), + source: Some(error), + }); + // Even an unsuccessful flush must settle queued blocking file work before + // the stage owner may reclaim bytes. Dropping Tokio File is insufficient. + drop(file.into_std().await); + settled_result(outcome, settlement) +} + type TorchCleanupCompletion = Shared>>>; type TorchChildReceipt = tokio::sync::watch::Receiver>>>; @@ -403,6 +501,7 @@ pub struct VersionInstaller { progress_tracker: Arc>, /// Cancellation flag. cancel_flag: Arc, + shutdown_flag: Arc, torch_control: Arc, torch_cleanup: Arc, torch_attempt_lock: Mutex<()>, @@ -436,6 +535,7 @@ impl VersionInstaller { metadata_manager, progress_tracker, cancel_flag, + shutdown_flag: Arc::new(AtomicBool::new(false)), torch_control: Arc::new(TorchInstallControl::new()), torch_cleanup: Arc::new(TorchCleanupTasks::default()), torch_attempt_lock: Mutex::new(()), @@ -459,6 +559,26 @@ impl VersionInstaller { children } + pub(crate) fn with_shutdown_flag(mut self, flag: Arc) -> Self { + self.shutdown_flag = flag; + self + } + + async fn send_progress(&self, sender: &mpsc::Sender, update: ProgressUpdate) { + tokio::select! { + _ = sender.send(update) => {}, + _ = super::wait_for_install_cancel(self.shutdown_flag.clone()) => {}, + _ = super::wait_for_install_cancel(self.cancel_flag.clone()) => {}, + } + } + + async fn wait_for_cancellation(&self) { + tokio::select! { + _ = super::wait_for_install_cancel(self.shutdown_flag.clone()) => {}, + _ = super::wait_for_install_cancel(self.cancel_flag.clone()) => {}, + } + } + pub(crate) fn with_torch_control(mut self, control: Arc) -> Self { self.torch_control = control; self @@ -651,6 +771,27 @@ impl VersionInstaller { release: &GitHubRelease, progress_tx: mpsc::Sender, ) -> Result<()> { + let _attempt = + self.torch_attempt_lock + .try_lock() + .map_err(|_| PumasError::InstallationFailed { + message: "llama.cpp installation already active".into(), + })?; + self.torch_control.start(); + let result = self + .install_llama_cpp_binary_inner(tag, release, progress_tx) + .await; + self.torch_control.finish(); + result + } + + async fn install_llama_cpp_binary_inner( + &self, + tag: &str, + release: &GitHubRelease, + progress_tx: mpsc::Sender, + ) -> Result<()> { + Self::validate_native_tag(tag)?; info!("Starting llama.cpp binary installation for {}", tag); let asset = self.select_llama_cpp_asset(&release.assets)?; @@ -681,40 +822,46 @@ impl VersionInstaller { ); } - let cache_downloads = self - .launcher_root - .join("launcher-data") - .join("cache") - .join("downloads"); - fs::create_dir_all(&cache_downloads) + // No digest is supplied by GitHubAsset. Filename and size are not + // integrity evidence: do not reuse or mutate retained download caches. + let versions = self.versions_dir(); + fs::create_dir_all(&versions) .await - .map_err(|e| PumasError::Io { - message: format!("Failed to create download cache directory: {}", e), - path: Some(cache_downloads.clone()), - source: Some(e), - })?; - - let archive_path = cache_downloads.join(&asset_name); - let cache_valid = self - .is_cached_download_valid(&archive_path, &asset_name, total_size) - .await?; - + .map_err(|e| PumasError::io_with_path(e, &versions))?; + let custody = Arc::new( + tokio::task::spawn_blocking(move || NativeInstallWorkspace::create(&versions)) + .await + .map_err(|error| { + PumasError::Other(format!("Failed to join native staging: {error}")) + })??, + ); + let archive_path = custody.path().join("archive"); let result = self .do_llama_cpp_install( tag, release, download_url, - total_size, &asset_name, &archive_path, - cache_valid, + custody.clone(), &progress_tx, ) .await; - if result.is_err() { - let _ = fs::remove_file(&archive_path).await; - } + // Stage reclamation can traverse a large archive. The supervised + // installation awaits that blocking cleanup before releasing admission. + let cleanup = tokio::task::spawn_blocking(move || { + Arc::try_unwrap(custody) + .map_err(|workspace| PumasError::InstallationFailed { + message: format!("Native workspace remains in use; retained stage requires reconciliation: {}", workspace.path().display()), + })? + .cleanup() + }) + .await + .unwrap_or_else(|error| Err( + PumasError::Other(format!("Failed to join native cleanup: {error}")) + )); + let result = settled_result(result, cleanup); { let mut tracker = self.progress_tracker.write().await; @@ -754,13 +901,15 @@ impl VersionInstaller { Some("Using cached download"), ); } - let _ = progress_tx - .send(ProgressUpdate::Download { + self.send_progress( + progress_tx, + ProgressUpdate::Download { downloaded_bytes: total_size, total_bytes: Some(total_size), speed_bytes_per_sec: None, - }) - .await; + }, + ) + .await; } else { self.download_archive(download_url, archive_path, progress_tx) .await?; @@ -788,12 +937,14 @@ impl VersionInstaller { Some("Extracting binary..."), ); } - let _ = progress_tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + progress_tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Extract, message: "Extracting binary...".to_string(), - }) - .await; + }, + ) + .await; let archive_path = archive_path.to_path_buf(); let version_dir_for_extract = version_dir.clone(); @@ -826,40 +977,6 @@ impl VersionInstaller { Ok(()) } - async fn is_cached_download_valid( - &self, - archive_path: &Path, - asset_name: &str, - total_size: u64, - ) -> Result { - if !path_exists(archive_path).await? { - return Ok(false); - } - - match fs::metadata(archive_path).await { - Ok(meta) if meta.len() == total_size => { - info!( - "Using cached download: {} ({} bytes)", - asset_name, total_size - ); - Ok(true) - } - Ok(meta) => { - info!( - "Cached download size mismatch ({} != {}), re-downloading", - meta.len(), - total_size - ); - let _ = fs::remove_file(archive_path).await; - Ok(false) - } - Err(_) => { - let _ = fs::remove_file(archive_path).await; - Ok(false) - } - } - } - /// Execute llama.cpp installation steps. #[allow(clippy::too_many_arguments)] async fn do_llama_cpp_install( @@ -867,55 +984,26 @@ impl VersionInstaller { tag: &str, release: &GitHubRelease, download_url: &str, - total_size: u64, asset_name: &str, archive_path: &Path, - cache_valid: bool, + custody: Arc, progress_tx: &mpsc::Sender, ) -> Result<()> { - self.check_cancelled()?; - - if cache_valid { - { - let mut tracker = self.progress_tracker.write().await; - tracker.update_stage( - InstallationStage::Download, - 100.0, - Some("Using cached download"), - ); - } - let _ = progress_tx - .send(ProgressUpdate::Download { - downloaded_bytes: total_size, - total_bytes: Some(total_size), - speed_bytes_per_sec: None, - }) - .await; - } else { - self.download_archive(download_url, archive_path, progress_tx) - .await?; - } - - self.check_cancelled()?; - + Self::validate_native_tag(tag)?; let version_dir = self.versions_dir().join(tag); + // Replacement requires a durable directory/metadata transaction owned + // elsewhere. Never delete or replace retained output here. if path_exists(&version_dir).await? { - fs::remove_dir_all(&version_dir) - .await - .map_err(|e| PumasError::Io { - message: format!("Failed to remove existing version directory: {}", e), - path: Some(version_dir.clone()), - source: Some(e), - })?; + return Err(PumasError::VersionAlreadyInstalled { tag: tag.into() }); } - fs::create_dir_all(&version_dir) + self.check_cancelled()?; + self.download_archive(download_url, archive_path, progress_tx) + .await?; + self.check_cancelled()?; + let stage = custody.path().join("output"); + fs::create_dir(&stage) .await - .map_err(|e| PumasError::Io { - message: format!("Failed to create version directory: {}", e), - path: Some(version_dir.clone()), - source: Some(e), - })?; - + .map_err(|e| PumasError::io_with_path(e, &stage))?; { let mut tracker = self.progress_tracker.write().await; tracker.update_stage( @@ -924,39 +1012,79 @@ impl VersionInstaller { Some("Extracting binary archive..."), ); } - let _ = progress_tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + progress_tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Extract, - message: "Extracting binary archive...".to_string(), - }) - .await; - + message: "Extracting binary archive...".into(), + }, + ) + .await; let archive_path = archive_path.to_path_buf(); - let version_dir_for_extract = version_dir.clone(); - let asset_name = asset_name.to_string(); + let extracted = stage.clone(); + let asset_name = asset_name.to_owned(); + // The blocking worker holds custody even if a direct caller drops its + // future. The manager never aborts its registered installation task. + let held = custody.clone(); tokio::task::spawn_blocking(move || { - Self::extract_llama_cpp_binary(&archive_path, &version_dir_for_extract, &asset_name) + let _held = held; + Self::extract_llama_cpp_binary(&archive_path, &extracted, &asset_name) }) .await .map_err(|e| { - PumasError::Other(format!("Failed to join llama.cpp extraction task: {}", e)) + PumasError::Other(format!("Failed to join llama.cpp extraction task: {e}")) })??; - - { - let mut tracker = self.progress_tracker.write().await; - tracker.update_stage( - InstallationStage::Extract, - 100.0, - Some("Extraction complete"), - ); - } - self.check_cancelled()?; - - self.finalize_llama_cpp_installation(tag, release, &version_dir, progress_tx) + self.finalize_llama_cpp_installation(tag, release, &stage, custody, progress_tx) .await?; + Ok(()) + } + + fn publish_native_stage( + stage: &Path, + destination: &Path, + finalize: impl FnOnce() -> std::result::Result<(), (PumasError, bool)>, + ) -> Result<()> { + match std::fs::symlink_metadata(destination) { + Ok(_) => { + return Err(PumasError::InstallationFailed { + message: "Native version destination already exists".into(), + }) + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(PumasError::io_with_path(error, destination)), + } + std::fs::rename(stage, destination) + .map_err(|e| PumasError::io_with_path(e, destination))?; + if let Err((error, can_withdraw)) = finalize() { + if !can_withdraw { + return Err(error); + } + // Only this attempt's newly published directory may be withdrawn. + std::fs::rename(destination, stage).map_err(|rollback| { + PumasError::InstallationFailed { + message: format!( + "Native finalization failed: {error}; stage withdrawal failed: {rollback}" + ), + } + })?; + return Err(error); + } + Ok(()) + } - info!("llama.cpp installation of {} completed successfully", tag); + fn validate_native_tag(tag: &str) -> Result<()> { + if tag.is_empty() + || !tag.bytes().all(|byte| { + byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+') + }) + || tag == "." + || tag == ".." + { + return Err(PumasError::InstallationFailed { + message: "Invalid native version tag".into(), + }); + } Ok(()) } @@ -1168,10 +1296,20 @@ impl VersionInstaller { }); } + Self::validate_native_output(version_dir)?; let server_binary = Self::find_named_binary(version_dir, &["llama-server", "server"])? .ok_or_else(|| PumasError::InstallationFailed { message: "Could not find llama-server in extracted archive".to_string(), })?; + if std::fs::metadata(&server_binary) + .map_err(|e| PumasError::io_with_path(e, &server_binary))? + .len() + == 0 + { + return Err(PumasError::InstallationFailed { + message: "Extracted llama.cpp server is empty".into(), + }); + } let launch_binary = Self::install_llama_cpp_launch_binary(version_dir, &server_binary)?; Self::make_binary_executable(&launch_binary)?; @@ -1182,6 +1320,31 @@ impl VersionInstaller { Ok(()) } + fn validate_native_output(root: &Path) -> Result<()> { + let canonical_root = root + .canonicalize() + .map_err(|e| PumasError::io_with_path(e, root))?; + for entry in walkdir::WalkDir::new(root).follow_links(false) { + let entry = entry.map_err(|e| PumasError::InstallationFailed { + message: format!("Invalid native output: {e}"), + })?; + if entry.file_type().is_symlink() { + let target = entry + .path() + .canonicalize() + .map_err(|e| PumasError::io_with_path(e, entry.path()))?; + let link = std::fs::read_link(entry.path()) + .map_err(|e| PumasError::io_with_path(e, entry.path()))?; + if link.is_absolute() || !target.starts_with(&canonical_root) || !target.is_file() { + return Err(PumasError::InstallationFailed { + message: "Native archive contains an unsafe link".into(), + }); + } + } + } + Ok(()) + } + fn install_llama_cpp_launch_binary(version_dir: &Path, source: &Path) -> Result { let binary_name = if cfg!(windows) { "llama-server.exe" @@ -1220,9 +1383,14 @@ impl VersionInstaller { source.display() ), })?; + let relative_dir = binary_dir.strip_prefix(version_dir).map_err(|_| { + PumasError::InstallationFailed { + message: "llama.cpp binary is outside staged output".into(), + } + })?; let wrapper = format!( - "#!/bin/sh\nBINARY_DIR={}\nexport LD_LIBRARY_PATH=\"$BINARY_DIR${{LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}}\"\nexec \"$BINARY_DIR/{}\" \"$@\"\n", - shell_single_quote(&binary_dir.to_string_lossy()), + "#!/bin/sh\nROOT=$(CDPATH= cd -- \"$(dirname -- \"$0\")/..\" && pwd) || exit 1\nBINARY_DIR=\"$ROOT\"/{}\nexport LD_LIBRARY_PATH=\"$BINARY_DIR${{LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}}\"\nexec \"$BINARY_DIR/{}\" \"$@\"\n", + shell_single_quote(&relative_dir.to_string_lossy()), binary_file.to_string_lossy().replace('"', "\\\"") ); std::fs::write(&final_path, wrapper).map_err(|e| PumasError::Io { @@ -1460,12 +1628,14 @@ impl VersionInstaller { Some("Finalizing installation..."), ); } - let _ = progress_tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + progress_tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Setup, message: "Finalizing installation...".to_string(), - }) - .await; + }, + ) + .await; // Find the download URL for metadata let download_url = release @@ -1504,11 +1674,13 @@ impl VersionInstaller { Some("Installation complete"), ); } - let _ = progress_tx - .send(ProgressUpdate::Setup { + self.send_progress( + progress_tx, + ProgressUpdate::Setup { message: "Installation complete".to_string(), - }) - .await; + }, + ) + .await; info!("Ollama installation of {} finalized", tag); Ok(()) @@ -1519,7 +1691,8 @@ impl VersionInstaller { &self, tag: &str, release: &GitHubRelease, - _version_dir: &Path, + stage: &Path, + custody: Arc, progress_tx: &mpsc::Sender, ) -> Result<()> { info!("Finalizing llama.cpp installation for {}", tag); @@ -1532,17 +1705,17 @@ impl VersionInstaller { Some("Finalizing installation..."), ); } - let _ = progress_tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + progress_tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Setup, message: "Finalizing installation...".to_string(), - }) - .await; + }, + ) + .await; - let (download_url, size) = self - .select_llama_cpp_asset(&release.assets) - .map(|asset| (Some(asset.download_url.clone()), Some(asset.size))) - .unwrap_or((None, release.archive_size)); + let selected = self.select_llama_cpp_asset(&release.assets)?; + let (download_url, size) = (Some(selected.download_url.clone()), Some(selected.size)); let metadata = InstalledVersionMetadata { path: tag.to_string(), @@ -1558,8 +1731,40 @@ impl VersionInstaller { dependencies_installed: Some(true), }; - self.metadata_manager - .update_installed_version(tag, metadata, Some(self.app_id))?; + self.check_cancelled()?; + if !self.torch_control.try_begin_publication() { + return Err(PumasError::InstallationFailed { + message: "Installation cancelled before publication".into(), + }); + } + let stage = stage.to_path_buf(); + let destination = self.versions_dir().join(tag); + let manager = self.metadata_manager.clone(); + let app_id = self.app_id; + let tag = tag.to_owned(); + tokio::task::spawn_blocking(move || { + let _custody = custody; + if manager.get_installed_version(&tag, Some(app_id))?.is_some() { + return Err(PumasError::VersionAlreadyInstalled { tag }); + } + Self::publish_native_stage(&stage, &destination, || { + match manager.update_installed_version(&tag, metadata, Some(app_id)) { + Ok(()) => Ok(()), + Err(error) => { + // atomic_write_json may fail after replacement. Withdraw + // output only when the metadata owner proves no entry. + match manager.get_installed_version(&tag, Some(app_id)) { + Ok(None) => Err((error, true)), + _ => Err((PumasError::InstallationFailed { + message: format!("Native metadata publication failed with retained complete output; reconciliation required: {error}"), + }, false)), + } + } + } + }) + }) + .await + .map_err(|e| PumasError::Other(format!("Failed to join native publication: {e}")))??; { let mut tracker = self.progress_tracker.write().await; @@ -1569,13 +1774,15 @@ impl VersionInstaller { Some("Installation complete"), ); } - let _ = progress_tx - .send(ProgressUpdate::Setup { + self.send_progress( + progress_tx, + ProgressUpdate::Setup { message: "Installation complete".to_string(), - }) - .await; + }, + ) + .await; - info!("llama.cpp installation of {} finalized", tag); + info!("llama.cpp installation finalized"); Ok(()) } @@ -1596,12 +1803,14 @@ impl VersionInstaller { Some("Starting download..."), ); } - let _ = progress_tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + progress_tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Download, message: "Starting download...".to_string(), - }) - .await; + }, + ) + .await; // Create HTTP client with appropriate timeouts for large downloads // - connect_timeout: time to establish connection (15s is fine) @@ -1618,7 +1827,12 @@ impl VersionInstaller { // Start download with retry let mut response = None; for attempt in 1..=InstallationConfig::DOWNLOAD_RETRY_ATTEMPTS { - match client.get(url).send().await { + self.check_cancelled()?; + let request = tokio::select! { + result = client.get(url).send() => result, + _ = self.wait_for_cancellation() => return Err(Self::cancellation_error()), + }; + match request { Ok(resp) => { if resp.status().is_success() { response = Some(resp); @@ -1639,7 +1853,10 @@ impl VersionInstaller { cause: Some(e.to_string()), }); } - tokio::time::sleep(std::time::Duration::from_secs(2u64.pow(attempt))).await; + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(2u64.pow(attempt))) => {}, + _ = self.wait_for_cancellation() => return Err(Self::cancellation_error()), + } } } } @@ -1660,60 +1877,71 @@ impl VersionInstaller { source: Some(e), })?; - // Download with progress - let mut downloaded: u64 = 0; - let mut stream = response.bytes_stream(); - let start_time = std::time::Instant::now(); + // Cancellation interrupts network waits only. File operations and + // their settlement remain awaited before the stage can be reclaimed. + let transfer = async { + let mut downloaded: u64 = 0; + let mut stream = response.bytes_stream(); + let start_time = std::time::Instant::now(); - use futures::StreamExt; - while let Some(chunk) = stream.next().await { - // Check cancellation - self.check_cancelled()?; + use futures::StreamExt; + loop { + let chunk = tokio::select! { + chunk = stream.next() => chunk, + _ = self.wait_for_cancellation() => { + return Err(Self::cancellation_error()); + }, + }; + let Some(chunk) = chunk else { + break; + }; + // Check cancellation + self.check_cancelled()?; + + let chunk = chunk.map_err(|e| PumasError::Network { + message: format!("Error reading download chunk: {}", e), + cause: Some(e.to_string()), + })?; - let chunk = chunk.map_err(|e| PumasError::Network { - message: format!("Error reading download chunk: {}", e), - cause: Some(e.to_string()), - })?; + file.write_all(&chunk).await.map_err(|e| PumasError::Io { + message: format!("Failed to write to archive: {}", e), + path: Some(archive_path.to_path_buf()), + source: Some(e), + })?; - file.write_all(&chunk).await.map_err(|e| PumasError::Io { - message: format!("Failed to write to archive: {}", e), - path: Some(archive_path.to_path_buf()), - source: Some(e), - })?; + downloaded += chunk.len() as u64; - downloaded += chunk.len() as u64; + // Calculate speed + let elapsed = start_time.elapsed().as_secs_f64(); + let speed = if elapsed > 0.0 { + Some(downloaded as f64 / elapsed) + } else { + None + }; - // Calculate speed - let elapsed = start_time.elapsed().as_secs_f64(); - let speed = if elapsed > 0.0 { - Some(downloaded as f64 / elapsed) - } else { - None - }; + // Update progress + { + let mut tracker = self.progress_tracker.write().await; + tracker.update_download_progress(downloaded, total_size, speed); + } - // Update progress - { - let mut tracker = self.progress_tracker.write().await; - tracker.update_download_progress(downloaded, total_size, speed); + self.send_progress( + progress_tx, + ProgressUpdate::Download { + downloaded_bytes: downloaded, + total_bytes: total_size, + speed_bytes_per_sec: speed, + }, + ) + .await; } - let _ = progress_tx - .send(ProgressUpdate::Download { - downloaded_bytes: downloaded, - total_bytes: total_size, - speed_bytes_per_sec: speed, - }) - .await; + self.check_cancelled()?; + Ok(downloaded) } + .await; - // Tokio can acknowledge the final write while its blocking file work - // is still queued. Checksum and extraction readers reopen this path, - // so finish all writes before reporting the download complete. - file.flush().await.map_err(|e| PumasError::Io { - message: format!("Failed to flush archive: {}", e), - path: Some(archive_path.to_path_buf()), - source: Some(e), - })?; + let downloaded = settle_archive_file(file, archive_path, transfer).await?; // Add to completed items { @@ -1831,12 +2059,14 @@ impl VersionInstaller { Some("Finalizing installation..."), ); } - let _ = progress_tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + progress_tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Setup, message: "Finalizing installation...".to_string(), - }) - .await; + }, + ) + .await; // Create metadata entry let metadata = InstalledVersionMetadata { @@ -1900,26 +2130,32 @@ impl VersionInstaller { Some("Installation complete"), ); } - let _ = progress_tx - .send(ProgressUpdate::Setup { + self.send_progress( + progress_tx, + ProgressUpdate::Setup { message: "Installation complete".to_string(), - }) - .await; + }, + ) + .await; info!("Installation of {} finalized", tag); Ok(()) } fn check_cancelled(&self) -> Result<()> { - if self.cancel_flag.load(Ordering::SeqCst) { - Err(PumasError::InstallationFailed { - message: "Installation cancelled by user".to_string(), - }) + if self.cancel_flag.load(Ordering::SeqCst) || self.shutdown_flag.load(Ordering::SeqCst) { + Err(Self::cancellation_error()) } else { Ok(()) } } + fn cancellation_error() -> PumasError { + PumasError::InstallationFailed { + message: "Installation cancelled".into(), + } + } + fn versions_dir(&self) -> PathBuf { self.launcher_root.join(self.app_id.versions_dir_name()) } @@ -2066,14 +2302,17 @@ mod tests { assert_eq!(launch_binary, version_dir.join("bin/llama-server")); let wrapper = std::fs::read_to_string(&launch_binary).unwrap(); assert!(wrapper.contains("LD_LIBRARY_PATH")); - assert!(wrapper.contains(archive_dir.to_string_lossy().as_ref())); + assert!(wrapper.contains("llama-b9090")); + assert!(!wrapper.contains(version_dir.to_string_lossy().as_ref())); let mode = std::fs::metadata(&launch_binary) .unwrap() .permissions() .mode(); assert_eq!(mode & 0o111, 0o111); let unrelated_cwd = tempfile::tempdir().unwrap(); - let output = std::process::Command::new(&launch_binary) + let published = absolute_root.join("published"); + std::fs::rename(&version_dir, &published).unwrap(); + let output = std::process::Command::new(published.join("bin/llama-server")) .current_dir(unrelated_cwd.path()) .output() .unwrap(); @@ -2085,6 +2324,253 @@ mod tests { assert_eq!(output.stdout, b"owned-wrapper-fixture"); } + #[test] + fn native_publication_failure_preserves_stage_and_existing_output() { + let root = tempfile::tempdir().unwrap(); + let stage = root.path().join("stage"); + let destination = root.path().join("version"); + std::fs::create_dir(&stage).unwrap(); + std::fs::write(stage.join("complete"), "new").unwrap(); + let error = VersionInstaller::publish_native_stage(&stage, &destination, || { + Err((PumasError::Other("metadata failure".into()), true)) + }) + .unwrap_err(); + assert!(error.to_string().contains("metadata failure")); + assert!(!destination.exists()); + assert_eq!(std::fs::read(stage.join("complete")).unwrap(), b"new"); + std::fs::create_dir(&destination).unwrap(); + std::fs::write(destination.join("retained"), "old").unwrap(); + assert!( + VersionInstaller::publish_native_stage(&stage, &destination, || panic!( + "must not finalize existing output" + )) + .is_err() + ); + assert_eq!(std::fs::read(destination.join("retained")).unwrap(), b"old"); + } + + #[test] + fn native_uncertain_metadata_publication_retains_complete_output() { + let root = tempfile::tempdir().unwrap(); + let stage = root.path().join("stage"); + let destination = root.path().join("version"); + std::fs::create_dir(&stage).unwrap(); + std::fs::write(stage.join("complete"), "owned").unwrap(); + assert!( + VersionInstaller::publish_native_stage(&stage, &destination, || { + Err((PumasError::Other("uncertain publication".into()), false)) + }) + .is_err() + ); + assert!(!stage.exists()); + assert_eq!( + std::fs::read(destination.join("complete")).unwrap(), + b"owned" + ); + } + + #[cfg(unix)] + #[test] + fn native_publication_preserves_a_dangling_destination_link() { + let root = tempfile::tempdir().unwrap(); + let stage = root.path().join("stage"); + let destination = root.path().join("version"); + std::fs::create_dir(&stage).unwrap(); + std::os::unix::fs::symlink("missing", &destination).unwrap(); + assert!( + VersionInstaller::publish_native_stage(&stage, &destination, || { + panic!("must not finalize over retained link") + }) + .is_err() + ); + assert_eq!( + std::fs::read_link(&destination).unwrap(), + Path::new("missing") + ); + assert!(stage.exists()); + } + + #[tokio::test] + async fn native_progress_backpressure_ends_on_cancel_and_shutdown() { + let root = tempfile::tempdir().unwrap(); + let cancelled = Arc::new(AtomicBool::new(false)); + let shutting_down = Arc::new(AtomicBool::new(false)); + let installer = VersionInstaller::new( + root.path().to_path_buf(), + AppId::LlamaCpp, + Arc::new(MetadataManager::new(root.path())), + Arc::new(RwLock::new(InstallationProgressTracker::new( + root.path().to_path_buf(), + ))), + cancelled.clone(), + ) + .with_shutdown_flag(shutting_down.clone()); + let (sender, _receiver) = mpsc::channel(1); + sender + .send(ProgressUpdate::Setup { + message: "full".into(), + }) + .await + .unwrap(); + for flag in [&cancelled, &shutting_down] { + let blocked = installer.send_progress( + &sender, + ProgressUpdate::Setup { + message: "pending".into(), + }, + ); + tokio::pin!(blocked); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut blocked) + .await + .is_err() + ); + flag.store(true, Ordering::SeqCst); + tokio::time::timeout(std::time::Duration::from_secs(1), blocked) + .await + .unwrap(); + flag.store(false, Ordering::SeqCst); + } + } + + #[test] + fn native_invalid_archive_cannot_touch_retained_output() { + let root = tempfile::tempdir().unwrap(); + let retained = root.path().join("version"); + std::fs::create_dir(&retained).unwrap(); + std::fs::write(retained.join("retained"), "old").unwrap(); + let stage = tempfile::tempdir_in(root.path()).unwrap(); + let archive = root.path().join("archive"); + std::fs::write(&archive, "invalid tar").unwrap(); + assert!(VersionInstaller::extract_llama_cpp_binary( + &archive, + stage.path(), + "native.tar.gz" + ) + .is_err()); + assert_eq!(std::fs::read(retained.join("retained")).unwrap(), b"old"); + } + + #[cfg(unix)] + #[test] + fn native_output_rejects_external_and_directory_links() { + let root = tempfile::tempdir().unwrap(); + let external = tempfile::tempdir().unwrap(); + std::fs::write(external.path().join("server"), "retained").unwrap(); + std::os::unix::fs::symlink( + external.path().join("server"), + root.path().join("llama-server"), + ) + .unwrap(); + assert!(VersionInstaller::validate_native_output(root.path()).is_err()); + std::fs::remove_file(root.path().join("llama-server")).unwrap(); + std::os::unix::fs::symlink(".", root.path().join("loop")).unwrap(); + assert!(VersionInstaller::validate_native_output(root.path()).is_err()); + } + + #[tokio::test] + async fn native_worker_retains_custody_after_waiter_is_cancelled() { + let root = tempfile::tempdir().unwrap(); + let custody = Arc::new(NativeInstallWorkspace::create(root.path()).unwrap()); + let path = custody.path().to_path_buf(); + std::fs::write(path.join("archive"), "owned").unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let worker_custody = custody.clone(); + let worker = tokio::task::spawn_blocking(move || { + let _held = worker_custody; + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + assert_eq!( + std::fs::read(_held.path().join("archive")).unwrap(), + b"owned" + ); + }); + entered_rx.await.unwrap(); + drop(custody); + worker.abort(); // spawn_blocking is already running and cannot be aborted. + assert!(path.exists()); + assert!(NativeInstallWorkspace::create(root.path()).is_err()); + release_tx.send(()).unwrap(); + worker.await.unwrap(); + // Cancellation loses the receipt, so implicit destruction preserves + // bytes for reconciliation even after the blocking worker has settled. + assert!(path.exists()); + let next = NativeInstallWorkspace::create(root.path()).unwrap(); + next.cleanup().unwrap(); + std::fs::remove_dir_all(path).unwrap(); + } + + #[test] + fn native_cleanup_failure_reports_and_retains_its_path() { + let root = tempfile::tempdir().unwrap(); + let custody = NativeInstallWorkspace::create(root.path()).unwrap(); + let path = custody.path().to_owned(); + std::fs::remove_dir(&path).unwrap(); + std::fs::write(&path, "retained uncertainty").unwrap(); + let error = custody.cleanup().unwrap_err().to_string(); + assert!(error.contains("cleanup incomplete")); + assert!(error.contains(&path.display().to_string())); + assert_eq!(std::fs::read(&path).unwrap(), b"retained uncertainty"); + NativeInstallWorkspace::create(root.path()) + .unwrap() + .cleanup() + .unwrap(); + } + + #[test] + fn native_cancelled_archive_settles_queued_writes_before_reclamation() { + let root = tempfile::tempdir().unwrap(); + let custody = NativeInstallWorkspace::create(root.path()).unwrap(); + let path = custody.path().join("archive"); + let file = std::fs::File::create(&path).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + runtime.block_on(async { + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let blocker = tokio::task::spawn_blocking(move || { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }); + entered_rx.await.unwrap(); + let mut file = fs::File::from_std(file); + // Tokio accepts this buffer while its blocking write is queued + // behind the occupied worker. Cancellation is then observed. + file.write_all(b"pending native bytes").await.unwrap(); + let write_path = path.clone(); + let mut settlement = tokio::spawn(async move { + settle_archive_file::<()>( + file, + &write_path, + Err(VersionInstaller::cancellation_error()), + ) + .await + }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), &mut settlement) + .await + .is_err() + ); + assert!(path.exists()); + assert!(NativeVersionsLock::try_acquire(root.path()).is_err()); + release_tx.send(()).unwrap(); + blocker.await.unwrap(); + assert!(settlement + .await + .unwrap() + .unwrap_err() + .to_string() + .contains("cancelled")); + assert_eq!(std::fs::read(&path).unwrap(), b"pending native bytes"); + custody.cleanup().unwrap(); + assert!(!path.exists()); + }); + } + #[test] fn llama_cpp_asset_selection_prefers_linux_vulkan_when_gpu_is_available() { let assets = vec![ diff --git a/rust/crates/pumas-app-manager/src/version_manager/mod.rs b/rust/crates/pumas-app-manager/src/version_manager/mod.rs index 80a25d27..85d07d1f 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/mod.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/mod.rs @@ -91,6 +91,35 @@ async fn path_exists(path: &Path) -> Result { .map_err(|err| PumasError::io_with_path(err, path)) } +type InstallationTask = tokio::task::JoinHandle>; +type InstallationShutdown = futures::future::Shared< + futures::future::BoxFuture<'static, std::result::Result<(), Arc>>, +>; + +#[derive(Default)] +struct InstallationTasks { + tasks: Vec, + failures: Vec, +} + +impl InstallationTasks { + fn harvest_finished(&mut self) { + use futures::FutureExt; + for mut task in std::mem::take(&mut self.tasks) { + if task.is_finished() { + match (&mut task).now_or_never() { + Some(Ok(Ok(()))) => {} + Some(Ok(Err(error))) => self.failures.push(error), + Some(Err(error)) => self.failures.push(error.to_string()), + None => self.tasks.push(task), + } + } else { + self.tasks.push(task); + } + } + } +} + async fn wait_for_install_cancel(cancel_flag: Arc) { loop { if cancel_flag.load(Ordering::SeqCst) { @@ -162,6 +191,8 @@ pub struct VersionManager { torch_control: Arc, torch_cleanup: Arc, torch_shutting_down: Arc, + installation_tasks: Arc>, + installation_shutdown: Arc>>, #[cfg(test)] torch_admission_pause: Option>, torch_previews: torch_preview::TorchPreviews, @@ -301,6 +332,8 @@ impl VersionManager { torch_control: Arc::new(installer::TorchInstallControl::new()), torch_cleanup: Arc::new(installer::TorchCleanupTasks::default()), torch_shutting_down: Arc::new(AtomicBool::new(false)), + installation_tasks: Arc::new(std::sync::Mutex::new(InstallationTasks::default())), + installation_shutdown: Arc::new(Mutex::new(None)), #[cfg(test)] torch_admission_pause: None, torch_previews: Arc::new(Mutex::new(Default::default())), @@ -699,7 +732,9 @@ impl VersionManager { return Ok(false); } - if self.app_id == AppId::Torch && !self.torch_control.request_cancel() { + if matches!(self.app_id, AppId::Torch | AppId::LlamaCpp) + && !self.torch_control.request_cancel() + { return Ok(false); } @@ -717,28 +752,94 @@ impl VersionManager { Ok(true) } - /// Stop admitting Torch cleanup work and wait for the current attempt and - /// every cleanup task owned by this manager before server shutdown ends. + /// Close admission and drain every supported installation. A retained + /// receipt survives cancelled shutdown waiters and preserves terminal failures. + /// + /// Installation failures, task panics and cleanup errors are returned after + /// draining and remain observable on repeated calls. Publication that already + /// won the cancellation race is allowed to finish. Owners must await this + /// before shutting down the Tokio runtime. + pub async fn shutdown_installations(&self) -> Result<()> { + use futures::FutureExt; + let completion = { + let mut shutdown = self.installation_shutdown.lock().await; + if let Some(completion) = &*shutdown { + completion.clone() + } else { + { + let _installing = self.installing_tag.lock().await; + self.torch_shutting_down.store(true, Ordering::SeqCst); + } + let manager = self.clone(); + let worker = + tokio::spawn(async move { + // Cancellation itself may wait for progress state. Keep + // it inside the retained worker, so dropping a waiter + // cannot stop an already-started shutdown. + let mut errors = Vec::new(); + if let Err(error) = manager.cancel_installation().await { + errors.push(error.to_string()); + } + let _install_guard = manager.install_lock.lock().await; + let registered = + std::mem::take(&mut *manager.installation_tasks.lock().map_err( + |_| Arc::new("Installation task registry poisoned".into()), + )?); + errors.extend(registered.failures); + for task in registered.tasks { + match task.await { + Ok(Ok(())) => {} + Ok(Err(error)) => errors.push(error), + Err(error) => errors.push(error.to_string()), + } + } + let _lifecycle_guard = manager.lifecycle_lock.lock().await; + if let Err(error) = manager.state.write().await.shutdown_mutations().await { + errors.push(error.to_string()); + } + if manager.app_id == AppId::Torch { + manager.torch_cleanup.close(); + if let Err(error) = manager.torch_cleanup.drain().await { + errors.push(error.to_string()); + } + if let Err(error) = manager.torch_cleanup.drain_child_slots().await { + errors.push(error.to_string()); + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(Arc::new(errors.join("; "))) + } + }); + let completion = async move { + worker + .await + .unwrap_or_else(|error| Err(Arc::new(error.to_string()))) + } + .boxed() + .shared(); + *shutdown = Some(completion.clone()); + completion + } + }; + completion + .await + .map_err(|error| PumasError::InstallationFailed { + message: (*error).clone(), + }) + } + + /// Compatibility entry point; drains every supported runtime. pub async fn shutdown_torch_cleanup(&self) -> Result<()> { - if self.app_id != AppId::Torch { - return Ok(()); - } - { - let _installing = self.installing_tag.lock().await; - self.torch_shutting_down.store(true, Ordering::SeqCst); - } - let _ = self.cancel_installation().await?; - let _install_guard = self.install_lock.lock().await; - self.torch_cleanup.close(); - let tasks = self.torch_cleanup.drain().await; - let children = self.torch_cleanup.drain_child_slots().await; - tasks?; - children + self.shutdown_installations().await } /// Install a version with progress channel. /// /// Returns a channel receiver for progress updates. + /// Dropping the receiver does not cancel admitted work; use cancellation or + /// `shutdown_installations` to settle the manager's owned installation. pub async fn install_version(&self, tag: &str) -> Result> { self.install_version_with_preview(tag, None).await } @@ -766,9 +867,9 @@ impl VersionManager { // Acquire install lock let install_guard = self.install_lock.clone().lock_owned().await; - if self.app_id == AppId::Torch && self.torch_shutting_down.load(Ordering::SeqCst) { + if self.torch_shutting_down.load(Ordering::SeqCst) { return Err(PumasError::InstallationFailed { - message: "Torch version manager is shutting down".into(), + message: "Version manager is shutting down".into(), }); } // Release lookup and package resolution are part of installation work for @@ -784,9 +885,9 @@ impl VersionManager { pause.reached.notify_one(); pause.resume.acquire().await.unwrap().forget(); } - if self.app_id == AppId::Torch && self.torch_shutting_down.load(Ordering::SeqCst) { + if self.torch_shutting_down.load(Ordering::SeqCst) { return Err(PumasError::InstallationFailed { - message: "Torch version manager is shutting down".into(), + message: "Version manager is shutting down".into(), }); } let torch_selection = if self.app_id == AppId::Torch { @@ -831,15 +932,24 @@ impl VersionManager { // Commit admission under the same lock used by cancellation. Shutdown // may begin during release resolution, before an installing tag exists. + let mut installing = self.installing_tag.lock().await; + // Registration is part of admission. A poisoned registry cannot leave + // a task running without its completion capability. + let mut registered = + self.installation_tasks + .lock() + .map_err(|_| PumasError::InstallationFailed { + message: "Installation task registry poisoned".into(), + })?; + registered.harvest_finished(); { - let mut installing = self.installing_tag.lock().await; - if self.app_id == AppId::Torch && self.torch_shutting_down.load(Ordering::SeqCst) { + if self.torch_shutting_down.load(Ordering::SeqCst) { return Err(PumasError::InstallationFailed { - message: "Torch version manager is shutting down".into(), + message: "Version manager is shutting down".into(), }); } self.cancel_flag.store(false, Ordering::SeqCst); - if self.app_id == AppId::Torch { + if matches!(self.app_id, AppId::Torch | AppId::LlamaCpp) { self.torch_control.start(); } *installing = Some(tag.to_string()); @@ -857,7 +967,8 @@ impl VersionManager { self.cancel_flag.clone(), ) .with_torch_control(self.torch_control.clone()) - .with_torch_cleanup(self.torch_cleanup.clone()); + .with_torch_cleanup(self.torch_cleanup.clone()) + .with_shutdown_flag(self.torch_shutting_down.clone()); #[cfg(test)] let installer = if let Some(pause) = &self.torch_publication_pause { installer.with_torch_publication_pause(pause.clone()) @@ -879,8 +990,9 @@ impl VersionManager { let torch_control = self.torch_control.clone(); let app_id = self.app_id; let manager = self.clone(); + let task_registry = self.installation_tasks.clone(); - tokio::spawn(async move { + let task = tokio::spawn(async move { let _install_guard = install_guard; if app_id == AppId::Torch { let mut tracker = progress_tracker.write().await; @@ -928,7 +1040,7 @@ impl VersionManager { } .await; - if app_id == AppId::Torch { + if matches!(app_id, AppId::Torch | AppId::LlamaCpp) { torch_control.finish(); } @@ -952,22 +1064,34 @@ impl VersionManager { tracker.set_error(&error.to_string()); tracker.complete_installation(false); } - let _ = tx - .send(match result { + let terminal = result.as_ref().map(|_| ()).map_err(ToString::to_string); + tokio::select! { + _ = tx.send(match result { Ok(_) => ProgressUpdate::Completed { success: true }, - Err(e) => ProgressUpdate::Error { - message: e.to_string(), + Err(e) => ProgressUpdate::Error { message: e.to_string() }, + }) => {}, + _ = wait_for_install_cancel(manager.torch_shutting_down.clone()) => {}, + } + // Keep existing delayed progress cleanup, with observed lifecycle. + let mut registered = task_registry + .lock() + .map_err(|_| "Installation task registry poisoned".to_owned())?; + registered.harvest_finished(); + let cleanup = tokio::spawn(async move { + tokio::select! { + _ = tokio::time::sleep(Duration::from_secs(5)) => { + progress_tracker.write().await.clear_completed_state_async().await; }, - }) - .await; - - // Schedule progress state cleanup after frontend has time to poll final status - tokio::spawn(async move { - tokio::time::sleep(Duration::from_secs(5)).await; - let mut tracker = progress_tracker.write().await; - tracker.clear_completed_state_async().await; + _ = wait_for_install_cancel(manager.torch_shutting_down.clone()) => {}, + } + Ok(()) }); + registered.tasks.push(cleanup); + terminal }); + registered.tasks.push(task); + drop(registered); + drop(installing); Ok(rx) } @@ -1083,10 +1207,21 @@ impl VersionManager { pub async fn remove_version(&self, tag: &str) -> Result { let _install_guard = self.install_lock.lock().await; let _lifecycle_guard = self.lifecycle_lock.lock().await; + let native_versions_lock = if self.app_id == AppId::LlamaCpp { + Some(installer::NativeVersionsLock::acquire(self.versions_dir()).await?) + } else { + None + }; let torch_versions_lock = self.acquire_torch_versions_lock_for_mutation().await?; // Another backend may have changed metadata while this manager was open. if let Some(lock) = &torch_versions_lock { self.state.write().await.refresh_with_lock(lock).await?; + } else if let Some(lock) = &native_versions_lock { + self.state + .write() + .await + .refresh_with_native_lock(lock) + .await?; } self.ensure_torch_stopped(tag).await?; // Check if installed @@ -1116,43 +1251,28 @@ impl VersionManager { } // Keep directory removal and metadata deletion in one leased worker, so - // cancelling this waiter cannot publish a half-removed Torch version. + // cancelling this waiter cannot release mutation admission while the + // filesystem and metadata effects are still running. let version_path = self.version_path(tag); + let mutations = self.state.read().await.mutation_tasks(); + let metadata = self.metadata_manager.clone(); + let removed_tag = tag.to_owned(); + let app_id = self.app_id; + let remove = move || { + info!("Removing version directory: {}", version_path.display()); + match std::fs::remove_dir_all(&version_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(PumasError::io_with_path(error, &version_path)), + } + metadata.remove_installed_version(&removed_tag, Some(app_id)) + }; if let Some(lock) = &torch_versions_lock { - let lease = lock.clone(); - let metadata = self.metadata_manager.clone(); - let removed_tag = tag.to_owned(); - tokio::task::spawn_blocking(move || { - let _lease = lease; - info!("Removing version directory: {}", version_path.display()); - match std::fs::remove_dir_all(&version_path) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => { - return Err(PumasError::Io { - message: format!("Failed to remove version directory: {}", error), - path: Some(version_path), - source: Some(error), - }); - } - } - metadata.remove_installed_version(&removed_tag, Some(AppId::Torch)) - }) - .await - .map_err(|error| PumasError::Other(format!("Torch removal task failed: {error}")))??; + mutations.leased_transaction(lock, remove).await?; + } else if let Some(lock) = &native_versions_lock { + mutations.leased_transaction(lock, remove).await?; } else { - if path_exists(&version_path).await? { - info!("Removing version directory: {}", version_path.display()); - fs::remove_dir_all(&version_path) - .await - .map_err(|error| PumasError::Io { - message: format!("Failed to remove version directory: {}", error), - path: Some(version_path), - source: Some(error), - })?; - } - self.metadata_manager - .remove_installed_version(tag, Some(self.app_id))?; + mutations.leased_transaction(&(), remove).await?; } #[cfg(test)] @@ -1166,6 +1286,8 @@ impl VersionManager { let mut state = self.state.write().await; if let Some(lock) = &torch_versions_lock { state.refresh_with_lock(lock).await?; + } else if let Some(lock) = &native_versions_lock { + state.refresh_with_native_lock(lock).await?; } else { state.refresh().await?; } @@ -1295,6 +1417,539 @@ mod tests { (manager, temp_dir) } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + async fn native_archive_fixture( + root: &Path, + ) -> ( + tokio::task::JoinHandle<()>, + tokio::sync::oneshot::Receiver<()>, + tokio::sync::oneshot::Sender, + ) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let payload = b"#!/bin/sh\nprintf 'native-fixture'\n"; + let compressed = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + let mut archive = tar::Builder::new(compressed); + let mut header = tar::Header::new_gnu(); + header.set_size(payload.len() as u64); + header.set_mode(0o755); + header.set_cksum(); + archive + .append_data(&mut header, "distribution/llama-server", &payload[..]) + .unwrap(); + let bytes = archive.into_inner().unwrap().finish().unwrap(); + let size = bytes.len() as u64; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/archive", listener.local_addr().unwrap()); + let releases = pumas_library::network::ReleasesCache::new( + root.join("launcher-data/cache"), + Duration::from_secs(3600), + ); + releases + .set_disk( + AppId::LlamaCpp.github_repo(), + &[pumas_library::network::GitHubRelease { + tag_name: "b1234".into(), + name: "Native fixture".into(), + published_at: "2026-09-29T00:00:00Z".into(), + body: None, + tarball_url: None, + zipball_url: None, + prerelease: false, + assets: vec![pumas_library::network::GitHubAsset { + name: "llama-b1234-bin-ubuntu-x64.tar.gz".into(), + size, + download_url: url, + content_type: Some("application/gzip".into()), + }], + html_url: "https://github.com/ggml-org/llama.cpp/releases/tag/b1234".into(), + total_size: Some(size), + archive_size: Some(size), + dependencies_size: None, + }], + ) + .unwrap(); + let (entered, observed) = tokio::sync::oneshot::channel(); + let (release, wait) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + assert!(stream.read(&mut request).await.unwrap() > 0); + entered.send(()).unwrap(); + if wait.await.unwrap() { + stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + stream.write_all(&bytes).await.unwrap(); + } + }); + (server, observed, release) + } + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn ollama_shutdown_cancels_stalled_headers_and_body() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + for send_headers in [false, true] { + let root = TempDir::new().unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/archive", listener.local_addr().unwrap()); + let releases = pumas_library::network::ReleasesCache::new( + root.path().join("launcher-data/cache"), + Duration::from_secs(3600), + ); + releases + .set_disk( + AppId::Ollama.github_repo(), + &[pumas_library::network::GitHubRelease { + tag_name: "v0.1.2".into(), + name: "Ollama fixture".into(), + published_at: "2026-09-29T00:00:00Z".into(), + body: None, + tarball_url: None, + zipball_url: None, + prerelease: false, + assets: vec![pumas_library::network::GitHubAsset { + name: "ollama-linux-amd64.tgz".into(), + size: 100, + download_url: url, + content_type: Some("application/gzip".into()), + }], + html_url: "https://github.com/ollama/ollama/releases/tag/v0.1.2".into(), + total_size: Some(100), + archive_size: Some(100), + dependencies_size: None, + }], + ) + .unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + assert!(stream.read(&mut request).await.unwrap() > 0); + if send_headers { + stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nx", + ) + .await + .unwrap(); + } + entered_tx.send(()).unwrap(); + release_rx.await.unwrap(); + }); + let manager = VersionManager::new(root.path(), AppId::Ollama) + .await + .unwrap(); + let mut updates = manager.install_version("v0.1.2").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), entered_rx) + .await + .unwrap() + .unwrap(); + if send_headers { + // Prove that body handling has started, rather than merely + // testing the header cancellation branch twice. + tokio::time::timeout(Duration::from_secs(2), async { + while let Some(update) = updates.recv().await { + if matches!(update, ProgressUpdate::Download { downloaded_bytes, .. } if downloaded_bytes > 0) { + return; + } + } + panic!("Ollama transfer ended without receiving its body"); + }) + .await + .unwrap(); + } + let error = + tokio::time::timeout(Duration::from_secs(2), manager.shutdown_installations()) + .await + .unwrap() + .unwrap_err() + .to_string(); + assert!(error.contains("cancelled"), "{error}"); + assert_eq!( + manager + .shutdown_installations() + .await + .unwrap_err() + .to_string(), + error + ); + assert!(!manager.version_path("v0.1.2").exists()); + assert!(manager + .metadata_manager + .get_installed_version("v0.1.2", Some(AppId::Ollama)) + .unwrap() + .is_none()); + release_tx.send(()).unwrap(); + server.await.unwrap(); + } + } + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn native_manager_removal_shares_installation_admission() { + let root = TempDir::new().unwrap(); + let manager = VersionManager::new(root.path(), AppId::LlamaCpp) + .await + .unwrap(); + for tag in ["b1234+cpu", "b1235+cpu"] { + std::fs::create_dir(manager.version_path(tag)).unwrap(); + std::fs::write(manager.version_path(tag).join("llama-server"), "complete").unwrap(); + manager + .state + .write() + .await + .add_installed_version( + tag, + pumas_library::metadata::InstalledVersionMetadata { + path: tag.into(), + release_tag: tag.into(), + ..Default::default() + }, + ) + .unwrap(); + } + manager.set_active_version("b1235+cpu").await.unwrap(); + let lease = installer::NativeVersionsLock::try_acquire(&manager.versions_dir()).unwrap(); + assert!(manager.remove_version("b1234+cpu").await.is_err()); + assert!(manager.version_path("b1234+cpu/llama-server").exists()); + assert!(manager + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_some()); + drop(lease); + assert!(manager.remove_version("b1234+cpu").await.unwrap()); + assert!(!manager.version_path("b1234+cpu").exists()); + assert!(manager + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_none()); + assert!(manager + .metadata_manager + .get_installed_version("b1235+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_some()); + manager.shutdown_installations().await.unwrap(); + } + + #[test] + fn native_removal_shutdown_observes_cancelled_waiter_completion_and_failure() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap(); + runtime.block_on(async { + for fail_removal in [false, true] { + let root = TempDir::new().unwrap(); + let mut manager = VersionManager::new(root.path(), AppId::LlamaCpp) + .await + .unwrap(); + for tag in ["b1234+cpu", "b1235+cpu"] { + std::fs::create_dir(manager.version_path(tag)).unwrap(); + std::fs::write(manager.version_path(tag).join("llama-server"), "complete") + .unwrap(); + manager + .state + .write() + .await + .add_installed_version( + tag, + pumas_library::metadata::InstalledVersionMetadata { + path: tag.into(), + release_tag: tag.into(), + ..Default::default() + }, + ) + .unwrap(); + } + manager.set_active_version("b1235+cpu").await.unwrap(); + let pause = Arc::new(RemovalPause { + entered: tokio::sync::Notify::new(), + proceed: tokio::sync::Notify::new(), + }); + manager.removal_pause = Some(pause.clone()); + let owner = manager.state.read().await.mutation_tasks(); + let removing = manager.clone(); + let waiter = + tokio::spawn(async move { removing.remove_version("b1234+cpu").await }); + pause.entered.notified().await; + if fail_removal { + std::fs::remove_dir_all(manager.version_path("b1234+cpu")).unwrap(); + std::fs::write(manager.version_path("b1234+cpu"), "retained").unwrap(); + } + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let blocker = tokio::task::spawn_blocking(move || { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }); + entered_rx.await.unwrap(); + pause.proceed.notify_one(); + tokio::time::timeout(Duration::from_secs(2), async { + while !owner.has_active_tasks() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let shutting = manager.clone(); + let mut shutdown = + tokio::spawn(async move { shutting.shutdown_installations().await }); + assert!( + tokio::time::timeout(Duration::from_millis(30), &mut shutdown) + .await + .is_err() + ); + assert!(manager.version_path("b1234+cpu").exists()); + assert!( + installer::NativeVersionsLock::try_acquire(&manager.versions_dir()).is_err() + ); + // Cancel the first shutdown waiter as well. Its retained drain + // still owns the queued removal and observes its terminal result. + shutdown.abort(); + assert!(shutdown.await.unwrap_err().is_cancelled()); + release_tx.send(()).unwrap(); + blocker.await.unwrap(); + let outcome = manager + .shutdown_installations() + .await + .map_err(|error| error.to_string()); + if fail_removal { + assert!(outcome.as_ref().unwrap_err().contains("b1234+cpu")); + assert_eq!( + std::fs::read(manager.version_path("b1234+cpu")).unwrap(), + b"retained" + ); + } else { + assert!(outcome.is_ok()); + assert!(!manager.version_path("b1234+cpu").exists()); + } + assert_eq!( + manager + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_some(), + fail_removal + ); + assert_eq!( + manager + .shutdown_installations() + .await + .map_err(|error| error.to_string()), + outcome + ); + assert!(manager.set_default_version(None).await.is_err()); + } + }); + } + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn native_archive_publishes_complete_output_and_reopens_metadata() { + let root = TempDir::new().unwrap(); + let (server, observed, release) = native_archive_fixture(root.path()).await; + let manager = VersionManager::new(root.path(), AppId::LlamaCpp) + .await + .unwrap(); + let mut updates = manager.install_version("b1234+cpu").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .unwrap() + .unwrap(); + assert!(!manager.version_path("b1234+cpu").exists()); + assert!(manager + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_none()); + release.send(true).unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match updates.recv().await.unwrap() { + ProgressUpdate::Completed { success: true } => break, + ProgressUpdate::Error { message } => { + panic!("Native installation failed: {message}") + } + _ => {} + } + } + }) + .await + .unwrap(); + server.await.unwrap(); + manager.shutdown_installations().await.unwrap(); + let output = + std::process::Command::new(manager.version_path("b1234+cpu").join("bin/llama-server")) + .current_dir(root.path()) + .output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(output.stdout, b"native-fixture"); + let reopened = VersionManager::new(root.path(), AppId::LlamaCpp) + .await + .unwrap(); + assert_eq!( + reopened.get_installed_versions().await.unwrap(), + vec!["b1234+cpu"] + ); + assert!(std::fs::read_dir(manager.versions_dir()) + .unwrap() + .all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".llama-install-") + })); + reopened.shutdown_installations().await.unwrap(); + } + + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn native_shutdown_cancels_stalled_transfer_and_retains_failed_outcome() { + let root = TempDir::new().unwrap(); + let (server, observed, release) = native_archive_fixture(root.path()).await; + let manager = VersionManager::new(root.path(), AppId::LlamaCpp) + .await + .unwrap(); + let _updates = manager.install_version("b1234+cpu").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .unwrap() + .unwrap(); + let error = tokio::time::timeout(Duration::from_secs(2), manager.shutdown_installations()) + .await + .unwrap() + .unwrap_err() + .to_string(); + assert!(error.contains("cancelled")); + assert_eq!( + manager + .shutdown_installations() + .await + .unwrap_err() + .to_string(), + error + ); + release.send(false).unwrap(); + server.await.unwrap(); + assert!(!manager.version_path("b1234+cpu").exists()); + assert!(manager + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_none()); + assert!(std::fs::read_dir(manager.versions_dir()) + .unwrap() + .all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".llama-install-") + })); + assert!(!manager.is_installing().await); + assert!(manager.get_installation_progress().await.unwrap().success == Some(false)); + } + + #[tokio::test] + async fn native_shutdown_drains_registered_tasks_and_retains_failure() { + let (manager, _root) = create_test_manager().await; + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + entered_tx.send(()).unwrap(); + release_rx.await.unwrap(); + Err("native terminal failure".into()) + }); + manager.installation_tasks.lock().unwrap().tasks.push(task); + entered_rx.await.unwrap(); + let shutting = manager.clone(); + let waiter = tokio::spawn(async move { shutting.shutdown_installations().await }); + while !manager.torch_shutting_down.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let repeated = manager.shutdown_installations(); + tokio::pin!(repeated); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut repeated) + .await + .is_err() + ); + release_tx.send(()).unwrap(); + let error = repeated.await.unwrap_err().to_string(); + assert!(error.contains("native terminal failure")); + assert_eq!( + manager + .shutdown_installations() + .await + .unwrap_err() + .to_string(), + error + ); + assert!(manager.install_version("v-new").await.is_err()); + } + + #[tokio::test] + async fn native_cancelled_shutdown_waiter_does_not_interrupt_cancellation() { + let (manager, _root) = create_test_manager().await; + *manager.installing_tag.lock().await = Some("owned-attempt".into()); + let tracker = manager.progress_tracker.write().await; + let (completed, observed) = tokio::sync::oneshot::channel(); + let flag = manager.cancel_flag.clone(); + manager + .installation_tasks + .lock() + .unwrap() + .tasks + .push(tokio::spawn(async move { + wait_for_install_cancel(flag).await; + completed.send(()).unwrap(); + Ok(()) + })); + let owner = manager.clone(); + let waiter = tokio::spawn(async move { owner.shutdown_installations().await }); + tokio::time::timeout(Duration::from_secs(2), async { + while !manager.cancel_flag.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + drop(tracker); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .unwrap() + .unwrap(); + // The retained worker drains without requiring another shutdown caller. + tokio::time::timeout(Duration::from_secs(2), async { + while !manager.installation_tasks.lock().unwrap().tasks.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + manager.shutdown_installations().await.unwrap(); + } + async fn create_torch_test_manager() -> (VersionManager, TempDir) { let root = TempDir::new().unwrap(); let manager = VersionManager::new(root.path(), AppId::Torch) diff --git a/rust/crates/pumas-app-manager/src/version_manager/ollama.rs b/rust/crates/pumas-app-manager/src/version_manager/ollama.rs index 8174e43b..8cf85ae3 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/ollama.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/ollama.rs @@ -5,6 +5,7 @@ use crate::version_manager::progress::ProgressUpdate; use crate::version_manager::state::VersionState; +use futures::FutureExt; use pumas_library::config::{AppId, InstallationConfig}; use pumas_library::metadata::{InstalledVersionMetadata, MetadataManager}; use pumas_library::models::InstallationStage; @@ -15,6 +16,7 @@ use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::sync::Mutex as StdMutex; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, Mutex, RwLock}; @@ -27,6 +29,11 @@ async fn path_exists(path: &Path) -> Result { } /// Ollama version manager specialized for binary-only installation. +/// +/// Await construction and retain an owner until `shutdown` completes before +/// dropping its storage or stopping the Tokio runtime. Clones share admission +/// and the shutdown receipt; dropping an operation waiter does not cancel it. +#[derive(Clone)] pub struct OllamaVersionManager { /// Root directory for launcher data. launcher_root: PathBuf, @@ -44,6 +51,14 @@ pub struct OllamaVersionManager { install_lock: Arc>, /// Currently installing tag. installing_tag: Arc>>, + shutdown_flag: Arc, + activities: Arc>, +} + +#[derive(Default)] +struct OllamaActivities { + tasks: super::InstallationTasks, + completion: Option, } impl OllamaVersionManager { @@ -65,9 +80,105 @@ impl OllamaVersionManager { cancel_flag: Arc::new(AtomicBool::new(false)), install_lock: Arc::new(Mutex::new(())), installing_tag: Arc::new(RwLock::new(None)), + shutdown_flag: Arc::new(AtomicBool::new(false)), + activities: Arc::new(StdMutex::new(OllamaActivities::default())), }) } + fn ensure_open(&self) -> Result<()> { + if self.shutdown_flag.load(Ordering::SeqCst) { + return Err(PumasError::Other( + "Ollama version manager is shutting down".into(), + )); + } + Ok(()) + } + + async fn owned_activity( + &self, + operation: impl std::future::Future> + Send + 'static, + ) -> Result<()> { + let (sender, receiver) = tokio::sync::oneshot::channel(); + { + let mut activities = self + .activities + .lock() + .map_err(|_| PumasError::Other("Ollama activity registry poisoned".into()))?; + self.ensure_open()?; + activities.tasks.harvest_finished(); + let task = tokio::spawn(async move { + let result = operation.await; + let terminal = result.as_ref().map(|_| ()).map_err(ToString::to_string); + let _ = sender.send(result); + terminal + }); + activities.tasks.tasks.push(task); + } + receiver.await.map_err(|error| { + PumasError::Other(format!("Ollama activity lost its result: {error}")) + })? + } + + async fn send_progress(&self, sender: &mpsc::Sender, update: ProgressUpdate) { + tokio::select! { + _ = sender.send(update) => {}, + _ = super::wait_for_install_cancel(self.shutdown_flag.clone()) => {}, + } + } + + /// Close mutation admission, request installation cancellation, and drain + /// admitted install/removal activities before draining state mutations. + /// Started work is joined, including work whose waiter was cancelled. + /// Repeated calls retain failures, including task panics. Existing network + /// waits and filesystem settlement may delay shutdown; elapsed time never + /// substitutes for completion. Cancelled shutdown waiters may safely retry. + pub async fn shutdown(&self) -> Result<()> { + let completion = { + let mut activities = self + .activities + .lock() + .map_err(|_| PumasError::Other("Ollama activity registry poisoned".into()))?; + if let Some(completion) = &activities.completion { + completion.clone() + } else { + self.shutdown_flag.store(true, Ordering::SeqCst); + self.cancel_flag.store(true, Ordering::SeqCst); + let registered = std::mem::take(&mut activities.tasks); + let owner = self.clone(); + let supervisor = tokio::spawn(async move { + let mut failures = registered.failures; + for task in registered.tasks { + match task.await { + Ok(Ok(())) => {} + Ok(Err(error)) => failures.push(error), + Err(error) => failures.push(error.to_string()), + } + } + if let Err(error) = owner.state.write().await.shutdown_mutations().await { + failures.push(error.to_string()); + } + if failures.is_empty() { + Ok(()) + } else { + Err(Arc::new(failures.join("; "))) + } + }); + let completion = async move { + supervisor + .await + .unwrap_or_else(|error| Err(Arc::new(error.to_string()))) + } + .boxed() + .shared(); + activities.completion = Some(completion.clone()); + completion + } + }; + completion + .await + .map_err(|error| PumasError::Other(format!("Ollama shutdown failed: {error}"))) + } + /// Get the versions directory. fn versions_dir(&self) -> PathBuf { self.launcher_root.join(self.app_id.versions_dir_name()) @@ -177,6 +288,17 @@ impl OllamaVersionManager { &self, tag: &str, progress_tx: Option>, + ) -> Result<()> { + let owner = self.clone(); + let tag = tag.to_owned(); + self.owned_activity(async move { owner.install_version_owned(&tag, progress_tx).await }) + .await + } + + async fn install_version_owned( + &self, + tag: &str, + progress_tx: Option>, ) -> Result<()> { // Acquire installation lock let _lock = self.install_lock.lock().await; @@ -184,12 +306,15 @@ impl OllamaVersionManager { // Set installing tag { let mut installing = self.installing_tag.write().await; + let _admission = self + .activities + .lock() + .map_err(|_| PumasError::Other("Ollama activity registry poisoned".into()))?; + self.ensure_open()?; *installing = Some(tag.to_string()); + self.cancel_flag.store(false, Ordering::SeqCst); } - // Reset cancel flag - self.cancel_flag.store(false, Ordering::SeqCst); - let result = self .install_version_internal(tag, progress_tx.clone()) .await; @@ -202,11 +327,13 @@ impl OllamaVersionManager { // Send completion status if let Some(tx) = progress_tx { - let _ = tx - .send(ProgressUpdate::Completed { + self.send_progress( + &tx, + ProgressUpdate::Completed { success: result.is_ok(), - }) - .await; + }, + ) + .await; } result @@ -231,12 +358,14 @@ impl OllamaVersionManager { // Send stage update if let Some(ref tx) = progress_tx { - let _ = tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Download, message: format!("Fetching release {}", tag), - }) - .await; + }, + ) + .await; } // Fetch releases and find the matching one @@ -264,12 +393,14 @@ impl OllamaVersionManager { let archive_path = version_path.join(&asset.name); if let Some(ref tx) = progress_tx { - let _ = tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Download, message: format!("Downloading {}", asset.name), - }) - .await; + }, + ) + .await; } self.download_file(download_url, &archive_path, progress_tx.clone()) @@ -280,12 +411,14 @@ impl OllamaVersionManager { // Extract and set up if let Some(ref tx) = progress_tx { - let _ = tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Extract, message: "Extracting binary".to_string(), - }) - .await; + }, + ) + .await; } self.extract_binary(&archive_path, &version_path).await?; @@ -297,12 +430,14 @@ impl OllamaVersionManager { // Record in metadata if let Some(ref tx) = progress_tx { - let _ = tx - .send(ProgressUpdate::StageChanged { + self.send_progress( + tx, + ProgressUpdate::StageChanged { stage: InstallationStage::Setup, message: "Recording installation".to_string(), - }) - .await; + }, + ) + .await; } // Create metadata and update state @@ -368,33 +503,53 @@ impl OllamaVersionManager { .await .map_err(|e| PumasError::io_with_path(e, dest))?; - use futures::StreamExt; - while let Some(chunk) = stream.next().await { - self.check_cancelled()?; - - let chunk = chunk.map_err(|e| PumasError::Network { - message: format!("Error reading download: {}", e), - cause: Some(e.to_string()), - })?; - - file.write_all(&chunk) - .await - .map_err(|e| PumasError::io_with_path(e, dest))?; + let transfer: Result<()> = async { + use futures::StreamExt; + while let Some(chunk) = stream.next().await { + self.check_cancelled()?; - downloaded += chunk.len() as u64; + let chunk = chunk.map_err(|e| PumasError::Network { + message: format!("Error reading download: {}", e), + cause: Some(e.to_string()), + })?; - if let Some(ref tx) = progress_tx { - let _ = tx - .send(ProgressUpdate::Download { - downloaded_bytes: downloaded, - total_bytes: total_size, - speed_bytes_per_sec: None, - }) + file.write_all(&chunk) + .await + .map_err(|e| PumasError::io_with_path(e, dest))?; + + downloaded += chunk.len() as u64; + + if let Some(ref tx) = progress_tx { + self.send_progress( + tx, + ProgressUpdate::Download { + downloaded_bytes: downloaded, + total_bytes: total_size, + speed_bytes_per_sec: None, + }, + ) .await; + } } - } - Ok(()) + Ok(()) + } + .await; + // Settle this activity's existing file writes before its terminal + // receipt, including cancellation/error exits. Dropping Tokio File + // alone can leave blocking writes active after wrapper shutdown. + let settlement = file + .flush() + .await + .map_err(|error| PumasError::io_with_path(error, dest)); + drop(file.into_std().await); + match (transfer, settlement) { + (result, Ok(())) => result, + (Ok(()), Err(error)) => Err(error), + (Err(error), Err(settlement)) => Err(PumasError::Other(format!( + "{error}; Ollama file settlement failed: {settlement}" + ))), + } } /// Extract the binary from archive. @@ -687,6 +842,7 @@ impl OllamaVersionManager { /// Set active version. pub async fn set_active_version(&self, tag: &str) -> Result<()> { let mut state = self.state.write().await; + self.ensure_open()?; state.set_active_version(tag).await?; Ok(()) } @@ -700,12 +856,22 @@ impl OllamaVersionManager { /// Set default version. pub async fn set_default_version(&self, tag: Option<&str>) -> Result<()> { let mut state = self.state.write().await; + self.ensure_open()?; state.set_default_version(tag).await?; Ok(()) } /// Uninstall a version. pub async fn uninstall_version(&self, tag: &str) -> Result<()> { + let owner = self.clone(); + let tag = tag.to_owned(); + self.owned_activity(async move { owner.uninstall_version_owned(&tag).await }) + .await + } + + async fn uninstall_version_owned(&self, tag: &str) -> Result<()> { + let _install = self.install_lock.lock().await; + self.ensure_open()?; let version_path = self.version_path(tag); if path_exists(&version_path).await? { @@ -745,6 +911,201 @@ impl OllamaVersionManager { mod tests { use super::*; + async fn test_manager() -> (OllamaVersionManager, tempfile::TempDir) { + let root = tempfile::tempdir().unwrap(); + let metadata = Arc::new(MetadataManager::new(root.path())); + metadata.ensure_directories().unwrap(); + let github = Arc::new(GitHubClient::new(root.path().join("launcher-data/cache")).unwrap()); + let manager = OllamaVersionManager::new(root.path().to_owned(), metadata, github) + .await + .unwrap(); + std::fs::create_dir_all(manager.version_path("v1")).unwrap(); + std::fs::write(manager.get_binary_path("v1"), "complete").unwrap(); + manager + .state + .write() + .await + .add_installed_version( + "v1", + InstalledVersionMetadata { + path: "v1".into(), + release_tag: "v1".into(), + ..Default::default() + }, + ) + .unwrap(); + (manager, root) + } + + fn single_blocking_worker_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .max_blocking_threads(1) + .build() + .unwrap() + } + + async fn occupy_blocking_worker() -> (std::sync::mpsc::Sender<()>, tokio::task::JoinHandle<()>) + { + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let worker = tokio::task::spawn_blocking(move || { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + }); + entered_rx.await.unwrap(); + (release_tx, worker) + } + + #[test] + fn ollama_wrapper_shutdown_drains_cancelled_state_mutation_and_retains_outcome() { + single_blocking_worker_runtime().block_on(async { + for fail_mutation in [false, true] { + let (manager, root) = test_manager().await; + let state_owner = manager.state.read().await.mutation_tasks(); + let (release, blocker) = occupy_blocking_worker().await; + let selecting = manager.clone(); + let waiter = + tokio::spawn(async move { selecting.set_default_version(Some("v1")).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while !state_owner.has_active_tasks() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + if fail_mutation { + let path = root + .path() + .join("launcher-data/metadata/versions-ollama.json"); + std::fs::remove_file(&path).unwrap(); + std::fs::create_dir(&path).unwrap(); + } + let owner = manager.clone(); + let mut shutdown = tokio::spawn(async move { owner.shutdown().await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), &mut shutdown) + .await + .is_err() + ); + shutdown.abort(); + assert!(shutdown.await.unwrap_err().is_cancelled()); + release.send(()).unwrap(); + blocker.await.unwrap(); + let outcome = manager.shutdown().await.map_err(|error| error.to_string()); + assert_eq!(outcome.is_err(), fail_mutation); + if fail_mutation { + assert!(outcome + .as_ref() + .unwrap_err() + .contains("versions-ollama.json")); + } else { + assert_eq!( + MetadataManager::new(root.path()) + .load_versions(Some(AppId::Ollama)) + .unwrap() + .default_version + .as_deref(), + Some("v1") + ); + } + assert_eq!( + manager.shutdown().await.map_err(|error| error.to_string()), + outcome + ); + assert!(manager.set_active_version("v1").await.is_err()); + assert!(manager.set_default_version(None).await.is_err()); + assert!(manager.install_version("late", None).await.is_err()); + assert!(manager.uninstall_version("v1").await.is_err()); + assert!(manager.get_binary_path("v1").exists()); + assert!(!manager.version_path("late").exists()); + } + }); + } + + #[test] + fn ollama_wrapper_shutdown_settles_removal_after_waiter_cancellation() { + single_blocking_worker_runtime().block_on(async { + for fail_removal in [false, true] { + let (manager, root) = test_manager().await; + if fail_removal { + std::fs::remove_dir_all(manager.version_path("v1")).unwrap(); + std::fs::write(manager.version_path("v1"), "retained").unwrap(); + } + let (release, blocker) = occupy_blocking_worker().await; + let removing = manager.clone(); + let waiter = tokio::spawn(async move { removing.uninstall_version("v1").await }); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while manager.install_lock.try_lock().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let shutdown = manager.shutdown(); + tokio::pin!(shutdown); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), &mut shutdown) + .await + .is_err() + ); + assert!(manager.version_path("v1").exists()); + release.send(()).unwrap(); + blocker.await.unwrap(); + let outcome = shutdown.await.map_err(|error| error.to_string()); + assert_eq!(outcome.is_err(), fail_removal); + assert_eq!(manager.version_path("v1").exists(), fail_removal); + assert_eq!( + MetadataManager::new(root.path()) + .get_installed_version("v1", Some(AppId::Ollama)) + .unwrap() + .is_some(), + fail_removal + ); + assert_eq!( + manager.shutdown().await.map_err(|error| error.to_string()), + outcome + ); + assert!(manager.uninstall_version("v1").await.is_err()); + } + }); + } + + #[tokio::test] + async fn ollama_wrapper_shutdown_rejects_queued_install_before_cancellation_reset() { + let (manager, _root) = test_manager().await; + let held = manager.install_lock.lock().await; + let installing = manager.clone(); + let waiter = tokio::spawn(async move { installing.install_version("queued", None).await }); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while manager.activities.lock().unwrap().tasks.tasks.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let shutdown = manager.shutdown(); + tokio::pin!(shutdown); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), &mut shutdown) + .await + .is_err() + ); + drop(held); + let error = shutdown.await.unwrap_err().to_string(); + assert!(error.contains("shutting down")); + assert!(manager.cancel_flag.load(Ordering::SeqCst)); + assert!(!manager.version_path("queued").exists()); + assert_eq!(manager.shutdown().await.unwrap_err().to_string(), error); + assert!(manager.install_version("late", None).await.is_err()); + } + #[test] fn test_binary_name() { let name = OllamaVersionManager::binary_name(); diff --git a/rust/crates/pumas-app-manager/src/version_manager/state.rs b/rust/crates/pumas-app-manager/src/version_manager/state.rs index 76218ee4..796fb837 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/state.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/state.rs @@ -3,14 +3,16 @@ //! Manages the state of installed, active, and default versions. //! Handles state persistence and validation. -use super::installer::TorchVersionsLock; +use super::installer::{NativeVersionsLock, TorchVersionsLock}; use crate::version_manager::ValidationResult; +use futures::FutureExt; use pumas_library::config::AppId; use pumas_library::metadata::{InstalledVersionMetadata, MetadataManager}; use pumas_library::{PumasError, Result}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::sync::Mutex as StdMutex; use tokio::fs; use tracing::{debug, info, warn}; @@ -36,21 +38,128 @@ fn legacy_llama_cpp_sycl_replacements( .collect() } -async fn torch_metadata_transaction( - lock: &TorchVersionsLock, - work: impl FnOnce() -> Result + Send + 'static, -) -> Result { - let lease = lock.clone(); - tokio::task::spawn_blocking(move || { - let _lease = lease; - work() - }) - .await - .map_err(|error| PumasError::Other(format!("Torch metadata task failed: {error}")))? +#[derive(Default)] +pub(crate) struct StateMutationTasks { + state: StdMutex, +} + +#[derive(Default)] +struct StateMutationLifecycle { + closed: bool, + tasks: super::InstallationTasks, + completion: Option, +} + +impl StateMutationTasks { + #[cfg(test)] + pub(crate) fn has_active_tasks(&self) -> bool { + self.state + .lock() + .unwrap() + .tasks + .tasks + .iter() + .any(|task| !task.is_finished()) + } + fn ensure_open(&self) -> Result<()> { + let state = self + .state + .lock() + .map_err(|_| PumasError::Other("State mutation registry poisoned".into()))?; + if state.closed { + return Err(PumasError::Other( + "Version state mutation admission closed".into(), + )); + } + Ok(()) + } + + pub(crate) async fn leased_transaction( + &self, + lock: &L, + work: impl FnOnce() -> Result + Send + 'static, + ) -> Result { + let (sender, receiver) = tokio::sync::oneshot::channel(); + { + let mut state = self + .state + .lock() + .map_err(|_| PumasError::Other("State mutation registry poisoned".into()))?; + if state.closed { + return Err(PumasError::Other( + "Version state mutation admission closed".into(), + )); + } + state.tasks.harvest_finished(); + let lease = lock.clone(); + // Register before any suspension. The receiver belongs to this + // caller; the lifecycle owns the worker's independent receipt. + let task = tokio::task::spawn_blocking(move || { + let result = work(); + drop(lease); + let terminal = result.as_ref().map(|_| ()).map_err(ToString::to_string); + let _ = sender.send(result); + terminal + }); + state.tasks.tasks.push(task); + } + receiver.await.map_err(|error| { + PumasError::Other(format!("Version metadata task lost its result: {error}")) + })? + } + + async fn shutdown(&self) -> Result<()> { + let completion = { + let mut state = self + .state + .lock() + .map_err(|_| PumasError::Other("State mutation registry poisoned".into()))?; + state.closed = true; + if let Some(completion) = &state.completion { + completion.clone() + } else { + let registered = std::mem::take(&mut state.tasks); + let supervisor = tokio::spawn(async move { + let mut failures = registered.failures; + for task in registered.tasks { + match task.await { + Ok(Ok(())) => {} + Ok(Err(error)) => failures.push(error), + Err(error) => failures.push(error.to_string()), + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(Arc::new(failures.join("; "))) + } + }); + let completion = async move { + supervisor + .await + .unwrap_or_else(|error| Err(Arc::new(error.to_string()))) + } + .boxed() + .shared(); + state.completion = Some(completion.clone()); + completion + } + }; + completion.await.map_err(|error| { + PumasError::Other(format!("Version state mutation drain failed: {error}")) + }) + } } /// Tracks the state of all versions. +/// +/// Native llama.cpp mutations and legacy normalization share the installation +/// lease. A competing mutation returns an error; started blocking workers keep +/// their lease if the caller stops waiting. Cached read accessors are snapshots. +/// Direct owners must retain this state and await `shutdown_mutations` before +/// dropping it or stopping its runtime. Construction must be awaited to completion. pub struct VersionState { + mutation_tasks: Arc, /// Root directory for launcher. launcher_root: PathBuf, /// Application ID. @@ -68,6 +177,16 @@ pub struct VersionState { } impl VersionState { + pub(crate) fn mutation_tasks(&self) -> Arc { + self.mutation_tasks.clone() + } + + /// Close mutation admission and observe registered mutation workers, including workers + /// whose callers stopped waiting. Repeated calls retain the same outcome. + pub async fn shutdown_mutations(&mut self) -> Result<()> { + self.mutation_tasks.shutdown().await + } + fn torch_versions_lock(&self) -> Result> { if self.app_id != AppId::Torch { return Ok(None); @@ -78,6 +197,15 @@ impl VersionState { TorchVersionsLock::try_acquire(&versions_dir).map_err(PumasError::from)?, )) } + async fn native_versions_lock(&self) -> Result> { + if self.app_id != AppId::LlamaCpp { + return Ok(None); + } + NativeVersionsLock::acquire(self.launcher_root.join(self.app_id.versions_dir_name())) + .await + .map(Some) + } + async fn load_versions_metadata(&self) -> Result { let metadata_manager = self.metadata_manager.clone(); let app_id = self.app_id; @@ -91,34 +219,14 @@ impl VersionState { })? } - async fn set_last_selected_version_metadata(&self, tag: Option) -> Result<()> { - let metadata_manager = self.metadata_manager.clone(); - let app_id = self.app_id; - tokio::task::spawn_blocking(move || { - metadata_manager.set_last_selected_version(tag.as_deref(), Some(app_id)) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join version-state last-selected write task: {}", - err - )) - })? - } - async fn set_default_version_metadata(&self, tag: Option) -> Result<()> { let metadata_manager = self.metadata_manager.clone(); let app_id = self.app_id; - tokio::task::spawn_blocking(move || { - metadata_manager.set_default_version(tag.as_deref(), Some(app_id)) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join version-state default-version write task: {}", - err - )) - })? + self.mutation_tasks + .leased_transaction(&(), move || { + metadata_manager.set_default_version(tag.as_deref(), Some(app_id)) + }) + .await } /// Create a new version state tracker. @@ -128,6 +236,7 @@ impl VersionState { metadata_manager: Arc, ) -> Result { let mut state = Self { + mutation_tasks: Arc::new(StateMutationTasks::default()), launcher_root: launcher_root.to_path_buf(), app_id, metadata_manager, @@ -143,6 +252,7 @@ impl VersionState { /// Initialize state from metadata and filesystem. async fn initialize(&mut self) -> Result<()> { + let native_lock = self.native_versions_lock().await?; if self.app_id == AppId::Torch { let versions_dir = self.launcher_root.join(self.app_id.versions_dir_name()); let metadata = self.metadata_manager.clone(); @@ -156,7 +266,8 @@ impl VersionState { Err(error) => warn!(%error, "Torch cleanup recovery task failed"), } } - self.normalize_llama_cpp_legacy_sycl_variants().await?; + self.normalize_llama_cpp_legacy_sycl_variants(native_lock.as_ref()) + .await?; // A selection writes the marker before its metadata commit. Startup // must not accept that marker while another backend owns the write. @@ -278,17 +389,32 @@ impl VersionState { /// Refresh state from disk. pub async fn refresh(&mut self) -> Result<()> { + self.mutation_tasks.ensure_open()?; + let native_lock = self.native_versions_lock().await?; let lock = self.torch_versions_lock()?; - self.refresh_inner(lock.as_ref()).await + self.refresh_inner(lock.as_ref(), native_lock.as_ref()) + .await } pub(crate) async fn refresh_with_lock(&mut self, lock: &TorchVersionsLock) -> Result<()> { debug_assert_eq!(self.app_id, AppId::Torch); - self.refresh_inner(Some(lock)).await + self.refresh_inner(Some(lock), None).await } - async fn refresh_inner(&mut self, _lock: Option<&TorchVersionsLock>) -> Result<()> { - self.normalize_llama_cpp_legacy_sycl_variants().await?; + pub(crate) async fn refresh_with_native_lock( + &mut self, + lock: &NativeVersionsLock, + ) -> Result<()> { + self.refresh_inner(None, Some(lock)).await + } + + async fn refresh_inner( + &mut self, + _lock: Option<&TorchVersionsLock>, + native_lock: Option<&NativeVersionsLock>, + ) -> Result<()> { + self.normalize_llama_cpp_legacy_sycl_variants(native_lock) + .await?; let versions = self.load_versions_metadata().await?; let installed_tags: HashSet = versions.installed.keys().cloned().collect(); @@ -317,123 +443,70 @@ impl VersionState { Ok(()) } - async fn normalize_llama_cpp_legacy_sycl_variants(&self) -> Result<()> { + async fn normalize_llama_cpp_legacy_sycl_variants( + &self, + native_lock: Option<&NativeVersionsLock>, + ) -> Result<()> { if self.app_id != AppId::LlamaCpp { return Ok(()); } - + let lock = native_lock.ok_or_else(|| PumasError::InstallationFailed { + message: "Native metadata mutation lease absent".into(), + })?; let metadata_manager = self.metadata_manager.clone(); - let versions = tokio::task::spawn_blocking(move || { - let mut versions = metadata_manager.load_versions(Some(AppId::LlamaCpp))?; - let replacements = legacy_llama_cpp_sycl_replacements(&versions.installed); - if replacements.is_empty() { - return Ok::<_, PumasError>((versions, replacements)); - } - - for (old_tag, new_tag) in &replacements { - let Some(mut metadata) = versions.installed.remove(old_tag) else { - continue; - }; - metadata.path = new_tag.clone(); - metadata.release_tag = new_tag.clone(); - versions - .installed - .entry(new_tag.clone()) - .or_insert(metadata); - if versions.last_selected_version.as_ref() == Some(old_tag) { - versions.last_selected_version = Some(new_tag.clone()); + let versions_dir = self.launcher_root.join(self.app_id.versions_dir_name()); + let active_file = super::active_version_path(&self.launcher_root, self.app_id); + self.mutation_tasks + .leased_transaction(lock, move || { + let mut versions = metadata_manager.load_versions(Some(AppId::LlamaCpp))?; + let replacements = legacy_llama_cpp_sycl_replacements(&versions.installed); + if replacements.is_empty() { + return Ok(()); } - if versions.default_version.as_ref() == Some(old_tag) { - versions.default_version = Some(new_tag.clone()); + for (old_tag, new_tag) in &replacements { + let Some(mut metadata) = versions.installed.remove(old_tag) else { + continue; + }; + metadata.path = new_tag.clone(); + metadata.release_tag = new_tag.clone(); + versions + .installed + .entry(new_tag.clone()) + .or_insert(metadata); + if versions.last_selected_version.as_ref() == Some(old_tag) { + versions.last_selected_version = Some(new_tag.clone()); + } + if versions.default_version.as_ref() == Some(old_tag) { + versions.default_version = Some(new_tag.clone()); + } } - } - metadata_manager.save_versions(&versions, Some(AppId::LlamaCpp))?; - Ok((versions, replacements)) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join llama.cpp SYCL metadata migration task: {}", - err - )) - })??; - - let (_versions, replacements) = versions; - for (old_tag, new_tag) in replacements { - self.rename_version_dir_if_needed(&old_tag, &new_tag) - .await?; - self.rewrite_active_version_if_needed(&old_tag, &new_tag) - .await?; - } - - Ok(()) - } - - async fn rename_version_dir_if_needed(&self, old_tag: &str, new_tag: &str) -> Result<()> { - let versions_dir = self.launcher_root.join(self.app_id.versions_dir_name()); - let old_path = versions_dir.join(old_tag); - let new_path = versions_dir.join(new_tag); - if !fs::try_exists(&old_path) - .await - .map_err(|e| PumasError::Io { - message: format!("Failed to check legacy version directory: {}", e), - path: Some(old_path.clone()), - source: Some(e), - })? - || fs::try_exists(&new_path) - .await - .map_err(|e| PumasError::Io { - message: format!("Failed to check replacement version directory: {}", e), - path: Some(new_path.clone()), - source: Some(e), - })? - { - return Ok(()); - } - - fs::rename(&old_path, &new_path) - .await - .map_err(|e| PumasError::Io { - message: format!( - "Failed to rename legacy llama.cpp SYCL version directory: {}", - e - ), - path: Some(old_path), - source: Some(e), + metadata_manager.save_versions(&versions, Some(AppId::LlamaCpp))?; + for (old_tag, new_tag) in replacements { + let old_path = versions_dir.join(&old_tag); + let new_path = versions_dir.join(&new_tag); + if old_path + .try_exists() + .map_err(|error| PumasError::io_with_path(error, &old_path))? + && !new_path + .try_exists() + .map_err(|error| PumasError::io_with_path(error, &new_path))? + { + std::fs::rename(&old_path, &new_path) + .map_err(|error| PumasError::io_with_path(error, &old_path))?; + } + match std::fs::read_to_string(&active_file) { + Ok(active) if active.trim() == old_tag => { + std::fs::write(&active_file, new_tag) + .map_err(|error| PumasError::io_with_path(error, &active_file))?; + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(PumasError::io_with_path(error, &active_file)), + } + } + Ok(()) }) - } - - async fn rewrite_active_version_if_needed(&self, old_tag: &str, new_tag: &str) -> Result<()> { - let active_file = super::active_version_path(&self.launcher_root, self.app_id); - if !fs::try_exists(&active_file) - .await - .map_err(|e| PumasError::Io { - message: format!("Failed to check active version file: {}", e), - path: Some(active_file.clone()), - source: Some(e), - })? - { - return Ok(()); - } - - let active_tag = fs::read_to_string(&active_file) .await - .map_err(|e| PumasError::Io { - message: format!("Failed to read active version file: {}", e), - path: Some(active_file.clone()), - source: Some(e), - })?; - if active_tag.trim() != old_tag { - return Ok(()); - } - - fs::write(&active_file, new_tag) - .await - .map_err(|e| PumasError::Io { - message: format!("Failed to update active version file: {}", e), - path: Some(active_file), - source: Some(e), - }) } // ======================================== @@ -486,11 +559,15 @@ impl VersionState { /// Set the active version. pub async fn set_active_version(&mut self, tag: &str) -> Result { + self.mutation_tasks.ensure_open()?; + let native_lock = self.native_versions_lock().await?; let lock = self.torch_versions_lock()?; - if lock.is_some() { - self.refresh_inner(lock.as_ref()).await?; + if lock.is_some() || native_lock.is_some() { + self.refresh_inner(lock.as_ref(), native_lock.as_ref()) + .await?; } - self.set_active_version_inner(tag, lock.as_ref()).await + self.set_active_version_inner(tag, lock.as_ref(), native_lock.as_ref()) + .await } pub(crate) async fn set_active_version_with_lock( @@ -498,13 +575,14 @@ impl VersionState { tag: &str, lock: &TorchVersionsLock, ) -> Result { - self.set_active_version_inner(tag, Some(lock)).await + self.set_active_version_inner(tag, Some(lock), None).await } async fn set_active_version_inner( &mut self, tag: &str, lock: Option<&TorchVersionsLock>, + native_lock: Option<&NativeVersionsLock>, ) -> Result { if !self.is_installed(tag) { return Err(PumasError::VersionNotFound { @@ -515,35 +593,51 @@ impl VersionState { if self.app_id == AppId::Torch { let metadata = self.metadata_manager.clone(); let selected = tag.to_owned(); - torch_metadata_transaction(lock.expect("Torch selection lock required"), move || { - let previous = match std::fs::read(&active_file) { - Ok(bytes) => Some(bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => return Err(PumasError::io_with_path(error, &active_file)), - }; - std::fs::write(&active_file, &selected) - .map_err(|error| PumasError::io_with_path(error, &active_file))?; - if let Err(error) = - metadata.set_last_selected_version(Some(&selected), Some(AppId::Torch)) - { - match previous { - Some(bytes) => { - let _ = std::fs::write(&active_file, bytes); - } - None => { - let _ = std::fs::remove_file(&active_file); + self.mutation_tasks + .leased_transaction(lock.expect("Torch selection lock required"), move || { + let previous = match std::fs::read(&active_file) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(PumasError::io_with_path(error, &active_file)), + }; + std::fs::write(&active_file, &selected) + .map_err(|error| PumasError::io_with_path(error, &active_file))?; + if let Err(error) = + metadata.set_last_selected_version(Some(&selected), Some(AppId::Torch)) + { + match previous { + Some(bytes) => { + let _ = std::fs::write(&active_file, bytes); + } + None => { + let _ = std::fs::remove_file(&active_file); + } } + return Err(error); } - return Err(error); - } - Ok(()) - }) - .await?; + Ok(()) + }) + .await?; + } else if let Some(lock) = native_lock { + let metadata = self.metadata_manager.clone(); + let selected = tag.to_owned(); + self.mutation_tasks + .leased_transaction(lock, move || { + std::fs::write(&active_file, &selected) + .map_err(|error| PumasError::io_with_path(error, &active_file))?; + metadata.set_last_selected_version(Some(&selected), Some(AppId::LlamaCpp)) + }) + .await?; } else { - fs::write(&active_file, tag) - .await - .map_err(|error| PumasError::io_with_path(error, &active_file))?; - self.set_last_selected_version_metadata(Some(tag.to_string())) + let selected = tag.to_owned(); + let metadata = self.metadata_manager.clone(); + let app_id = self.app_id; + self.mutation_tasks + .leased_transaction(&(), move || { + std::fs::write(&active_file, &selected) + .map_err(|error| PumasError::io_with_path(error, &active_file))?; + metadata.set_last_selected_version(Some(&selected), Some(app_id)) + }) .await?; } self.active_version = Some(tag.to_string()); @@ -554,11 +648,15 @@ impl VersionState { /// Set the default version. pub async fn set_default_version(&mut self, tag: Option<&str>) -> Result { + self.mutation_tasks.ensure_open()?; + let native_lock = self.native_versions_lock().await?; let lock = self.torch_versions_lock()?; - if lock.is_some() { - self.refresh_inner(lock.as_ref()).await?; + if lock.is_some() || native_lock.is_some() { + self.refresh_inner(lock.as_ref(), native_lock.as_ref()) + .await?; } - self.set_default_version_inner(tag, lock.as_ref()).await + self.set_default_version_inner(tag, lock.as_ref(), native_lock.as_ref()) + .await } pub(crate) async fn set_default_version_with_lock( @@ -566,13 +664,14 @@ impl VersionState { tag: Option<&str>, lock: &TorchVersionsLock, ) -> Result { - self.set_default_version_inner(tag, Some(lock)).await + self.set_default_version_inner(tag, Some(lock), None).await } async fn set_default_version_inner( &mut self, tag: Option<&str>, lock: Option<&TorchVersionsLock>, + native_lock: Option<&NativeVersionsLock>, ) -> Result { if let Some(t) = tag { if !self.is_installed(t) { @@ -583,10 +682,19 @@ impl VersionState { if self.app_id == AppId::Torch { let metadata = self.metadata_manager.clone(); let selected = tag.map(str::to_owned); - torch_metadata_transaction(lock.expect("Torch selection lock required"), move || { - metadata.set_default_version(selected.as_deref(), Some(AppId::Torch)) - }) - .await?; + self.mutation_tasks + .leased_transaction(lock.expect("Torch selection lock required"), move || { + metadata.set_default_version(selected.as_deref(), Some(AppId::Torch)) + }) + .await?; + } else if let Some(lock) = native_lock { + let metadata = self.metadata_manager.clone(); + let selected = tag.map(str::to_owned); + self.mutation_tasks + .leased_transaction(lock, move || { + metadata.set_default_version(selected.as_deref(), Some(AppId::LlamaCpp)) + }) + .await?; } else { self.set_default_version_metadata(tag.map(String::from)) .await?; @@ -604,27 +712,28 @@ impl VersionState { debug_assert_eq!(self.app_id, AppId::Torch); let marker = super::active_version_path(&self.launcher_root, self.app_id); let metadata = self.metadata_manager.clone(); - torch_metadata_transaction(lock, move || { - let previous = match std::fs::read(&marker) { - Ok(bytes) => Some(bytes), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => return Err(PumasError::io_with_path(error, &marker)), - }; - match std::fs::remove_file(&marker) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => return Err(PumasError::io_with_path(error, &marker)), - } - if let Err(error) = metadata.set_last_selected_version(None, Some(AppId::Torch)) { - if let Some(bytes) = previous { - let _ = std::fs::write(&marker, bytes); + self.mutation_tasks + .leased_transaction(lock, move || { + let previous = match std::fs::read(&marker) { + Ok(bytes) => Some(bytes), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(PumasError::io_with_path(error, &marker)), + }; + match std::fs::remove_file(&marker) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(PumasError::io_with_path(error, &marker)), } - return Err(error); - } - Ok(()) - }) - .await?; - self.refresh_inner(Some(lock)).await + if let Err(error) = metadata.set_last_selected_version(None, Some(AppId::Torch)) { + if let Some(bytes) = previous { + let _ = std::fs::write(&marker, bytes); + } + return Err(error); + } + Ok(()) + }) + .await?; + self.refresh_inner(Some(lock), None).await } /// Add a new installed version. @@ -633,9 +742,17 @@ impl VersionState { tag: &str, metadata: InstalledVersionMetadata, ) -> Result<()> { + self.mutation_tasks.ensure_open()?; let _lock = self.torch_versions_lock()?; - if self.app_id == AppId::Torch { - let versions = self.metadata_manager.load_versions(Some(AppId::Torch))?; + let _native_lock = if self.app_id == AppId::LlamaCpp { + Some(NativeVersionsLock::try_acquire( + &self.launcher_root.join(self.app_id.versions_dir_name()), + )?) + } else { + None + }; + if matches!(self.app_id, AppId::Torch | AppId::LlamaCpp) { + let versions = self.metadata_manager.load_versions(Some(self.app_id))?; self.installed_tags = versions.installed.keys().cloned().collect(); self.installed_metadata = versions.installed; self.default_version = versions.default_version; @@ -652,11 +769,14 @@ impl VersionState { /// Remove an installed version. pub async fn remove_installed_version(&mut self, tag: &str) -> Result<()> { + self.mutation_tasks.ensure_open()?; + let native_lock = self.native_versions_lock().await?; let lock = self.torch_versions_lock()?; - if lock.is_some() { - self.refresh_inner(lock.as_ref()).await?; + if lock.is_some() || native_lock.is_some() { + self.refresh_inner(lock.as_ref(), native_lock.as_ref()) + .await?; } - self.remove_installed_version_inner(tag, lock.as_ref()) + self.remove_installed_version_inner(tag, lock.as_ref(), native_lock.as_ref()) .await } @@ -664,9 +784,30 @@ impl VersionState { &mut self, tag: &str, _lock: Option<&TorchVersionsLock>, + native_lock: Option<&NativeVersionsLock>, ) -> Result<()> { - self.metadata_manager - .remove_installed_version(tag, Some(self.app_id))?; + if let Some(lock) = native_lock { + let metadata = self.metadata_manager.clone(); + let removed = tag.to_owned(); + let marker = super::active_version_path(&self.launcher_root, self.app_id); + let remove_marker = self.active_version.as_deref() == Some(tag); + self.mutation_tasks + .leased_transaction(lock, move || { + metadata.remove_installed_version(&removed, Some(AppId::LlamaCpp))?; + if remove_marker { + match std::fs::remove_file(&marker) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(PumasError::io_with_path(error, &marker)), + } + } + Ok(()) + }) + .await?; + } else { + self.metadata_manager + .remove_installed_version(tag, Some(self.app_id))?; + } self.installed_tags.remove(tag); self.installed_metadata.remove(tag); @@ -675,17 +816,22 @@ impl VersionState { self.active_version = None; // Clear .active-version file let active_file = super::active_version_path(&self.launcher_root, self.app_id); - if self.app_id == AppId::Torch { + if native_lock.is_some() { + // Marker removal settled in the leased metadata worker. + } else if self.app_id == AppId::Torch { match std::fs::remove_file(&active_file) { Ok(()) => {} Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(PumasError::io_with_path(error, &active_file)), } - } else if fs::try_exists(&active_file) - .await - .map_err(|error| PumasError::io_with_path(error, &active_file))? - { - let _ = fs::remove_file(&active_file).await; + } else { + self.mutation_tasks + .leased_transaction(&(), move || match std::fs::remove_file(&active_file) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(PumasError::io_with_path(error, &active_file)), + }) + .await?; } } @@ -704,16 +850,21 @@ impl VersionState { /// Validate all installations and remove incomplete ones. pub async fn validate_installations(&mut self) -> Result { + self.mutation_tasks.ensure_open()?; + let native_lock = self.native_versions_lock().await?; let lock = self.torch_versions_lock()?; - if lock.is_some() { - self.refresh_inner(lock.as_ref()).await?; + if lock.is_some() || native_lock.is_some() { + self.refresh_inner(lock.as_ref(), native_lock.as_ref()) + .await?; } - self.validate_installations_inner(lock.as_ref()).await + self.validate_installations_inner(lock.as_ref(), native_lock.as_ref()) + .await } async fn validate_installations_inner( &mut self, lock: Option<&TorchVersionsLock>, + native_lock: Option<&NativeVersionsLock>, ) -> Result { let versions_dir = self.launcher_root.join(self.app_id.versions_dir_name()); let mut removed_tags = Vec::new(); @@ -754,7 +905,8 @@ impl VersionState { "Removing stale metadata entry for incomplete installation: {}", tag ); - self.remove_installed_version_inner(tag, lock).await?; + self.remove_installed_version_inner(tag, lock, native_lock) + .await?; // NOTE: We no longer delete files automatically to prevent data loss // Orphaned directories will be reported but not deleted } @@ -982,6 +1134,180 @@ mod tests { create_test_state_for_app(AppId::Torch).await } + #[tokio::test] + async fn native_public_state_mutations_share_installation_admission() { + let (mut state, root) = create_test_state_for_app(AppId::LlamaCpp).await; + let versions = root.path().join(AppId::LlamaCpp.versions_dir_name()); + std::fs::create_dir(versions.join("b1234+cpu")).unwrap(); + std::fs::write(versions.join("b1234+cpu/llama-server"), "complete").unwrap(); + let metadata = InstalledVersionMetadata { + path: "b1234+cpu".into(), + release_tag: "b1234".into(), + ..Default::default() + }; + state + .add_installed_version("b1234+cpu", metadata.clone()) + .unwrap(); + state.set_active_version("b1234+cpu").await.unwrap(); + state.set_default_version(Some("b1234+cpu")).await.unwrap(); + let before = state + .metadata_manager + .load_versions(Some(AppId::LlamaCpp)) + .unwrap(); + let marker = super::super::active_version_path(root.path(), AppId::LlamaCpp); + let before_marker = std::fs::read(&marker).unwrap(); + let lease = NativeVersionsLock::try_acquire(&versions).unwrap(); + assert!( + VersionState::new(root.path(), AppId::LlamaCpp, state.metadata_manager.clone()) + .await + .is_err() + ); + assert!(state.refresh().await.is_err()); + assert!(state.set_active_version("b1234+cpu").await.is_err()); + assert!(state.set_default_version(None).await.is_err()); + assert!(state.add_installed_version("b1235+cpu", metadata).is_err()); + assert!(state.remove_installed_version("b1234+cpu").await.is_err()); + assert!(state.validate_installations().await.is_err()); + let after = state + .metadata_manager + .load_versions(Some(AppId::LlamaCpp)) + .unwrap(); + assert_eq!( + serde_json::to_value(before).unwrap(), + serde_json::to_value(after).unwrap() + ); + assert_eq!(std::fs::read(&marker).unwrap(), before_marker); + assert!(versions.join("b1234+cpu/llama-server").exists()); + drop(lease); + state.set_default_version(None).await.unwrap(); + state.remove_installed_version("b1234+cpu").await.unwrap(); + assert!(state + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn native_metadata_worker_holds_lease_after_waiter_is_cancelled() { + let root = TempDir::new().unwrap(); + let versions = root.path().join(AppId::LlamaCpp.versions_dir_name()); + let lease = NativeVersionsLock::try_acquire(&versions).unwrap(); + let metadata = Arc::new(MetadataManager::new(root.path())); + metadata.ensure_directories().unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (finished_tx, finished_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let owner = Arc::new(StateMutationTasks::default()); + let worker_owner = owner.clone(); + let waiter = tokio::spawn(async move { + worker_owner + .leased_transaction(&lease, move || { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + metadata.update_installed_version( + "b1234+cpu", + InstalledVersionMetadata::default(), + Some(AppId::LlamaCpp), + )?; + finished_tx.send(()).unwrap(); + Ok(()) + }) + .await + }); + entered_rx.await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert!(NativeVersionsLock::try_acquire(&versions).is_err()); + release_tx.send(()).unwrap(); + finished_rx.await.unwrap(); + owner.shutdown().await.unwrap(); + // The notification precedes lease release; await actual admission. + tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + if let Ok(lease) = NativeVersionsLock::try_acquire(&versions) { + drop(lease); + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(MetadataManager::new(root.path()) + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_some()); + } + + #[tokio::test] + async fn state_mutation_shutdown_drains_cancelled_waiters_and_retains_failures() { + for terminal in ["success", "error", "panic"] { + let (mut state, root) = create_test_state_for_app(AppId::LlamaCpp).await; + let versions = root.path().join(AppId::LlamaCpp.versions_dir_name()); + let lease = NativeVersionsLock::try_acquire(&versions).unwrap(); + let owner = state.mutation_tasks(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let effect = root.path().join("terminal-effect"); + let worker_effect = effect.clone(); + let waiter = tokio::spawn(async move { + owner + .leased_transaction(&lease, move || { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + std::fs::write(worker_effect, terminal).unwrap(); + match terminal { + "error" => Err(PumasError::Other("owned mutation failure".into())), + "panic" => panic!("owned mutation panic"), + _ => Ok(()), + } + }) + .await + }); + entered_rx.await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + let outcome = { + let drain = state.shutdown_mutations(); + tokio::pin!(drain); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(30), &mut drain) + .await + .is_err() + ); + assert!(!effect.exists()); + assert!(NativeVersionsLock::try_acquire(&versions).is_err()); + release_tx.send(()).unwrap(); + drain.await.map_err(|error| error.to_string()) + }; + assert_eq!( + state + .shutdown_mutations() + .await + .map_err(|error| error.to_string()), + outcome + ); + assert!(state.set_default_version(None).await.is_err()); + assert!(state + .add_installed_version("after-close", InstalledVersionMetadata::default()) + .is_err()); + assert_eq!(std::fs::read_to_string(&effect).unwrap(), terminal); + if terminal == "success" { + outcome.unwrap(); + } else { + assert!(outcome + .unwrap_err() + .to_string() + .contains(if terminal == "error" { + "owned mutation failure" + } else { + "owned mutation panic" + })); + } + } + } + #[tokio::test] async fn test_empty_state() { let (state, _temp) = create_test_state().await; diff --git a/rust/crates/pumas-rpc/src/server.rs b/rust/crates/pumas-rpc/src/server.rs index 4525e681..399a4842 100644 --- a/rust/crates/pumas-rpc/src/server.rs +++ b/rust/crates/pumas-rpc/src/server.rs @@ -342,13 +342,19 @@ pub async fn start_server( result = serving => result.map_err(anyhow::Error::from), _ = shutdown.changed() => Ok(()), }; - let torch_cleanup = async { + let installation_cleanup = async { #[cfg(feature = "inference-plugins")] { let managers = state.version_managers.read().await; let mut errors = Vec::new(); - for manager in managers.values() { - if let Err(error) = manager.shutdown_torch_cleanup().await { + let outcomes = futures::future::join_all( + managers + .values() + .map(VersionManager::shutdown_installations), + ) + .await; + for outcome in outcomes { + if let Err(error) = outcome { errors.push(error.to_string()); } } @@ -358,7 +364,7 @@ pub async fn start_server( } Ok::<(), anyhow::Error>(()) }; - let (owners, runtimes, torch_cleanup) = tokio::join!( + let (owners, runtimes, installation_cleanup) = tokio::join!( drain_server_owners( server_result, async { @@ -377,7 +383,7 @@ pub async fn start_server( state.api.shutdown_conversions(), ), state.api.stop_all_managed_runtime_profiles(), - torch_cleanup, + installation_cleanup, ); let runtimes = runtimes.map_err(anyhow::Error::from).and_then(|summary| { if summary.errors.is_empty() { @@ -386,11 +392,13 @@ pub async fn start_server( Err(anyhow::anyhow!(summary.errors.join("; "))) } }); - let owners = match (owners, torch_cleanup) { + let owners = match (owners, installation_cleanup) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) => Err(error), - (Ok(()), Err(error)) => Err(anyhow::anyhow!("Torch cleanup: {error}")), - (Err(owners), Err(error)) => Err(anyhow::anyhow!("{owners}; Torch cleanup: {error}")), + (Ok(()), Err(error)) => Err(anyhow::anyhow!("Installation cleanup: {error}")), + (Err(owners), Err(error)) => { + Err(anyhow::anyhow!("{owners}; Installation cleanup: {error}")) + } }; match (owners, runtimes) { (Ok(()), Ok(())) => Ok(()), @@ -873,6 +881,44 @@ mod tests { } } + #[cfg(feature = "inference-plugins")] + #[tokio::test] + async fn real_server_shutdown_closes_native_installation_admission() { + let root = TempDir::new().unwrap(); + let api = crate::handlers::test_support::build_test_api_with_hf(root.path()).await; + let manager = VersionManager::new(root.path(), AppId::LlamaCpp) + .await + .unwrap(); + let managers = HashMap::from([("llama-cpp".into(), manager.clone())]); + let sizes = SizeCalculator::new_with_cache(root.path().join("launcher-data/cache")).await; + let plugins = PluginLoader::new_async(root.path().join("launcher-data/plugins")) + .await + .unwrap(); + let server = match start_server( + api, + managers, + sizes, + plugins, + LoopbackHost::parse("127.0.0.1").unwrap(), + 0, + ) + .await + { + Ok(server) => server, + Err(error) if is_socket_bind_permission_error(&error) => { + eprintln!( + "Native shutdown integration unavailable: socket bind not permitted ({error})" + ); + return; + } + Err(error) => panic!("Native shutdown server failed: {error:#}"), + }; + server.shutdown().await.unwrap(); + server.shutdown().await.unwrap(); + let error = manager.install_version("b1234").await.unwrap_err(); + assert!(error.to_string().contains("shutting down")); + } + #[tokio::test] async fn test_server_starts() { let temp_dir = TempDir::new().unwrap(); From 2193236087f549df9b98eefa39ada440d9012c96 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 18:40:34 -0700 Subject: [PATCH 08/20] docs(acquisition): record q1 candidate evidence --- .../artifact-acquisition/execution-ledger.md | 21 +++++++++ docs/plans/artifact-acquisition/plan.md | 2 +- .../reports/architecture-review.md | 14 ++++++ .../reports/coding-standards-mcp-usability.md | 45 +++++++++++++++++++ .../reports/write-sets.md | 2 +- 5 files changed, 82 insertions(+), 2 deletions(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 88b6618b..b8349767 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -66,3 +66,24 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - This sub-slice is authored atop Q1 source `98c19910faeebbb75056ddd604b99554305b14a8`. `gh pr checks 7` could not reach GitHub (`error connecting to api.github.com`); a direct PR-page fetch also returned a cache miss. The last prior observation for the pushed `98c19910…` head was Build run `36652117684` in progress, so no pass is claimed for this candidate. The user-owned `docs/breif/future.md` remains untouched and untracked. - The isolated native proposal remains unintegrated. Its GPT-6.1 Sol High reviewer confirmed the four initial custody findings were addressed, then found a P2: canceled metadata transaction/removal waiters left blocking workers outside the shutdown receipt. The contributor reports that diff `642c96dde04fb49a133c56081c814abacc06b08c5959b87239fe755de700157a` registers these tasks with `VersionState`, drains them through manager shutdown, and exposes an explicit direct-owner drain. Independent follow-up review confirmed the P2 closed for `VersionManager` and direct `VersionState` owners and found no new P0/P1/P2 in that scope. It also identified the public `OllamaVersionManager` wrapper as an adjacent drain owner; a read-only tree search found no in-repository production caller, but its public API owns the same state. The exact worker write set is therefore expanded to include `version_manager/ollama.rs` only for its owned mutation drain and a focused regression. The reviewer explicitly did not extend the guarantee to that wrapper yet. The worker's exact-candidate tests report 254 app-manager, 265 RPC, 17 integration and 2 intent-integration cases passing, plus targeted canceled-waiter and static checks. Its native downloader remains independent, and AQ-HTTP is not ready. - Q1/AQ-HTTP remains in progress/not ready. The shared durable owner/store, actual HF import and native consumer cutover, source resolver consumption, lifecycle reopen across handoff, desktop path, real source and all AC01–AC10/AC15/AC16/AC18 claims remain pending. + +### Custody candidate review and integration update + +- The public `OllamaVersionManager` drain repair was completed on the exact five-file candidate diff `a4eeb6e655c72d3125499f11a4cbee52b8dc093c47a983387bac24dde55672e1`, based on `8b96cab5cbb7ff5104d6e33692538d3680b32fc8`. The contributor's exact-candidate evidence: three wrapper shutdown tests; 257 app-manager, 265 RPC, 17 integration and 2 intent-integration tests; both crates' all-target/all-feature Clippy; RPC no-default-features check; formatting and diff checks. A root rerun on the exact candidate passed all three wrapper shutdown regressions. These are native custody tests, not shared-acquisition or Q1 consumer acceptance. +- Final independent GPT-6.1 Sol High review verified the same diff digest and found no new P0/P1/P2. It confirmed closure of the public Ollama wrapper drain finding. Active install shutdown and wrapper panic paths remain source-supported rather than independently runtime-tested; release-discovery activity is outside the shutdown receipt. Existing network waits may still delay shutdown. Shared acquisition and AQ-HTTP remain unaccepted. +- The worker committed `560cec716211c64cdfc04bc737c8bda94266fcf2` as `fix(runtime): retain native installer effects through shutdown`. The primary integrator verified its base was the exact ancestor `8b96cab5…`, reviewed the complete five-file range and disjoint write set, then integrated it with a no-ff merge. Q1 branch merge commit is `48ad1ba374593f4619fb22f0044377914b268381`; composed app-manager/RPC tests and Clippy passed before push. The worker commit remains reachable through the merge. +- The final store-cutover design review selected no second store: move existing task/effect custody into acquisition, then change the active writer and `LibraryMutationAuthority` inventory reader together. `DownloadDestinationRoot` remains model-specific because it creates a library marker and carries model deletion semantics; neutral workspace capability operations must be extracted without fake model IDs. The actual v4->v5 migration writes during load and spans ordinary, quarantine, hidden-admission and admitted-revocation snapshots. Exact record IDs, queue predecessors/releases, revocations, pending/verified cleanup distinctions and positive post-reopen confirmation must be preserved. Live retained-state population and old-writer isolation remain unknown; only source work and disposable migration fixtures can proceed. +- PR #7 was updated through the REST API at helper head `096d180372d8c694948ea41329d9a82dc3fbd371`. Its hosted Workflow/release check `36654655099` was pending; CodeRabbit review was skipped because the PR is draft. The newly merged head `48ad1ba374593f4619fb22f0044377914b268381` passes local composed tests and static checks but has not yet been pushed or checked by CI. The branch is two commits ahead of origin before these plan-record updates are committed. + +### Composed candidate at `48ad1ba374593f4619fb22f0044377914b268381` + +- Exact candidate repository tests: `cargo test --manifest-path rust/Cargo.toml --locked --all-targets -p pumas-app-manager -p pumas-rpc` passed: 257 app-manager tests, 265 RPC unit tests, 17 integration tests, and 2 intent-integration tests; 13 tests were ignored. The two ignored live Hugging Face intent cases are not counted as evidence. +- Exact candidate checks: `cargo clippy --manifest-path rust/Cargo.toml --locked -p pumas-app-manager -p pumas-rpc --all-targets --all-features -- -D warnings` passed; `cargo check --manifest-path rust/Cargo.toml --locked -p pumas-rpc --all-targets --no-default-features` passed with existing dead-code warnings in untouched files; `cargo fmt --manifest-path rust/Cargo.toml --all -- --check` passed. The Q1 lifecycle fixture command `cargo test --manifest-path rust/Cargo.toml --locked -p pumas-library model_library::hf::lifecycle::tests:: -- --test-threads=1` passed 37 tests on this candidate. Earlier on the exact `096d180…` helper candidate, pumas-library Clippy with `-D warnings`, no-default-features check, and formatting also passed. +- These are local Rust tests, controlled/disposable filesystem cases, and static checks. No live Hugging Face import, shared llama.cpp acquisition/extraction, model generation, desktop workflow, retained live-root migration, real S3, or network-denied Torch install has been executed. The existing Ollama source/archive loop remains independent of acquisition. +- The primary branch still requires a push and current hosted checks for this exact head; the prior hosted run belongs to an older SHA and skipped conditional jobs are not passes. AQ-HTTP remains not ready and no runtime slice may rely on it. + +### Coding-Standards route for the next shared-custody slice + +- Fresh route snapshot `snapshot:v1:2ea2f5e4-70d3-4a32-97c6-13a93989fb3f` selected 34 standards with zero unresolved fact categories after explicitly marking the unrelated framework fact known-absent. The first route omitted that fact and returned one required unresolved category; correcting the route preserved the same change scope. +- Focused reads were requested in three batches for 17 standards: Core, Architecture, Code Design, Concurrency, Rust Async/API, Persistence, Contracts/Evolution/Protocols, Resilience, Security/Untrusted Execution, Dependencies, Verification, Commit, and Concurrent Plan Integration. The combined tool presentation exceeded the response limit and was truncated, so exact follow-up policy reads will be kept smaller and limited to the owner boundaries actually changed. This route is guidance for the upcoming extraction; it is not a compliance result or implementation evidence. +- Repository inspection remains necessary to identify that HF's current task ID/projection scans and shutdown callback are owner-wide, and that its destination root creates model-library marker/deletion authority. The extraction slice will retain the current store and HF model policy while adding scoped custody under one source-neutral supervisor. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index 5be59422..92c4efad 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,7 +2,7 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The existing GitHub client now has an additive exact-tag resolver that maps live publisher asset identity/digest metadata to a verified manifest while leaving the public/cache DTO unchanged. The HF lifecycle owner now keys held root grants by physical-root identity, allowing one owner to retain distinct model/runtime roots and coalesce independently reopened handles; this is extraction preparation, not the shared acquisition cutover. An isolated native-custody proposal now has a candidate for shutdown drainage, metadata coordination, pending-file-I/O settlement, and fallible stage cleanup. Independent review confirmed managed `VersionManager` and direct `VersionState` worker drainage, then found the public `OllamaVersionManager` wrapper also owns `VersionState`; its exact adjacent write set is being added to close that drain boundary. The resolver is not yet consumed by the native installer, and shared durable handoff ownership, retained-store evolution, desktop path, and required real-source qualification remain pending. +**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The existing GitHub client has an additive exact-tag resolver that maps live publisher asset identity/digest metadata to a verified manifest while leaving the public/cache DTO unchanged. The HF lifecycle owner coalesces independently reopened handles for the same physical root while retaining distinct grants; this remains extraction preparation, not the shared acquisition cutover. Native installer shutdown custody is committed and integrated on the Q1 branch after repairs for metadata coordination, pending file-I/O settlement, fallible stage cleanup, and the public `OllamaVersionManager` wrapper's owned drain, with no remaining P0–P2 review findings in that custody scope. Its downloader remains independent. The next admitted slice moves the existing task/effect supervisor behind the acquisition boundary and scopes consumer operations while preserving HF model policy and retained state. The native installer does not yet consume the GitHub resolver or shared owner; durable handoff, retained-store evolution, desktop path, and required real-source qualification remain pending. **Exactly one next slice:** Continue **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. diff --git a/docs/plans/artifact-acquisition/reports/architecture-review.md b/docs/plans/artifact-acquisition/reports/architecture-review.md index edaaee23..e176d88f 100644 --- a/docs/plans/artifact-acquisition/reports/architecture-review.md +++ b/docs/plans/artifact-acquisition/reports/architecture-review.md @@ -87,3 +87,17 @@ The shared contract closes the prerequisite without pulling runtime execution in **Persistence:** the current versioned JSON store already owns attempt/admission/release and uncertain publication. Generalize its proven mechanism and explicitly version records; use one ledger authority for migrated acquisitions. A new database/service is not selected merely because the output is generic. If real contention/transaction/volume requirements invalidate that mechanism, record the deciding evidence and re-plan at the persistence owner before changing the store. **Package tooling:** use the current supported report/resolution mechanism and a real local-input install probe. Test a valid wheel set plus an alternate same-version wheel and a missing dependency with network denied. That determines the exact handoff before independent adapter registration depends on it; it does not require a universal lockfile or private pip API. + +## Q1 source-state reinspection — 2026-09-29 + +Read-only review of the current durable state and consumer code refined the extraction sequence; it did not change the shared-owner acceptance boundary. Promote existing task/effect custody and destination coordination into `pumas-core::acquisition` while preserving HF behavior, inject that owner into the ordinary HF workflow, and extract capability-backed neutral workspace operations before native acquisition is connected. The native installer then consumes the same owner and retains its verified-file lease through extraction and cleanup. Do not create a neutral side store while HF continues writing active transfer state. + +The current `DownloadDestinationRoot` is not itself the neutral workspace contract: opening it writes a `.pumas-library-id.json` marker, and `DownloadRecoveryDestination` embeds model IDs and deletion-index semantics. Keep those model-specific checks in a model wrapper while extracting only bounded capability operations and physical identity needed by acquisition. + +`LibraryMutationAuthority::require_no_download_custody` reads hidden admissions, queue owners and quarantines. Its inventory reader must change with the sole acquisition-store writer; otherwise transfer migration could silently remove the exclusion protecting model-library mutations. Actual deployed retained-state population and retirement/isolation of older writers remain unknown. Implementation and disposable-state migration fixtures may proceed, but live retained-state mutation remains blocked. + +The durable cutover must preserve the supported version-4 and version-5 populations and their current refusal semantics. Version 4 is upgraded to 5 during store load by adding explicit `revision: null` in ordinary downloads, quarantine snapshots, hidden admission snapshots, and admitted-revocation snapshots. Preserve all active/hidden/released identities and exact ordering/ownership facts; do not infer a pinned immutable source for legacy null revisions. Reopening verified-cleanup records still requires this process's positive confirmation, and Pending cleanup is never replayed automatically. + +The current preparatory multi-root change is reviewed as a physical-identity bookkeeping improvement only. It does not turn a model-root capability into runtime storage authority and is not evidence that a shared acquisition owner exists. The multi-root review found no blocker, while noting inactive root slots need pruning or bounded admission once dynamic multi-root use is introduced. + +A follow-up lifecycle review found that sharing the current `DownloadTaskOwner` unchanged is unsafe: task ID scans and finished/projection lookup are global, HF shutdown closes the entire owner, and the first shutdown callback supplies a single final projection. The selected supervisor therefore needs consumer-scoped handles for lookup, cancellation, projection, and close/drain while retaining every Tokio handle under one supervisor. Those scoped handles filter authority; they are not separate task owners. Current-generation checks remain scoped to the owning consumer operation. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index 73eaf723..5ed51d90 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -99,6 +99,51 @@ This report records agent experience using the Coding-Standards MCP during Acqui - **Smallest sufficient workflow:** Reuse retained route facts, route the focused shutdown change, read Rust Async and Concurrency, then inspect the exact diff and tests. - **Recommendations:** Keep the minimal policy-only workflow easy to repeat and make source review context explicit. A concise owner inventory would help identify wrappers that must delegate to a newly introduced lifecycle owner. +## Native-custody contributor — public Ollama wrapper drain + +- **Useful calls:** The contributor reused its retained standards snapshot, routed the wrapper lifecycle boundary to 26 standards with zero unresolved categories, then read Rust Async, Concurrency and Rust API. Those obligations clarified that the public wrapper also had to close admission and retain completion receipts for its owned `VersionState` mutations. +- **Confusing or redundant steps:** No new broad discovery was necessary. The route still carried relation metadata; scoped reads were sufficient. +- **Missing context:** The MCP did not identify that `OllamaVersionManager` owns private state and runs removal/download activity inline. Repository source established the activity and shutdown order. +- **Where the contributor left MCP:** The wrapper receipt, cancellation behavior, file-write settlement, regression tests and exact-candidate Cargo results came from source inspection and execution in the worker worktree. The MCP supplied obligations only. +- **Smallest sufficient workflow:** Reuse facts and snapshot, route the newly discovered owner boundary, read async/concurrency/API, then inspect and test the exact candidate. +- **Recommendations:** Show the exact affected owner boundary and prior relevant reads in a compact view; offer section-scoped policy reads. Keep repository evidence explicitly outside the MCP compliance claim. + +## Native-custody independent reviewer — final wrapper candidate + +- **Useful calls:** The reviewer reused registered facts, routed the final wrapper scope to 27 standards with no unresolved questions, and read Concurrency and Rust Async. This was sufficient to assess close-admission ordering, completion ownership after waiter cancellation, and shutdown draining. +- **Confusing or redundant steps:** No extra discovery call was needed. Source inspection remained necessary to check the actual wrapper receipt and queued-install cancellation reset. +- **Missing context:** The MCP did not supply the candidate digest, activity registry, or exact call graph; repository/Git inspection did. +- **Where the reviewer left MCP:** Findings, exact source evidence, tests inspected, and the disposition of the previous wrapper limitation came from read-only source/diff review. The reviewer ran no tests. +- **Smallest sufficient workflow:** Reuse the prior route, read the two lifecycle policies, and inspect the exact diff and named regression tests. +- **Recommendations:** Make candidate identity and affected owner paths easy to provide as non-authoritative review context. Keep it clear that policy routing does not inspect or approve source. + +## Q1 store-cutover reviewer — source-neutral lifecycle design + +- **Useful calls:** The reviewer discovered routing facts, routed the Q1 persistence/concurrency/contract/async/replay/architecture/verification slice, and read focused selected policies. The route completed with 28 standards and no unresolved questions. It helped expose the single-writer, supported-state, no-assumed-compatibility, supervised-effect, and claim-scoped-evidence requirements. +- **Confusing or redundant steps:** Seven whole-policy reads produced excessive output and truncation; the reviewer had to retrieve obscured concurrency/async/replay bodies separately. Text and structured forms repeated content. +- **Missing context:** The MCP did not know the existing v4-to-v5 migration-on-load behavior, model-library marker semantics, hidden custody inventory, mutation-authority reader, or composition boundaries. Repository inspection supplied these facts. +- **Where the reviewer left MCP:** Exact migration populations, unknown deployed-root scope, consumer ownership and proposed cutover were established from source and plan inspection. No files were edited and no tests were run. +- **Smallest sufficient workflow:** One `routing_facts`, one concise `route`, one focused `read_many`; schema discovery only when the tool contract actually requires it. +- **Recommendations:** Keep route output free of repeated relation graphs; support section-scoped policy reads; distinguish applicability from repository compliance; avoid duplicate policy text in structured and textual results; provide examples for selecting routing facts. + +## Primary integrator — merged native-custody candidate and shared-owner boundary + +- **Useful calls:** For the merged native custody scope, I refreshed `routing_facts`, routed 32 standards with no unresolved categories under snapshot `snapshot:v1:37f8f667-71b0-4580-9084-4d202fdcc04c`, and read focused Architecture, Concurrency, Rust Async, Rust API, Persistence, Contract Evolution, Resilience, Verification and Commit standards. The final route captured the actual app-manager/RPC, async shutdown, persistence-adjacent and verification boundaries. Reading Concurrency/Async and Architecture again after lifecycle review exposed that one supervisor needs consumer-scoped handles rather than sharing the current HF owner unchanged. +- **Confusing or redundant steps:** The route summary is compact only without policy bodies; focused `read_many` still returns whole policy text and repeated handles/relationship context. The first larger batch truncated output, so I split further review into smaller groups. +- **Missing context:** The MCP did not know the current owner scans global task IDs/projections or that HF Drop supplies the owner-wide shutdown callback. Repository source and the exact reviewer identified that coupling. It also did not know that the downloaded model root initializer writes a model library marker. +- **Where I left MCP:** Candidate identities, Git ancestry, staged scope, source ownership, tests, feature checks, and CI state were verified with repository and GitHub tools. MCP routing did not inspect source or supply acceptance evidence. +- **Smallest sufficient workflow:** Discover or reuse the routing facts; route without inline policy content; read only Architecture, Concurrency, Rust Async/API, Persistence/Evolution, Verification, Resilience and Commit obligations implicated by the actual boundary; inspect source and evidence; run a final route on the completed candidate. +- **Recommendations:** Support policy-section reads and compact route deltas; keep candidate/source context clearly marked non-authoritative; expose the already-read standards list; and state explicitly that MCP applicability is neither source inspection nor compliance evidence. + +### Primary integrator — shared-custody slice routing + +- **Useful calls:** A fresh `routing_facts` snapshot and a concise route selected 34 applicable standards with zero unresolved facts after marking unrelated framework routing known-absent. The selected closure covered Rust core/API/async, acquisition ownership, concurrency, security/trust, persistence-adjacent behavior, verification and commit workflow. +- **Confusing or redundant steps:** Omitting the unrelated `routing.frameworks` fact left it as a required unresolved category; the router needs an explicit known-absent marker. A single `read_many` across 17 selected policies produced a very large repeated result and was truncated by the client, so future reads need smaller batches. Relationship and policy metadata is repeated around full policy text. +- **Missing context:** The MCP did not identify that the current HF supervisor globally scans operation IDs/projections, that HF shutdown closes the whole owner, or that `DownloadDestinationRoot` creates model-specific marker/deletion authority. Source and reviewer inspection supplied those ownership facts. +- **Where I left MCP:** Git ancestry, exact write sets, existing task APIs, migration-on-load behavior, current CI, and all executed test evidence come from Git, repository source, and command results. The route supplies obligations only. +- **Smallest sufficient workflow:** Declare every routing fact, route the actual slice, read focused Architecture, Concurrency, Rust Async/API, Persistence/Evolution, Security, Verification and Commit policies, then inspect/test the exact implementation and route it again before handoff. +- **Recommendations:** Treat omitted facts as unknown only when the fact is genuinely unknown; make known-absent explicit in concise examples. Support policy-section reads and compact incremental route output, and avoid repeating relation graphs in every result. Preserve a clear boundary between applicable standards and source compliance evidence. + ## Participation The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, and shared-owner design reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index d734d0a9..88ae9740 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -17,7 +17,7 @@ These are the admitted exact paths/closed path families. The actual source files The current root-owned extraction-preparation sub-slice is limited to `rust/crates/pumas-core/src/model_library/{download_recovery.rs,hf/lifecycle.rs}` and the co-located lifecycle regression. It adds a physical-root equality key and lets the existing supervised lifecycle owner hold distinct root grants concurrently while coalescing independently reopened handles for the same root. It does not move the lifecycle owner, add a transfer/persistence authority, or advance AQ-HTTP. -The isolated native-custody worker's original four-file set is expanded by one exact adjacent file after review found the public `OllamaVersionManager` retains the shared `VersionState` but exposes no drain for its mutation registry. Its only permitted `ollama.rs` work is to close/drain that owned state lifecycle and add a focused regression; no Ollama downloader, source, archive, publication, or server-owner redesign is admitted. The same routed Rust async/concurrency/public-API obligations apply. +The isolated native-custody worker proposal is complete and integrated from `work/q1-native-custody` commit `560cec71` into the Q1 branch. Its exact five-file write set is `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs,ollama.rs,state.rs}` and `rust/crates/pumas-rpc/src/server.rs`. The `ollama.rs` extension closes the public wrapper's owned state-mutation drain. It repairs native install/removal custody, metadata coordination, pending file-I/O settlement, and fallible stage cleanup; it does not change the source/download selection algorithm or replace the independent Ollama downloader. Shared acquisition consumer cutover remains outstanding. **Tests:** proposed `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`, plus existing co-located GitHub metadata, HF lifecycle/recovery, and installer regression tests. Fixtures must reach the owner under test with independent expected byte/effect outcomes. From 584b5f45a17a80cd584b16ce77f7be5d9688a9b8 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 19:03:46 -0700 Subject: [PATCH 09/20] refactor(acquisition): scope shared task custody by consumer Move task, nested effect, predecessor drain, and projection custody into one acquisition supervisor. Isolate consumer scopes and register each terminal projection before admission so scoped and global shutdown retain repeatable outcomes. Keep model root grants, destination queues, and admission matching in the HF policy facade. Retain opaque effect leases through observation and leave persistent download formats unchanged. --- rust/crates/pumas-core/src/acquisition/mod.rs | 7 +- .../src/acquisition/task_custody.rs | 4592 +++++++++++++++ .../src/model_library/hf/download.rs | 31 +- .../src/model_library/hf/lifecycle.rs | 5024 +++-------------- .../pumas-core/src/model_library/hf/mod.rs | 16 +- 5 files changed, 5292 insertions(+), 4378 deletions(-) create mode 100644 rust/crates/pumas-core/src/acquisition/task_custody.rs diff --git a/rust/crates/pumas-core/src/acquisition/mod.rs b/rust/crates/pumas-core/src/acquisition/mod.rs index bc4bfabd..cbdbe197 100644 --- a/rust/crates/pumas-core/src/acquisition/mod.rs +++ b/rust/crates/pumas-core/src/acquisition/mod.rs @@ -1,12 +1,13 @@ -//! Source-neutral descriptions of selected artifact bytes. +//! Source-neutral artifact selection, HTTP access, and asynchronous custody. //! //! This module owns validated selection data and the shared HTTP response and -//! body-streaming protocol. Durable custody, lifecycle admission, publication, -//! and consumer settlement still belong to their current production owners. +//! body-streaming protocol plus consumer-scoped task/effect supervision. Durable +//! acquisition state and consumer publication remain with their current owners. mod github_release; mod http; mod manifest; +pub(crate) mod task_custody; pub(crate) use http::{ open_http_artifact, stream_http_artifact, HttpArtifactSink, HttpAttemptHost, HttpBodyOutcome, diff --git a/rust/crates/pumas-core/src/acquisition/task_custody.rs b/rust/crates/pumas-core/src/acquisition/task_custody.rs new file mode 100644 index 00000000..c7d4809b --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/task_custody.rs @@ -0,0 +1,4592 @@ +//! Source-neutral ownership of artifact acquisition tasks and effects. +//! +//! Request futures prepare work, but this module owns every installed Tokio +//! handle. Installation is synchronous and gated so state and task custody can +//! be committed together before work starts. Opaque allocation identities +//! prevent an old task from observing or removing its successor. + +use std::any::Any; +use std::collections::HashMap; +use std::fmt; +use std::future::Future; +use std::ops::{Deref, DerefMut}; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; +use std::sync::MutexGuard; +use std::sync::{Arc, Mutex, Weak}; + +use futures::FutureExt; +use tokio::sync::{oneshot, Notify}; +use tokio::task::JoinHandle; + +type FallibleBlockingReceiver = + oneshot::Receiver, String>>; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TaskRole { + Invocation, + AdmissionTransition, + RecoveryTransition, + Worker, + CancelFinalizer, + TerminalProjection, +} + +#[derive(Clone)] +pub(crate) struct TaskGeneration(Arc); + +impl PartialEq for TaskGeneration { + fn eq(&self, other: &Self) -> bool { + self.matches(other) + } +} + +impl Eq for TaskGeneration {} + +impl TaskGeneration { + pub(crate) fn new() -> Self { + Self(Arc::new(Notify::new())) + } + + pub(crate) fn wake_pause(&self) { + self.0.notify_waiters(); + } + + pub(crate) fn matches(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } + + fn key(&self) -> usize { + Arc::as_ptr(&self.0) as usize + } +} + +impl fmt::Debug for TaskGeneration { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_tuple("TaskGeneration") + .field(&Arc::as_ptr(&self.0)) + .finish() + } +} + +#[derive(Clone, Debug)] +pub(crate) struct TaskSnapshot { + pub(crate) role: TaskRole, + pub(crate) finished: bool, + pub(crate) outer_finished: bool, + pub(crate) started: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[repr(u8)] +enum TaskStartState { + Gated = 0, + Running = 1, + Abandoned = 2, +} + +impl TaskStartState { + fn load(state: &AtomicU8) -> Self { + match state.load(Ordering::Acquire) { + 0 => Self::Gated, + 1 => Self::Running, + _ => Self::Abandoned, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum TaskTerminal { + Completed, + Cancelled, + Panicked, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TaskObservation { + pub(crate) generation: TaskGeneration, + pub(crate) role: TaskRole, + pub(crate) terminal: TaskTerminal, + pub(crate) nested_failures: usize, + pub(crate) outer_finished_before_replacement: bool, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ProjectionOutcome { + Pending, + Committed, + RolledBack, + Failed, + Panicked, + Superseded, + Shutdown, +} + +#[derive(Debug)] +struct ProjectionCellState { + predecessor_ready: bool, + predecessor: Option, + outcome: ProjectionOutcome, + settled: bool, + failed: bool, + failure_projected: bool, +} + +#[derive(Debug)] +struct ProjectionCell { + state: Mutex, + inherited: Mutex>>, + notify: Notify, +} + +impl ProjectionCell { + fn new(predecessor_ready: bool) -> Self { + Self { + state: Mutex::new(ProjectionCellState { + predecessor_ready, + predecessor: None, + outcome: ProjectionOutcome::Pending, + settled: false, + failed: false, + failure_projected: false, + }), + inherited: Mutex::new(Vec::new()), + notify: Notify::new(), + } + } + + fn inherit(&self, cell: Arc) { + self.inherited + .lock() + .expect("acquisition inherited projection-cell lock poisoned") + .push(cell); + } + + fn record_predecessor(&self, observation: TaskObservation) { + { + let mut state = self + .state + .lock() + .expect("acquisition projection-cell lock poisoned"); + state.predecessor = Some(observation); + state.predecessor_ready = true; + } + self.notify.notify_waiters(); + } + + async fn wait_for_predecessor(&self) -> Option { + loop { + let notified = self.notify.notified(); + let ready = { + let state = self + .state + .lock() + .expect("acquisition projection-cell lock poisoned"); + state.predecessor_ready.then(|| state.predecessor.clone()) + }; + if let Some(predecessor) = ready { + return predecessor; + } + notified.await; + } + } + + fn settle(&self, outcome: ProjectionOutcome) { + { + let mut state = self + .state + .lock() + .expect("acquisition projection-cell lock poisoned"); + if state.outcome != ProjectionOutcome::Pending { + if matches!( + outcome, + ProjectionOutcome::Failed | ProjectionOutcome::Panicked + ) { + state.failed = true; + } + return; + } + if matches!( + outcome, + ProjectionOutcome::Failed | ProjectionOutcome::Panicked + ) { + state.failed = true; + } + state.outcome = outcome; + } + self.notify.notify_waiters(); + } + + fn outcome(&self) -> ProjectionOutcome { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .outcome + } + + async fn wait(&self) -> ProjectionOutcome { + loop { + let notified = self.notify.notified(); + let outcome = self.outcome(); + if outcome != ProjectionOutcome::Pending { + return outcome; + } + notified.await; + } + } + + fn mark_settled(&self) { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .settled = true; + } + + fn is_settled(&self) -> bool { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .settled + } + + fn mark_failed(&self) { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .failed = true; + } + + fn acknowledge_failure_projection(&self) { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .failure_projected = true; + let inherited = self + .inherited + .lock() + .expect("acquisition inherited projection-cell lock poisoned") + .clone(); + for cell in inherited { + cell.acknowledge_failure_projection(); + cell.mark_settled(); + } + self.notify.notify_waiters(); + } + + fn is_ready_to_settle(&self) -> bool { + let state = self + .state + .lock() + .expect("acquisition projection-cell lock poisoned"); + state.outcome != ProjectionOutcome::Pending && (!state.failed || state.failure_projected) + } + + fn has_unprojected_failure(&self) -> bool { + let state = self + .state + .lock() + .expect("acquisition projection-cell lock poisoned"); + state.outcome != ProjectionOutcome::Pending && state.failed && !state.failure_projected + } + + #[cfg(test)] + fn failure_projected(&self) -> bool { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .failure_projected + } + + fn failed(&self) -> bool { + self.state + .lock() + .expect("acquisition projection-cell lock poisoned") + .failed + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum BlockingTaskError { + StaleGeneration, + Join(String), + ResultChannelClosed, +} + +impl fmt::Display for BlockingTaskError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::StaleGeneration => formatter.write_str("task generation is no longer current"), + Self::Join(detail) => write!(formatter, "blocking task failed: {detail}"), + Self::ResultChannelClosed => formatter.write_str("blocking result channel closed"), + } + } +} + +struct NestedTask { + handle: JoinHandle<()>, + completion: Arc, + failure_kind: NestedFailureKind, +} + +enum NestedFailureKind { + Effect, + Predecessor, +} + +struct NestedCompletion { + finished: AtomicBool, + failed: AtomicBool, + notify: Notify, +} + +struct RetiredTask { + observer: JoinHandle, +} + +enum StartGate { + Work(oneshot::Sender<()>), + Custody(oneshot::Sender<()>), +} + +impl StartGate { + fn start(self) { + match self { + Self::Work(sender) | Self::Custody(sender) => { + let _ = sender.send(()); + } + } + } + + fn drain(self) { + if let Self::Custody(sender) = self { + let _ = sender.send(()); + } + } +} + +struct TaskEntry { + admission: Option<( + Arc, + tokio::sync::watch::Receiver, + )>, + generation: TaskGeneration, + role: TaskRole, + outer: JoinHandle<()>, + nested: Vec, + nested_failures_archived: usize, + predecessor_failures_archived: usize, + projection: Option>, + starts: Vec, + superseded_projection: Option>, + abort_on_start: Vec, + start_state: Arc, +} + +struct PreparedEntry { + download_id: String, + generation: TaskGeneration, + role: TaskRole, + start: oneshot::Sender<()>, + outer: JoinHandle<()>, + projection: Option>, + start_state: Arc, +} + +impl TaskEntry { + fn finished(&self) -> bool { + self.outer.is_finished() && self.nested.iter().all(|nested| nested.handle.is_finished()) + } + + fn reap_completed_nested(&mut self) { + let mut retained = Vec::with_capacity(self.nested.len()); + for mut nested in self.nested.drain(..) { + let observed = if nested.handle.is_finished() { + (&mut nested.handle).now_or_never() + } else { + None + }; + if let Some(result) = observed { + let failures = usize::from( + result.is_err() || nested.completion.failed.load(Ordering::Acquire), + ); + match nested.failure_kind { + NestedFailureKind::Effect => self.nested_failures_archived += failures, + NestedFailureKind::Predecessor => { + self.predecessor_failures_archived += failures + } + } + } else { + retained.push(nested); + } + } + self.nested = retained; + } +} + +#[cfg(test)] +type BlockingObserver = Arc; + +#[cfg(test)] +type TaskIdsObserver = Arc; + +#[cfg(test)] +type DrainObserver = Arc; + +#[cfg(test)] +type SnapshotObserver = Arc) + Send + Sync>; + +#[cfg(test)] +type CancellationCheckObserver = Arc; + +#[cfg(test)] +type CancelReplacementObserver = Arc; + +#[cfg(test)] +type WorkerProjectionObserver = Arc; + +#[cfg(test)] +type BlockingResultObserver = Arc; + +#[cfg(test)] +type BlockingFailureObserver = Arc bool + Send + Sync>; + +#[cfg(test)] +type AmbientAdmissionObserver = Arc; + +#[cfg(test)] +type ProjectionObserver = Arc; + +/// One owner for all consumer-scoped task and effect custody. A scope is an +/// access handle, never a separately populated supervisor or task registry. +#[derive(Default)] +pub(crate) struct TaskCustodyOwner { + state: Mutex, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +struct ScopeId(u64); + +type ScopeFinalizer = + Box Pin> + Send>> + Send>; + +#[derive(Default)] +struct SupervisorState { + closed: bool, + next_scope: u64, + scopes: HashMap, + shutdown: Option, + shutdown_driver: Option>, +} + +struct ScopeGuard<'a> { + supervisor: MutexGuard<'a, SupervisorState>, + identity: ScopeId, +} + +impl Deref for ScopeGuard<'_> { + type Target = ScopeState; + fn deref(&self) -> &ScopeState { + self.supervisor + .scopes + .get(&self.identity) + .expect("minted custody scope remains registered") + } +} + +impl DerefMut for ScopeGuard<'_> { + fn deref_mut(&mut self) -> &mut ScopeState { + self.supervisor + .scopes + .get_mut(&self.identity) + .expect("minted custody scope remains registered") + } +} + +pub(crate) struct AdmissionSnapshot { + pub(crate) operation_id: String, + pub(crate) generation: TaskGeneration, + pub(crate) metadata: Arc, + pub(crate) completed: tokio::sync::watch::Receiver, +} + +impl TaskCustodyOwner { + pub(crate) fn new() -> Self { + Self::default() + } + + /// Mint a scope and register its terminal projection before admitting work. + /// Scope identities are never removed or reused during this owner's life. + pub(crate) fn open_scope( + self: &Arc, + finalizer: F, + ) -> crate::Result> + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let mut state = self + .state + .lock() + .expect("acquisition task custody lock poisoned"); + if state.closed { + return Err(crate::PumasError::DownloadLifecycleClosed); + } + let identity = ScopeId(state.next_scope); + state.next_scope = state.next_scope.checked_add(1).ok_or_else(|| { + crate::PumasError::Other("Acquisition scope capacity exhausted".into()) + })?; + let scope = Arc::new(TaskScope { + owner: self.clone(), + identity, + retired_observations: AtomicUsize::new(0), + #[cfg(test)] + blocking_observer: Mutex::default(), + #[cfg(test)] + ids_observer: Mutex::default(), + #[cfg(test)] + drain_observer: Mutex::default(), + #[cfg(test)] + snapshot_observer: Mutex::default(), + #[cfg(test)] + cancellation_check_observer: Mutex::default(), + #[cfg(test)] + cancel_replacement_observer: Mutex::default(), + #[cfg(test)] + worker_projection_observer: Mutex::default(), + #[cfg(test)] + blocking_result_observer: Mutex::default(), + #[cfg(test)] + blocking_failure_observer: Mutex::default(), + #[cfg(test)] + ambient_admission_observer: Mutex::default(), + #[cfg(test)] + projection_observer: Mutex::default(), + }); + state.scopes.insert( + identity, + ScopeState { + finalizer: Some(Box::new(move || Box::pin(finalizer()))), + handle: Arc::downgrade(&scope), + ..ScopeState::default() + }, + ); + Ok(scope) + } + + /// Close every scope at one admission boundary. Each registered projection + /// runs once after its effects settle, and repeated callers share the result. + pub(crate) fn request_shutdown(self: &Arc) -> ShutdownReceipt { + let mut state = self + .state + .lock() + .expect("acquisition task custody lock poisoned"); + if let Some(receipt) = &state.shutdown { + return receipt.clone(); + } + state.closed = true; + let (result, receiver) = tokio::sync::watch::channel(None); + let receipt = ShutdownReceipt { result: receiver }; + state.shutdown = Some(receipt.clone()); + let mut receipts = Vec::with_capacity(state.scopes.len()); + let mut starts = Vec::with_capacity(state.scopes.len()); + for scope in state.scopes.values_mut() { + let keepalive: Arc = scope + .handle + .upgrade() + .map(|handle| handle as Arc) + .unwrap_or_else(|| self.clone()); + let (receipt, start) = begin_scope_shutdown(scope, keepalive); + receipts.push(receipt); + if let Some(start) = start { + starts.push(start); + } + } + match tokio::runtime::Handle::try_current() { + Ok(runtime) => { + let owner = self.clone(); + let (start, started) = oneshot::channel(); + state.shutdown_driver = Some(runtime.spawn(async move { + let _ = started.await; + let mut failures = 0; + for receipt in receipts { + failures += receipt.failures().await; + } + let _ = result.send(Some(failures)); + drop(owner); + })); + starts.push(start); + } + Err(_) => { + let _ = result.send(Some(1)); + } + } + drop(state); + for start in starts { + let _ = start.send(()); + } + receipt + } +} + +fn begin_scope_shutdown( + state: &mut ScopeState, + keepalive: Arc, +) -> (ShutdownReceipt, Option>) { + if let Some(receipt) = &state.shutdown { + return (receipt.clone(), None); + } + state.closed = true; + for entry in state.prepared.values() { + entry.outer.abort(); + } + for entry in state.tasks.values() { + entry.outer.abort(); + for abort in &entry.abort_on_start { + abort.abort(); + } + } + let (result, receiver) = tokio::sync::watch::channel(None); + let receipt = ShutdownReceipt { result: receiver }; + state.shutdown = Some(receipt.clone()); + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + let _ = result.send(Some(1)); + return (receipt, None); + }; + let prepared = std::mem::take(&mut state.prepared); + let tasks = std::mem::take(&mut state.tasks); + let retired = std::mem::take(&mut state.retired); + let finalizer = state + .finalizer + .take() + .expect("scope registers its finalizer before admission"); + let mut failures = state.retired_failures; + let (start, started) = oneshot::channel(); + state.shutdown_driver = Some(runtime.spawn(async move { + let _ = started.await; + for (_, entry) in prepared { + drop(entry.start); + if entry.outer.await.is_err_and(|error| error.is_panic()) { + failures += 1; + } + if let Some(cell) = entry.projection { + if std::panic::catch_unwind(AssertUnwindSafe(|| { + cell.settle(ProjectionOutcome::Shutdown) + })) + .is_err() + { + failures += 1; + } + } + } + let mut draining = Vec::new(); + for (_, mut entry) in tasks { + for gate in entry.starts.drain(..) { + gate.drain(); + } + draining.push(entry); + } + for entry in draining { + failures += drain_shutdown_entry(entry).await; + } + for task in retired { + failures += task.observer.await.unwrap_or(1); + } + if !matches!( + AssertUnwindSafe(async move { finalizer().await }) + .catch_unwind() + .await, + Ok(Ok(())) + ) { + failures += 1; + } + let _ = result.send(Some(failures)); + drop(keepalive); + })); + (receipt, Some(start)) +} + +pub(crate) struct TaskScope { + owner: Arc, + identity: ScopeId, + retired_observations: AtomicUsize, + #[cfg(test)] + blocking_observer: Mutex>, + #[cfg(test)] + ids_observer: Mutex>, + #[cfg(test)] + drain_observer: Mutex>, + #[cfg(test)] + snapshot_observer: Mutex>, + #[cfg(test)] + cancellation_check_observer: Mutex>, + #[cfg(test)] + cancel_replacement_observer: Mutex>, + #[cfg(test)] + worker_projection_observer: Mutex>, + #[cfg(test)] + blocking_result_observer: Mutex>, + #[cfg(test)] + blocking_failure_observer: Mutex>, + #[cfg(test)] + ambient_admission_observer: Mutex>, + #[cfg(test)] + projection_observer: Mutex>, +} + +#[derive(Default)] +struct ScopeState { + // Admission, ownership transfers, and shutdown capture share this mutex. + // Entries leave these populations only for another registered observer. + closed: bool, + tasks: HashMap, + prepared: HashMap, + retired: Vec, + retired_failures: usize, + shutdown: Option, + shutdown_driver: Option>, + finalizer: Option, + handle: Weak, +} + +struct InvocationWaiter { + owner: Arc, + id: String, + generation: TaskGeneration, +} + +impl Drop for InvocationWaiter { + fn drop(&mut self) { + let mut state = self.owner.lock_state(); + let start = if state + .tasks + .get(&self.id) + .is_some_and(|entry| entry.generation.matches(&self.generation)) + { + state + .tasks + .remove(&self.id) + .map(|entry| retire_entry(&mut state, entry)) + } else { + None + }; + drop(state); + if let Some(start) = start { + let _ = start.send(()); + } + } +} + +fn retire_entry(state: &mut ScopeState, mut entry: TaskEntry) -> oneshot::Sender<()> { + // The observer is registered in custody before this lock is released. + // It is never aborted: blocking descendants must remain owned through join. + entry.outer.abort(); + for abort in entry.abort_on_start.drain(..) { + abort.abort(); + } + let (start, started) = oneshot::channel(); + let observer = tokio::spawn(async move { + let _ = started.await; + for gate in entry.starts.drain(..) { + gate.drain(); + } + drain_shutdown_entry(entry).await + }); + state.retired.push(RetiredTask { observer }); + start +} + +async fn drain_shutdown_entry(entry: TaskEntry) -> usize { + let projection = entry.projection.clone(); + let superseded = entry.superseded_projection.clone(); + let role = entry.role; + let observed = AssertUnwindSafe(observe_entry(entry, role, false)) + .catch_unwind() + .await; + let mut failures = match observed { + Ok(observation) => { + observation.nested_failures + + usize::from(observation.terminal == TaskTerminal::Panicked) + } + Err(_) => 1, + }; + for cell in [projection, superseded].into_iter().flatten() { + // A broken receipt must not drop unrelated entries still awaiting + // drain. Retain failure without recovering poisoned projection state. + if std::panic::catch_unwind(AssertUnwindSafe(|| { + cell.settle(ProjectionOutcome::Shutdown) + })) + .is_err() + { + failures += 1; + } + } + failures +} + +#[derive(Clone)] +pub(crate) struct ShutdownReceipt { + result: tokio::sync::watch::Receiver>, +} + +impl ShutdownReceipt { + async fn failures(mut self) -> usize { + loop { + if let Some(failures) = *self.result.borrow_and_update() { + return failures; + } + if self.result.changed().await.is_err() { + return 1; + } + } + } + + pub(crate) async fn wait(mut self) -> crate::Result<()> { + loop { + if let Some(failures) = *self.result.borrow_and_update() { + return if failures == 0 { + Ok(()) + } else { + Err(crate::PumasError::DownloadShutdownFailed { failures }) + }; + } + if self.result.changed().await.is_err() { + return Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }); + } + } + } +} + +impl fmt::Debug for TaskScope { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("TaskScope") + .field("scope", &self.identity) + .field("task_count", &self.lock_state().tasks.len()) + .finish() + } +} + +pub(crate) struct TaskContext { + owner: Weak, + download_id: String, + generation: TaskGeneration, + projection_failure: Option>, + effect_lease: Option>, +} + +impl Clone for TaskContext { + fn clone(&self) -> Self { + Self { + owner: self.owner.clone(), + download_id: self.download_id.clone(), + generation: self.generation.clone(), + projection_failure: self.projection_failure.clone(), + effect_lease: self.effect_lease.clone(), + } + } +} + +pub(crate) struct PreparedTask { + owner: Weak, + download_id: String, + generation: TaskGeneration, + role: TaskRole, + start_state: Arc, + armed: bool, +} + +impl fmt::Debug for PreparedTask { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("PreparedTask") + .field("download_id", &self.download_id) + .field("generation", &self.generation) + .field("role", &self.role) + .finish_non_exhaustive() + } +} + +#[derive(Debug)] +pub(crate) struct InstalledTask { + owner: Arc, + download_id: String, + generation: TaskGeneration, + start_state: Arc, +} + +#[derive(Debug)] +pub(crate) struct PreparedProjection { + task: PreparedTask, + cell: Arc, +} + +pub(crate) struct InstalledProjection { + task: InstalledTask, + ticket: ProjectionTicket, +} + +#[derive(Clone)] +pub(crate) struct ProjectionTicket { + scope: Weak, + download_id: String, + generation: TaskGeneration, + cell: Arc, +} + +pub(crate) enum ProjectionTransition { + Started(InstalledProjection), + Existing(InstalledProjection), + NotReady, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ProjectionSettlement { + Pending, + FailureUnprojected, + Settled, + AlreadySettled, + StaleGeneration, + Missing, +} + +#[derive(Debug)] +pub(crate) enum CancelTransition { + Started(InstalledTask), + Existing(InstalledTask), + AlreadyRunning, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum CancelPredecessor { + Absent, + Observed(TaskObservation), +} + +impl TaskScope { + pub(crate) fn is_closed(&self) -> bool { + self.lock_state().closed + } + + pub(crate) fn ensure_open(&self) -> crate::Result<()> { + if self.is_closed() { + Err(crate::PumasError::DownloadLifecycleClosed) + } else { + Ok(()) + } + } + + /// Owns pre-task preparation independently of its caller. Dropping the + /// waiter cancels only this invocation's outer work; registered effects + /// remain in retired custody, and installed child generations are untouched. + pub(crate) async fn run_invocation( + self: &Arc, + operation: F, + ) -> crate::Result + where + T: Send + 'static, + F: FnOnce(TaskContext) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let id = format!("invocation-{}", uuid::Uuid::new_v4()); + let (sender, receiver) = oneshot::channel(); + let prepared = self.prepare( + id.clone(), + TaskRole::Invocation, + move |context| async move { + let result = operation(context).await; + let _ = sender.send(result); + }, + )?; + let generation = prepared.generation.clone(); + let installed = self + .install_gated(prepared) + .map_err(|_| crate::PumasError::DownloadLifecycleClosed)?; + let waiter = InvocationWaiter { + owner: self.clone(), + id, + generation, + }; + installed.start(); + let result = receiver.await.map_err(|_| { + if self.is_closed() { + crate::PumasError::DownloadLifecycleClosed + } else { + crate::PumasError::DownloadShutdownFailed { failures: 1 } + } + })?; + drop(waiter); + self.ensure_open()?; + result + } + + pub(crate) async fn shutdown(self: &Arc) -> crate::Result<()> { + self.request_shutdown().wait().await + } + + /// Close this scope, keeping the retained driver independent of waiters. + pub(crate) fn request_shutdown(self: &Arc) -> ShutdownReceipt { + let mut state = self.lock_state(); + let (receipt, start) = begin_scope_shutdown(&mut state, self.clone()); + drop(state); + if let Some(start) = start { + let _ = start.send(()); + } + receipt + } + + fn lock_state(&self) -> ScopeGuard<'_> { + ScopeGuard { + supervisor: self + .owner + .state + .lock() + .expect("acquisition task custody lock poisoned"), + identity: self.identity, + } + } + + #[cfg(test)] + fn new_test() -> Arc { + Arc::new(TaskCustodyOwner::new()) + .open_scope(|| async { Ok(()) }) + .unwrap() + } + + pub(crate) fn prepare( + self: &Arc, + download_id: String, + role: TaskRole, + work: F, + ) -> crate::Result + where + F: FnOnce(TaskContext) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + self.prepare_with_projection(download_id, role, None, work) + } + + fn prepare_with_projection( + self: &Arc, + download_id: String, + role: TaskRole, + projection: Option>, + work: F, + ) -> crate::Result + where + F: FnOnce(TaskContext) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let mut state = self.lock_state(); + if state.closed { + return Err(crate::PumasError::DownloadLifecycleClosed); + } + let generation = TaskGeneration::new(); + let context = TaskContext { + owner: Arc::downgrade(self), + download_id: download_id.clone(), + generation: generation.clone(), + projection_failure: None, + effect_lease: None, + }; + let (start, started) = oneshot::channel(); + let outer = tokio::spawn(async move { + if started.await.is_ok() { + work(context).await; + } + }); + let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); + state.prepared.insert( + generation.key(), + PreparedEntry { + download_id: download_id.clone(), + generation: generation.clone(), + role, + start, + outer, + projection: projection.clone(), + start_state: start_state.clone(), + }, + ); + Ok(PreparedTask { + owner: Arc::downgrade(self), + download_id, + generation, + role, + start_state, + armed: true, + }) + } + + pub(crate) fn prepare_projection( + self: &Arc, + download_id: String, + project: F, + project_panic: P, + ) -> crate::Result + where + F: FnOnce(TaskContext, Option) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + P: FnOnce(TaskContext) -> PFut + Send + 'static, + PFut: Future + Send + 'static, + { + let cell = Arc::new(ProjectionCell::new(true)); + let project_cell = cell.clone(); + let task = self.prepare_with_projection( + download_id, + TaskRole::TerminalProjection, + Some(cell.clone()), + move |mut context| async move { + context.projection_failure = Some(project_cell.clone()); + let predecessor = project_cell.wait_for_predecessor().await; + let project_context = context.clone(); + let outcome = + match AssertUnwindSafe( + async move { project(project_context, predecessor).await }, + ) + .catch_unwind() + .await + { + Ok(outcome) => outcome, + Err(_) => { + project_cell.mark_failed(); + let fallback = + AssertUnwindSafe(async move { project_panic(context).await }) + .catch_unwind() + .await; + if matches!(fallback, Ok(ProjectionOutcome::Failed)) { + project_cell.acknowledge_failure_projection(); + } + ProjectionOutcome::Panicked + } + }; + if outcome == ProjectionOutcome::Failed { + project_cell.mark_failed(); + } + if project_cell.failed() + && matches!( + outcome, + ProjectionOutcome::Committed | ProjectionOutcome::Failed + ) + { + project_cell.acknowledge_failure_projection(); + } + project_cell.settle(outcome); + }, + )?; + Ok(PreparedProjection { task, cell }) + } + + /// Installs a prepared task while its start gate remains closed. + /// + /// Dropping the returned token only marks its owner-held start lease as + /// abandoned. Callers rescue it after releasing any outer state guard. + pub(crate) fn install_gated( + self: &Arc, + mut prepared: PreparedTask, + ) -> std::result::Result { + if !prepared + .owner + .upgrade() + .is_some_and(|owner| Arc::ptr_eq(&owner, self)) + { + return Err(prepared); + } + let mut state = self.lock_state(); + if state.closed || state.tasks.contains_key(&prepared.download_id) { + return Err(prepared); + } + let Some(entry) = state.prepared.remove(&prepared.generation.key()) else { + return Err(prepared); + }; + let tasks = &mut state.tasks; + let download_id = entry.download_id.clone(); + let generation = entry.generation.clone(); + tasks.insert( + download_id.clone(), + TaskEntry { + admission: None, + generation: generation.clone(), + role: entry.role, + outer: entry.outer, + nested: Vec::new(), + nested_failures_archived: 0, + predecessor_failures_archived: 0, + projection: entry.projection, + starts: vec![StartGate::Work(entry.start)], + superseded_projection: None, + abort_on_start: Vec::new(), + start_state: entry.start_state, + }, + ); + drop(state); + prepared.armed = false; + Ok(InstalledTask { + owner: self.clone(), + download_id, + generation, + start_state: prepared.start_state.clone(), + }) + } + + pub(crate) fn install_projection_gated( + self: &Arc, + prepared: PreparedProjection, + ) -> std::result::Result { + let cell = prepared.cell.clone(); + match self.install_gated(prepared.task) { + Ok(task) => { + let ticket = ProjectionTicket { + scope: Arc::downgrade(self), + download_id: task.download_id.clone(), + generation: task.generation.clone(), + cell, + }; + Ok(InstalledProjection { task, ticket }) + } + Err(task) => Err(PreparedProjection { task, cell }), + } + } + + pub(crate) fn snapshot(&self, download_id: &str) -> Option { + let snapshot = self + .lock_state() + .tasks + .get(download_id) + .map(|entry| TaskSnapshot { + role: entry.role, + finished: entry.finished(), + outer_finished: entry.outer.is_finished(), + started: TaskStartState::load(&entry.start_state) == TaskStartState::Running, + }); + #[cfg(test)] + let observer = self + .snapshot_observer + .lock() + .expect("acquisition task snapshot observer lock poisoned") + .clone(); + #[cfg(test)] + if let Some(observer) = observer { + observer(download_id, snapshot.clone()); + } + snapshot + } + + pub(crate) fn active_worker_generation(&self, download_id: &str) -> Option { + self.lock_state() + .tasks + .get(download_id) + .filter(|entry| { + entry.role == TaskRole::Worker + && TaskStartState::load(&entry.start_state) == TaskStartState::Running + && !entry.outer.is_finished() + }) + .map(|entry| entry.generation.clone()) + } + + #[cfg(test)] + fn generation_for_test(&self, download_id: &str) -> Option { + self.lock_state() + .tasks + .get(download_id) + .map(|entry| entry.generation.clone()) + } + + #[cfg(test)] + fn nested_count_for_test(&self, download_id: &str) -> Option { + self.lock_state() + .tasks + .get(download_id) + .map(|entry| entry.nested.len()) + } + + #[cfg(test)] + pub(crate) fn outer_finished_for_test(&self, download_id: &str) -> bool { + self.lock_state() + .tasks + .get(download_id) + .is_some_and(|entry| entry.outer.is_finished()) + } + + #[cfg(test)] + fn prepared_count_for_test(&self) -> usize { + self.lock_state().prepared.len() + } + + pub(crate) fn contains(&self, download_id: &str) -> bool { + self.snapshot(download_id).is_some() + } + + pub(crate) fn generation_is_current( + &self, + download_id: &str, + generation: &TaskGeneration, + ) -> bool { + self.lock_state() + .tasks + .get(download_id) + .is_some_and(|entry| entry.generation.matches(generation)) + } + + fn generation_has_role( + &self, + download_id: &str, + generation: &TaskGeneration, + role: TaskRole, + ) -> bool { + self.lock_state() + .tasks + .get(download_id) + .is_some_and(|entry| entry.generation.matches(generation) && entry.role == role) + } + + /// Called under the download-state commit lock after durable confirmation. + pub(crate) fn promote_admission(&self, download_id: &str, generation: &TaskGeneration) -> bool { + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + let Some(entry) = tasks.get_mut(download_id) else { + return false; + }; + if !entry.generation.matches(generation) || entry.role != TaskRole::AdmissionTransition { + return false; + } + entry.role = TaskRole::Worker; + true + } + + pub(crate) fn bind_pending_admission( + &self, + download_id: &str, + generation: &TaskGeneration, + identity: Arc, + completed: tokio::sync::watch::Receiver, + ) { + let mut state = self.lock_state(); + let mut offered = Some((identity, completed)); + let previous = state + .tasks + .get_mut(download_id) + .filter(|entry| entry.generation.matches(generation)) + .and_then(|entry| std::mem::replace(&mut entry.admission, offered.take())); + // An opaque consumer payload may have its own destructor. Dispose of + // replaced metadata only after releasing the custody mutex. + drop(state); + drop(previous); + drop(offered); + } + + /// Capture immutable consumer metadata without invoking consumer code under + /// the custody mutex. The receiving scope owns interpretation and matching. + pub(crate) fn admission_snapshots(&self, role: TaskRole) -> Vec { + self.lock_state() + .tasks + .iter() + .filter_map(|(id, entry)| { + (entry.role == role) + .then(|| { + entry + .admission + .as_ref() + .map(|(metadata, completed)| AdmissionSnapshot { + operation_id: id.clone(), + generation: entry.generation.clone(), + metadata: metadata.clone(), + completed: completed.clone(), + }) + }) + .flatten() + }) + .collect() + } + + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.lock_state().tasks.is_empty() + } + + pub(crate) fn ids(&self) -> Vec { + let ids = self + .lock_state() + .tasks + .iter() + .filter(|(_, entry)| entry.role != TaskRole::Invocation) + .map(|(id, _)| id.clone()) + .collect(); + #[cfg(test)] + let observer = self + .ids_observer + .lock() + .expect("acquisition task IDs observer lock poisoned") + .clone(); + #[cfg(test)] + if let Some(observer) = observer { + observer(); + } + ids + } + + /// Starts a caller-independent finalizer after synchronously replacing and + /// aborting the current generation. A separately retained observer drains + /// predecessor custody even if the finalizer is aborted before `finish`. + pub(crate) fn begin_cancel( + self: &Arc, + download_id: &str, + finish: F, + ) -> crate::Result + where + F: FnOnce(TaskContext, CancelPredecessor) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + #[cfg(test)] + { + let observer = self + .cancel_replacement_observer + .lock() + .expect("acquisition cancel-replacement observer lock poisoned") + .clone(); + if let Some(observer) = observer { + observer(); + } + } + let mut state = self.lock_state(); + if state.closed { + return Err(crate::PumasError::DownloadLifecycleClosed); + } + let tasks = &mut state.tasks; + if let Some(current) = tasks.get_mut(download_id) { + if current.role == TaskRole::CancelFinalizer && !current.finished() { + return Ok( + if TaskStartState::load(¤t.start_state) == TaskStartState::Running { + CancelTransition::AlreadyRunning + } else { + CancelTransition::Existing(InstalledTask { + owner: self.clone(), + download_id: download_id.to_string(), + generation: current.generation.clone(), + start_state: current.start_state.clone(), + }) + }, + ); + } + } + let mut current = tasks.remove(download_id); + let outer_finished_before_replacement = current + .as_ref() + .is_some_and(|entry| entry.outer.is_finished()); + let mut abort_on_start: Vec<_> = current + .as_ref() + .map(|entry| entry.outer.abort_handle()) + .into_iter() + .collect(); + if let Some(entry) = current.as_mut() { + abort_on_start.append(&mut entry.abort_on_start); + } + let superseded_projection = current.as_ref().and_then(|entry| { + entry + .projection + .clone() + .or_else(|| entry.superseded_projection.clone()) + }); + let predecessor_starts = current + .as_mut() + .map(|entry| std::mem::take(&mut entry.starts)) + .unwrap_or_default(); + + let generation = TaskGeneration::new(); + let context = TaskContext { + owner: Arc::downgrade(self), + download_id: download_id.to_string(), + generation: generation.clone(), + projection_failure: superseded_projection.clone(), + effect_lease: None, + }; + let (start, started) = oneshot::channel(); + let (predecessor_start, predecessor_started) = oneshot::channel(); + let (predecessor_sender, predecessor_receiver) = oneshot::channel(); + let predecessor_completion = Arc::new(NestedCompletion { + finished: AtomicBool::new(false), + failed: AtomicBool::new(false), + notify: Notify::new(), + }); + let predecessor_observer = tokio::spawn(observe_cancellation_predecessor( + current, + outer_finished_before_replacement, + predecessor_started, + predecessor_completion.clone(), + predecessor_sender, + )); + let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); + let outer = tokio::spawn(async move { + if started.await.is_err() { + return; + } + let Ok(predecessor) = predecessor_receiver.await else { + // Observer failure remains owned by the nested task; it must + // never authorize cleanup as an absent predecessor. + return; + }; + finish(context, predecessor).await; + }); + tasks.insert( + download_id.to_string(), + TaskEntry { + admission: None, + generation: generation.clone(), + role: TaskRole::CancelFinalizer, + outer, + nested: vec![NestedTask { + handle: predecessor_observer, + completion: predecessor_completion, + failure_kind: NestedFailureKind::Predecessor, + }], + nested_failures_archived: 0, + predecessor_failures_archived: 0, + projection: None, + starts: predecessor_starts + .into_iter() + .chain(std::iter::once(StartGate::Custody(predecessor_start))) + .chain(std::iter::once(StartGate::Work(start))) + .collect(), + superseded_projection, + abort_on_start, + start_state: start_state.clone(), + }, + ); + drop(state); + Ok(CancelTransition::Started(InstalledTask { + owner: self.clone(), + download_id: download_id.to_string(), + generation, + start_state, + })) + } + + #[cfg(test)] + pub(crate) async fn observe_finished(&self, download_id: &str) -> Option { + let entry = { + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + if !tasks.get(download_id).is_some_and(TaskEntry::finished) { + return None; + } + tasks.remove(download_id) + }?; + let role = entry.role; + Some(observe_entry(entry, role, false).await) + } + + #[cfg(test)] + pub(crate) async fn observe_finished_generation( + &self, + download_id: &str, + generation: &TaskGeneration, + ) -> Option { + let entry = { + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + if !tasks + .get(download_id) + .is_some_and(|entry| entry.generation.matches(generation) && entry.finished()) + { + return None; + } + tasks.remove(download_id) + }?; + let role = entry.role; + Some(observe_entry(entry, role, false).await) + } + + pub(crate) fn finished_or_projecting_ids(&self) -> Vec { + self.lock_state() + .tasks + .iter() + .filter_map(|(download_id, entry)| { + (entry.role != TaskRole::Invocation + && (entry.role == TaskRole::TerminalProjection || entry.finished())) + .then_some(download_id.clone()) + }) + .collect() + } + + /// Replaces one fully finished generation with a start-gated projection + /// owner under the same download ID. The predecessor is observed by a + /// nested owner task, so its failure remains visible if cancellation + /// supersedes the projector before state projection begins. + pub(crate) fn begin_finished_projection( + self: &Arc, + download_id: &str, + inherit_failure: bool, + project: F, + project_panic: P, + ) -> crate::Result + where + F: FnOnce(TaskContext, Option) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + P: FnOnce(TaskContext) -> PFut + Send + 'static, + PFut: Future + Send + 'static, + { + let mut state = self.lock_state(); + if state.closed { + return Err(crate::PumasError::DownloadLifecycleClosed); + } + let tasks = &mut state.tasks; + let Some(current) = tasks.get_mut(download_id) else { + return Ok(ProjectionTransition::NotReady); + }; + if current.role == TaskRole::TerminalProjection { + let cell = current + .projection + .clone() + .expect("terminal projection owns a projection cell"); + let generation = current.generation.clone(); + return Ok(ProjectionTransition::Existing(InstalledProjection { + task: InstalledTask { + owner: self.clone(), + download_id: download_id.to_string(), + generation: generation.clone(), + start_state: current.start_state.clone(), + }, + ticket: ProjectionTicket { + scope: Arc::downgrade(self), + download_id: download_id.to_string(), + generation, + cell, + }, + })); + } + if !current.finished() { + return Ok(ProjectionTransition::NotReady); + } + + let predecessor = tasks + .remove(download_id) + .expect("finished predecessor remained present"); + let predecessor_role = predecessor.role; + let inherited_projection = predecessor.superseded_projection.clone(); + let generation = TaskGeneration::new(); + let cell = Arc::new(ProjectionCell::new(false)); + if let Some(inherited) = inherited_projection { + if inherited.failed() { + cell.mark_failed(); + } + cell.inherit(inherited); + } + if inherit_failure { + cell.mark_failed(); + } + let context = TaskContext { + owner: Arc::downgrade(self), + download_id: download_id.to_string(), + generation: generation.clone(), + projection_failure: Some(cell.clone()), + effect_lease: None, + }; + let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); + + let predecessor_cell = cell.clone(); + let predecessor_completion = Arc::new(NestedCompletion { + finished: AtomicBool::new(false), + failed: AtomicBool::new(false), + notify: Notify::new(), + }); + let predecessor_completion_task = predecessor_completion.clone(); + let (predecessor_start, predecessor_started) = oneshot::channel(); + let predecessor_observer = tokio::spawn(async move { + if predecessor_started.await.is_err() { + return; + } + let observation = observe_entry(predecessor, predecessor_role, false).await; + if observation.terminal == TaskTerminal::Panicked || observation.nested_failures > 0 { + predecessor_completion_task + .failed + .store(true, Ordering::Release); + } + predecessor_cell.record_predecessor(observation); + predecessor_completion_task + .finished + .store(true, Ordering::Release); + predecessor_completion_task.notify.notify_waiters(); + }); + + let project_cell = cell.clone(); + let (project_start, project_started) = oneshot::channel(); + let outer = tokio::spawn(async move { + if project_started.await.is_err() { + return; + } + let predecessor = project_cell.wait_for_predecessor().await; + let project_context = context.clone(); + let outcome = + match AssertUnwindSafe(async move { project(project_context, predecessor).await }) + .catch_unwind() + .await + { + Ok(outcome) => outcome, + Err(_) => { + project_cell.mark_failed(); + let fallback = + AssertUnwindSafe(async move { project_panic(context).await }) + .catch_unwind() + .await; + if matches!(fallback, Ok(ProjectionOutcome::Failed)) { + project_cell.acknowledge_failure_projection(); + } + ProjectionOutcome::Panicked + } + }; + if outcome == ProjectionOutcome::Failed { + project_cell.mark_failed(); + } + if project_cell.failed() + && matches!( + outcome, + ProjectionOutcome::Committed | ProjectionOutcome::Failed + ) + { + project_cell.acknowledge_failure_projection(); + } + project_cell.settle(outcome); + }); + + tasks.insert( + download_id.to_string(), + TaskEntry { + admission: None, + generation: generation.clone(), + role: TaskRole::TerminalProjection, + outer, + nested: vec![NestedTask { + handle: predecessor_observer, + completion: predecessor_completion, + failure_kind: NestedFailureKind::Effect, + }], + nested_failures_archived: 0, + predecessor_failures_archived: 0, + projection: Some(cell.clone()), + starts: vec![ + StartGate::Custody(predecessor_start), + StartGate::Work(project_start), + ], + superseded_projection: None, + abort_on_start: Vec::new(), + start_state: start_state.clone(), + }, + ); + drop(state); + + let ticket = ProjectionTicket { + scope: Arc::downgrade(self), + download_id: download_id.to_string(), + generation: generation.clone(), + cell, + }; + Ok(ProjectionTransition::Started(InstalledProjection { + task: InstalledTask { + owner: self.clone(), + download_id: download_id.to_string(), + generation, + start_state, + }, + ticket, + })) + } + + pub(crate) fn settle_projection(&self, ticket: &ProjectionTicket) -> ProjectionSettlement { + if !ticket + .scope + .upgrade() + .is_some_and(|scope| std::ptr::eq(scope.as_ref(), self)) + { + return ProjectionSettlement::StaleGeneration; + } + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + let Some(entry) = tasks.get(&ticket.download_id) else { + return if ticket.cell.is_settled() { + ProjectionSettlement::AlreadySettled + } else { + ProjectionSettlement::Missing + }; + }; + let matches = entry.role == TaskRole::TerminalProjection + && entry.generation.matches(&ticket.generation); + if !matches { + return ProjectionSettlement::StaleGeneration; + } + if ticket.cell.has_unprojected_failure() { + return ProjectionSettlement::FailureUnprojected; + } + if !ticket.cell.is_ready_to_settle() { + return ProjectionSettlement::Pending; + } + if !entry.finished() { + return ProjectionSettlement::Pending; + } + ticket.cell.mark_settled(); + let start = tasks + .remove(&ticket.download_id) + .map(|entry| retire_entry(&mut state, entry)); + drop(state); + if let Some(start) = start { + let _ = start.send(()); + } + ProjectionSettlement::Settled + } + + fn promote_generation( + &self, + download_id: &str, + generation: &TaskGeneration, + role: TaskRole, + ) -> bool { + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + let Some(entry) = tasks.get_mut(download_id) else { + return false; + }; + if !entry.generation.matches(generation) { + return false; + } + entry.role = role; + true + } + + fn start_generation(&self, download_id: &str, generation: &TaskGeneration) -> bool { + let (aborts, superseded_projection, starts) = { + let mut state = self.lock_state(); + if state.closed { + return false; + } + let tasks = &mut state.tasks; + let Some(entry) = tasks.get_mut(download_id) else { + return false; + }; + if !entry.generation.matches(generation) { + return false; + } + entry + .start_state + .store(TaskStartState::Running as u8, Ordering::Release); + ( + std::mem::take(&mut entry.abort_on_start), + entry.superseded_projection.clone(), + std::mem::take(&mut entry.starts), + ) + }; + for abort in aborts { + abort.abort(); + } + if let Some(cell) = superseded_projection { + cell.settle(ProjectionOutcome::Superseded); + } + for start in starts { + start.start(); + } + true + } + + /// Runs after outer state locks are released. Abandoned projectors and + /// finalizers are safe to start because they retain required predecessor + /// custody; abandoned workers are removed and aborted without claiming + /// that they ever ran. + pub(crate) fn rescue_abandoned(&self) { + self.reap_retired(); + let mut state = self.lock_state(); + if state.closed { + return; + } + let mut retired_starts = Vec::new(); + let keys = state + .prepared + .iter() + .filter_map(|(key, entry)| { + (TaskStartState::load(&entry.start_state) == TaskStartState::Abandoned) + .then_some(*key) + }) + .collect::>(); + for key in keys { + if let Some(entry) = state.prepared.remove(&key) { + entry.outer.abort(); + let (start, started) = oneshot::channel(); + let observer = tokio::spawn(async move { + let _ = started.await; + drop(entry.start); + let failures = + usize::from(entry.outer.await.is_err_and(|error| error.is_panic())); + if let Some(cell) = entry.projection { + cell.settle(ProjectionOutcome::Shutdown); + } + failures + }); + state.retired.push(RetiredTask { observer }); + retired_starts.push(start); + } + } + let mut starts = Vec::new(); + let mut removals = Vec::new(); + for (id, entry) in &state.tasks { + if TaskStartState::load(&entry.start_state) != TaskStartState::Abandoned { + continue; + } + if matches!( + entry.role, + TaskRole::TerminalProjection | TaskRole::CancelFinalizer + ) { + starts.push((id.clone(), entry.generation.clone())); + } else { + removals.push(id.clone()); + } + } + for id in removals { + if let Some(entry) = state.tasks.remove(&id) { + retired_starts.push(retire_entry(&mut state, entry)); + } + } + drop(state); + for start in retired_starts { + let _ = start.send(()); + } + for (id, generation) in starts { + self.start_generation(&id, &generation); + } + } + + fn reap_retired(&self) { + let mut state = self.lock_state(); + while let Some(index) = state + .retired + .iter() + .position(|task| task.observer.is_finished()) + { + let mut task = state.retired.swap_remove(index); + match (&mut task.observer).now_or_never() { + Some(result) => { + state.retired_failures += result.unwrap_or(1); + self.retired_observations.fetch_add(1, Ordering::AcqRel); + } + None => { + state.retired.push(task); + break; + } + } + } + } + + #[cfg(test)] + pub(crate) fn outstanding_retired_for_test(&self) -> usize { + self.reap_retired(); + self.lock_state().retired.len() + } + + #[cfg(test)] + fn retired_observations_for_test(&self) -> usize { + self.retired_observations.load(Ordering::Acquire) + } + + #[cfg(test)] + fn register_blocking( + self: &Arc, + download_id: &str, + generation: &TaskGeneration, + operation: &'static str, + function: F, + ) -> std::result::Result>, BlockingTaskError> + where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + { + self.register_blocking_with_failure( + download_id, + generation, + operation, + function, + |_| false, + None, + ) + } + + fn register_fallible_blocking( + self: &Arc, + download_id: &str, + generation: &TaskGeneration, + operation: &'static str, + function: F, + effect_lease: Option>, + ) -> std::result::Result, BlockingTaskError> + where + T: Send + 'static, + E: Send + 'static, + F: FnOnce() -> std::result::Result + Send + 'static, + { + self.register_blocking_with_failure( + download_id, + generation, + operation, + function, + std::result::Result::is_err, + effect_lease, + ) + } + + fn register_blocking_with_failure( + self: &Arc, + download_id: &str, + generation: &TaskGeneration, + operation: &'static str, + function: F, + failed: C, + effect_lease: Option>, + ) -> std::result::Result>, BlockingTaskError> + where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + C: FnOnce(&T) -> bool + Send + 'static, + { + #[cfg(not(test))] + let _ = operation; + let mut state = self.lock_state(); + if state.closed { + return Err(BlockingTaskError::StaleGeneration); + } + let tasks = &mut state.tasks; + let Some(entry) = tasks.get_mut(download_id) else { + return Err(BlockingTaskError::StaleGeneration); + }; + if !entry.generation.matches(generation) { + return Err(BlockingTaskError::StaleGeneration); + } + entry.reap_completed_nested(); + + #[cfg(test)] + let blocking_observer = self + .blocking_observer + .lock() + .expect("acquisition blocking observer lock poisoned") + .clone(); + let (start_sender, start_receiver) = oneshot::channel(); + let (result_sender, result_receiver) = oneshot::channel(); + let completion = Arc::new(NestedCompletion { + finished: AtomicBool::new(false), + failed: AtomicBool::new(false), + notify: Notify::new(), + }); + let completion_in_observer = completion.clone(); + #[cfg(test)] + let result_observer = self + .blocking_result_observer + .lock() + .expect("acquisition blocking-result observer lock poisoned") + .clone(); + let observer = tokio::spawn(async move { + let closure_grant = effect_lease.clone(); + let result = if start_receiver.await.is_ok() { + tokio::task::spawn_blocking(move || { + let _grant = closure_grant; + #[cfg(test)] + if let Some(observer) = blocking_observer { + observer(operation); + } + function() + }) + .await + } else { + return; + } + .map_err(|error| { + completion_in_observer.failed.store(true, Ordering::Release); + error.to_string() + }); + if result.as_ref().is_ok_and(failed) { + completion_in_observer.failed.store(true, Ordering::Release); + } + #[cfg(test)] + if let Some(observer) = result_observer { + observer(operation); + } + let _ = result_sender.send(result); + completion_in_observer + .finished + .store(true, Ordering::Release); + completion_in_observer.notify.notify_waiters(); + drop(effect_lease); + }); + entry.nested.push(NestedTask { + handle: observer, + completion, + failure_kind: NestedFailureKind::Effect, + }); + drop(state); + let _ = start_sender.send(()); + Ok(result_receiver) + } + + #[cfg(test)] + pub(crate) fn set_blocking_observer(&self, observer: Option) { + *self + .blocking_observer + .lock() + .expect("acquisition blocking observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_ids_observer(&self, observer: Option) { + *self + .ids_observer + .lock() + .expect("acquisition task IDs observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_drain_observer(&self, observer: Option) { + *self + .drain_observer + .lock() + .expect("acquisition task drain observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_snapshot_observer(&self, observer: Option) { + *self + .snapshot_observer + .lock() + .expect("acquisition task snapshot observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_cancellation_check_observer( + &self, + observer: Option, + ) { + *self + .cancellation_check_observer + .lock() + .expect("acquisition cancellation-check observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_cancel_replacement_observer( + &self, + observer: Option, + ) { + *self + .cancel_replacement_observer + .lock() + .expect("acquisition cancel-replacement observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_worker_projection_observer( + &self, + observer: Option, + ) { + *self + .worker_projection_observer + .lock() + .expect("acquisition worker-projection observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_blocking_result_observer(&self, observer: Option) { + *self + .blocking_result_observer + .lock() + .expect("acquisition blocking-result observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_blocking_failure_observer(&self, observer: Option) { + *self + .blocking_failure_observer + .lock() + .expect("acquisition blocking-failure observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_ambient_admission_observer( + &self, + observer: Option, + ) { + *self + .ambient_admission_observer + .lock() + .expect("acquisition ambient-admission observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn set_projection_observer(&self, observer: Option) { + *self + .projection_observer + .lock() + .expect("acquisition projection observer lock poisoned") = observer; + } + + #[cfg(test)] + pub(crate) fn observe_ambient_admission(&self, operation: &'static str, download_id: &str) { + let observer = self + .ambient_admission_observer + .lock() + .expect("acquisition ambient-admission observer lock poisoned") + .clone(); + if let Some(observer) = observer { + observer(operation, download_id); + } + } + + async fn drain_blocking_generation( + &self, + download_id: &str, + generation: &TaskGeneration, + ) -> std::result::Result { + let (_archived_failures, completions) = { + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + let Some(entry) = tasks.get_mut(download_id) else { + return Err(BlockingTaskError::StaleGeneration); + }; + if !entry.generation.matches(generation) { + return Err(BlockingTaskError::StaleGeneration); + } + entry.reap_completed_nested(); + ( + entry.nested_failures_archived, + entry + .nested + .iter() + .map(|nested| nested.completion.clone()) + .collect::>(), + ) + }; + #[cfg(test)] + let observer = self + .drain_observer + .lock() + .expect("acquisition task drain observer lock poisoned") + .clone(); + #[cfg(test)] + if let Some(observer) = observer { + observer(); + } + let _ = wait_for_nested(&completions).await; + loop { + let completed = { + let mut state = self.lock_state(); + let tasks = &mut state.tasks; + let Some(entry) = tasks.get_mut(download_id) else { + return Err(BlockingTaskError::StaleGeneration); + }; + if !entry.generation.matches(generation) { + return Err(BlockingTaskError::StaleGeneration); + } + entry.reap_completed_nested(); + if entry.nested.is_empty() { + // Predecessor failures remain terminal provenance, not a + // new failure of this generation's cleanup effects. + Some(entry.nested_failures_archived) + } else { + None + } + }; + if let Some(failures) = completed { + return Ok(failures); + } + tokio::task::yield_now().await; + } + } +} + +impl TaskContext { + /// A consumer-provided capability is retained by each registered effect + /// through completion observation. This grants no interpretation authority. + pub(crate) fn with_effect_lease(&self, lease: Option>) -> Self { + let mut context = self.clone(); + context.effect_lease = lease; + context + } + + pub(crate) fn shares_scope(&self, other: &Self) -> bool { + Weak::ptr_eq(&self.owner, &other.owner) + } + + /// Registers an async effect whose internal work must survive cancellation + /// of the invoking future. Like blocking effects, this observer is joined, + /// never aborted, by lifecycle shutdown. + pub(crate) async fn run_fallible_async_named( + &self, + _operation: &'static str, + function: F, + ) -> std::result::Result, BlockingTaskError> + where + T: Send + 'static, + E: Send + 'static, + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let owner = self + .owner + .upgrade() + .ok_or(BlockingTaskError::StaleGeneration)?; + let receiver = { + let mut state = owner.lock_state(); + if state.closed { + return Err(BlockingTaskError::StaleGeneration); + } + let entry = state + .tasks + .get_mut(&self.download_id) + .filter(|entry| entry.generation.matches(&self.generation)) + .ok_or(BlockingTaskError::StaleGeneration)?; + entry.reap_completed_nested(); + let (start, started) = oneshot::channel(); + let (sender, receiver) = oneshot::channel(); + let completion = Arc::new(NestedCompletion { + finished: AtomicBool::new(false), + failed: AtomicBool::new(false), + notify: Notify::new(), + }); + let observed = completion.clone(); + let effect_lease = self.effect_lease.clone(); + let handle = tokio::spawn(async move { + let _ = started.await; + let result = AssertUnwindSafe(async move { function().await }) + .catch_unwind() + .await + .map_err(|_| "owned async effect panicked".to_string()); + if !matches!(&result, Ok(Ok(_))) { + observed.failed.store(true, Ordering::Release); + } + let _ = sender.send(result); + observed.finished.store(true, Ordering::Release); + observed.notify.notify_waiters(); + drop(effect_lease); + }); + entry.nested.push(NestedTask { + handle, + completion, + failure_kind: NestedFailureKind::Effect, + }); + drop(state); + let _ = start.send(()); + receiver + }; + receiver + .await + .map_err(|_| BlockingTaskError::ResultChannelClosed)? + .map_err(BlockingTaskError::Join) + } + + pub(crate) async fn pause_requested(&self, pause_flag: &AtomicBool) { + loop { + let notified = self.generation.0.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + if pause_flag.load(Ordering::Acquire) { + return; + } + notified.await; + } + } + + pub(crate) fn download_id(&self) -> &str { + &self.download_id + } + + pub(crate) fn generation(&self) -> &TaskGeneration { + &self.generation + } + + pub(crate) fn is_current_role(&self, role: TaskRole) -> bool { + self.owner.upgrade().is_some_and(|owner| { + owner.generation_has_role(&self.download_id, &self.generation, role) + }) + } + + pub(crate) fn promote_role(&self, role: TaskRole) -> bool { + self.owner.upgrade().is_some_and(|owner| { + owner.promote_generation(&self.download_id, &self.generation, role) + }) + } + + /// Completes custody transferred from a superseded terminal projector. + /// A failed cell is acknowledged only after the finalizer has published + /// its fail-closed terminal state. + pub(crate) fn complete_transferred_projection(&self, failure_projected: bool) -> bool { + let Some(cell) = &self.projection_failure else { + return false; + }; + if cell.failed() { + if !failure_projected { + return false; + } + cell.acknowledge_failure_projection(); + } + cell.mark_settled(); + true + } + + #[cfg(test)] + pub(crate) async fn run_blocking( + &self, + function: F, + ) -> std::result::Result + where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + { + self.run_blocking_named("unnamed", function).await + } + + #[cfg(test)] + pub(crate) fn register_blocking_without_wait_for_test( + &self, + operation: &'static str, + function: F, + ) -> std::result::Result<(), BlockingTaskError> + where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + { + let owner = self + .owner + .upgrade() + .ok_or(BlockingTaskError::StaleGeneration)?; + let receiver = + owner.register_blocking(&self.download_id, &self.generation, operation, function)?; + drop(receiver); + Ok(()) + } + + /// Exercises owned blocking success/panic observation without a domain error. + pub(crate) async fn run_blocking_named( + &self, + operation: &'static str, + function: F, + ) -> std::result::Result + where + T: Send + 'static, + F: FnOnce() -> T + Send + 'static, + { + let owner = self + .owner + .upgrade() + .ok_or(BlockingTaskError::StaleGeneration)?; + let receiver = owner.register_blocking_with_failure( + &self.download_id, + &self.generation, + operation, + function, + |_| false, + self.effect_lease.clone(), + )?; + receiver + .await + .map_err(|_| BlockingTaskError::ResultChannelClosed)? + .map_err(BlockingTaskError::Join) + } + + pub(crate) async fn run_fallible_blocking_named( + &self, + operation: &'static str, + function: F, + ) -> std::result::Result, BlockingTaskError> + where + T: Send + 'static, + E: Send + 'static, + F: FnOnce() -> std::result::Result + Send + 'static, + { + let owner = self + .owner + .upgrade() + .ok_or(BlockingTaskError::StaleGeneration)?; + let receiver = owner.register_fallible_blocking( + &self.download_id, + &self.generation, + operation, + function, + self.effect_lease.clone(), + )?; + receiver + .await + .map_err(|_| BlockingTaskError::ResultChannelClosed)? + .map_err(BlockingTaskError::Join) + } + + pub(crate) async fn drain_blocking(&self) -> std::result::Result { + let owner = self + .owner + .upgrade() + .ok_or(BlockingTaskError::StaleGeneration)?; + owner + .drain_blocking_generation(&self.download_id, &self.generation) + .await + } + + #[cfg(test)] + pub(crate) fn should_fail_blocking_operation(&self, operation: &'static str) -> bool { + self.owner.upgrade().is_some_and(|owner| { + owner + .blocking_failure_observer + .lock() + .expect("acquisition blocking-failure observer lock poisoned") + .as_ref() + .is_some_and(|observer| observer(operation)) + }) + } + + #[cfg(test)] + pub(crate) fn observe_projection(&self, projection: &'static str) { + if let Some(owner) = self.owner.upgrade() { + let observer = owner + .projection_observer + .lock() + .expect("acquisition projection observer lock poisoned") + .clone(); + if let Some(observer) = observer { + observer(projection); + } + } + } + + #[cfg(test)] + pub(crate) fn observe_cancellation_check(&self) { + if let Some(owner) = self.owner.upgrade() { + let observer = owner + .cancellation_check_observer + .lock() + .expect("acquisition cancellation-check observer lock poisoned") + .clone(); + if let Some(observer) = observer { + observer(); + } + } + } + + #[cfg(test)] + pub(crate) fn observe_worker_projection(&self, projection: &'static str) { + if let Some(owner) = self.owner.upgrade() { + let observer = owner + .worker_projection_observer + .lock() + .expect("acquisition worker-projection observer lock poisoned") + .clone(); + if let Some(observer) = observer { + observer(projection); + } + } + } +} + +impl InstalledTask { + pub(crate) fn generation(&self) -> &TaskGeneration { + &self.generation + } + + pub(crate) fn start(self) { + let _ = self + .owner + .start_generation(&self.download_id, &self.generation); + } +} + +impl Drop for InstalledTask { + fn drop(&mut self) { + let _ = self.start_state.compare_exchange( + TaskStartState::Gated as u8, + TaskStartState::Abandoned as u8, + Ordering::AcqRel, + Ordering::Acquire, + ); + } +} + +impl Drop for PreparedTask { + fn drop(&mut self) { + if !self.armed { + return; + } + let _ = self.start_state.compare_exchange( + TaskStartState::Gated as u8, + TaskStartState::Abandoned as u8, + Ordering::AcqRel, + Ordering::Acquire, + ); + } +} + +impl InstalledProjection { + pub(crate) fn start(self) -> ProjectionTicket { + let Self { task, ticket } = self; + task.start(); + ticket + } +} + +impl ProjectionTicket { + pub(crate) async fn wait(&self) -> ProjectionOutcome { + self.cell.wait().await + } + + #[cfg(test)] + pub(crate) fn failure_projected_for_test(&self) -> bool { + self.cell.failure_projected() + } + + #[cfg(test)] + pub(crate) fn settled_for_test(&self) -> bool { + self.cell.is_settled() + } +} + +async fn observe_cancellation_predecessor( + current: Option, + outer_finished_before_replacement: bool, + started: oneshot::Receiver<()>, + completion: Arc, + receipt: oneshot::Sender, +) { + let started = started.await.is_ok(); + let predecessor = match current { + Some(current) => { + if !started { + current.outer.abort(); + } + let role = current.role; + match AssertUnwindSafe(observe_entry( + current, + role, + outer_finished_before_replacement, + )) + .catch_unwind() + .await + { + Ok(observation) => { + completion.failed.store( + !started + || observation.terminal == TaskTerminal::Panicked + || observation.nested_failures > 0, + Ordering::Release, + ); + Some(CancelPredecessor::Observed(observation)) + } + Err(_) => { + completion.failed.store(true, Ordering::Release); + None + } + } + } + None => { + completion.failed.store(!started, Ordering::Release); + Some(CancelPredecessor::Absent) + } + }; + completion.finished.store(true, Ordering::Release); + completion.notify.notify_waiters(); + if started { + if let Some(predecessor) = predecessor { + let _ = receipt.send(predecessor); + } + } +} + +async fn observe_entry( + mut entry: TaskEntry, + role: TaskRole, + outer_finished_before_replacement: bool, +) -> TaskObservation { + let generation = entry.generation.clone(); + let terminal = match entry.outer.await { + Ok(()) => TaskTerminal::Completed, + Err(error) if error.is_cancelled() => TaskTerminal::Cancelled, + Err(_) => TaskTerminal::Panicked, + }; + let nested_failures = entry.nested_failures_archived + + entry.predecessor_failures_archived + + observe_nested(entry.nested.drain(..).collect()).await; + // Drain owned effects before reading failure provenance, whose poisoned + // bookkeeping must not cause an observer to detach unfinished work. + let projection_failed = entry.projection.as_ref().is_some_and(|cell| cell.failed()) + || entry + .superseded_projection + .as_ref() + .is_some_and(|cell| cell.failed()); + let nested_failures = nested_failures + usize::from(projection_failed); + TaskObservation { + generation, + role, + terminal, + nested_failures, + outer_finished_before_replacement, + } +} + +async fn observe_nested(nested: Vec) -> usize { + let mut failures = 0; + for nested in nested { + let join_failed = nested.handle.await.is_err(); + if join_failed || nested.completion.failed.load(Ordering::Acquire) { + failures += 1; + } + } + failures +} + +async fn wait_for_nested(completions: &[Arc]) -> usize { + for completion in completions { + loop { + let notified = completion.notify.notified(); + if completion.finished.load(Ordering::Acquire) { + break; + } + notified.await; + } + } + completions + .iter() + .filter(|completion| completion.failed.load(Ordering::Acquire)) + .count() +} + +#[cfg(test)] +mod tests { + #[tokio::test] + async fn shutdown_rejects_work_whose_start_gate_was_already_extracted() { + let owner = TaskScope::new_test(); + let ran = Arc::new(AtomicBool::new(false)); + let marker = ran.clone(); + let prepared = owner + .prepare("in-flight".into(), TaskRole::Worker, move |_| async move { + marker.store(true, Ordering::Release); + }) + .unwrap(); + let installed = owner.install_gated(prepared).unwrap(); + // This is start_generation's in-flight custody after its coordination + // lock is released but before its gate sends reach the outer task. + let gates = { + let mut state = owner.lock_state(); + let entry = state.tasks.get_mut("in-flight").unwrap(); + entry + .start_state + .store(TaskStartState::Running as u8, Ordering::Release); + std::mem::take(&mut entry.starts) + }; + let receipt = owner.request_shutdown(); + for gate in gates { + gate.start(); + } + drop(installed); + receipt.wait().await.unwrap(); + assert!(!ran.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn shutdown_starts_gated_predecessor_custody_but_never_cancel_cleanup() { + let owner = TaskScope::new_test(); + let (entered, entered_rx) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let worker = owner + .prepare( + "worker".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_fallible_blocking_named("held predecessor", move || { + let _ = entered.send(()); + released.recv().unwrap(); + Ok::<_, ()>(()) + }) + .await; + }, + ) + .unwrap(); + owner.install_gated(worker).unwrap().start(); + entered_rx.await.unwrap(); + let cleanup_ran = Arc::new(AtomicBool::new(false)); + let cleanup_marker = cleanup_ran.clone(); + let finalizer = owner + .begin_cancel("worker", move |_, _| async move { + cleanup_marker.store(true, Ordering::Release); + }) + .unwrap(); + let receipt = owner.request_shutdown(); + drop(finalizer); + let mut waiting = Box::pin(receipt.wait()); + assert!(futures::poll!(&mut waiting).is_pending()); + assert!(!cleanup_ran.load(Ordering::Acquire)); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(2), waiting) + .await + .unwrap() + .unwrap(); + assert!(!cleanup_ran.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn shutdown_keeps_async_effect_error_and_panic_after_caller_disappears() { + for panic in [false, true] { + let owner = TaskScope::new_test(); + let (entered, entered_rx) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let caller_owner = owner.clone(); + let caller = tokio::spawn(async move { + caller_owner + .run_invocation(move |context| async move { + let _ = context + .run_fallible_async_named("failing async effect", move || async move { + let _ = entered.send(()); + released.await.unwrap(); + assert!(!panic, "injected async effect panic"); + Err::<(), _>("injected async effect error") + }) + .await; + Ok(()) + }) + .await + }); + entered_rx.await.unwrap(); + caller.abort(); + let _ = caller.await; + let receipt = owner.request_shutdown(); + release.send(()).unwrap(); + assert!(matches!( + receipt.wait().await, + Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }) + )); + } + } + + #[tokio::test] + async fn shutdown_closes_prepared_and_installed_work_without_starting_it() { + let owner = TaskScope::new_test(); + let ran = Arc::new(AtomicUsize::new(0)); + let prepared = owner + .prepare("prepared".into(), TaskRole::Worker, { + let ran = ran.clone(); + move |_| async move { + ran.fetch_add(1, Ordering::SeqCst); + } + }) + .unwrap(); + let installed = owner + .install_gated( + owner + .prepare("installed".into(), TaskRole::Worker, { + let ran = ran.clone(); + move |_| async move { + ran.fetch_add(1, Ordering::SeqCst); + } + }) + .unwrap(), + ) + .unwrap(); + let projection = owner + .install_projection_gated( + owner + .prepare_projection( + "projection".into(), + |_, _| async { panic!("gated projection must not run") }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap(), + ) + .unwrap(); + let ticket = projection.ticket.clone(); + let receipt = owner.request_shutdown(); + assert!(owner.is_closed()); + assert!(owner.install_gated(prepared).is_err()); + installed.start(); + drop(projection); + assert!(matches!( + owner.prepare("late".into(), TaskRole::Worker, |_| async {}), + Err(crate::PumasError::DownloadLifecycleClosed) + )); + assert!(matches!( + owner.begin_cancel("late", |_, _| async {}), + Err(crate::PumasError::DownloadLifecycleClosed) + )); + receipt.wait().await.unwrap(); + assert_eq!(ticket.wait().await, ProjectionOutcome::Shutdown); + assert_eq!(ran.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn shutdown_retains_cancelled_invocation_effect_and_shared_failure_receipt() { + for outcome in 0..3 { + let completed = Arc::new(AtomicBool::new(false)); + let projected = Arc::new(AtomicUsize::new(0)); + let final_projected = projected.clone(); + let final_completed = completed.clone(); + let owner = Arc::new(TaskCustodyOwner::new()) + .open_scope(move || async move { + assert!(final_completed.load(Ordering::Acquire)); + final_projected.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .unwrap(); + let weak = Arc::downgrade(&owner); + let (entered, entered_rx) = oneshot::channel(); + let (release, released) = std::sync::mpsc::channel(); + let effect_completed = completed.clone(); + let caller_owner = owner.clone(); + let caller = tokio::spawn(async move { + caller_owner + .run_invocation(move |context| async move { + context + .run_fallible_blocking_named("shutdown held effect", move || { + let _ = entered.send(()); + released.recv().unwrap(); + effect_completed.store(true, Ordering::Release); + match outcome { + 0 => Ok(()), + 1 => Err("effect failed"), + _ => panic!("effect panicked"), + } + }) + .await + .map_err(|_| crate::PumasError::DownloadShutdownFailed { failures: 1 })? + .map_err(|_| crate::PumasError::DownloadShutdownFailed { failures: 1 }) + }) + .await + }); + entered_rx.await.unwrap(); + caller.abort(); + let _ = caller.await; + let receipt = owner.request_shutdown(); + let repeated = owner.request_shutdown(); + let mut waiter = Box::pin(receipt.clone().wait()); + assert!(futures::poll!(&mut waiter).is_pending()); + drop(waiter); + drop(owner); + assert!( + weak.upgrade().is_some(), + "driver retains the lifecycle owner" + ); + assert!(!completed.load(Ordering::Acquire)); + release.send(()).unwrap(); + let result = receipt.wait().await; + let repeat_result = repeated.wait().await; + if outcome == 0 { + result.unwrap(); + repeat_result.unwrap(); + } else { + assert!(matches!( + result, + Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }) + )); + assert!(matches!( + repeat_result, + Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }) + )); + } + assert_eq!(projected.load(Ordering::SeqCst), 1); + tokio::time::timeout(Duration::from_secs(2), async { + while weak.upgrade().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + } + + #[tokio::test] + async fn shutdown_drains_owned_async_effect_after_invocation_abort() { + let owner = TaskScope::new_test(); + let (entered, entered_rx) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let caller_owner = owner.clone(); + let caller = tokio::spawn(async move { + caller_owner + .run_invocation(move |context| async move { + context + .run_fallible_async_named("held async effect", move || async move { + let _ = entered.send(()); + released.await.unwrap(); + Ok::<_, ()>(()) + }) + .await + .unwrap() + .unwrap(); + Ok(()) + }) + .await + }); + entered_rx.await.unwrap(); + let receipt = owner.request_shutdown(); + assert!(matches!( + caller.await.unwrap(), + Err(crate::PumasError::DownloadLifecycleClosed) + )); + let mut pending = Box::pin(receipt.clone().wait()); + assert!(futures::poll!(&mut pending).is_pending()); + release.send(()).unwrap(); + pending.await.unwrap(); + } + + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; + + #[tokio::test] + async fn opaque_admission_payloads_are_disposed_outside_the_custody_mutex() { + struct Payload { + owner: Weak, + dropped: Arc, + } + impl Drop for Payload { + fn drop(&mut self) { + let owner = self.owner.upgrade().unwrap(); + assert!( + owner.state.try_lock().is_ok(), + "consumer destructors may re-enter custody" + ); + self.dropped.store(true, Ordering::Release); + } + } + let owner = Arc::new(TaskCustodyOwner::new()); + let scope = owner.open_scope(|| async { Ok(()) }).unwrap(); + let task = scope + .prepare( + "admission".into(), + TaskRole::AdmissionTransition, + |_| async {}, + ) + .unwrap(); + let generation = task.generation.clone(); + let installed = scope.install_gated(task).unwrap(); + let (_, completed) = tokio::sync::watch::channel(false); + let dropped = Arc::new(AtomicBool::new(false)); + scope.bind_pending_admission( + "admission", + &generation, + Arc::new(Payload { + owner: Arc::downgrade(&owner), + dropped: dropped.clone(), + }), + completed.clone(), + ); + scope.bind_pending_admission("admission", &generation, Arc::new(()), completed.clone()); + assert!(dropped.load(Ordering::Acquire)); + dropped.store(false, Ordering::Release); + scope.bind_pending_admission( + "missing", + &generation, + Arc::new(Payload { + owner: Arc::downgrade(&owner), + dropped: dropped.clone(), + }), + completed, + ); + assert!(dropped.load(Ordering::Acquire)); + drop(installed); + owner.request_shutdown().wait().await.unwrap(); + } + + #[tokio::test] + async fn duplicate_operation_ids_are_isolated_by_consumer_scope() { + let owner = Arc::new(TaskCustodyOwner::new()); + let hf = owner.open_scope(|| async { Ok(()) }).unwrap(); + let native = owner.open_scope(|| async { Ok(()) }).unwrap(); + let (hf_ready, hf_context) = oneshot::channel(); + let (native_ready, native_context) = oneshot::channel(); + for (scope, ready) in [(&hf, hf_ready), (&native, native_ready)] { + let task = scope + .prepare( + "same-operation".into(), + TaskRole::Worker, + move |context| async move { + ready.send(context).ok(); + std::future::pending::<()>().await; + }, + ) + .unwrap(); + scope.install_gated(task).unwrap().start(); + } + let hf_context = hf_context.await.unwrap(); + let native_context = native_context.await.unwrap(); + assert!(hf_context.is_current_role(TaskRole::Worker)); + assert!(native_context.is_current_role(TaskRole::Worker)); + assert!(!hf.generation_is_current("same-operation", native_context.generation())); + hf.request_shutdown().wait().await.unwrap(); + assert!(!hf_context.is_current_role(TaskRole::Worker)); + assert!(native_context.is_current_role(TaskRole::Worker)); + owner.request_shutdown().wait().await.unwrap(); + } + + #[tokio::test] + async fn scoped_scans_and_tokens_cannot_observe_or_replace_another_consumer() { + let owner = Arc::new(TaskCustodyOwner::new()); + let hf = owner.open_scope(|| async { Ok(()) }).unwrap(); + let native = owner.open_scope(|| async { Ok(()) }).unwrap(); + let task = native + .prepare( + "native-only".into(), + TaskRole::AdmissionTransition, + |_| async {}, + ) + .unwrap(); + let generation = task.generation.clone(); + let installed = native.install_gated(task).unwrap(); + let (_, completed) = tokio::sync::watch::channel(false); + native.bind_pending_admission( + "native-only", + &generation, + Arc::new("native metadata"), + completed, + ); + assert!(hf.ids().is_empty()); + assert!(hf.finished_or_projecting_ids().is_empty()); + assert!(hf + .admission_snapshots(TaskRole::AdmissionTransition) + .is_empty()); + assert_eq!( + native + .admission_snapshots(TaskRole::AdmissionTransition) + .len(), + 1 + ); + assert!(!hf.contains("native-only")); + let foreign = native + .prepare("foreign-prepared".into(), TaskRole::Worker, |_| async {}) + .unwrap(); + let foreign = hf.install_gated(foreign).unwrap_err(); + native.install_gated(foreign).unwrap().start(); + let projection = native + .prepare_projection( + "native-projection".into(), + |_, _| async { ProjectionOutcome::Committed }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap(); + let ticket = native.install_projection_gated(projection).unwrap().start(); + assert_eq!( + hf.settle_projection(&ticket), + ProjectionSettlement::StaleGeneration + ); + let transition = hf + .begin_cancel("native-only", |_, predecessor| async move { + assert_eq!(predecessor, CancelPredecessor::Absent); + }) + .unwrap(); + assert!(start_cancel(transition)); + assert!(native.generation_is_current("native-only", &generation)); + drop(installed); + owner.request_shutdown().wait().await.unwrap(); + } + + #[tokio::test] + async fn scoped_close_drains_effects_and_projection_while_peer_continues() { + let owner = Arc::new(TaskCustodyOwner::new()); + let completed = Arc::new(AtomicBool::new(false)); + let projected = Arc::new(AtomicUsize::new(0)); + let completion = completed.clone(); + let projection = projected.clone(); + let hf = owner + .open_scope(move || async move { + assert!(completion.load(Ordering::Acquire)); + projection.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .unwrap(); + let native = owner.open_scope(|| async { Ok(()) }).unwrap(); + let (entered, ready) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let marker = completed.clone(); + let task = hf + .prepare( + "held-hf-effect".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_fallible_async_named("held scoped effect", move || async move { + entered.send(()).unwrap(); + released.await.unwrap(); + marker.store(true, Ordering::Release); + Ok::<_, ()>(()) + }) + .await; + }, + ) + .unwrap(); + hf.install_gated(task).unwrap().start(); + ready.await.unwrap(); + let receipt = hf.request_shutdown(); + let mut waiting = Box::pin(receipt.clone().wait()); + assert!(futures::poll!(&mut waiting).is_pending()); + assert_eq!(projected.load(Ordering::SeqCst), 0); + assert_eq!(native.run_invocation(|_| async { Ok(7) }).await.unwrap(), 7); + release.send(()).unwrap(); + waiting.await.unwrap(); + hf.request_shutdown().wait().await.unwrap(); + assert_eq!(projected.load(Ordering::SeqCst), 1); + assert!(!native.is_closed()); + owner.request_shutdown().wait().await.unwrap(); + assert_eq!(projected.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn global_shutdown_drains_all_scopes_and_repeats_aggregate_failures() { + let owner = Arc::new(TaskCustodyOwner::new()); + let projected = Arc::new(AtomicUsize::new(0)); + let mut releases = Vec::new(); + let mut scopes = Vec::new(); + for index in 0..2 { + let completed = Arc::new(AtomicBool::new(false)); + let final_completed = completed.clone(); + let projected = projected.clone(); + let weak_owner = Arc::downgrade(&owner); + let scope = owner + .open_scope(move || async move { + assert!(final_completed.load(Ordering::Acquire)); + // Re-entry would deadlock if finalizers ran under the owner mutex. + assert!(matches!( + weak_owner + .upgrade() + .unwrap() + .open_scope(|| async { Ok(()) }), + Err(crate::PumasError::DownloadLifecycleClosed) + )); + projected.fetch_add(1, Ordering::SeqCst); + Err(crate::PumasError::Other(format!( + "scope {index} projection failed" + ))) + }) + .unwrap(); + let (entered, ready) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let task = scope + .prepare( + "same-held-effect".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_fallible_async_named("global held effect", move || async move { + entered.send(()).unwrap(); + released.await.unwrap(); + completed.store(true, Ordering::Release); + Err::<(), _>("scope effect failed") + }) + .await; + }, + ) + .unwrap(); + scope.install_gated(task).unwrap().start(); + ready.await.unwrap(); + releases.push(release); + scopes.push(scope); + } + let receipt = owner.request_shutdown(); + assert!(scopes.iter().all(|scope| scope.is_closed())); + let mut waiting = Box::pin(receipt.clone().wait()); + assert!(futures::poll!(&mut waiting).is_pending()); + for release in releases { + release.send(()).unwrap(); + } + for receipt in [receipt, owner.request_shutdown()] { + assert!(matches!( + receipt.wait().await, + Err(crate::PumasError::DownloadShutdownFailed { failures: 4 }) + )); + } + assert_eq!(projected.load(Ordering::SeqCst), 2); + for scope in scopes { + assert!(matches!( + scope.request_shutdown().wait().await, + Err(crate::PumasError::DownloadShutdownFailed { failures: 2 }) + )); + } + } + + #[tokio::test] + async fn stale_scoped_context_cannot_submit_effects_after_close() { + let owner = Arc::new(TaskCustodyOwner::new()); + let scope = owner.open_scope(|| async { Ok(()) }).unwrap(); + let (sender, context) = oneshot::channel(); + let task = scope + .prepare( + "stale-context".into(), + TaskRole::Worker, + move |context| async move { + sender.send(context).ok(); + std::future::pending::<()>().await; + }, + ) + .unwrap(); + scope.install_gated(task).unwrap().start(); + let context = context.await.unwrap(); + scope.request_shutdown().wait().await.unwrap(); + let ran = Arc::new(AtomicBool::new(false)); + let marker = ran.clone(); + assert_eq!( + context + .run_blocking(move || marker.store(true, Ordering::Release)) + .await, + Err(BlockingTaskError::StaleGeneration) + ); + let marker = ran.clone(); + assert_eq!( + context + .run_fallible_async_named("stale async", move || async move { + marker.store(true, Ordering::Release); + Ok::<_, ()>(()) + }) + .await, + Err(BlockingTaskError::StaleGeneration) + ); + assert!(!ran.load(Ordering::Acquire)); + owner.request_shutdown().wait().await.unwrap(); + } + + #[tokio::test] + async fn cancelling_global_shutdown_waiter_retains_drain_and_projection() { + let owner = Arc::new(TaskCustodyOwner::new()); + let projected = Arc::new(AtomicBool::new(false)); + let marker = projected.clone(); + let scope = owner + .open_scope(move || async move { + marker.store(true, Ordering::Release); + Ok(()) + }) + .unwrap(); + let (entered, ready) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let task = scope + .prepare( + "retained-effect".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_fallible_async_named("retained global effect", move || async move { + entered.send(()).unwrap(); + released.await.unwrap(); + Ok::<_, ()>(()) + }) + .await; + }, + ) + .unwrap(); + scope.install_gated(task).unwrap().start(); + ready.await.unwrap(); + let receipt = owner.request_shutdown(); + let mut waiting = Box::pin(receipt.clone().wait()); + assert!(futures::poll!(&mut waiting).is_pending()); + drop(waiting); + drop(scope); + let weak = Arc::downgrade(&owner); + drop(owner); + assert!(weak.upgrade().is_some()); + assert!(!projected.load(Ordering::Acquire)); + release.send(()).unwrap(); + receipt.wait().await.unwrap(); + assert!(projected.load(Ordering::Acquire)); + } + + fn start_cancel(transition: CancelTransition) -> bool { + match transition { + CancelTransition::Started(finalizer) | CancelTransition::Existing(finalizer) => { + finalizer.start(); + } + CancelTransition::AlreadyRunning => {} + } + true + } + + async fn acknowledge_failed_projection_through_cancel( + owner: &Arc, + download_id: &str, + ticket: &ProjectionTicket, + ) { + assert_eq!( + owner.settle_projection(ticket), + ProjectionSettlement::FailureUnprojected + ); + assert!(owner.contains(download_id)); + let (acknowledged_sender, acknowledged) = oneshot::channel(); + let transition = owner + .begin_cancel(download_id, move |context, predecessor| async move { + let CancelPredecessor::Observed(observation) = predecessor else { + panic!("failed projector must remain predecessor custody"); + }; + assert!(observation.nested_failures > 0); + assert!(context.complete_transferred_projection(true)); + let _ = acknowledged_sender.send(()); + }) + .unwrap(); + let finalizer = match transition { + CancelTransition::Started(finalizer) => finalizer, + _ => panic!("failed projector must be replaced by one finalizer"), + }; + finalizer.start(); + assert_eq!( + owner.settle_projection(ticket), + ProjectionSettlement::StaleGeneration + ); + tokio::time::timeout(Duration::from_secs(1), acknowledged) + .await + .expect("finalizer must acknowledge transferred failure") + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot(download_id) + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("acknowledging finalizer must reach terminal Join state"); + assert!(ticket.failure_projected_for_test()); + assert!(ticket.settled_for_test()); + assert!(owner.observe_finished(download_id).await.is_some()); + assert!(!owner.contains(download_id)); + assert_eq!( + owner.settle_projection(ticket), + ProjectionSettlement::AlreadySettled + ); + } + + #[tokio::test] + async fn prepared_task_runs_only_after_owned_install_and_start() { + let owner = TaskScope::new_test(); + let ran = Arc::new(AtomicBool::new(false)); + let ran_in_task = ran.clone(); + let prepared = owner + .prepare( + "download".to_string(), + TaskRole::Worker, + move |_| async move { + ran_in_task.store(true, Ordering::SeqCst); + }, + ) + .unwrap(); + + tokio::task::yield_now().await; + assert!(!ran.load(Ordering::SeqCst)); + + let installed = owner.install_gated(prepared).unwrap(); + let generation = installed.generation().clone(); + assert!(owner.snapshot("download").is_some_and(|snapshot| { + owner + .generation_for_test("download") + .is_some_and(|current| current.matches(&generation)) + && snapshot.role == TaskRole::Worker + && !snapshot.finished + })); + assert!(!ran.load(Ordering::SeqCst)); + + installed.start(); + tokio::time::timeout(Duration::from_secs(1), async { + while !ran.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .expect("installed task should start"); + } + + #[tokio::test] + async fn dropping_unstarted_install_generation_matches_cleanup() { + for role in [TaskRole::Worker, TaskRole::RecoveryTransition] { + let owner = TaskScope::new_test(); + let ran = Arc::new(AtomicBool::new(false)); + let ran_in_task = ran.clone(); + let prepared = owner + .prepare("download".to_string(), role, move |_| async move { + ran_in_task.store(true, Ordering::SeqCst); + }) + .unwrap(); + let installed = owner.install_gated(prepared).unwrap(); + assert!(owner.contains("download")); + drop(installed); + assert!(owner.contains("download")); + assert!(!ran.load(Ordering::SeqCst)); + + owner.rescue_abandoned(); + assert!(!owner.contains("download")); + tokio::time::timeout(Duration::from_secs(1), async { + while owner.outstanding_retired_for_test() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("abandoned installed wrapper must be observed to terminal"); + assert_eq!(owner.retired_observations_for_test(), 1); + assert!(!ran.load(Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn abandoned_projection_is_rescued_without_signalling_from_drop() { + let owner = TaskScope::new_test(); + let ran = Arc::new(AtomicBool::new(false)); + let ran_in_projection = ran.clone(); + let prepared = owner + .prepare_projection( + "download".to_string(), + move |_, predecessor| { + let ran = ran_in_projection.clone(); + async move { + assert!(predecessor.is_none()); + ran.store(true, Ordering::SeqCst); + ProjectionOutcome::Committed + } + }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap(); + let InstalledProjection { task, ticket } = + owner.install_projection_gated(prepared).unwrap(); + + // Token destruction is CAS-only. It cannot release the gate while an + // outer state guard may still be unwinding. + drop(task); + tokio::task::yield_now().await; + assert!(!ran.load(Ordering::SeqCst)); + assert!(owner.contains("download")); + + owner.rescue_abandoned(); + assert_eq!(ticket.wait().await, ProjectionOutcome::Committed); + assert!(ran.load(Ordering::SeqCst)); + while owner.settle_projection(&ticket) == ProjectionSettlement::Pending { + tokio::task::yield_now().await; + } + assert_eq!( + owner.settle_projection(&ticket), + ProjectionSettlement::AlreadySettled + ); + } + + #[tokio::test] + async fn abandoned_prepared_collision_is_inert_until_explicit_rescue() { + for rejected_role in [TaskRole::Worker, TaskRole::RecoveryTransition] { + let owner = TaskScope::new_test(); + let first = owner + .prepare("download".to_string(), TaskRole::Worker, |_| async { + std::future::pending::<()>().await; + }) + .unwrap(); + owner.install_gated(first).unwrap().start(); + + let ran = Arc::new(AtomicBool::new(false)); + let ran_in_task = ran.clone(); + let second = owner + .prepare("download".to_string(), rejected_role, move |_| async move { + ran_in_task.store(true, Ordering::SeqCst); + }) + .unwrap(); + let rejected = owner.install_gated(second).unwrap_err(); + drop(rejected); + tokio::task::yield_now().await; + assert!(!ran.load(Ordering::SeqCst)); + assert_eq!(owner.prepared_count_for_test(), 1); + + owner.rescue_abandoned(); + assert_eq!(owner.prepared_count_for_test(), 0); + tokio::time::timeout(Duration::from_secs(1), async { + while owner.outstanding_retired_for_test() != 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("abandoned prepared wrapper must be observed to terminal"); + assert_eq!(owner.retired_observations_for_test(), 1); + assert!(!ran.load(Ordering::SeqCst)); + } + } + + #[tokio::test] + async fn projection_settlement_distinguishes_duplicate_stale_and_missing() { + let owner = TaskScope::new_test(); + let prepared = owner + .prepare_projection( + "projection".to_string(), + |_, _| async { ProjectionOutcome::Committed }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap(); + let ticket = owner.install_projection_gated(prepared).unwrap().start(); + assert_eq!( + owner.settle_projection(&ticket), + ProjectionSettlement::Pending + ); + assert_eq!(ticket.wait().await, ProjectionOutcome::Committed); + while owner.settle_projection(&ticket) == ProjectionSettlement::Pending { + tokio::task::yield_now().await; + } + assert_eq!( + owner.settle_projection(&ticket), + ProjectionSettlement::AlreadySettled + ); + + let missing_cell = Arc::new(ProjectionCell::new(true)); + missing_cell.settle(ProjectionOutcome::Committed); + let missing = ProjectionTicket { + scope: Arc::downgrade(&owner), + download_id: "missing".to_string(), + generation: TaskGeneration::new(), + cell: missing_cell, + }; + assert_eq!( + owner.settle_projection(&missing), + ProjectionSettlement::Missing + ); + + let stale_cell = Arc::new(ProjectionCell::new(true)); + stale_cell.settle(ProjectionOutcome::Committed); + let stale = ProjectionTicket { + scope: Arc::downgrade(&owner), + download_id: "successor".to_string(), + generation: TaskGeneration::new(), + cell: stale_cell, + }; + let successor = owner + .prepare("successor".to_string(), TaskRole::Worker, |_| async { + std::future::pending::<()>().await; + }) + .unwrap(); + owner.install_gated(successor).unwrap().start(); + assert_eq!( + owner.settle_projection(&stale), + ProjectionSettlement::StaleGeneration + ); + assert!(owner.contains("successor")); + } + + #[tokio::test] + async fn projection_catches_call_and_poll_panics_for_both_constructors() { + let owner = TaskScope::new_test(); + + let call = owner + .prepare_projection( + "ownerless-call".to_string(), + |_, _| { + panic!("call-time projection panic"); + #[allow(unreachable_code)] + std::future::ready(ProjectionOutcome::Committed) + }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap(); + let call_ticket = owner.install_projection_gated(call).unwrap().start(); + assert_eq!(call_ticket.wait().await, ProjectionOutcome::Panicked); + + let poll = owner + .prepare_projection( + "ownerless-poll".to_string(), + |_, _| async { + panic!("poll-time projection panic"); + }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap(); + let poll_ticket = owner.install_projection_gated(poll).unwrap().start(); + assert_eq!(poll_ticket.wait().await, ProjectionOutcome::Panicked); + + for (id, call_time) in [("finished-call", true), ("finished-poll", false)] { + let predecessor = owner + .prepare(id.to_string(), TaskRole::Worker, |_| async {}) + .unwrap(); + owner.install_gated(predecessor).unwrap().start(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner.snapshot(id).is_some_and(|task| task.finished) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let transition = if call_time { + owner + .begin_finished_projection( + id, + false, + |_, _| { + panic!("call-time finished projection panic"); + #[allow(unreachable_code)] + std::future::ready(ProjectionOutcome::Committed) + }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap() + } else { + owner + .begin_finished_projection( + id, + false, + |_, _| async { + panic!("poll-time finished projection panic"); + }, + |_| async { ProjectionOutcome::Failed }, + ) + .unwrap() + }; + let ProjectionTransition::Started(projection) = transition else { + panic!("finished predecessor should install a projector"); + }; + let ticket = projection.start(); + assert_eq!(ticket.wait().await, ProjectionOutcome::Panicked); + } + } + + #[tokio::test] + async fn fallback_panics_remain_owned_until_cancel_acknowledges_failure() { + let owner = TaskScope::new_test(); + + let ownerless = owner + .prepare_projection( + "ownerless-double-panic".to_string(), + |_, _| { + panic!("call-time primary projection panic"); + #[allow(unreachable_code)] + std::future::ready(ProjectionOutcome::Committed) + }, + |_| { + panic!("call-time fallback projection panic"); + #[allow(unreachable_code)] + std::future::ready(ProjectionOutcome::Failed) + }, + ) + .unwrap(); + let InstalledProjection { task, ticket } = + owner.install_projection_gated(ownerless).unwrap(); + let abandoned_waiter_ticket = ticket.clone(); + let abandoned_waiter = tokio::spawn(async move { + let _ = abandoned_waiter_ticket.wait().await; + }); + abandoned_waiter.abort(); + let _ = abandoned_waiter.await; + task.start(); + assert_eq!(ticket.wait().await, ProjectionOutcome::Panicked); + acknowledge_failed_projection_through_cancel(&owner, "ownerless-double-panic", &ticket) + .await; + + let predecessor = owner + .prepare( + "finished-double-panic".to_string(), + TaskRole::Worker, + |_| async {}, + ) + .unwrap(); + owner.install_gated(predecessor).unwrap().start(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("finished-double-panic") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let transition = owner + .begin_finished_projection( + "finished-double-panic", + false, + |_, _| async { + panic!("poll-time primary projection panic"); + }, + |_| async { + panic!("poll-time fallback projection panic"); + }, + ) + .unwrap(); + let ProjectionTransition::Started(projection) = transition else { + panic!("finished predecessor should install a projector"); + }; + let ticket = projection.start(); + assert_eq!(ticket.wait().await, ProjectionOutcome::Panicked); + acknowledge_failed_projection_through_cancel(&owner, "finished-double-panic", &ticket) + .await; + } + + #[tokio::test] + async fn superseded_failed_projection_is_unacked_until_finalizer_projection() { + let owner = TaskScope::new_test(); + let (fallback_reached_sender, fallback_reached) = oneshot::channel(); + let (_fallback_release_sender, fallback_release) = oneshot::channel::<()>(); + let prepared = owner + .prepare_projection( + "transferred-failure".to_string(), + |_, _| async { + panic!("primary projection panic"); + }, + move |_| async move { + let _ = fallback_reached_sender.send(()); + let _ = fallback_release.await; + ProjectionOutcome::RolledBack + }, + ) + .unwrap(); + let ticket = owner.install_projection_gated(prepared).unwrap().start(); + tokio::time::timeout(Duration::from_secs(1), fallback_reached) + .await + .expect("primary panic must enter fallback") + .unwrap(); + assert!(!ticket.failure_projected_for_test()); + + let (finalizer_reached_sender, finalizer_reached) = oneshot::channel(); + let (allow_projection_sender, allow_projection) = oneshot::channel(); + let transition = owner + .begin_cancel( + "transferred-failure", + move |context, predecessor| async move { + let CancelPredecessor::Observed(observation) = predecessor else { + panic!("superseded projector must remain predecessor custody"); + }; + assert!(observation.nested_failures > 0); + let _ = finalizer_reached_sender.send(()); + let _ = allow_projection.await; + assert!(context.complete_transferred_projection(true)); + }, + ) + .unwrap(); + let CancelTransition::Started(finalizer) = transition else { + panic!("projection must be replaced by one finalizer"); + }; + finalizer.start(); + assert_eq!(ticket.wait().await, ProjectionOutcome::Superseded); + tokio::time::timeout(Duration::from_secs(1), finalizer_reached) + .await + .expect("finalizer must observe the failed projector") + .unwrap(); + assert!(!ticket.failure_projected_for_test()); + assert!(!ticket.settled_for_test()); + allow_projection_sender.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !ticket.failure_projected_for_test() || !ticket.settled_for_test() { + tokio::task::yield_now().await; + } + }) + .await + .expect("finalizer must acknowledge only after its terminal projection"); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("transferred-failure") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert!(owner + .observe_finished("transferred-failure") + .await + .is_some()); + assert!(!owner.contains("transferred-failure")); + } + + #[tokio::test] + async fn finished_and_panicked_tasks_are_observed_once() { + let owner = TaskScope::new_test(); + let prepared = owner + .prepare("panic".to_string(), TaskRole::Worker, |_| async { + panic!("sentinel panic"); + }) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("panic") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + + let observation = owner.observe_finished("panic").await.unwrap(); + assert_eq!(observation.role, TaskRole::Worker); + assert_eq!(observation.terminal, TaskTerminal::Panicked); + assert_eq!(observation.nested_failures, 0); + assert!(owner.observe_finished("panic").await.is_none()); + } + + #[tokio::test] + async fn cancel_finalizer_drains_registered_blocking_work_before_terminal_callback() { + let owner = TaskScope::new_test(); + let (blocking_started_sender, blocking_started) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + let prepared = owner + .prepare( + "download".to_string(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_blocking(move || { + let _ = blocking_started_sender.send(()); + let _ = release.recv(); + }) + .await; + }, + ) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + blocking_started.await.unwrap(); + + let finalized = Arc::new(AtomicBool::new(false)); + let finalized_in_task = finalized.clone(); + assert!(start_cancel( + owner + .begin_cancel("download", move |_context, predecessor| async move { + let CancelPredecessor::Observed(observation) = predecessor else { + panic!("installed worker must be observed"); + }; + assert_eq!(observation.terminal, TaskTerminal::Cancelled); + finalized_in_task.store(true, Ordering::SeqCst); + },) + .unwrap() + )); + tokio::task::yield_now().await; + assert!(!finalized.load(Ordering::SeqCst)); + + release_sender.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !finalized.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .expect("finalizer should wait for nested blocking work"); + } + + #[tokio::test] + async fn stale_generation_cannot_observe_or_remove_cancel_successor() { + let owner = TaskScope::new_test(); + let prepared = owner + .prepare("download".to_string(), TaskRole::Worker, |_| async { + std::future::pending::<()>().await; + }) + .unwrap(); + let installed = owner.install_gated(prepared).unwrap(); + let stale_generation = installed.generation().clone(); + installed.start(); + let (finish_sender, finish_receiver) = oneshot::channel(); + let transition = owner + .begin_cancel("download", move |_context, _| async move { + let _ = finish_receiver.await; + }) + .unwrap(); + let CancelTransition::Started(finalizer) = transition else { + panic!("worker should transition to a finalizer"); + }; + finalizer.start(); + let successor = owner.generation_for_test("download").unwrap(); + + assert!(owner + .observe_finished_generation("download", &stale_generation) + .await + .is_none()); + assert!(owner.snapshot("download").is_some_and(|snapshot| { + owner + .generation_for_test("download") + .is_some_and(|current| current.matches(&successor)) + && snapshot.role == TaskRole::CancelFinalizer + })); + finish_sender.send(()).unwrap(); + } + + #[tokio::test] + async fn repeated_cancel_keeps_one_finalizer_owner() { + let owner = TaskScope::new_test(); + let prepared = owner + .prepare("download".to_string(), TaskRole::Worker, |_| async { + std::future::pending::<()>().await; + }) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + let count = Arc::new(AtomicUsize::new(0)); + let count_in_finalizer = count.clone(); + let (finish_sender, finish_receiver) = oneshot::channel(); + let first = owner + .begin_cancel("download", move |_context, _| async move { + count_in_finalizer.fetch_add(1, Ordering::SeqCst); + let _ = finish_receiver.await; + }) + .unwrap(); + let CancelTransition::Started(finalizer) = first else { + panic!("first cancellation should install a finalizer"); + }; + finalizer.start(); + let first_generation = owner.generation_for_test("download").unwrap(); + let second = owner.begin_cancel("download", |_, _| async {}).unwrap(); + let CancelTransition::AlreadyRunning = second else { + panic!("repeat cancellation must not replace the finalizer"); + }; + let second_generation = owner.generation_for_test("download").unwrap(); + assert!(first_generation.matches(&second_generation)); + finish_sender.send(()).unwrap(); + tokio::task::yield_now().await; + assert_eq!(count.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn aborted_finalizer_retains_predecessor_drain_and_failure() { + for outcome in ["success", "error", "panic"] { + let owner = TaskScope::new_test(); + let (entered_sender, entered) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + let prepared = owner + .prepare( + "download".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_fallible_blocking_named( + "held cancellation predecessor", + move || -> Result<(), &'static str> { + entered_sender.send(()).unwrap(); + release.recv().unwrap(); + match outcome { + "success" => Ok(()), + "error" => Err("predecessor failure"), + _ => panic!("predecessor panic"), + } + }, + ) + .await; + }, + ) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + entered.await.unwrap(); + let finished = Arc::new(AtomicBool::new(false)); + let finished_in_finalizer = finished.clone(); + let CancelTransition::Started(finalizer) = owner + .begin_cancel("download", move |_, _| async move { + finished_in_finalizer.store(true, Ordering::Release); + }) + .unwrap() + else { + panic!("worker must receive a finalizer") + }; + let generation = finalizer.generation().clone(); + finalizer.start(); + owner.lock_state().tasks["download"].outer.abort(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner.outer_finished_for_test("download") { + tokio::task::yield_now().await; + } + }) + .await + .expect("finalizer outer must observe cancellation"); + let prematurely_finished = owner.snapshot("download").unwrap().finished; + let premature_observation = owner + .observe_finished_generation("download", &generation) + .await; + release_sender.send(()).unwrap(); + assert!(!prematurely_finished, "predecessor still held: {outcome}"); + assert!(premature_observation.is_none()); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("predecessor observation must drain after release"); + let observation = owner + .observe_finished_generation("download", &generation) + .await + .unwrap(); + assert_eq!(observation.terminal, TaskTerminal::Cancelled); + assert_eq!( + observation.nested_failures, + usize::from(outcome != "success") + ); + assert!(!finished.load(Ordering::Acquire)); + } + } + + #[tokio::test] + async fn predecessor_failure_is_retained_without_failing_new_cleanup_effects() { + for abort_after_delivery in [false, true] { + let owner = TaskScope::new_test(); + let (entered_sender, entered) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + let prepared = owner + .prepare( + "download".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_fallible_blocking_named( + "failed predecessor", + move || -> Result<(), &'static str> { + entered_sender.send(()).unwrap(); + release.recv().unwrap(); + Err("predecessor failed") + }, + ) + .await; + }, + ) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + entered.await.unwrap(); + let (delivered_sender, delivered) = oneshot::channel(); + let (finish_sender, finish) = oneshot::channel(); + let CancelTransition::Started(finalizer) = owner + .begin_cancel("download", move |context, predecessor| async move { + let CancelPredecessor::Observed(observation) = predecessor else { + panic!("predecessor must be observed") + }; + assert_eq!(observation.nested_failures, 1); + context + .run_fallible_blocking_named("successful cleanup", || { + Ok::<_, &'static str>(()) + }) + .await + .unwrap() + .unwrap(); + assert_eq!(context.drain_blocking().await, Ok(0)); + delivered_sender.send(()).unwrap(); + let _ = finish.await; + }) + .unwrap() + else { + panic!("worker must receive a finalizer") + }; + let generation = finalizer.generation().clone(); + finalizer.start(); + release_sender.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), delivered) + .await + .expect("predecessor failure must not prevent cleanup drain") + .unwrap(); + if abort_after_delivery { + owner.lock_state().tasks["download"].outer.abort(); + } else { + finish_sender.send(()).unwrap(); + } + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("finalizer must become observable"); + let observation = owner + .observe_finished_generation("download", &generation) + .await + .unwrap(); + assert_eq!(observation.nested_failures, 1); + assert_eq!( + observation.terminal, + if abort_after_delivery { + TaskTerminal::Cancelled + } else { + TaskTerminal::Completed + } + ); + } + } + + #[tokio::test] + async fn failed_predecessor_observation_closes_receipt_only_after_effect_drain() { + let owner = TaskScope::new_test(); + let (entered_sender, entered) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + let prepared = owner + .prepare( + "download".into(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_blocking(move || { + entered_sender.send(()).unwrap(); + release.recv().unwrap(); + }) + .await; + }, + ) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + entered.await.unwrap(); + let mut predecessor = owner.lock_state().tasks.remove("download").unwrap(); + let outer_abort = predecessor.outer.abort_handle(); + outer_abort.abort(); + tokio::time::timeout(Duration::from_secs(1), async { + while !outer_abort.is_finished() { + tokio::task::yield_now().await; + } + }) + .await + .expect("predecessor outer must finish before observing its nested work"); + let poisoned = Arc::new(ProjectionCell::new(false)); + assert!(std::panic::catch_unwind(AssertUnwindSafe(|| { + let _guard = poisoned.state.lock().unwrap(); + panic!("poison predecessor provenance"); + })) + .is_err()); + // Exercise the observer's own bookkeeping-failure path without placing + // a poisoned cell in a live successor's unrelated projection state. + predecessor.projection = Some(poisoned); + let completion = Arc::new(NestedCompletion { + finished: AtomicBool::new(false), + failed: AtomicBool::new(false), + notify: Notify::new(), + }); + let (start, started) = oneshot::channel(); + let (receipt_sender, mut receipt) = oneshot::channel(); + let mut observer = Box::pin(observe_cancellation_predecessor( + Some(predecessor), + false, + started, + completion.clone(), + receipt_sender, + )); + start.send(()).unwrap(); + let observation_while_held = futures::poll!(&mut observer); + let finished_while_held = completion.finished.load(Ordering::Acquire); + let receipt_while_held = receipt.try_recv(); + release_sender.send(()).unwrap(); + assert!(observation_while_held.is_pending()); + tokio::time::timeout(Duration::from_secs(1), observer) + .await + .expect("failed observer must drain before completing"); + assert!(!finished_while_held); + assert!(matches!( + receipt_while_held, + Err(oneshot::error::TryRecvError::Empty) + )); + assert!(completion.finished.load(Ordering::Acquire)); + assert!(completion.failed.load(Ordering::Acquire)); + assert!( + receipt.await.is_err(), + "failed observation must not synthesize Absent" + ); + } + + #[tokio::test] + async fn blocking_panic_is_retained_when_outer_receiver_is_cancelled() { + let owner = TaskScope::new_test(); + let (blocking_started_sender, blocking_started) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + let prepared = owner + .prepare( + "download".to_string(), + TaskRole::Worker, + move |context| async move { + let _ = context + .run_blocking(move || { + let _ = blocking_started_sender.send(()); + let _ = release.recv(); + panic!("nested sentinel panic"); + }) + .await; + }, + ) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + blocking_started.await.unwrap(); + + let (observed_sender, observed) = oneshot::channel(); + assert!(start_cancel( + owner + .begin_cancel("download", move |_context, predecessor| async move { + let _ = observed_sender.send(predecessor); + },) + .unwrap() + )); + release_sender.send(()).unwrap(); + let CancelPredecessor::Observed(observation) = observed.await.unwrap() else { + panic!("installed worker must be observed"); + }; + assert_eq!(observation.terminal, TaskTerminal::Cancelled); + assert_eq!(observation.nested_failures, 1); + } + + #[tokio::test] + async fn finished_unobserved_finalizer_is_replaced_and_observed_once() { + let owner = TaskScope::new_test(); + let prepared = owner + .prepare("download".to_string(), TaskRole::Worker, |_| async {}) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + let count = Arc::new(AtomicUsize::new(0)); + let count_in_finalizer = count.clone(); + assert!(start_cancel( + owner + .begin_cancel("download", move |_, _| async move { + count_in_finalizer.fetch_add(1, Ordering::SeqCst); + },) + .unwrap() + )); + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + + let (predecessor_sender, predecessor) = oneshot::channel(); + let replacement = owner + .begin_cancel("download", move |_, predecessor| async move { + let _ = predecessor_sender.send(predecessor); + }) + .unwrap(); + let CancelTransition::Started(replacement) = replacement else { + panic!("finished finalizer must be replaced by an observing finalizer"); + }; + replacement.start(); + let CancelPredecessor::Observed(observation) = predecessor.await.unwrap() else { + panic!("replacement must observe the finished finalizer"); + }; + assert_eq!(observation.role, TaskRole::CancelFinalizer); + assert_eq!(observation.terminal, TaskTerminal::Completed); + assert_eq!(count.load(Ordering::SeqCst), 1); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + owner.observe_finished("download").await.unwrap().role, + TaskRole::CancelFinalizer + ); + assert!(owner.observe_finished("download").await.is_none()); + } + + #[tokio::test] + async fn cancelling_an_outer_drain_keeps_nested_custody_with_the_finalizer() { + let owner = TaskScope::new_test(); + let (drain_sender, drain_receiver) = oneshot::channel(); + let prepared = owner + .prepare( + "download".to_string(), + TaskRole::Worker, + move |context| async move { + let _ = drain_receiver.await; + let _ = context.drain_blocking().await; + }, + ) + .unwrap(); + let installed = owner.install_gated(prepared).unwrap(); + let generation = installed.generation().clone(); + installed.start(); + + let (blocking_started_sender, blocking_started) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + let result = owner + .register_blocking( + "download", + &generation, + "drain cancellation sentinel", + move || { + let _ = blocking_started_sender.send(()); + let _ = release.recv(); + }, + ) + .unwrap(); + drop(result); + blocking_started.await.unwrap(); + let (drain_started_sender, drain_started) = oneshot::channel(); + let drain_started_sender = Arc::new(Mutex::new(Some(drain_started_sender))); + owner.set_drain_observer(Some(Arc::new(move || { + if let Some(sender) = drain_started_sender.lock().unwrap().take() { + let _ = sender.send(()); + } + }))); + drain_sender.send(()).unwrap(); + drain_started.await.unwrap(); + assert_eq!(owner.nested_count_for_test("download"), Some(1)); + + let terminal = Arc::new(AtomicBool::new(false)); + let terminal_in_finalizer = terminal.clone(); + assert!(start_cancel( + owner + .begin_cancel("download", move |_, _| async move { + terminal_in_finalizer.store(true, Ordering::SeqCst); + },) + .unwrap() + )); + tokio::task::yield_now().await; + let terminal_before_release = terminal.load(Ordering::SeqCst); + release_sender.send(()).unwrap(); + owner.set_drain_observer(None); + + assert!( + !terminal_before_release, + "cancelling a drain must not detach its registered blocking owner" + ); + tokio::time::timeout(Duration::from_secs(1), async { + while !terminal.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .expect("the same runtime must drive the finalizer after nested release"); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("finalizer should reach an observable terminal outcome"); + let observation = owner.observe_finished("download").await.unwrap(); + assert_eq!(observation.role, TaskRole::CancelFinalizer); + assert_eq!(observation.terminal, TaskTerminal::Completed); + assert_eq!(observation.nested_failures, 0); + } + + #[tokio::test] + async fn finished_outer_is_not_observable_until_registered_blocking_work_finishes() { + let owner = TaskScope::new_test(); + let (outer_release_sender, outer_release) = oneshot::channel(); + let prepared = owner + .prepare( + "download".to_string(), + TaskRole::Worker, + move |_| async move { + let _ = outer_release.await; + }, + ) + .unwrap(); + let installed = owner.install_gated(prepared).unwrap(); + let generation = installed.generation().clone(); + let (blocking_started_sender, blocking_started) = oneshot::channel(); + let (release_sender, release) = std::sync::mpsc::channel(); + // Register the real held blocking work synchronously through the + // owner/generation before allowing the outer future to complete. + let nested_result = owner + .register_blocking( + "download", + &generation, + "finished outer held operation", + move || { + let _ = blocking_started_sender.send(()); + let _ = release.recv(); + }, + ) + .unwrap(); + installed.start(); + tokio::time::timeout(Duration::from_secs(1), blocking_started) + .await + .expect("registered blocking work should start") + .unwrap(); + outer_release_sender.send(()).unwrap(); + tokio::task::yield_now().await; + + assert!(owner + .snapshot("download") + .is_some_and(|snapshot| { !snapshot.finished && snapshot.role == TaskRole::Worker })); + assert!(owner.observe_finished("download").await.is_none()); + assert!(owner.contains("download")); + + release_sender.send(()).unwrap(); + nested_result.await.unwrap().unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner + .snapshot("download") + .is_some_and(|snapshot| snapshot.finished) + { + tokio::task::yield_now().await; + } + }) + .await + .expect("nested completion should make the owner observable"); + let observation = owner + .observe_finished("download") + .await + .expect("finished nested work must be observable"); + assert_eq!(observation.role, TaskRole::Worker); + assert_eq!(observation.terminal, TaskTerminal::Completed); + assert_eq!(observation.nested_failures, 0); + assert!(!owner.contains("download")); + } + + #[tokio::test] + async fn completed_nested_work_is_reaped_without_losing_failure_evidence() { + let owner = TaskScope::new_test(); + let owner_in_task = owner.clone(); + let (metrics_sender, metrics) = oneshot::channel(); + let prepared = owner + .prepare( + "download".to_string(), + TaskRole::Worker, + move |context| async move { + let mut retained_max = 0; + for index in 0..512 { + let result = context + .run_blocking(move || { + if index == 0 { + panic!("archived nested failure sentinel"); + } + }) + .await; + if index == 0 { + assert!(matches!(result, Err(BlockingTaskError::Join(_)))); + } else { + result.unwrap(); + } + retained_max = retained_max.max( + owner_in_task + .nested_count_for_test("download") + .unwrap_or_default(), + ); + } + let drained_failures = context.drain_blocking().await.unwrap(); + let _ = metrics_sender.send((retained_max, drained_failures)); + std::future::pending::<()>().await; + }, + ) + .unwrap(); + owner.install_gated(prepared).unwrap().start(); + + let (retained_max, drained_failures) = metrics.await.unwrap(); + assert!( + retained_max <= 2, + "sequential blocking operations must retain only the current and terminalizing observer" + ); + assert_eq!(drained_failures, 1); + + let (predecessor_sender, predecessor) = oneshot::channel(); + assert!(start_cancel( + owner + .begin_cancel("download", move |_, predecessor| async move { + let _ = predecessor_sender.send(predecessor); + },) + .unwrap() + )); + let CancelPredecessor::Observed(observation) = predecessor.await.unwrap() else { + panic!("the worker remains owned until cancellation"); + }; + assert_eq!(observation.nested_failures, 1); + } + + #[tokio::test] + async fn state_only_cancellation_does_not_fabricate_a_worker_observation() { + let owner = TaskScope::new_test(); + let owner_in_finalizer = owner.clone(); + let (observation_sender, observation) = oneshot::channel(); + assert!(start_cancel( + owner + .begin_cancel("state-only", move |_, observation| async move { + assert!(owner_in_finalizer.contains("state-only")); + let _ = observation_sender.send(observation); + },) + .unwrap() + )); + + assert_eq!(observation.await.unwrap(), CancelPredecessor::Absent); + } +} diff --git a/rust/crates/pumas-core/src/model_library/hf/download.rs b/rust/crates/pumas-core/src/model_library/hf/download.rs index 75e36936..f892e595 100644 --- a/rust/crates/pumas-core/src/model_library/hf/download.rs +++ b/rust/crates/pumas-core/src/model_library/hf/download.rs @@ -2362,11 +2362,7 @@ impl HuggingFaceClient { /// Permanently close download admission and observe owned work to completion. /// Cancelling one waiter does not cancel the shared drain or its result. pub async fn shutdown_downloads(&self) -> Result<()> { - let downloads = self.downloads.clone(); - let publications = self.download_publications.clone(); - self.download_tasks - .shutdown(move || project_download_shutdown(downloads, publications)) - .await + self.download_tasks.shutdown().await } async fn reconcile_download_reads(&self) { @@ -3422,12 +3418,11 @@ impl HuggingFaceClient { let (admission_completed, admission_completion) = tokio::sync::watch::channel(false); let admission_identity = super::lifecycle::PendingAdmissionIdentity { destination: destination.identity(), - repo_id: request.repo_id.clone(), - revision: revision.clone(), - files: files - .iter() - .map(|file| (file.filename.clone(), file.size, file.sha256.clone())) - .collect(), + selection: super::acquisition_source::manifest_for_download( + &request.repo_id, + &revision, + &files, + )?, }; let installed = { let downloads = self.downloads.write().await; @@ -5624,12 +5619,11 @@ impl HuggingFaceClient { if let RecoveryLaunchPlan::Existing { download_id } = &launch_plan { let admission_identity = super::lifecycle::PendingAdmissionIdentity { destination: verified.destination.identity(), - repo_id: verified.repo_id.clone(), - revision: DownloadRevision::legacy_main(), - files: files - .iter() - .map(|file| (file.filename.clone(), file.size, file.sha256.clone())) - .collect(), + selection: super::acquisition_source::manifest_for_download( + &verified.repo_id, + &DownloadRevision::legacy_main(), + &files, + )?, }; let (admission_completed, admission_completion) = tokio::sync::watch::channel(false); let transition_download_id = download_id.clone(); @@ -15076,7 +15070,8 @@ mod tests { .unwrap() .unwrap(); let weak_client = Arc::downgrade(&client); - let weak_owner = Arc::downgrade(&client.download_tasks); + // The HF facade owns model policy; its scope owns the actual drain. + let weak_owner = Arc::downgrade(&**client.download_tasks); if explicit_shutdown { let shutdown_client = client.clone(); let shutdown = diff --git a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs index 22d0f59e..e86a3f04 100644 --- a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs +++ b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs @@ -1,78 +1,24 @@ -//! Ownership of asynchronous HuggingFace download work. -//! -//! Request futures prepare work, but this module owns every installed Tokio -//! handle. Installation is synchronous and gated so state and task custody can -//! be committed together before work starts. Opaque allocation identities -//! prevent an old task from observing or removing its successor. +//! Model download authority layered over the shared acquisition task owner. +//! Root grants, destination queues, and model admission matching remain here; +//! task/effect/projection handles belong only to acquisition custody. use std::collections::{HashMap, HashSet, VecDeque}; -use std::fmt; use std::future::Future; -use std::panic::AssertUnwindSafe; -#[cfg(test)] -use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering}; +use std::ops::Deref; use std::sync::{Arc, Mutex, Weak}; +use tokio::sync::Notify; -use futures::FutureExt; -use tokio::sync::{oneshot, Notify}; -use tokio::task::JoinHandle; - +use crate::acquisition::task_custody::{self, TaskCustodyOwner, TaskScope}; +pub(super) use crate::acquisition::task_custody::{ + CancelPredecessor, CancelTransition, InstalledProjection, PreparedTask, ProjectionOutcome, + ProjectionSettlement, ProjectionTransition, TaskGeneration, TaskObservation, TaskRole, + TaskSnapshot, TaskTerminal, +}; +use crate::acquisition::ArtifactManifest; use crate::model_library::download_recovery::{ DestinationIdentity, DestinationRootIdentity, DownloadDestinationRoot, RootExecutionGrant, }; -type FallibleBlockingReceiver = - oneshot::Receiver, String>>; - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum TaskRole { - Invocation, - AdmissionTransition, - RecoveryTransition, - Worker, - CancelFinalizer, - TerminalProjection, -} - -#[derive(Clone)] -pub(super) struct TaskGeneration(Arc); - -impl PartialEq for TaskGeneration { - fn eq(&self, other: &Self) -> bool { - self.matches(other) - } -} - -impl Eq for TaskGeneration {} - -impl TaskGeneration { - fn new() -> Self { - Self(Arc::new(Notify::new())) - } - - pub(super) fn wake_pause(&self) { - self.0.notify_waiters(); - } - - pub(super) fn matches(&self, other: &Self) -> bool { - Arc::ptr_eq(&self.0, &other.0) - } - - fn key(&self) -> usize { - Arc::as_ptr(&self.0) as usize - } -} - -impl fmt::Debug for TaskGeneration { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_tuple("TaskGeneration") - .field(&Arc::as_ptr(&self.0)) - .finish() - } -} - #[derive(Clone)] struct DestinationClaim { download_id: String, @@ -352,649 +298,319 @@ impl DestinationExecutionOwner { } } -#[derive(Clone, Debug)] -pub(super) struct TaskSnapshot { - pub(super) role: TaskRole, - pub(super) finished: bool, - pub(super) outer_finished: bool, - pub(super) started: bool, +#[derive(Default)] +struct ModelRootGrants { + grants: Mutex>, + changed: Notify, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -#[repr(u8)] -enum TaskStartState { - Gated = 0, - Running = 1, - Abandoned = 2, +#[derive(Default)] +struct RootGrantSlot { + grant: Weak, + acquiring: bool, } -impl TaskStartState { - fn load(state: &AtomicU8) -> Self { - match state.load(Ordering::Acquire) { - 0 => Self::Gated, - 1 => Self::Running, - _ => Self::Abandoned, - } - } +enum GrantAcquisition { + Reuse(Arc), + Open, + Wait, } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum TaskTerminal { - Completed, - Cancelled, - Panicked, +/// HF policy facade; all task state is held in its shared custody scope. +pub(super) struct DownloadTaskOwner { + scope: Arc, + // Standalone clients construct their own supervisor; an injected supervisor + // remains with its composition owner and only HF's scope is closed here. + owned_supervisor: Option>, + roots: ModelRootGrants, } -#[derive(Clone, Debug, Eq, PartialEq)] -pub(super) struct TaskObservation { - pub(super) generation: TaskGeneration, - pub(super) role: TaskRole, - pub(super) terminal: TaskTerminal, - pub(super) nested_failures: usize, - pub(super) outer_finished_before_replacement: bool, +impl std::fmt::Debug for DownloadTaskOwner { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("DownloadTaskOwner") + .field("scope", &self.scope) + .finish_non_exhaustive() + } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum ProjectionOutcome { - Pending, - Committed, - RolledBack, - Failed, - Panicked, - Superseded, - Shutdown, +impl Deref for DownloadTaskOwner { + type Target = Arc; + fn deref(&self) -> &Self::Target { + &self.scope + } } -#[derive(Debug)] -struct ProjectionCellState { - predecessor_ready: bool, - predecessor: Option, - outcome: ProjectionOutcome, - settled: bool, - failed: bool, - failure_projected: bool, +#[derive(Clone, PartialEq, Eq)] +pub(super) struct PendingAdmissionIdentity { + pub(super) destination: DestinationIdentity, + pub(super) selection: ArtifactManifest, } -#[derive(Debug)] -struct ProjectionCell { - state: Mutex, - inherited: Mutex>>, - notify: Notify, +#[derive(Clone)] +pub(crate) struct TaskContext { + inner: task_custody::TaskContext, + model_owner: Weak, + root_grant: Option>, } -impl ProjectionCell { - fn new(predecessor_ready: bool) -> Self { - Self { - state: Mutex::new(ProjectionCellState { - predecessor_ready, - predecessor: None, - outcome: ProjectionOutcome::Pending, - settled: false, - failed: false, - failure_projected: false, - }), - inherited: Mutex::new(Vec::new()), - notify: Notify::new(), - } - } - - fn inherit(&self, cell: Arc) { - self.inherited - .lock() - .expect("HF inherited projection-cell lock poisoned") - .push(cell); +impl Deref for TaskContext { + type Target = task_custody::TaskContext; + fn deref(&self) -> &Self::Target { + &self.inner } +} - fn record_predecessor(&self, observation: TaskObservation) { - { - let mut state = self.state.lock().expect("HF projection-cell lock poisoned"); - state.predecessor = Some(observation); - state.predecessor_ready = true; +impl TaskContext { + fn wrap(inner: task_custody::TaskContext, model_owner: Weak) -> Self { + Self { + inner, + model_owner, + root_grant: None, } - self.notify.notify_waiters(); } - - async fn wait_for_predecessor(&self) -> Option { - loop { - let notified = self.notify.notified(); - let ready = { - let state = self.state.lock().expect("HF projection-cell lock poisoned"); - state.predecessor_ready.then(|| state.predecessor.clone()) - }; - if let Some(predecessor) = ready { - return predecessor; - } - notified.await; - } + /// Retain existing native exclusion across a nested owned library effect. + /// The library validates this grant against its configured root. + pub(crate) fn held_root_execution_grant(&self) -> crate::Result> { + self.root_grant + .clone() + .ok_or_else(|| crate::PumasError::Config { + message: "Download root execution grant is unavailable".into(), + }) } - fn settle(&self, outcome: ProjectionOutcome) { - { - let mut state = self.state.lock().expect("HF projection-cell lock poisoned"); - if state.outcome != ProjectionOutcome::Pending { - if matches!( - outcome, - ProjectionOutcome::Failed | ProjectionOutcome::Panicked - ) { - state.failed = true; - } - return; - } - if matches!( - outcome, - ProjectionOutcome::Failed | ProjectionOutcome::Panicked - ) { - state.failed = true; - } - state.outcome = outcome; - } - self.notify.notify_waiters(); + /// Scope physical exclusion to mutation, not to historical task entries. + /// Acquisition itself is retained work: a cancelled waiter cannot strand + /// the in-progress slot or detach a newly opened native lock. + pub(crate) async fn with_root_grant( + &self, + root: DownloadDestinationRoot, + ) -> crate::Result { + let owner = self + .model_owner + .upgrade() + .ok_or(crate::PumasError::DownloadLifecycleClosed)?; + let context = self.clone(); + let grant = self + .run_fallible_async_named("acquire download root grant", move || async move { + // A refused acquisition has no protected effects and must not poison + // shutdown as a failed effect. Panics and observation failures still do. + Ok::<_, std::convert::Infallible>(owner.acquire_root_grant(&context, root).await) + }) + .await + .map_err(|error| { + crate::PumasError::Other(format!("Download root grant observation failed: {error}")) + })? + .expect("infallible acquisition envelope")?; + let mut scoped = self.clone(); + scoped.inner = scoped.inner.with_effect_lease(Some(grant.clone())); + scoped.root_grant = Some(grant); + Ok(scoped) } - fn outcome(&self) -> ProjectionOutcome { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .outcome + pub(crate) fn without_root_grant(&self) -> Self { + let mut context = self.clone(); + context.inner = context.inner.with_effect_lease(None); + context.root_grant = None; + context } - async fn wait(&self) -> ProjectionOutcome { - loop { - let notified = self.notify.notified(); - let outcome = self.outcome(); - if outcome != ProjectionOutcome::Pending { - return outcome; - } - notified.await; - } + /// Preserve the receiving generation while transferring protected custody. + pub(crate) fn inherit_root_grant(&self, source: &Self) -> Self { + assert!( + self.inner.shares_scope(&source.inner), + "root grant transfer requires the same task owner" + ); + let mut context = self.clone(); + context.inner = context.inner.with_effect_lease( + source + .root_grant + .clone() + .map(|grant| grant as Arc), + ); + context.root_grant = source.root_grant.clone(); + context } +} - fn mark_settled(&self) { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .settled = true; +impl DownloadTaskOwner { + #[cfg(test)] + pub(super) fn new() -> Self { + Self::new_with_finalizer(|| async { Ok(()) }) } - fn is_settled(&self) -> bool { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .settled + pub(super) fn new_with_finalizer(finalizer: F) -> Self + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let supervisor = Arc::new(TaskCustodyOwner::new()); + let mut owner = Self::with_supervisor(supervisor.clone(), finalizer) + .expect("a fresh task supervisor admits its first scope"); + owner.owned_supervisor = Some(supervisor); + owner } - fn mark_failed(&self) { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .failed = true; + pub(super) fn with_supervisor( + supervisor: Arc, + finalizer: F, + ) -> crate::Result + where + F: FnOnce() -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + Ok(Self { + scope: supervisor.open_scope(finalizer)?, + owned_supervisor: None, + roots: ModelRootGrants::default(), + }) } - fn acknowledge_failure_projection(&self) { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .failure_projected = true; - let inherited = self - .inherited - .lock() - .expect("HF inherited projection-cell lock poisoned") - .clone(); - for cell in inherited { - cell.acknowledge_failure_projection(); - cell.mark_settled(); - } - self.notify.notify_waiters(); + pub(super) fn request_shutdown(self: &Arc) -> task_custody::ShutdownReceipt { + let receipt = self.scope.request_shutdown(); + self.owned_supervisor + .as_ref() + .map_or(receipt, |owner| owner.request_shutdown()) } - fn is_ready_to_settle(&self) -> bool { - let state = self.state.lock().expect("HF projection-cell lock poisoned"); - state.outcome != ProjectionOutcome::Pending && (!state.failed || state.failure_projected) + pub(super) async fn shutdown(self: &Arc) -> crate::Result<()> { + if self.owned_supervisor.is_some() { + self.request_shutdown().wait().await + } else { + self.scope.shutdown().await + } } - fn has_unprojected_failure(&self) -> bool { - let state = self.state.lock().expect("HF projection-cell lock poisoned"); - state.outcome != ProjectionOutcome::Pending && state.failed && !state.failure_projected + pub(super) async fn run_invocation( + self: &Arc, + operation: F, + ) -> crate::Result + where + T: Send + 'static, + F: FnOnce(TaskContext) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let owner = Arc::downgrade(self); + self.scope + .run_invocation(move |context| operation(TaskContext::wrap(context, owner))) + .await } - #[cfg(test)] - fn failure_projected(&self) -> bool { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .failure_projected + pub(super) fn prepare( + self: &Arc, + id: String, + role: TaskRole, + work: F, + ) -> crate::Result + where + F: FnOnce(TaskContext) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let owner = Arc::downgrade(self); + self.scope.prepare(id, role, move |context| { + work(TaskContext::wrap(context, owner)) + }) } - fn failed(&self) -> bool { - self.state - .lock() - .expect("HF projection-cell lock poisoned") - .failed + pub(super) fn prepare_projection( + self: &Arc, + id: String, + project: F, + project_panic: P, + ) -> crate::Result + where + F: FnOnce(TaskContext, Option) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + P: FnOnce(TaskContext) -> PFut + Send + 'static, + PFut: Future + Send + 'static, + { + let owner = Arc::downgrade(self); + let panic_owner = owner.clone(); + self.scope.prepare_projection( + id, + move |context, predecessor| project(TaskContext::wrap(context, owner), predecessor), + move |context| project_panic(TaskContext::wrap(context, panic_owner)), + ) } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) enum BlockingTaskError { - StaleGeneration, - Join(String), - ResultChannelClosed, -} -impl fmt::Display for BlockingTaskError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::StaleGeneration => formatter.write_str("task generation is no longer current"), - Self::Join(detail) => write!(formatter, "blocking task failed: {detail}"), - Self::ResultChannelClosed => formatter.write_str("blocking result channel closed"), - } + pub(super) fn begin_cancel( + self: &Arc, + id: &str, + finish: F, + ) -> crate::Result + where + F: FnOnce(TaskContext, CancelPredecessor) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + { + let owner = Arc::downgrade(self); + self.scope.begin_cancel(id, move |context, predecessor| { + finish(TaskContext::wrap(context, owner), predecessor) + }) } -} - -struct NestedTask { - handle: JoinHandle<()>, - completion: Arc, - failure_kind: NestedFailureKind, -} - -enum NestedFailureKind { - Effect, - Predecessor, -} - -struct NestedCompletion { - finished: AtomicBool, - failed: AtomicBool, - notify: Notify, -} - -struct RetiredTask { - observer: JoinHandle, -} - -enum StartGate { - Work(oneshot::Sender<()>), - Custody(oneshot::Sender<()>), -} -impl StartGate { - fn start(self) { - match self { - Self::Work(sender) | Self::Custody(sender) => { - let _ = sender.send(()); - } - } + pub(super) fn begin_finished_projection( + self: &Arc, + id: &str, + inherit_failure: bool, + project: F, + project_panic: P, + ) -> crate::Result + where + F: FnOnce(TaskContext, Option) -> Fut + Send + 'static, + Fut: Future + Send + 'static, + P: FnOnce(TaskContext) -> PFut + Send + 'static, + PFut: Future + Send + 'static, + { + let owner = Arc::downgrade(self); + let panic_owner = owner.clone(); + self.scope.begin_finished_projection( + id, + inherit_failure, + move |context, predecessor| project(TaskContext::wrap(context, owner), predecessor), + move |context| project_panic(TaskContext::wrap(context, panic_owner)), + ) } - fn drain(self) { - if let Self::Custody(sender) = self { - let _ = sender.send(()); - } + pub(super) fn bind_pending_admission( + &self, + id: &str, + generation: &TaskGeneration, + identity: PendingAdmissionIdentity, + completed: tokio::sync::watch::Receiver, + ) { + self.scope + .bind_pending_admission(id, generation, Arc::new(identity), completed); } -} - -struct TaskEntry { - admission: Option<(PendingAdmissionIdentity, tokio::sync::watch::Receiver)>, - generation: TaskGeneration, - role: TaskRole, - outer: JoinHandle<()>, - nested: Vec, - nested_failures_archived: usize, - predecessor_failures_archived: usize, - projection: Option>, - starts: Vec, - superseded_projection: Option>, - abort_on_start: Vec, - start_state: Arc, -} - -#[derive(Clone, PartialEq, Eq)] -pub(super) struct PendingAdmissionIdentity { - pub(super) destination: crate::model_library::download_recovery::DestinationIdentity, - pub(super) repo_id: String, - pub(super) revision: crate::model_library::artifact_identity::DownloadRevision, - pub(super) files: Vec<(String, Option, Option)>, -} - -struct PreparedEntry { - download_id: String, - generation: TaskGeneration, - role: TaskRole, - start: oneshot::Sender<()>, - outer: JoinHandle<()>, - projection: Option>, - start_state: Arc, -} -impl TaskEntry { - fn finished(&self) -> bool { - self.outer.is_finished() && self.nested.iter().all(|nested| nested.handle.is_finished()) + pub(super) fn pending_admission( + &self, + identity: &PendingAdmissionIdentity, + ) -> Option<(String, tokio::sync::watch::Receiver)> { + self.scope + .admission_snapshots(TaskRole::AdmissionTransition) + .into_iter() + .find_map(|snapshot| { + (snapshot.metadata.downcast_ref::() == Some(identity)) + .then_some((snapshot.operation_id, snapshot.completed)) + }) } - fn reap_completed_nested(&mut self) { - let mut retained = Vec::with_capacity(self.nested.len()); - for mut nested in self.nested.drain(..) { - let observed = if nested.handle.is_finished() { - (&mut nested.handle).now_or_never() - } else { - None - }; - if let Some(result) = observed { - let failures = usize::from( - result.is_err() || nested.completion.failed.load(Ordering::Acquire), - ); - match nested.failure_kind { - NestedFailureKind::Effect => self.nested_failures_archived += failures, - NestedFailureKind::Predecessor => { - self.predecessor_failures_archived += failures - } - } - } else { - retained.push(nested); - } - } - self.nested = retained; + pub(super) fn pending_recovery_admission( + &self, + id: &str, + identity: &PendingAdmissionIdentity, + ) -> Option<(TaskGeneration, tokio::sync::watch::Receiver)> { + self.scope + .admission_snapshots(TaskRole::RecoveryTransition) + .into_iter() + .find_map(|snapshot| { + (snapshot.operation_id == id + && snapshot.metadata.downcast_ref::() + == Some(identity)) + .then_some((snapshot.generation, snapshot.completed)) + }) } -} - -#[cfg(test)] -type BlockingObserver = Arc; -#[cfg(test)] -type TaskIdsObserver = Arc; - -#[cfg(test)] -type DrainObserver = Arc; - -#[cfg(test)] -type SnapshotObserver = Arc) + Send + Sync>; - -#[cfg(test)] -type CancellationCheckObserver = Arc; - -#[cfg(test)] -type CancelReplacementObserver = Arc; - -#[cfg(test)] -type WorkerProjectionObserver = Arc; - -#[cfg(test)] -type BlockingResultObserver = Arc; - -#[cfg(test)] -type BlockingFailureObserver = Arc bool + Send + Sync>; - -#[cfg(test)] -type AmbientAdmissionObserver = Arc; - -#[cfg(test)] -type ProjectionObserver = Arc; - -#[derive(Default)] -pub(super) struct DownloadTaskOwner { - state: Mutex, - root_grant_changed: Notify, - retired_observations: AtomicUsize, - #[cfg(test)] - blocking_observer: Mutex>, - #[cfg(test)] - ids_observer: Mutex>, - #[cfg(test)] - drain_observer: Mutex>, - #[cfg(test)] - snapshot_observer: Mutex>, - #[cfg(test)] - cancellation_check_observer: Mutex>, - #[cfg(test)] - cancel_replacement_observer: Mutex>, - #[cfg(test)] - worker_projection_observer: Mutex>, - #[cfg(test)] - blocking_result_observer: Mutex>, - #[cfg(test)] - blocking_failure_observer: Mutex>, - #[cfg(test)] - ambient_admission_observer: Mutex>, - #[cfg(test)] - projection_observer: Mutex>, -} - -#[derive(Default)] -struct OwnerState { - // Admission, ownership transfers, and shutdown capture share this mutex. - // Entries leave these populations only for another registered observer. - closed: bool, - root_grants: HashMap, - tasks: HashMap, - prepared: HashMap, - retired: Vec, - retired_failures: usize, - shutdown: Option, - shutdown_driver: Option>, -} - -#[derive(Default)] -struct RootGrantSlot { - grant: Weak, - acquiring: bool, -} - -struct InvocationWaiter { - owner: Arc, - id: String, - generation: TaskGeneration, -} - -enum GrantAcquisition { - Reuse(Arc), - Open, - Wait, -} - -impl Drop for InvocationWaiter { - fn drop(&mut self) { - let mut state = self - .owner - .state - .lock() - .expect("HF task owner lock poisoned"); - let start = if state - .tasks - .get(&self.id) - .is_some_and(|entry| entry.generation.matches(&self.generation)) - { - state - .tasks - .remove(&self.id) - .map(|entry| retire_entry(&mut state, entry)) - } else { - None - }; - drop(state); - if let Some(start) = start { - let _ = start.send(()); - } - } -} - -fn retire_entry(state: &mut OwnerState, mut entry: TaskEntry) -> oneshot::Sender<()> { - // The observer is registered in custody before this lock is released. - // It is never aborted: blocking descendants must remain owned through join. - entry.outer.abort(); - for abort in entry.abort_on_start.drain(..) { - abort.abort(); - } - let (start, started) = oneshot::channel(); - let observer = tokio::spawn(async move { - let _ = started.await; - for gate in entry.starts.drain(..) { - gate.drain(); - } - drain_shutdown_entry(entry).await - }); - state.retired.push(RetiredTask { observer }); - start -} - -async fn drain_shutdown_entry(entry: TaskEntry) -> usize { - let projection = entry.projection.clone(); - let superseded = entry.superseded_projection.clone(); - let role = entry.role; - let observed = AssertUnwindSafe(observe_entry(entry, role, false)) - .catch_unwind() - .await; - let mut failures = match observed { - Ok(observation) => { - observation.nested_failures - + usize::from(observation.terminal == TaskTerminal::Panicked) - } - Err(_) => 1, - }; - for cell in [projection, superseded].into_iter().flatten() { - // A broken receipt must not drop unrelated entries still awaiting - // drain. Retain failure without recovering poisoned projection state. - if std::panic::catch_unwind(AssertUnwindSafe(|| { - cell.settle(ProjectionOutcome::Shutdown) - })) - .is_err() - { - failures += 1; - } - } - failures -} - -#[derive(Clone)] -pub(super) struct ShutdownReceipt { - result: tokio::sync::watch::Receiver>, -} - -impl ShutdownReceipt { - pub(super) async fn wait(mut self) -> crate::Result<()> { - loop { - if let Some(failures) = *self.result.borrow_and_update() { - return if failures == 0 { - Ok(()) - } else { - Err(crate::PumasError::DownloadShutdownFailed { failures }) - }; - } - if self.result.changed().await.is_err() { - return Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }); - } - } - } -} - -impl fmt::Debug for DownloadTaskOwner { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - let count = self - .state - .lock() - .map(|state| state.tasks.len()) - .unwrap_or_default(); - formatter - .debug_struct("DownloadTaskOwner") - .field("task_count", &count) - .finish() - } -} - -pub(crate) struct TaskContext { - owner: Weak, - download_id: String, - generation: TaskGeneration, - projection_failure: Option>, - root_grant: Option>, -} - -impl Clone for TaskContext { - fn clone(&self) -> Self { - Self { - owner: self.owner.clone(), - download_id: self.download_id.clone(), - generation: self.generation.clone(), - projection_failure: self.projection_failure.clone(), - root_grant: self.root_grant.clone(), - } - } -} - -pub(super) struct PreparedTask { - owner: Weak, - download_id: String, - generation: TaskGeneration, - role: TaskRole, - start_state: Arc, - armed: bool, -} - -impl fmt::Debug for PreparedTask { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("PreparedTask") - .field("download_id", &self.download_id) - .field("generation", &self.generation) - .field("role", &self.role) - .finish_non_exhaustive() - } -} - -#[derive(Debug)] -pub(super) struct InstalledTask { - owner: Arc, - download_id: String, - generation: TaskGeneration, - start_state: Arc, -} - -#[derive(Debug)] -pub(super) struct PreparedProjection { - task: PreparedTask, - cell: Arc, -} - -pub(super) struct InstalledProjection { - task: InstalledTask, - ticket: ProjectionTicket, -} - -#[derive(Clone)] -pub(super) struct ProjectionTicket { - download_id: String, - generation: TaskGeneration, - cell: Arc, -} - -pub(super) enum ProjectionTransition { - Started(InstalledProjection), - Existing(InstalledProjection), - NotReady, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum ProjectionSettlement { - Pending, - FailureUnprojected, - Settled, - AlreadySettled, - StaleGeneration, - Missing, -} - -#[derive(Debug)] -pub(super) enum CancelTransition { - Started(InstalledTask), - Existing(InstalledTask), - AlreadyRunning, -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub(super) enum CancelPredecessor { - Absent, - Observed(TaskObservation), -} - -impl DownloadTaskOwner { async fn acquire_root_grant( self: &Arc, context: &TaskContext, @@ -1002,15 +618,19 @@ impl DownloadTaskOwner { ) -> crate::Result> { let identity = root.grant_identity(); loop { - let changed = self.root_grant_changed.notified(); + let changed = self.roots.changed.notified(); tokio::pin!(changed); changed.as_mut().enable(); let acquisition = { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { + let mut state = self + .roots + .grants + .lock() + .expect("HF root grant cache lock poisoned"); + if self.scope.is_closed() { return Err(crate::PumasError::DownloadLifecycleClosed); } - let slot = state.root_grants.entry(identity).or_default(); + let slot = state.entry(identity).or_default(); if let Some(grant) = slot.grant.upgrade() { GrantAcquisition::Reuse(grant) } else if slot.acquiring { @@ -1051,3746 +671,452 @@ impl DownloadTaskOwner { }) .and_then(|result| result); { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let slot = state.root_grants.entry(identity).or_default(); + let mut state = self + .roots + .grants + .lock() + .expect("HF root grant cache lock poisoned"); + let slot = state.entry(identity).or_default(); if let Ok(grant) = &result { slot.grant = Arc::downgrade(grant); } slot.acquiring = false; } - self.root_grant_changed.notify_waiters(); + self.roots.changed.notify_waiters(); return result; } } } } +} - pub(super) fn is_closed(&self) -> bool { - self.state - .lock() - .expect("HF task owner lock poisoned") - .closed - } - - pub(super) fn ensure_open(&self) -> crate::Result<()> { - if self.is_closed() { - Err(crate::PumasError::DownloadLifecycleClosed) - } else { - Ok(()) - } - } - - /// Owns pre-task preparation independently of its caller. Dropping the - /// waiter cancels only this invocation's outer work; registered effects - /// remain in retired custody, and installed child generations are untouched. - pub(super) async fn run_invocation( - self: &Arc, - operation: F, - ) -> crate::Result - where - T: Send + 'static, - F: FnOnce(TaskContext) -> Fut + Send + 'static, - Fut: Future> + Send + 'static, - { - let id = format!("invocation-{}", uuid::Uuid::new_v4()); - let (sender, receiver) = oneshot::channel(); - let prepared = self.prepare( - id.clone(), - TaskRole::Invocation, - move |context| async move { - let result = operation(context).await; - let _ = sender.send(result); +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::time::Duration; + use tokio::sync::oneshot; + #[tokio::test] + async fn admission_matching_uses_ordered_validated_selection_and_weak_legacy_revision() { + use crate::model_library::artifact_identity::DownloadRevision; + use crate::model_library::hf::types::FileToDownload; + let supervisor = Arc::new(TaskCustodyOwner::new()); + let hf = Arc::new( + DownloadTaskOwner::with_supervisor(supervisor.clone(), || async { Ok(()) }).unwrap(), + ); + let native = supervisor.open_scope(|| async { Ok(()) }).unwrap(); + let temp = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(temp.path()).unwrap(); + let destination = root.resolve(Path::new("model")).unwrap().identity(); + let files = vec![ + FileToDownload { + filename: "config.json".into(), + size: None, + sha256: None, }, - )?; - let generation = prepared.generation.clone(); - let installed = self - .install_gated(prepared) - .map_err(|_| crate::PumasError::DownloadLifecycleClosed)?; - let waiter = InvocationWaiter { - owner: self.clone(), - id, - generation, + FileToDownload { + filename: "weights.bin".into(), + size: Some(4), + sha256: Some("A".repeat(64)), + }, + ]; + let selection = super::super::acquisition_source::manifest_for_download( + "org/model", + &DownloadRevision::legacy_main(), + &files, + ) + .unwrap(); + assert_eq!( + selection.source().revision().strength(), + crate::acquisition::RevisionStrength::Weak + ); + assert!(!selection.permits_resume(0)); + assert!(selection.permits_resume(1)); + let identity = PendingAdmissionIdentity { + destination: destination.clone(), + selection: selection.clone(), }; - installed.start(); - let result = receiver.await.map_err(|_| { - if self.is_closed() { - crate::PumasError::DownloadLifecycleClosed - } else { - crate::PumasError::DownloadShutdownFailed { failures: 1 } - } - })?; - drop(waiter); - self.ensure_open()?; - result + let task = hf + .prepare( + "hf-admission".into(), + TaskRole::AdmissionTransition, + |_| async { std::future::pending::<()>().await }, + ) + .unwrap(); + let installed = hf.install_gated(task).unwrap(); + let generation = installed.generation().clone(); + let (_, completed) = tokio::sync::watch::channel(false); + hf.bind_pending_admission("hf-admission", &generation, identity.clone(), completed); + assert_eq!(hf.pending_admission(&identity).unwrap().0, "hf-admission"); + assert!(native + .admission_snapshots(TaskRole::AdmissionTransition) + .is_empty()); + let mut canonical_files = files.clone(); + canonical_files[1].sha256 = Some("a".repeat(64)); + let canonical = PendingAdmissionIdentity { + destination: destination.clone(), + selection: super::super::acquisition_source::manifest_for_download( + "org/model", + &DownloadRevision::legacy_main(), + &canonical_files, + ) + .unwrap(), + }; + assert!( + hf.pending_admission(&canonical).is_some(), + "the validated manifest owns digest normalization" + ); + canonical_files.reverse(); + let reordered = PendingAdmissionIdentity { + destination: destination.clone(), + selection: super::super::acquisition_source::manifest_for_download( + "org/model", + &DownloadRevision::legacy_main(), + &canonical_files, + ) + .unwrap(), + }; + assert!(hf.pending_admission(&reordered).is_none()); + let pinned = + DownloadRevision::from_commit("0123456789abcdef0123456789abcdef01234567").unwrap(); + let pinned = PendingAdmissionIdentity { + destination: destination.clone(), + selection: super::super::acquisition_source::manifest_for_download( + "org/model", + &pinned, + &files, + ) + .unwrap(), + }; + assert!(hf.pending_admission(&pinned).is_none()); + let another_destination = PendingAdmissionIdentity { + destination: root.resolve(Path::new("another-model")).unwrap().identity(), + selection, + }; + assert!(hf.pending_admission(&another_destination).is_none()); + let mut invalid = files; + invalid[1].sha256 = Some("invalid-digest".into()); + assert!(super::super::acquisition_source::manifest_for_download( + "org/model", + &DownloadRevision::legacy_main(), + &invalid + ) + .is_err()); + drop(installed); + hf.shutdown().await.unwrap(); + assert!(!native.is_closed(), "HF closes only its injected scope"); + supervisor.request_shutdown().wait().await.unwrap(); } - pub(super) async fn shutdown(self: &Arc, final_projection: F) -> crate::Result<()> - where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future> + Send + 'static, - { - self.request_shutdown(final_projection).wait().await + #[tokio::test] + async fn abandoned_prepared_mutation_retains_root_until_owned_drain() { + let temp = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(temp.path()).unwrap(); + let phase_root = root.clone(); + let owner = Arc::new(DownloadTaskOwner::new()); + let preparing_owner = owner.clone(); + let ran = Arc::new(AtomicBool::new(false)); + let work_ran = ran.clone(); + let prepared = owner + .run_invocation(move |context| async move { + let protected = context.with_root_grant(phase_root).await?; + preparing_owner.prepare( + "protected-prepared".into(), + TaskRole::Worker, + move |context| async move { + let _context = context.inherit_root_grant(&protected); + work_ran.store(true, Ordering::Release); + }, + ) + }) + .await + .unwrap(); + assert!(matches!( + root.try_acquire_execution_grant(), + Err(crate::PumasError::DownloadRootBusy) + )); + drop(prepared); + owner.shutdown().await.unwrap(); + assert!(!ran.load(Ordering::Acquire)); + root.try_acquire_execution_grant().unwrap(); } - /// Permanently closes admission and retains one drain driver. The first - /// projection callback wins; every waiter observes the same outcome after - /// all captured effects and final projection have completed. This is not - /// persisted cleanup confirmation or permission to release queue ownership. - pub(super) fn request_shutdown(self: &Arc, final_projection: F) -> ShutdownReceipt - where - F: FnOnce() -> Fut + Send + 'static, - Fut: Future> + Send + 'static, - { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if let Some(receipt) = &state.shutdown { - return receipt.clone(); - } - state.closed = true; - // Abort requests do not poll user work. Issue them at the admission - // boundary so an already-extracted Work gate cannot start an outer - // after closure while the retained driver is still awaiting scheduling. - for entry in state.prepared.values() { - entry.outer.abort(); - } - for entry in state.tasks.values() { - entry.outer.abort(); - for abort in &entry.abort_on_start { - abort.abort(); - } - } - let (result, receiver) = tokio::sync::watch::channel(None); - let receipt = ShutdownReceipt { result: receiver }; - state.shutdown = Some(receipt.clone()); - let Ok(runtime) = tokio::runtime::Handle::try_current() else { - // A search-only client may be dropped outside any executor. This - // requests closure but cannot claim an async projection was observed. - let _ = result.send(Some(1)); - return receipt; - }; - let prepared = std::mem::take(&mut state.prepared); - let tasks = std::mem::take(&mut state.tasks); - let retired = std::mem::take(&mut state.retired); - let mut failures = state.retired_failures; - let owner = self.clone(); - let (start, started) = oneshot::channel(); - state.shutdown_driver = Some(runtime.spawn(async move { - let _ = started.await; - for (_, entry) in prepared { - drop(entry.start); - if entry.outer.await.is_err_and(|error| error.is_panic()) { - failures += 1; - } - if let Some(cell) = entry.projection { - if std::panic::catch_unwind(AssertUnwindSafe(|| { - cell.settle(ProjectionOutcome::Shutdown) - })) - .is_err() - { - failures += 1; - } - } - } - let mut draining = Vec::new(); - for (_, mut entry) in tasks { - for gate in entry.starts.drain(..) { - gate.drain(); + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn root_grant_outlives_completed_blocking_work_until_result_observation() { + let temp = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(temp.path()).unwrap(); + let phase_root = root.clone(); + let owner = Arc::new(DownloadTaskOwner::new()); + let (entered, ready) = oneshot::channel(); + let entered = Mutex::new(Some(entered)); + let (release, held) = std::sync::mpsc::channel(); + let held = Mutex::new(held); + owner.set_blocking_result_observer(Some(Arc::new(move |operation| { + if operation == "protected completed write" { + let sender = entered.lock().unwrap().take(); + if let Some(sender) = sender { + let _ = sender.send(()); + held.lock().unwrap().recv().unwrap(); } - draining.push(entry); - } - for entry in draining { - failures += drain_shutdown_entry(entry).await; } - for task in retired { - failures += task.observer.await.unwrap_or(1); - } - if !matches!( - AssertUnwindSafe(async move { final_projection().await }) - .catch_unwind() - .await, - Ok(Ok(())) - ) { - failures += 1; - } - let _ = result.send(Some(failures)); - drop(owner); - })); - drop(state); - let _ = start.send(()); - receipt - } - pub(super) fn new() -> Self { - Self::default() - } - - pub(super) fn prepare( - self: &Arc, - download_id: String, - role: TaskRole, - work: F, - ) -> crate::Result - where - F: FnOnce(TaskContext) -> Fut + Send + 'static, - Fut: Future + Send + 'static, - { - self.prepare_with_projection(download_id, role, None, work) - } - - fn prepare_with_projection( - self: &Arc, - download_id: String, - role: TaskRole, - projection: Option>, - work: F, - ) -> crate::Result - where - F: FnOnce(TaskContext) -> Fut + Send + 'static, - Fut: Future + Send + 'static, - { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return Err(crate::PumasError::DownloadLifecycleClosed); - } - let generation = TaskGeneration::new(); - let context = TaskContext { - owner: Arc::downgrade(self), - download_id: download_id.clone(), - generation: generation.clone(), - projection_failure: None, - root_grant: None, - }; - let (start, started) = oneshot::channel(); - let outer = tokio::spawn(async move { - if started.await.is_ok() { - work(context).await; - } - }); - let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); - state.prepared.insert( - generation.key(), - PreparedEntry { - download_id: download_id.clone(), - generation: generation.clone(), - role, - start, - outer, - projection: projection.clone(), - start_state: start_state.clone(), - }, - ); - Ok(PreparedTask { - owner: Arc::downgrade(self), - download_id, - generation, - role, - start_state, - armed: true, - }) - } - - pub(super) fn prepare_projection( - self: &Arc, - download_id: String, - project: F, - project_panic: P, - ) -> crate::Result - where - F: FnOnce(TaskContext, Option) -> Fut + Send + 'static, - Fut: Future + Send + 'static, - P: FnOnce(TaskContext) -> PFut + Send + 'static, - PFut: Future + Send + 'static, - { - let cell = Arc::new(ProjectionCell::new(true)); - let project_cell = cell.clone(); - let task = self.prepare_with_projection( - download_id, - TaskRole::TerminalProjection, - Some(cell.clone()), - move |mut context| async move { - context.projection_failure = Some(project_cell.clone()); - let predecessor = project_cell.wait_for_predecessor().await; - let project_context = context.clone(); - let outcome = - match AssertUnwindSafe( - async move { project(project_context, predecessor).await }, - ) - .catch_unwind() - .await - { - Ok(outcome) => outcome, - Err(_) => { - project_cell.mark_failed(); - let fallback = - AssertUnwindSafe(async move { project_panic(context).await }) - .catch_unwind() - .await; - if matches!(fallback, Ok(ProjectionOutcome::Failed)) { - project_cell.acknowledge_failure_projection(); - } - ProjectionOutcome::Panicked - } - }; - if outcome == ProjectionOutcome::Failed { - project_cell.mark_failed(); - } - if project_cell.failed() - && matches!( - outcome, - ProjectionOutcome::Committed | ProjectionOutcome::Failed - ) - { - project_cell.acknowledge_failure_projection(); - } - project_cell.settle(outcome); - }, - )?; - Ok(PreparedProjection { task, cell }) - } - - /// Installs a prepared task while its start gate remains closed. - /// - /// Dropping the returned token only marks its owner-held start lease as - /// abandoned. Callers rescue it after releasing any outer state guard. - pub(super) fn install_gated( - self: &Arc, - mut prepared: PreparedTask, - ) -> std::result::Result { - if !prepared - .owner - .upgrade() - .is_some_and(|owner| Arc::ptr_eq(&owner, self)) - { - return Err(prepared); - } - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed || state.tasks.contains_key(&prepared.download_id) { - return Err(prepared); - } - let Some(entry) = state.prepared.remove(&prepared.generation.key()) else { - return Err(prepared); - }; - let tasks = &mut state.tasks; - let download_id = entry.download_id.clone(); - let generation = entry.generation.clone(); - tasks.insert( - download_id.clone(), - TaskEntry { - admission: None, - generation: generation.clone(), - role: entry.role, - outer: entry.outer, - nested: Vec::new(), - nested_failures_archived: 0, - predecessor_failures_archived: 0, - projection: entry.projection, - starts: vec![StartGate::Work(entry.start)], - superseded_projection: None, - abort_on_start: Vec::new(), - start_state: entry.start_state, - }, - ); - drop(state); - prepared.armed = false; - Ok(InstalledTask { - owner: self.clone(), - download_id, - generation, - start_state: prepared.start_state.clone(), - }) - } - - pub(super) fn install_projection_gated( - self: &Arc, - prepared: PreparedProjection, - ) -> std::result::Result { - let cell = prepared.cell.clone(); - match self.install_gated(prepared.task) { - Ok(task) => { - let ticket = ProjectionTicket { - download_id: task.download_id.clone(), - generation: task.generation.clone(), - cell, - }; - Ok(InstalledProjection { task, ticket }) - } - Err(task) => Err(PreparedProjection { task, cell }), - } - } - - pub(super) fn snapshot(&self, download_id: &str) -> Option { - let snapshot = self - .state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .map(|entry| TaskSnapshot { - role: entry.role, - finished: entry.finished(), - outer_finished: entry.outer.is_finished(), - started: TaskStartState::load(&entry.start_state) == TaskStartState::Running, - }); - #[cfg(test)] - let observer = self - .snapshot_observer - .lock() - .expect("HF task snapshot observer lock poisoned") - .clone(); - #[cfg(test)] - if let Some(observer) = observer { - observer(download_id, snapshot.clone()); - } - snapshot - } - - pub(super) fn active_worker_generation(&self, download_id: &str) -> Option { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .filter(|entry| { - entry.role == TaskRole::Worker - && TaskStartState::load(&entry.start_state) == TaskStartState::Running - && !entry.outer.is_finished() - }) - .map(|entry| entry.generation.clone()) - } - - #[cfg(test)] - fn generation_for_test(&self, download_id: &str) -> Option { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .map(|entry| entry.generation.clone()) - } - - #[cfg(test)] - fn nested_count_for_test(&self, download_id: &str) -> Option { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .map(|entry| entry.nested.len()) - } - - #[cfg(test)] - pub(super) fn outer_finished_for_test(&self, download_id: &str) -> bool { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .is_some_and(|entry| entry.outer.is_finished()) - } - - #[cfg(test)] - fn prepared_count_for_test(&self) -> usize { - self.state - .lock() - .expect("HF task owner lock poisoned") - .prepared - .len() - } - - pub(super) fn contains(&self, download_id: &str) -> bool { - self.snapshot(download_id).is_some() - } - - pub(super) fn generation_is_current( - &self, - download_id: &str, - generation: &TaskGeneration, - ) -> bool { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .is_some_and(|entry| entry.generation.matches(generation)) - } - - fn generation_has_role( - &self, - download_id: &str, - generation: &TaskGeneration, - role: TaskRole, - ) -> bool { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .is_some_and(|entry| entry.generation.matches(generation) && entry.role == role) - } - - /// Called under the download-state commit lock after durable confirmation. - pub(super) fn promote_admission(&self, download_id: &str, generation: &TaskGeneration) -> bool { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - let Some(entry) = tasks.get_mut(download_id) else { - return false; - }; - if !entry.generation.matches(generation) || entry.role != TaskRole::AdmissionTransition { - return false; - } - entry.role = TaskRole::Worker; - true - } - - pub(super) fn bind_pending_admission( - &self, - download_id: &str, - generation: &TaskGeneration, - identity: PendingAdmissionIdentity, - completed: tokio::sync::watch::Receiver, - ) { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - if let Some(entry) = tasks - .get_mut(download_id) - .filter(|entry| entry.generation.matches(generation)) - { - entry.admission = Some((identity, completed)); - } - } - - pub(super) fn pending_admission( - &self, - identity: &PendingAdmissionIdentity, - ) -> Option<(String, tokio::sync::watch::Receiver)> { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .iter() - .find_map(|(id, entry)| { - if entry.role != TaskRole::AdmissionTransition { - return None; - } - entry - .admission - .as_ref() - .filter(|(key, _)| key == identity) - .map(|(_, completion)| (id.clone(), completion.clone())) - }) - } - - pub(super) fn pending_recovery_admission( - &self, - download_id: &str, - identity: &PendingAdmissionIdentity, - ) -> Option<(TaskGeneration, tokio::sync::watch::Receiver)> { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .get(download_id) - .filter(|entry| entry.role == TaskRole::RecoveryTransition) - .and_then(|entry| { - entry - .admission - .as_ref() - .filter(|(key, _)| key == identity) - .map(|(_, completion)| (entry.generation.clone(), completion.clone())) - }) - } - - #[cfg(test)] - pub(super) fn is_empty(&self) -> bool { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .is_empty() - } - - pub(super) fn ids(&self) -> Vec { - let ids = self - .state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .iter() - .filter(|(_, entry)| entry.role != TaskRole::Invocation) - .map(|(id, _)| id.clone()) - .collect(); - #[cfg(test)] - let observer = self - .ids_observer - .lock() - .expect("HF task IDs observer lock poisoned") - .clone(); - #[cfg(test)] - if let Some(observer) = observer { - observer(); - } - ids - } - - /// Starts a caller-independent finalizer after synchronously replacing and - /// aborting the current generation. A separately retained observer drains - /// predecessor custody even if the finalizer is aborted before `finish`. - pub(super) fn begin_cancel( - self: &Arc, - download_id: &str, - finish: F, - ) -> crate::Result - where - F: FnOnce(TaskContext, CancelPredecessor) -> Fut + Send + 'static, - Fut: Future + Send + 'static, - { - #[cfg(test)] - { - let observer = self - .cancel_replacement_observer - .lock() - .expect("HF cancel-replacement observer lock poisoned") - .clone(); - if let Some(observer) = observer { - observer(); - } - } - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return Err(crate::PumasError::DownloadLifecycleClosed); - } - let tasks = &mut state.tasks; - if let Some(current) = tasks.get_mut(download_id) { - if current.role == TaskRole::CancelFinalizer && !current.finished() { - return Ok( - if TaskStartState::load(¤t.start_state) == TaskStartState::Running { - CancelTransition::AlreadyRunning - } else { - CancelTransition::Existing(InstalledTask { - owner: self.clone(), - download_id: download_id.to_string(), - generation: current.generation.clone(), - start_state: current.start_state.clone(), - }) - }, - ); - } - } - let mut current = tasks.remove(download_id); - let outer_finished_before_replacement = current - .as_ref() - .is_some_and(|entry| entry.outer.is_finished()); - let mut abort_on_start: Vec<_> = current - .as_ref() - .map(|entry| entry.outer.abort_handle()) - .into_iter() - .collect(); - if let Some(entry) = current.as_mut() { - abort_on_start.append(&mut entry.abort_on_start); - } - let superseded_projection = current.as_ref().and_then(|entry| { - entry - .projection - .clone() - .or_else(|| entry.superseded_projection.clone()) - }); - let predecessor_starts = current - .as_mut() - .map(|entry| std::mem::take(&mut entry.starts)) - .unwrap_or_default(); - - let generation = TaskGeneration::new(); - let context = TaskContext { - owner: Arc::downgrade(self), - download_id: download_id.to_string(), - generation: generation.clone(), - projection_failure: superseded_projection.clone(), - root_grant: None, - }; - let (start, started) = oneshot::channel(); - let (predecessor_start, predecessor_started) = oneshot::channel(); - let (predecessor_sender, predecessor_receiver) = oneshot::channel(); - let predecessor_completion = Arc::new(NestedCompletion { - finished: AtomicBool::new(false), - failed: AtomicBool::new(false), - notify: Notify::new(), - }); - let predecessor_observer = tokio::spawn(observe_cancellation_predecessor( - current, - outer_finished_before_replacement, - predecessor_started, - predecessor_completion.clone(), - predecessor_sender, - )); - let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); - let outer = tokio::spawn(async move { - if started.await.is_err() { - return; - } - let Ok(predecessor) = predecessor_receiver.await else { - // Observer failure remains owned by the nested task; it must - // never authorize cleanup as an absent predecessor. - return; - }; - finish(context, predecessor).await; - }); - tasks.insert( - download_id.to_string(), - TaskEntry { - admission: None, - generation: generation.clone(), - role: TaskRole::CancelFinalizer, - outer, - nested: vec![NestedTask { - handle: predecessor_observer, - completion: predecessor_completion, - failure_kind: NestedFailureKind::Predecessor, - }], - nested_failures_archived: 0, - predecessor_failures_archived: 0, - projection: None, - starts: predecessor_starts - .into_iter() - .chain(std::iter::once(StartGate::Custody(predecessor_start))) - .chain(std::iter::once(StartGate::Work(start))) - .collect(), - superseded_projection, - abort_on_start, - start_state: start_state.clone(), - }, - ); - drop(state); - Ok(CancelTransition::Started(InstalledTask { - owner: self.clone(), - download_id: download_id.to_string(), - generation, - start_state, - })) - } - - #[cfg(test)] - pub(super) async fn observe_finished(&self, download_id: &str) -> Option { - let entry = { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - if !tasks.get(download_id).is_some_and(TaskEntry::finished) { - return None; - } - tasks.remove(download_id) - }?; - let role = entry.role; - Some(observe_entry(entry, role, false).await) - } - - #[cfg(test)] - pub(super) async fn observe_finished_generation( - &self, - download_id: &str, - generation: &TaskGeneration, - ) -> Option { - let entry = { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - if !tasks - .get(download_id) - .is_some_and(|entry| entry.generation.matches(generation) && entry.finished()) - { - return None; - } - tasks.remove(download_id) - }?; - let role = entry.role; - Some(observe_entry(entry, role, false).await) - } - - pub(super) fn finished_or_projecting_ids(&self) -> Vec { - self.state - .lock() - .expect("HF task owner lock poisoned") - .tasks - .iter() - .filter_map(|(download_id, entry)| { - (entry.role != TaskRole::Invocation - && (entry.role == TaskRole::TerminalProjection || entry.finished())) - .then_some(download_id.clone()) - }) - .collect() - } - - /// Replaces one fully finished generation with a start-gated projection - /// owner under the same download ID. The predecessor is observed by a - /// nested owner task, so its failure remains visible if cancellation - /// supersedes the projector before state projection begins. - pub(super) fn begin_finished_projection( - self: &Arc, - download_id: &str, - inherit_failure: bool, - project: F, - project_panic: P, - ) -> crate::Result - where - F: FnOnce(TaskContext, Option) -> Fut + Send + 'static, - Fut: Future + Send + 'static, - P: FnOnce(TaskContext) -> PFut + Send + 'static, - PFut: Future + Send + 'static, - { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return Err(crate::PumasError::DownloadLifecycleClosed); - } - let tasks = &mut state.tasks; - let Some(current) = tasks.get_mut(download_id) else { - return Ok(ProjectionTransition::NotReady); - }; - if current.role == TaskRole::TerminalProjection { - let cell = current - .projection - .clone() - .expect("terminal projection owns a projection cell"); - let generation = current.generation.clone(); - return Ok(ProjectionTransition::Existing(InstalledProjection { - task: InstalledTask { - owner: self.clone(), - download_id: download_id.to_string(), - generation: generation.clone(), - start_state: current.start_state.clone(), - }, - ticket: ProjectionTicket { - download_id: download_id.to_string(), - generation, - cell, - }, - })); - } - if !current.finished() { - return Ok(ProjectionTransition::NotReady); - } - - let predecessor = tasks - .remove(download_id) - .expect("finished predecessor remained present"); - let predecessor_role = predecessor.role; - let inherited_projection = predecessor.superseded_projection.clone(); - let generation = TaskGeneration::new(); - let cell = Arc::new(ProjectionCell::new(false)); - if let Some(inherited) = inherited_projection { - if inherited.failed() { - cell.mark_failed(); - } - cell.inherit(inherited); - } - if inherit_failure { - cell.mark_failed(); - } - let context = TaskContext { - owner: Arc::downgrade(self), - download_id: download_id.to_string(), - generation: generation.clone(), - projection_failure: Some(cell.clone()), - root_grant: None, - }; - let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); - - let predecessor_cell = cell.clone(); - let predecessor_completion = Arc::new(NestedCompletion { - finished: AtomicBool::new(false), - failed: AtomicBool::new(false), - notify: Notify::new(), - }); - let predecessor_completion_task = predecessor_completion.clone(); - let (predecessor_start, predecessor_started) = oneshot::channel(); - let predecessor_observer = tokio::spawn(async move { - if predecessor_started.await.is_err() { - return; - } - let observation = observe_entry(predecessor, predecessor_role, false).await; - if observation.terminal == TaskTerminal::Panicked || observation.nested_failures > 0 { - predecessor_completion_task - .failed - .store(true, Ordering::Release); - } - predecessor_cell.record_predecessor(observation); - predecessor_completion_task - .finished - .store(true, Ordering::Release); - predecessor_completion_task.notify.notify_waiters(); - }); - - let project_cell = cell.clone(); - let (project_start, project_started) = oneshot::channel(); - let outer = tokio::spawn(async move { - if project_started.await.is_err() { - return; - } - let predecessor = project_cell.wait_for_predecessor().await; - let project_context = context.clone(); - let outcome = - match AssertUnwindSafe(async move { project(project_context, predecessor).await }) - .catch_unwind() - .await - { - Ok(outcome) => outcome, - Err(_) => { - project_cell.mark_failed(); - let fallback = - AssertUnwindSafe(async move { project_panic(context).await }) - .catch_unwind() - .await; - if matches!(fallback, Ok(ProjectionOutcome::Failed)) { - project_cell.acknowledge_failure_projection(); - } - ProjectionOutcome::Panicked - } - }; - if outcome == ProjectionOutcome::Failed { - project_cell.mark_failed(); - } - if project_cell.failed() - && matches!( - outcome, - ProjectionOutcome::Committed | ProjectionOutcome::Failed - ) - { - project_cell.acknowledge_failure_projection(); - } - project_cell.settle(outcome); - }); - - tasks.insert( - download_id.to_string(), - TaskEntry { - admission: None, - generation: generation.clone(), - role: TaskRole::TerminalProjection, - outer, - nested: vec![NestedTask { - handle: predecessor_observer, - completion: predecessor_completion, - failure_kind: NestedFailureKind::Effect, - }], - nested_failures_archived: 0, - predecessor_failures_archived: 0, - projection: Some(cell.clone()), - starts: vec![ - StartGate::Custody(predecessor_start), - StartGate::Work(project_start), - ], - superseded_projection: None, - abort_on_start: Vec::new(), - start_state: start_state.clone(), - }, - ); - drop(state); - - let ticket = ProjectionTicket { - download_id: download_id.to_string(), - generation: generation.clone(), - cell, - }; - Ok(ProjectionTransition::Started(InstalledProjection { - task: InstalledTask { - owner: self.clone(), - download_id: download_id.to_string(), - generation, - start_state, - }, - ticket, - })) - } - - pub(super) fn settle_projection(&self, ticket: &ProjectionTicket) -> ProjectionSettlement { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - let Some(entry) = tasks.get(&ticket.download_id) else { - return if ticket.cell.is_settled() { - ProjectionSettlement::AlreadySettled - } else { - ProjectionSettlement::Missing - }; - }; - let matches = entry.role == TaskRole::TerminalProjection - && entry.generation.matches(&ticket.generation); - if !matches { - return ProjectionSettlement::StaleGeneration; - } - if ticket.cell.has_unprojected_failure() { - return ProjectionSettlement::FailureUnprojected; - } - if !ticket.cell.is_ready_to_settle() { - return ProjectionSettlement::Pending; - } - if !entry.finished() { - return ProjectionSettlement::Pending; - } - ticket.cell.mark_settled(); - let start = tasks - .remove(&ticket.download_id) - .map(|entry| retire_entry(&mut state, entry)); - drop(state); - if let Some(start) = start { - let _ = start.send(()); - } - ProjectionSettlement::Settled - } - - fn promote_generation( - &self, - download_id: &str, - generation: &TaskGeneration, - role: TaskRole, - ) -> bool { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - let Some(entry) = tasks.get_mut(download_id) else { - return false; - }; - if !entry.generation.matches(generation) { - return false; - } - entry.role = role; - true - } - - fn start_generation(&self, download_id: &str, generation: &TaskGeneration) -> bool { - let (aborts, superseded_projection, starts) = { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return false; - } - let tasks = &mut state.tasks; - let Some(entry) = tasks.get_mut(download_id) else { - return false; - }; - if !entry.generation.matches(generation) { - return false; - } - entry - .start_state - .store(TaskStartState::Running as u8, Ordering::Release); - ( - std::mem::take(&mut entry.abort_on_start), - entry.superseded_projection.clone(), - std::mem::take(&mut entry.starts), - ) - }; - for abort in aborts { - abort.abort(); - } - if let Some(cell) = superseded_projection { - cell.settle(ProjectionOutcome::Superseded); - } - for start in starts { - start.start(); - } - true - } - - /// Runs after outer state locks are released. Abandoned projectors and - /// finalizers are safe to start because they retain required predecessor - /// custody; abandoned workers are removed and aborted without claiming - /// that they ever ran. - pub(super) fn rescue_abandoned(&self) { - self.reap_retired(); - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return; - } - let mut retired_starts = Vec::new(); - let keys = state - .prepared - .iter() - .filter_map(|(key, entry)| { - (TaskStartState::load(&entry.start_state) == TaskStartState::Abandoned) - .then_some(*key) - }) - .collect::>(); - for key in keys { - if let Some(entry) = state.prepared.remove(&key) { - entry.outer.abort(); - let (start, started) = oneshot::channel(); - let observer = tokio::spawn(async move { - let _ = started.await; - drop(entry.start); - let failures = - usize::from(entry.outer.await.is_err_and(|error| error.is_panic())); - if let Some(cell) = entry.projection { - cell.settle(ProjectionOutcome::Shutdown); - } - failures - }); - state.retired.push(RetiredTask { observer }); - retired_starts.push(start); - } - } - let mut starts = Vec::new(); - let mut removals = Vec::new(); - for (id, entry) in &state.tasks { - if TaskStartState::load(&entry.start_state) != TaskStartState::Abandoned { - continue; - } - if matches!( - entry.role, - TaskRole::TerminalProjection | TaskRole::CancelFinalizer - ) { - starts.push((id.clone(), entry.generation.clone())); - } else { - removals.push(id.clone()); - } - } - for id in removals { - if let Some(entry) = state.tasks.remove(&id) { - retired_starts.push(retire_entry(&mut state, entry)); - } - } - drop(state); - for start in retired_starts { - let _ = start.send(()); - } - for (id, generation) in starts { - self.start_generation(&id, &generation); - } - } - - fn reap_retired(&self) { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - while let Some(index) = state - .retired - .iter() - .position(|task| task.observer.is_finished()) - { - let mut task = state.retired.swap_remove(index); - match (&mut task.observer).now_or_never() { - Some(result) => { - state.retired_failures += result.unwrap_or(1); - self.retired_observations.fetch_add(1, Ordering::AcqRel); - } - None => { - state.retired.push(task); - break; - } - } - } - } - - #[cfg(test)] - pub(super) fn outstanding_retired_for_test(&self) -> usize { - self.reap_retired(); - self.state - .lock() - .expect("HF task owner lock poisoned") - .retired - .len() - } - - #[cfg(test)] - fn retired_observations_for_test(&self) -> usize { - self.retired_observations.load(Ordering::Acquire) - } - - #[cfg(test)] - fn register_blocking( - self: &Arc, - download_id: &str, - generation: &TaskGeneration, - operation: &'static str, - function: F, - ) -> std::result::Result>, BlockingTaskError> - where - T: Send + 'static, - F: FnOnce() -> T + Send + 'static, - { - self.register_blocking_with_failure( - download_id, - generation, - operation, - function, - |_| false, - None, - ) - } - - fn register_fallible_blocking( - self: &Arc, - download_id: &str, - generation: &TaskGeneration, - operation: &'static str, - function: F, - root_grant: Option>, - ) -> std::result::Result, BlockingTaskError> - where - T: Send + 'static, - E: Send + 'static, - F: FnOnce() -> std::result::Result + Send + 'static, - { - self.register_blocking_with_failure( - download_id, - generation, - operation, - function, - std::result::Result::is_err, - root_grant, - ) - } - - fn register_blocking_with_failure( - self: &Arc, - download_id: &str, - generation: &TaskGeneration, - operation: &'static str, - function: F, - failed: C, - root_grant: Option>, - ) -> std::result::Result>, BlockingTaskError> - where - T: Send + 'static, - F: FnOnce() -> T + Send + 'static, - C: FnOnce(&T) -> bool + Send + 'static, - { - #[cfg(not(test))] - let _ = operation; - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return Err(BlockingTaskError::StaleGeneration); - } - let tasks = &mut state.tasks; - let Some(entry) = tasks.get_mut(download_id) else { - return Err(BlockingTaskError::StaleGeneration); - }; - if !entry.generation.matches(generation) { - return Err(BlockingTaskError::StaleGeneration); - } - entry.reap_completed_nested(); - - #[cfg(test)] - let blocking_observer = self - .blocking_observer - .lock() - .expect("HF blocking observer lock poisoned") - .clone(); - let (start_sender, start_receiver) = oneshot::channel(); - let (result_sender, result_receiver) = oneshot::channel(); - let completion = Arc::new(NestedCompletion { - finished: AtomicBool::new(false), - failed: AtomicBool::new(false), - notify: Notify::new(), - }); - let completion_in_observer = completion.clone(); - #[cfg(test)] - let result_observer = self - .blocking_result_observer - .lock() - .expect("HF blocking-result observer lock poisoned") - .clone(); - let observer = tokio::spawn(async move { - let closure_grant = root_grant.clone(); - let result = if start_receiver.await.is_ok() { - tokio::task::spawn_blocking(move || { - let _grant = closure_grant; - #[cfg(test)] - if let Some(observer) = blocking_observer { - observer(operation); - } - function() - }) - .await - } else { - return; - } - .map_err(|error| { - completion_in_observer.failed.store(true, Ordering::Release); - error.to_string() - }); - if result.as_ref().is_ok_and(failed) { - completion_in_observer.failed.store(true, Ordering::Release); - } - #[cfg(test)] - if let Some(observer) = result_observer { - observer(operation); - } - let _ = result_sender.send(result); - completion_in_observer - .finished - .store(true, Ordering::Release); - completion_in_observer.notify.notify_waiters(); - drop(root_grant); - }); - entry.nested.push(NestedTask { - handle: observer, - completion, - failure_kind: NestedFailureKind::Effect, - }); - drop(state); - let _ = start_sender.send(()); - Ok(result_receiver) - } - - #[cfg(test)] - pub(super) fn set_blocking_observer(&self, observer: Option) { - *self - .blocking_observer - .lock() - .expect("HF blocking observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_ids_observer(&self, observer: Option) { - *self - .ids_observer - .lock() - .expect("HF task IDs observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_drain_observer(&self, observer: Option) { - *self - .drain_observer - .lock() - .expect("HF task drain observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_snapshot_observer(&self, observer: Option) { - *self - .snapshot_observer - .lock() - .expect("HF task snapshot observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_cancellation_check_observer( - &self, - observer: Option, - ) { - *self - .cancellation_check_observer - .lock() - .expect("HF cancellation-check observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_cancel_replacement_observer( - &self, - observer: Option, - ) { - *self - .cancel_replacement_observer - .lock() - .expect("HF cancel-replacement observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_worker_projection_observer( - &self, - observer: Option, - ) { - *self - .worker_projection_observer - .lock() - .expect("HF worker-projection observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_blocking_result_observer(&self, observer: Option) { - *self - .blocking_result_observer - .lock() - .expect("HF blocking-result observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_blocking_failure_observer(&self, observer: Option) { - *self - .blocking_failure_observer - .lock() - .expect("HF blocking-failure observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_ambient_admission_observer( - &self, - observer: Option, - ) { - *self - .ambient_admission_observer - .lock() - .expect("HF ambient-admission observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn set_projection_observer(&self, observer: Option) { - *self - .projection_observer - .lock() - .expect("HF projection observer lock poisoned") = observer; - } - - #[cfg(test)] - pub(super) fn observe_ambient_admission(&self, operation: &'static str, download_id: &str) { - let observer = self - .ambient_admission_observer - .lock() - .expect("HF ambient-admission observer lock poisoned") - .clone(); - if let Some(observer) = observer { - observer(operation, download_id); - } - } - - async fn drain_blocking_generation( - &self, - download_id: &str, - generation: &TaskGeneration, - ) -> std::result::Result { - let (_archived_failures, completions) = { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - let Some(entry) = tasks.get_mut(download_id) else { - return Err(BlockingTaskError::StaleGeneration); - }; - if !entry.generation.matches(generation) { - return Err(BlockingTaskError::StaleGeneration); - } - entry.reap_completed_nested(); - ( - entry.nested_failures_archived, - entry - .nested - .iter() - .map(|nested| nested.completion.clone()) - .collect::>(), - ) - }; - #[cfg(test)] - let observer = self - .drain_observer - .lock() - .expect("HF task drain observer lock poisoned") - .clone(); - #[cfg(test)] - if let Some(observer) = observer { - observer(); - } - let _ = wait_for_nested(&completions).await; - loop { - let completed = { - let mut state = self.state.lock().expect("HF task owner lock poisoned"); - let tasks = &mut state.tasks; - let Some(entry) = tasks.get_mut(download_id) else { - return Err(BlockingTaskError::StaleGeneration); - }; - if !entry.generation.matches(generation) { - return Err(BlockingTaskError::StaleGeneration); - } - entry.reap_completed_nested(); - if entry.nested.is_empty() { - // Predecessor failures remain terminal provenance, not a - // new failure of this generation's cleanup effects. - Some(entry.nested_failures_archived) - } else { - None - } - }; - if let Some(failures) = completed { - return Ok(failures); - } - tokio::task::yield_now().await; - } - } -} - -impl TaskContext { - /// Retain existing native exclusion across a nested owned library effect. - /// The library validates this grant against its configured root. - pub(crate) fn held_root_execution_grant(&self) -> crate::Result> { - self.root_grant - .clone() - .ok_or_else(|| crate::PumasError::Config { - message: "Download root execution grant is unavailable".into(), - }) - } - - /// Scope physical exclusion to mutation, not to historical task entries. - /// Acquisition itself is retained work: a cancelled waiter cannot strand - /// the in-progress slot or detach a newly opened native lock. - pub(crate) async fn with_root_grant( - &self, - root: DownloadDestinationRoot, - ) -> crate::Result { - let owner = self - .owner - .upgrade() - .ok_or(crate::PumasError::DownloadLifecycleClosed)?; - let context = self.clone(); - let grant = self - .run_fallible_async_named("acquire download root grant", move || async move { - // A refused acquisition has no protected effects and must not poison - // shutdown as a failed effect. Panics and observation failures still do. - Ok::<_, std::convert::Infallible>(owner.acquire_root_grant(&context, root).await) - }) - .await - .map_err(|error| { - crate::PumasError::Other(format!("Download root grant observation failed: {error}")) - })? - .expect("infallible acquisition envelope")?; - let mut scoped = self.clone(); - scoped.root_grant = Some(grant); - Ok(scoped) - } - - pub(crate) fn without_root_grant(&self) -> Self { - let mut context = self.clone(); - context.root_grant = None; - context - } - - /// Preserve the receiving generation while transferring protected custody. - pub(crate) fn inherit_root_grant(&self, source: &Self) -> Self { - assert!( - Weak::ptr_eq(&self.owner, &source.owner), - "root grant transfer requires the same task owner" - ); - let mut context = self.clone(); - context.root_grant = source.root_grant.clone(); - context - } - - /// Registers an async effect whose internal work must survive cancellation - /// of the invoking future. Like blocking effects, this observer is joined, - /// never aborted, by lifecycle shutdown. - pub(crate) async fn run_fallible_async_named( - &self, - _operation: &'static str, - function: F, - ) -> std::result::Result, BlockingTaskError> - where - T: Send + 'static, - E: Send + 'static, - F: FnOnce() -> Fut + Send + 'static, - Fut: Future> + Send + 'static, - { - let owner = self - .owner - .upgrade() - .ok_or(BlockingTaskError::StaleGeneration)?; - let receiver = { - let mut state = owner.state.lock().expect("HF task owner lock poisoned"); - if state.closed { - return Err(BlockingTaskError::StaleGeneration); - } - let entry = state - .tasks - .get_mut(&self.download_id) - .filter(|entry| entry.generation.matches(&self.generation)) - .ok_or(BlockingTaskError::StaleGeneration)?; - entry.reap_completed_nested(); - let (start, started) = oneshot::channel(); - let (sender, receiver) = oneshot::channel(); - let completion = Arc::new(NestedCompletion { - finished: AtomicBool::new(false), - failed: AtomicBool::new(false), - notify: Notify::new(), - }); - let observed = completion.clone(); - let root_grant = self.root_grant.clone(); - let handle = tokio::spawn(async move { - let _ = started.await; - let result = AssertUnwindSafe(async move { function().await }) - .catch_unwind() - .await - .map_err(|_| "owned async effect panicked".to_string()); - if !matches!(&result, Ok(Ok(_))) { - observed.failed.store(true, Ordering::Release); - } - let _ = sender.send(result); - observed.finished.store(true, Ordering::Release); - observed.notify.notify_waiters(); - drop(root_grant); - }); - entry.nested.push(NestedTask { - handle, - completion, - failure_kind: NestedFailureKind::Effect, - }); - drop(state); - let _ = start.send(()); - receiver - }; - receiver - .await - .map_err(|_| BlockingTaskError::ResultChannelClosed)? - .map_err(BlockingTaskError::Join) - } - - pub(super) async fn pause_requested(&self, pause_flag: &AtomicBool) { - loop { - let notified = self.generation.0.notified(); - tokio::pin!(notified); - notified.as_mut().enable(); - if pause_flag.load(Ordering::Acquire) { - return; - } - notified.await; - } - } - - pub(super) fn download_id(&self) -> &str { - &self.download_id - } - - pub(super) fn generation(&self) -> &TaskGeneration { - &self.generation - } - - pub(super) fn is_current_role(&self, role: TaskRole) -> bool { - self.owner.upgrade().is_some_and(|owner| { - owner.generation_has_role(&self.download_id, &self.generation, role) - }) - } - - pub(super) fn promote_role(&self, role: TaskRole) -> bool { - self.owner.upgrade().is_some_and(|owner| { - owner.promote_generation(&self.download_id, &self.generation, role) - }) - } - - /// Completes custody transferred from a superseded terminal projector. - /// A failed cell is acknowledged only after the finalizer has published - /// its fail-closed terminal state. - pub(super) fn complete_transferred_projection(&self, failure_projected: bool) -> bool { - let Some(cell) = &self.projection_failure else { - return false; - }; - if cell.failed() { - if !failure_projected { - return false; - } - cell.acknowledge_failure_projection(); - } - cell.mark_settled(); - true - } - - #[cfg(test)] - pub(super) async fn run_blocking( - &self, - function: F, - ) -> std::result::Result - where - T: Send + 'static, - F: FnOnce() -> T + Send + 'static, - { - self.run_blocking_named("unnamed", function).await - } - - #[cfg(test)] - pub(super) fn register_blocking_without_wait_for_test( - &self, - operation: &'static str, - function: F, - ) -> std::result::Result<(), BlockingTaskError> - where - T: Send + 'static, - F: FnOnce() -> T + Send + 'static, - { - let owner = self - .owner - .upgrade() - .ok_or(BlockingTaskError::StaleGeneration)?; - let receiver = - owner.register_blocking(&self.download_id, &self.generation, operation, function)?; - drop(receiver); - Ok(()) - } - - /// Exercises owned blocking success/panic observation without a domain error. - pub(super) async fn run_blocking_named( - &self, - operation: &'static str, - function: F, - ) -> std::result::Result - where - T: Send + 'static, - F: FnOnce() -> T + Send + 'static, - { - let owner = self - .owner - .upgrade() - .ok_or(BlockingTaskError::StaleGeneration)?; - let receiver = owner.register_blocking_with_failure( - &self.download_id, - &self.generation, - operation, - function, - |_| false, - self.root_grant.clone(), - )?; - receiver - .await - .map_err(|_| BlockingTaskError::ResultChannelClosed)? - .map_err(BlockingTaskError::Join) - } - - pub(crate) async fn run_fallible_blocking_named( - &self, - operation: &'static str, - function: F, - ) -> std::result::Result, BlockingTaskError> - where - T: Send + 'static, - E: Send + 'static, - F: FnOnce() -> std::result::Result + Send + 'static, - { - let owner = self - .owner - .upgrade() - .ok_or(BlockingTaskError::StaleGeneration)?; - let receiver = owner.register_fallible_blocking( - &self.download_id, - &self.generation, - operation, - function, - self.root_grant.clone(), - )?; - receiver - .await - .map_err(|_| BlockingTaskError::ResultChannelClosed)? - .map_err(BlockingTaskError::Join) - } - - pub(super) async fn drain_blocking(&self) -> std::result::Result { - let owner = self - .owner - .upgrade() - .ok_or(BlockingTaskError::StaleGeneration)?; - owner - .drain_blocking_generation(&self.download_id, &self.generation) - .await - } - - #[cfg(test)] - pub(super) fn should_fail_blocking_operation(&self, operation: &'static str) -> bool { - self.owner.upgrade().is_some_and(|owner| { - owner - .blocking_failure_observer - .lock() - .expect("HF blocking-failure observer lock poisoned") - .as_ref() - .is_some_and(|observer| observer(operation)) - }) - } - - #[cfg(test)] - pub(super) fn observe_projection(&self, projection: &'static str) { - if let Some(owner) = self.owner.upgrade() { - let observer = owner - .projection_observer - .lock() - .expect("HF projection observer lock poisoned") - .clone(); - if let Some(observer) = observer { - observer(projection); - } - } - } - - #[cfg(test)] - pub(super) fn observe_cancellation_check(&self) { - if let Some(owner) = self.owner.upgrade() { - let observer = owner - .cancellation_check_observer - .lock() - .expect("HF cancellation-check observer lock poisoned") - .clone(); - if let Some(observer) = observer { - observer(); - } - } - } - - #[cfg(test)] - pub(super) fn observe_worker_projection(&self, projection: &'static str) { - if let Some(owner) = self.owner.upgrade() { - let observer = owner - .worker_projection_observer - .lock() - .expect("HF worker-projection observer lock poisoned") - .clone(); - if let Some(observer) = observer { - observer(projection); - } - } - } -} - -impl InstalledTask { - pub(super) fn generation(&self) -> &TaskGeneration { - &self.generation - } - - pub(super) fn start(self) { - let _ = self - .owner - .start_generation(&self.download_id, &self.generation); - } -} - -impl Drop for InstalledTask { - fn drop(&mut self) { - let _ = self.start_state.compare_exchange( - TaskStartState::Gated as u8, - TaskStartState::Abandoned as u8, - Ordering::AcqRel, - Ordering::Acquire, - ); - } -} - -impl Drop for PreparedTask { - fn drop(&mut self) { - if !self.armed { - return; - } - let _ = self.start_state.compare_exchange( - TaskStartState::Gated as u8, - TaskStartState::Abandoned as u8, - Ordering::AcqRel, - Ordering::Acquire, - ); - } -} - -impl InstalledProjection { - pub(super) fn start(self) -> ProjectionTicket { - let Self { task, ticket } = self; - task.start(); - ticket - } -} - -impl ProjectionTicket { - pub(super) async fn wait(&self) -> ProjectionOutcome { - self.cell.wait().await - } - - #[cfg(test)] - pub(super) fn failure_projected_for_test(&self) -> bool { - self.cell.failure_projected() - } - - #[cfg(test)] - pub(super) fn settled_for_test(&self) -> bool { - self.cell.is_settled() - } -} - -async fn observe_cancellation_predecessor( - current: Option, - outer_finished_before_replacement: bool, - started: oneshot::Receiver<()>, - completion: Arc, - receipt: oneshot::Sender, -) { - let started = started.await.is_ok(); - let predecessor = match current { - Some(current) => { - if !started { - current.outer.abort(); - } - let role = current.role; - match AssertUnwindSafe(observe_entry( - current, - role, - outer_finished_before_replacement, - )) - .catch_unwind() - .await - { - Ok(observation) => { - completion.failed.store( - !started - || observation.terminal == TaskTerminal::Panicked - || observation.nested_failures > 0, - Ordering::Release, - ); - Some(CancelPredecessor::Observed(observation)) - } - Err(_) => { - completion.failed.store(true, Ordering::Release); - None - } - } - } - None => { - completion.failed.store(!started, Ordering::Release); - Some(CancelPredecessor::Absent) - } - }; - completion.finished.store(true, Ordering::Release); - completion.notify.notify_waiters(); - if started { - if let Some(predecessor) = predecessor { - let _ = receipt.send(predecessor); - } - } -} - -async fn observe_entry( - mut entry: TaskEntry, - role: TaskRole, - outer_finished_before_replacement: bool, -) -> TaskObservation { - let generation = entry.generation.clone(); - let terminal = match entry.outer.await { - Ok(()) => TaskTerminal::Completed, - Err(error) if error.is_cancelled() => TaskTerminal::Cancelled, - Err(_) => TaskTerminal::Panicked, - }; - let nested_failures = entry.nested_failures_archived - + entry.predecessor_failures_archived - + observe_nested(entry.nested.drain(..).collect()).await; - // Drain owned effects before reading failure provenance, whose poisoned - // bookkeeping must not cause an observer to detach unfinished work. - let projection_failed = entry.projection.as_ref().is_some_and(|cell| cell.failed()) - || entry - .superseded_projection - .as_ref() - .is_some_and(|cell| cell.failed()); - let nested_failures = nested_failures + usize::from(projection_failed); - TaskObservation { - generation, - role, - terminal, - nested_failures, - outer_finished_before_replacement, - } -} - -async fn observe_nested(nested: Vec) -> usize { - let mut failures = 0; - for nested in nested { - let join_failed = nested.handle.await.is_err(); - if join_failed || nested.completion.failed.load(Ordering::Acquire) { - failures += 1; - } - } - failures -} - -async fn wait_for_nested(completions: &[Arc]) -> usize { - for completion in completions { - loop { - let notified = completion.notify.notified(); - if completion.finished.load(Ordering::Acquire) { - break; - } - notified.await; - } - } - completions - .iter() - .filter(|completion| completion.failed.load(Ordering::Acquire)) - .count() -} - -#[cfg(test)] -mod tests { - #[tokio::test] - async fn abandoned_prepared_mutation_retains_root_until_owned_drain() { - let temp = tempfile::TempDir::new().unwrap(); - let root = DownloadDestinationRoot::open(temp.path()).unwrap(); - let phase_root = root.clone(); - let owner = Arc::new(DownloadTaskOwner::new()); - let preparing_owner = owner.clone(); - let ran = Arc::new(AtomicBool::new(false)); - let work_ran = ran.clone(); - let prepared = owner - .run_invocation(move |context| async move { - let protected = context.with_root_grant(phase_root).await?; - preparing_owner.prepare( - "protected-prepared".into(), - TaskRole::Worker, - move |context| async move { - let _context = context.inherit_root_grant(&protected); - work_ran.store(true, Ordering::Release); - }, - ) - }) - .await - .unwrap(); - assert!(matches!( - root.try_acquire_execution_grant(), - Err(crate::PumasError::DownloadRootBusy) - )); - drop(prepared); - owner.shutdown(|| async { Ok(()) }).await.unwrap(); - assert!(!ran.load(Ordering::Acquire)); - root.try_acquire_execution_grant().unwrap(); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn root_grant_outlives_completed_blocking_work_until_result_observation() { - let temp = tempfile::TempDir::new().unwrap(); - let root = DownloadDestinationRoot::open(temp.path()).unwrap(); - let phase_root = root.clone(); - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered, ready) = oneshot::channel(); - let entered = Mutex::new(Some(entered)); - let (release, held) = std::sync::mpsc::channel(); - let held = Mutex::new(held); - owner.set_blocking_result_observer(Some(Arc::new(move |operation| { - if operation == "protected completed write" { - let sender = entered.lock().unwrap().take(); - if let Some(sender) = sender { - let _ = sender.send(()); - held.lock().unwrap().recv().unwrap(); - } - } - }))); - let caller_owner = owner.clone(); - let caller = tokio::spawn(async move { - caller_owner - .run_invocation(move |context| async move { - let context = context.with_root_grant(phase_root).await?; - context - .run_fallible_blocking_named( - "protected completed write", - || Ok::<_, ()>(()), - ) - .await - .unwrap() - .unwrap(); - Ok(()) - }) - .await - }); - tokio::time::timeout(Duration::from_secs(3), ready) - .await - .unwrap() - .unwrap(); - caller.abort(); - let _ = caller.await; - let receipt = owner.request_shutdown(|| async { Ok(()) }); - let mut shutdown = Box::pin(receipt.wait()); - let pending = futures::poll!(&mut shutdown).is_pending(); - let contention = root.try_acquire_execution_grant(); - release.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(3), shutdown) - .await - .unwrap() - .unwrap(); - assert!(pending); - assert!(matches!( - contention, - Err(crate::PumasError::DownloadRootBusy) - )); - owner.set_blocking_result_observer(None); - root.try_acquire_execution_grant().unwrap(); - } - - #[tokio::test] - async fn scoped_root_grants_share_and_release_without_retiring_the_invocation() { - let temp = tempfile::TempDir::new().unwrap(); - let root = DownloadDestinationRoot::open(temp.path()).unwrap(); - let owner = Arc::new(DownloadTaskOwner::new()); - owner - .run_invocation(move |context| async move { - let (first, second) = tokio::join!( - context.with_root_grant(root.clone()), - context.with_root_grant(root.clone()), - ); - let first = first?; - let second = second?; - assert!(Arc::ptr_eq( - first.root_grant.as_ref().unwrap(), - second.root_grant.as_ref().unwrap() - )); - assert!(matches!( - root.try_acquire_execution_grant(), - Err(crate::PumasError::DownloadRootBusy) - )); - drop(first); - drop(second); - context.drain_blocking().await.unwrap(); - // This invocation is still registered and running: only its mutation - // scope, not its registry membership, determines native custody. - root.try_acquire_execution_grant()?; - Ok(()) - }) - .await - .unwrap(); - owner.shutdown(|| async { Ok(()) }).await.unwrap(); - } - - #[tokio::test] - async fn one_owner_acquires_distinct_physical_root_grants_concurrently() { - let model_root_dir = tempfile::TempDir::new().unwrap(); - let runtime_root_dir = tempfile::TempDir::new().unwrap(); - let model_root = DownloadDestinationRoot::open(model_root_dir.path()).unwrap(); - let reopened_model_root = DownloadDestinationRoot::open(model_root_dir.path()).unwrap(); - let runtime_root = DownloadDestinationRoot::open(runtime_root_dir.path()).unwrap(); - let owner = Arc::new(DownloadTaskOwner::new()); - owner - .run_invocation(move |context| async move { - let (model, reopened_model, runtime) = tokio::join!( - context.with_root_grant(model_root.clone()), - context.with_root_grant(reopened_model_root.clone()), - context.with_root_grant(runtime_root.clone()), - ); - let model = model?; - let reopened_model = reopened_model?; - let runtime = runtime?; - assert!(Arc::ptr_eq( - model.root_grant.as_ref().unwrap(), - reopened_model.root_grant.as_ref().unwrap() - )); - assert!(!Arc::ptr_eq( - model.root_grant.as_ref().unwrap(), - runtime.root_grant.as_ref().unwrap() - )); - assert!(matches!( - model_root.try_acquire_execution_grant(), - Err(crate::PumasError::DownloadRootBusy) - )); - assert!(matches!( - runtime_root.try_acquire_execution_grant(), - Err(crate::PumasError::DownloadRootBusy) - )); - drop(model); - drop(reopened_model); - drop(runtime); - context.drain_blocking().await.unwrap(); - model_root.try_acquire_execution_grant()?; - runtime_root.try_acquire_execution_grant()?; - Ok(()) - }) - .await - .unwrap(); - owner.shutdown(|| async { Ok(()) }).await.unwrap(); - } - - #[tokio::test] - async fn refused_root_grant_does_not_poison_shutdown() { - let temp = tempfile::TempDir::new().unwrap(); - let root = DownloadDestinationRoot::open(temp.path()).unwrap(); - let held = root.try_acquire_execution_grant().unwrap(); - let owner = Arc::new(DownloadTaskOwner::new()); - let outcome = owner - .run_invocation(move |context| async move { - context.with_root_grant(root).await.map(|_| ()) - }) - .await; - assert!(matches!(outcome, Err(crate::PumasError::DownloadRootBusy))); - owner.shutdown(|| async { Ok(()) }).await.unwrap(); - drop(held); - } - - #[tokio::test] - async fn root_grant_retains_cancelled_blocking_and_async_effects_until_observed() { - for asynchronous in [false, true] { - for failure in 0..3 { - let temp = tempfile::TempDir::new().unwrap(); - let root = DownloadDestinationRoot::open(temp.path()).unwrap(); - let effect_root = root.clone(); - let owner = Arc::new(DownloadTaskOwner::new()); - let caller_owner = owner.clone(); - let (entered, ready) = oneshot::channel(); - let (release, released) = oneshot::channel(); - let caller = tokio::spawn(async move { - caller_owner - .run_invocation(move |context| async move { - let context = context.with_root_grant(effect_root).await?; - if asynchronous { - let _ = context - .run_fallible_async_named( - "held protected async effect", - move || async move { - let _ = entered.send(()); - released.await.unwrap(); - assert_ne!( - failure, 2, - "injected protected async panic" - ); - if failure == 1 { - Err("protected async failure") - } else { - Ok(()) - } - }, - ) - .await; - } else { - let _ = context - .run_fallible_blocking_named( - "held protected blocking effect", - move || { - let _ = entered.send(()); - released.blocking_recv().unwrap(); - assert_ne!( - failure, 2, - "injected protected blocking panic" - ); - if failure == 1 { - Err("protected blocking failure") - } else { - Ok(()) - } - }, - ) - .await; - } - Ok(()) - }) - .await - }); - tokio::time::timeout(Duration::from_secs(3), ready) - .await - .unwrap() - .unwrap(); - caller.abort(); - let _ = caller.await; - let receipt = owner.request_shutdown(|| async { Ok(()) }); - let mut shutdown = Box::pin(receipt.wait()); - let pending = futures::poll!(&mut shutdown).is_pending(); - let contention = root.try_acquire_execution_grant(); - release.send(()).unwrap(); - let outcome = tokio::time::timeout(Duration::from_secs(3), shutdown) - .await - .unwrap(); - assert!(pending); - assert!(matches!( - contention, - Err(crate::PumasError::DownloadRootBusy) - )); - assert_eq!(outcome.is_err(), failure != 0); - root.try_acquire_execution_grant().unwrap(); - } - } - } - - #[tokio::test] - async fn shutdown_rejects_work_whose_start_gate_was_already_extracted() { - let owner = Arc::new(DownloadTaskOwner::new()); - let ran = Arc::new(AtomicBool::new(false)); - let marker = ran.clone(); - let prepared = owner - .prepare("in-flight".into(), TaskRole::Worker, move |_| async move { - marker.store(true, Ordering::Release); - }) - .unwrap(); - let installed = owner.install_gated(prepared).unwrap(); - // This is start_generation's in-flight custody after its coordination - // lock is released but before its gate sends reach the outer task. - let gates = { - let mut state = owner.state.lock().unwrap(); - let entry = state.tasks.get_mut("in-flight").unwrap(); - entry - .start_state - .store(TaskStartState::Running as u8, Ordering::Release); - std::mem::take(&mut entry.starts) - }; - let receipt = owner.request_shutdown(|| async { Ok(()) }); - for gate in gates { - gate.start(); - } - drop(installed); - receipt.wait().await.unwrap(); - assert!(!ran.load(Ordering::Acquire)); - } - - #[tokio::test] - async fn shutdown_starts_gated_predecessor_custody_but_never_cancel_cleanup() { - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered, entered_rx) = oneshot::channel(); - let (release, released) = std::sync::mpsc::channel(); - let worker = owner - .prepare( - "worker".into(), - TaskRole::Worker, - move |context| async move { - let _ = context - .run_fallible_blocking_named("held predecessor", move || { - let _ = entered.send(()); - released.recv().unwrap(); - Ok::<_, ()>(()) - }) - .await; - }, - ) - .unwrap(); - owner.install_gated(worker).unwrap().start(); - entered_rx.await.unwrap(); - let cleanup_ran = Arc::new(AtomicBool::new(false)); - let cleanup_marker = cleanup_ran.clone(); - let finalizer = owner - .begin_cancel("worker", move |_, _| async move { - cleanup_marker.store(true, Ordering::Release); - }) - .unwrap(); - let receipt = owner.request_shutdown(|| async { Ok(()) }); - drop(finalizer); - let mut waiting = Box::pin(receipt.wait()); - assert!(futures::poll!(&mut waiting).is_pending()); - assert!(!cleanup_ran.load(Ordering::Acquire)); - release.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(2), waiting) - .await - .unwrap() - .unwrap(); - assert!(!cleanup_ran.load(Ordering::Acquire)); - } - - #[tokio::test] - async fn shutdown_keeps_async_effect_error_and_panic_after_caller_disappears() { - for panic in [false, true] { - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered, entered_rx) = oneshot::channel(); - let (release, released) = oneshot::channel(); - let caller_owner = owner.clone(); - let caller = tokio::spawn(async move { - caller_owner - .run_invocation(move |context| async move { - let _ = context - .run_fallible_async_named("failing async effect", move || async move { - let _ = entered.send(()); - released.await.unwrap(); - assert!(!panic, "injected async effect panic"); - Err::<(), _>("injected async effect error") - }) - .await; - Ok(()) - }) - .await - }); - entered_rx.await.unwrap(); - caller.abort(); - let _ = caller.await; - let receipt = owner.request_shutdown(|| async { Ok(()) }); - release.send(()).unwrap(); - assert!(matches!( - receipt.wait().await, - Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }) - )); - } - } - - #[tokio::test] - async fn shutdown_closes_prepared_and_installed_work_without_starting_it() { - let owner = Arc::new(DownloadTaskOwner::new()); - let ran = Arc::new(AtomicUsize::new(0)); - let prepared = owner - .prepare("prepared".into(), TaskRole::Worker, { - let ran = ran.clone(); - move |_| async move { - ran.fetch_add(1, Ordering::SeqCst); - } - }) - .unwrap(); - let installed = owner - .install_gated( - owner - .prepare("installed".into(), TaskRole::Worker, { - let ran = ran.clone(); - move |_| async move { - ran.fetch_add(1, Ordering::SeqCst); - } - }) - .unwrap(), - ) - .unwrap(); - let projection = owner - .install_projection_gated( - owner - .prepare_projection( - "projection".into(), - |_, _| async { panic!("gated projection must not run") }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap(), - ) - .unwrap(); - let ticket = projection.ticket.clone(); - let receipt = owner.request_shutdown(|| async { Ok(()) }); - assert!(owner.is_closed()); - assert!(owner.install_gated(prepared).is_err()); - installed.start(); - drop(projection); - assert!(matches!( - owner.prepare("late".into(), TaskRole::Worker, |_| async {}), - Err(crate::PumasError::DownloadLifecycleClosed) - )); - assert!(matches!( - owner.begin_cancel("late", |_, _| async {}), - Err(crate::PumasError::DownloadLifecycleClosed) - )); - receipt.wait().await.unwrap(); - assert_eq!(ticket.wait().await, ProjectionOutcome::Shutdown); - assert_eq!(ran.load(Ordering::SeqCst), 0); - } - - #[tokio::test] - async fn shutdown_retains_cancelled_invocation_effect_and_shared_failure_receipt() { - for outcome in 0..3 { - let owner = Arc::new(DownloadTaskOwner::new()); - let weak = Arc::downgrade(&owner); - let (entered, entered_rx) = oneshot::channel(); - let (release, released) = std::sync::mpsc::channel(); - let completed = Arc::new(AtomicBool::new(false)); - let effect_completed = completed.clone(); - let caller_owner = owner.clone(); - let caller = tokio::spawn(async move { - caller_owner - .run_invocation(move |context| async move { - context - .run_fallible_blocking_named("shutdown held effect", move || { - let _ = entered.send(()); - released.recv().unwrap(); - effect_completed.store(true, Ordering::Release); - match outcome { - 0 => Ok(()), - 1 => Err("effect failed"), - _ => panic!("effect panicked"), - } - }) - .await - .map_err(|_| crate::PumasError::DownloadShutdownFailed { failures: 1 })? - .map_err(|_| crate::PumasError::DownloadShutdownFailed { failures: 1 }) - }) - .await - }); - entered_rx.await.unwrap(); - caller.abort(); - let _ = caller.await; - let projected = Arc::new(AtomicUsize::new(0)); - let final_projected = projected.clone(); - let final_completed = completed.clone(); - let receipt = owner.request_shutdown(move || async move { - assert!(final_completed.load(Ordering::Acquire)); - final_projected.fetch_add(1, Ordering::SeqCst); - Ok(()) - }); - let repeated = - owner.request_shutdown(|| async { panic!("only first projection executes") }); - let mut waiter = Box::pin(receipt.clone().wait()); - assert!(futures::poll!(&mut waiter).is_pending()); - drop(waiter); - drop(owner); - assert!( - weak.upgrade().is_some(), - "driver retains the lifecycle owner" - ); - assert!(!completed.load(Ordering::Acquire)); - release.send(()).unwrap(); - let result = receipt.wait().await; - let repeat_result = repeated.wait().await; - if outcome == 0 { - result.unwrap(); - repeat_result.unwrap(); - } else { - assert!(matches!( - result, - Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }) - )); - assert!(matches!( - repeat_result, - Err(crate::PumasError::DownloadShutdownFailed { failures: 1 }) - )); - } - assert_eq!(projected.load(Ordering::SeqCst), 1); - tokio::time::timeout(Duration::from_secs(2), async { - while weak.upgrade().is_some() { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - } - } - - #[tokio::test] - async fn shutdown_drains_owned_async_effect_after_invocation_abort() { - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered, entered_rx) = oneshot::channel(); - let (release, released) = oneshot::channel(); - let caller_owner = owner.clone(); - let caller = tokio::spawn(async move { - caller_owner - .run_invocation(move |context| async move { - context - .run_fallible_async_named("held async effect", move || async move { - let _ = entered.send(()); - released.await.unwrap(); - Ok::<_, ()>(()) - }) - .await - .unwrap() - .unwrap(); - Ok(()) - }) - .await - }); - entered_rx.await.unwrap(); - let receipt = owner.request_shutdown(|| async { Ok(()) }); - assert!(matches!( - caller.await.unwrap(), - Err(crate::PumasError::DownloadLifecycleClosed) - )); - let mut pending = Box::pin(receipt.clone().wait()); - assert!(futures::poll!(&mut pending).is_pending()); - release.send(()).unwrap(); - pending.await.unwrap(); - } - - fn queue_destination() -> (tempfile::TempDir, DestinationIdentity) { - let root = tempfile::TempDir::new().unwrap(); - let authority = - crate::model_library::download_recovery::DownloadDestinationRoot::open(root.path()) - .unwrap(); - let destination = authority.resolve(Path::new("model")).unwrap().identity(); - (root, destination) - } - - #[test] - fn released_destination_claim_cannot_be_resurrected_from_stale_inventory() { - let owner = super::DestinationExecutionOwner::new(); - let (_root, path) = queue_destination(); - let first = super::TaskGeneration::new(); - let second = super::TaskGeneration::new(); - assert!(owner.reserve( - path.clone(), - "first".into(), - super::DestinationDomain::Ambient, - first.clone() - )); - assert!(owner.release(&path, "first", super::DestinationDomain::Ambient, &first)); - assert!(!owner.reserve_dormant( - path.clone(), - "first".into(), - super::DestinationDomain::Ambient - )); - assert!(!owner.reserve( - path.clone(), - "first".into(), - super::DestinationDomain::Ambient, - second - )); - assert_eq!(owner.claim_count(&path), 0); - } - use super::*; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - use std::time::Duration; - - fn start_cancel(transition: CancelTransition) -> bool { - match transition { - CancelTransition::Started(finalizer) | CancelTransition::Existing(finalizer) => { - finalizer.start(); - } - CancelTransition::AlreadyRunning => {} - } - true - } - - async fn acknowledge_failed_projection_through_cancel( - owner: &Arc, - download_id: &str, - ticket: &ProjectionTicket, - ) { - assert_eq!( - owner.settle_projection(ticket), - ProjectionSettlement::FailureUnprojected - ); - assert!(owner.contains(download_id)); - let (acknowledged_sender, acknowledged) = oneshot::channel(); - let transition = owner - .begin_cancel(download_id, move |context, predecessor| async move { - let CancelPredecessor::Observed(observation) = predecessor else { - panic!("failed projector must remain predecessor custody"); - }; - assert!(observation.nested_failures > 0); - assert!(context.complete_transferred_projection(true)); - let _ = acknowledged_sender.send(()); - }) - .unwrap(); - let finalizer = match transition { - CancelTransition::Started(finalizer) => finalizer, - _ => panic!("failed projector must be replaced by one finalizer"), - }; - finalizer.start(); - assert_eq!( - owner.settle_projection(ticket), - ProjectionSettlement::StaleGeneration - ); - tokio::time::timeout(Duration::from_secs(1), acknowledged) - .await - .expect("finalizer must acknowledge transferred failure") - .unwrap(); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot(download_id) - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .expect("acknowledging finalizer must reach terminal Join state"); - assert!(ticket.failure_projected_for_test()); - assert!(ticket.settled_for_test()); - assert!(owner.observe_finished(download_id).await.is_some()); - assert!(!owner.contains(download_id)); - assert_eq!( - owner.settle_projection(ticket), - ProjectionSettlement::AlreadySettled - ); - } - - #[tokio::test] - async fn prepared_task_runs_only_after_owned_install_and_start() { - let owner = Arc::new(DownloadTaskOwner::new()); - let ran = Arc::new(AtomicBool::new(false)); - let ran_in_task = ran.clone(); - let prepared = owner - .prepare( - "download".to_string(), - TaskRole::Worker, - move |_| async move { - ran_in_task.store(true, Ordering::SeqCst); - }, - ) - .unwrap(); - - tokio::task::yield_now().await; - assert!(!ran.load(Ordering::SeqCst)); - - let installed = owner.install_gated(prepared).unwrap(); - let generation = installed.generation().clone(); - assert!(owner.snapshot("download").is_some_and(|snapshot| { - owner - .generation_for_test("download") - .is_some_and(|current| current.matches(&generation)) - && snapshot.role == TaskRole::Worker - && !snapshot.finished - })); - assert!(!ran.load(Ordering::SeqCst)); - - installed.start(); - tokio::time::timeout(Duration::from_secs(1), async { - while !ran.load(Ordering::SeqCst) { - tokio::task::yield_now().await; - } - }) - .await - .expect("installed task should start"); - } - - #[tokio::test] - async fn dropping_unstarted_install_generation_matches_cleanup() { - for role in [TaskRole::Worker, TaskRole::RecoveryTransition] { - let owner = Arc::new(DownloadTaskOwner::new()); - let ran = Arc::new(AtomicBool::new(false)); - let ran_in_task = ran.clone(); - let prepared = owner - .prepare("download".to_string(), role, move |_| async move { - ran_in_task.store(true, Ordering::SeqCst); - }) - .unwrap(); - let installed = owner.install_gated(prepared).unwrap(); - assert!(owner.contains("download")); - drop(installed); - assert!(owner.contains("download")); - assert!(!ran.load(Ordering::SeqCst)); - - owner.rescue_abandoned(); - assert!(!owner.contains("download")); - tokio::time::timeout(Duration::from_secs(1), async { - while owner.outstanding_retired_for_test() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("abandoned installed wrapper must be observed to terminal"); - assert_eq!(owner.retired_observations_for_test(), 1); - assert!(!ran.load(Ordering::SeqCst)); - } - } - - #[tokio::test] - async fn abandoned_projection_is_rescued_without_signalling_from_drop() { - let owner = Arc::new(DownloadTaskOwner::new()); - let ran = Arc::new(AtomicBool::new(false)); - let ran_in_projection = ran.clone(); - let prepared = owner - .prepare_projection( - "download".to_string(), - move |_, predecessor| { - let ran = ran_in_projection.clone(); - async move { - assert!(predecessor.is_none()); - ran.store(true, Ordering::SeqCst); - ProjectionOutcome::Committed - } - }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap(); - let InstalledProjection { task, ticket } = - owner.install_projection_gated(prepared).unwrap(); - - // Token destruction is CAS-only. It cannot release the gate while an - // outer state guard may still be unwinding. - drop(task); - tokio::task::yield_now().await; - assert!(!ran.load(Ordering::SeqCst)); - assert!(owner.contains("download")); - - owner.rescue_abandoned(); - assert_eq!(ticket.wait().await, ProjectionOutcome::Committed); - assert!(ran.load(Ordering::SeqCst)); - while owner.settle_projection(&ticket) == ProjectionSettlement::Pending { - tokio::task::yield_now().await; - } - assert_eq!( - owner.settle_projection(&ticket), - ProjectionSettlement::AlreadySettled - ); - } - - #[tokio::test] - async fn abandoned_prepared_collision_is_inert_until_explicit_rescue() { - for rejected_role in [TaskRole::Worker, TaskRole::RecoveryTransition] { - let owner = Arc::new(DownloadTaskOwner::new()); - let first = owner - .prepare("download".to_string(), TaskRole::Worker, |_| async { - std::future::pending::<()>().await; - }) - .unwrap(); - owner.install_gated(first).unwrap().start(); - - let ran = Arc::new(AtomicBool::new(false)); - let ran_in_task = ran.clone(); - let second = owner - .prepare("download".to_string(), rejected_role, move |_| async move { - ran_in_task.store(true, Ordering::SeqCst); - }) - .unwrap(); - let rejected = owner.install_gated(second).unwrap_err(); - drop(rejected); - tokio::task::yield_now().await; - assert!(!ran.load(Ordering::SeqCst)); - assert_eq!(owner.prepared_count_for_test(), 1); - - owner.rescue_abandoned(); - assert_eq!(owner.prepared_count_for_test(), 0); - tokio::time::timeout(Duration::from_secs(1), async { - while owner.outstanding_retired_for_test() != 0 { - tokio::task::yield_now().await; - } - }) - .await - .expect("abandoned prepared wrapper must be observed to terminal"); - assert_eq!(owner.retired_observations_for_test(), 1); - assert!(!ran.load(Ordering::SeqCst)); - } - } - - #[tokio::test] - async fn projection_settlement_distinguishes_duplicate_stale_and_missing() { - let owner = Arc::new(DownloadTaskOwner::new()); - let prepared = owner - .prepare_projection( - "projection".to_string(), - |_, _| async { ProjectionOutcome::Committed }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap(); - let ticket = owner.install_projection_gated(prepared).unwrap().start(); - assert_eq!( - owner.settle_projection(&ticket), - ProjectionSettlement::Pending - ); - assert_eq!(ticket.wait().await, ProjectionOutcome::Committed); - while owner.settle_projection(&ticket) == ProjectionSettlement::Pending { - tokio::task::yield_now().await; - } - assert_eq!( - owner.settle_projection(&ticket), - ProjectionSettlement::AlreadySettled - ); - - let missing_cell = Arc::new(ProjectionCell::new(true)); - missing_cell.settle(ProjectionOutcome::Committed); - let missing = ProjectionTicket { - download_id: "missing".to_string(), - generation: TaskGeneration::new(), - cell: missing_cell, - }; - assert_eq!( - owner.settle_projection(&missing), - ProjectionSettlement::Missing - ); - - let stale_cell = Arc::new(ProjectionCell::new(true)); - stale_cell.settle(ProjectionOutcome::Committed); - let stale = ProjectionTicket { - download_id: "successor".to_string(), - generation: TaskGeneration::new(), - cell: stale_cell, - }; - let successor = owner - .prepare("successor".to_string(), TaskRole::Worker, |_| async { - std::future::pending::<()>().await; - }) - .unwrap(); - owner.install_gated(successor).unwrap().start(); - assert_eq!( - owner.settle_projection(&stale), - ProjectionSettlement::StaleGeneration - ); - assert!(owner.contains("successor")); - } - - #[tokio::test] - async fn projection_catches_call_and_poll_panics_for_both_constructors() { - let owner = Arc::new(DownloadTaskOwner::new()); - - let call = owner - .prepare_projection( - "ownerless-call".to_string(), - |_, _| { - panic!("call-time projection panic"); - #[allow(unreachable_code)] - std::future::ready(ProjectionOutcome::Committed) - }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap(); - let call_ticket = owner.install_projection_gated(call).unwrap().start(); - assert_eq!(call_ticket.wait().await, ProjectionOutcome::Panicked); - - let poll = owner - .prepare_projection( - "ownerless-poll".to_string(), - |_, _| async { - panic!("poll-time projection panic"); - }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap(); - let poll_ticket = owner.install_projection_gated(poll).unwrap().start(); - assert_eq!(poll_ticket.wait().await, ProjectionOutcome::Panicked); - - for (id, call_time) in [("finished-call", true), ("finished-poll", false)] { - let predecessor = owner - .prepare(id.to_string(), TaskRole::Worker, |_| async {}) - .unwrap(); - owner.install_gated(predecessor).unwrap().start(); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner.snapshot(id).is_some_and(|task| task.finished) { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - let transition = if call_time { - owner - .begin_finished_projection( - id, - false, - |_, _| { - panic!("call-time finished projection panic"); - #[allow(unreachable_code)] - std::future::ready(ProjectionOutcome::Committed) - }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap() - } else { - owner - .begin_finished_projection( - id, - false, - |_, _| async { - panic!("poll-time finished projection panic"); - }, - |_| async { ProjectionOutcome::Failed }, - ) - .unwrap() - }; - let ProjectionTransition::Started(projection) = transition else { - panic!("finished predecessor should install a projector"); - }; - let ticket = projection.start(); - assert_eq!(ticket.wait().await, ProjectionOutcome::Panicked); - } - } - - #[tokio::test] - async fn fallback_panics_remain_owned_until_cancel_acknowledges_failure() { - let owner = Arc::new(DownloadTaskOwner::new()); - - let ownerless = owner - .prepare_projection( - "ownerless-double-panic".to_string(), - |_, _| { - panic!("call-time primary projection panic"); - #[allow(unreachable_code)] - std::future::ready(ProjectionOutcome::Committed) - }, - |_| { - panic!("call-time fallback projection panic"); - #[allow(unreachable_code)] - std::future::ready(ProjectionOutcome::Failed) - }, - ) - .unwrap(); - let InstalledProjection { task, ticket } = - owner.install_projection_gated(ownerless).unwrap(); - let abandoned_waiter_ticket = ticket.clone(); - let abandoned_waiter = tokio::spawn(async move { - let _ = abandoned_waiter_ticket.wait().await; - }); - abandoned_waiter.abort(); - let _ = abandoned_waiter.await; - task.start(); - assert_eq!(ticket.wait().await, ProjectionOutcome::Panicked); - acknowledge_failed_projection_through_cancel(&owner, "ownerless-double-panic", &ticket) - .await; - - let predecessor = owner - .prepare( - "finished-double-panic".to_string(), - TaskRole::Worker, - |_| async {}, - ) - .unwrap(); - owner.install_gated(predecessor).unwrap().start(); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("finished-double-panic") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - let transition = owner - .begin_finished_projection( - "finished-double-panic", - false, - |_, _| async { - panic!("poll-time primary projection panic"); - }, - |_| async { - panic!("poll-time fallback projection panic"); - }, - ) - .unwrap(); - let ProjectionTransition::Started(projection) = transition else { - panic!("finished predecessor should install a projector"); - }; - let ticket = projection.start(); - assert_eq!(ticket.wait().await, ProjectionOutcome::Panicked); - acknowledge_failed_projection_through_cancel(&owner, "finished-double-panic", &ticket) - .await; - } - - #[tokio::test] - async fn superseded_failed_projection_is_unacked_until_finalizer_projection() { - let owner = Arc::new(DownloadTaskOwner::new()); - let (fallback_reached_sender, fallback_reached) = oneshot::channel(); - let (_fallback_release_sender, fallback_release) = oneshot::channel::<()>(); - let prepared = owner - .prepare_projection( - "transferred-failure".to_string(), - |_, _| async { - panic!("primary projection panic"); - }, - move |_| async move { - let _ = fallback_reached_sender.send(()); - let _ = fallback_release.await; - ProjectionOutcome::RolledBack - }, - ) - .unwrap(); - let ticket = owner.install_projection_gated(prepared).unwrap().start(); - tokio::time::timeout(Duration::from_secs(1), fallback_reached) - .await - .expect("primary panic must enter fallback") - .unwrap(); - assert!(!ticket.failure_projected_for_test()); - - let (finalizer_reached_sender, finalizer_reached) = oneshot::channel(); - let (allow_projection_sender, allow_projection) = oneshot::channel(); - let transition = owner - .begin_cancel( - "transferred-failure", - move |context, predecessor| async move { - let CancelPredecessor::Observed(observation) = predecessor else { - panic!("superseded projector must remain predecessor custody"); - }; - assert!(observation.nested_failures > 0); - let _ = finalizer_reached_sender.send(()); - let _ = allow_projection.await; - assert!(context.complete_transferred_projection(true)); - }, - ) - .unwrap(); - let CancelTransition::Started(finalizer) = transition else { - panic!("projection must be replaced by one finalizer"); - }; - finalizer.start(); - assert_eq!(ticket.wait().await, ProjectionOutcome::Superseded); - tokio::time::timeout(Duration::from_secs(1), finalizer_reached) - .await - .expect("finalizer must observe the failed projector") - .unwrap(); - assert!(!ticket.failure_projected_for_test()); - assert!(!ticket.settled_for_test()); - allow_projection_sender.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(1), async { - while !ticket.failure_projected_for_test() || !ticket.settled_for_test() { - tokio::task::yield_now().await; - } - }) - .await - .expect("finalizer must acknowledge only after its terminal projection"); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("transferred-failure") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - assert!(owner - .observe_finished("transferred-failure") - .await - .is_some()); - assert!(!owner.contains("transferred-failure")); - } - - #[tokio::test] - async fn finished_and_panicked_tasks_are_observed_once() { - let owner = Arc::new(DownloadTaskOwner::new()); - let prepared = owner - .prepare("panic".to_string(), TaskRole::Worker, |_| async { - panic!("sentinel panic"); - }) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("panic") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - - let observation = owner.observe_finished("panic").await.unwrap(); - assert_eq!(observation.role, TaskRole::Worker); - assert_eq!(observation.terminal, TaskTerminal::Panicked); - assert_eq!(observation.nested_failures, 0); - assert!(owner.observe_finished("panic").await.is_none()); - } - - #[tokio::test] - async fn cancel_finalizer_drains_registered_blocking_work_before_terminal_callback() { - let owner = Arc::new(DownloadTaskOwner::new()); - let (blocking_started_sender, blocking_started) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - let prepared = owner - .prepare( - "download".to_string(), - TaskRole::Worker, - move |context| async move { - let _ = context - .run_blocking(move || { - let _ = blocking_started_sender.send(()); - let _ = release.recv(); - }) - .await; - }, - ) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - blocking_started.await.unwrap(); - - let finalized = Arc::new(AtomicBool::new(false)); - let finalized_in_task = finalized.clone(); - assert!(start_cancel( - owner - .begin_cancel("download", move |_context, predecessor| async move { - let CancelPredecessor::Observed(observation) = predecessor else { - panic!("installed worker must be observed"); - }; - assert_eq!(observation.terminal, TaskTerminal::Cancelled); - finalized_in_task.store(true, Ordering::SeqCst); - },) - .unwrap() - )); - tokio::task::yield_now().await; - assert!(!finalized.load(Ordering::SeqCst)); - - release_sender.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(1), async { - while !finalized.load(Ordering::SeqCst) { - tokio::task::yield_now().await; - } - }) - .await - .expect("finalizer should wait for nested blocking work"); - } - - #[tokio::test] - async fn stale_generation_cannot_observe_or_remove_cancel_successor() { - let owner = Arc::new(DownloadTaskOwner::new()); - let prepared = owner - .prepare("download".to_string(), TaskRole::Worker, |_| async { - std::future::pending::<()>().await; - }) - .unwrap(); - let installed = owner.install_gated(prepared).unwrap(); - let stale_generation = installed.generation().clone(); - installed.start(); - let (finish_sender, finish_receiver) = oneshot::channel(); - let transition = owner - .begin_cancel("download", move |_context, _| async move { - let _ = finish_receiver.await; - }) - .unwrap(); - let CancelTransition::Started(finalizer) = transition else { - panic!("worker should transition to a finalizer"); - }; - finalizer.start(); - let successor = owner.generation_for_test("download").unwrap(); - - assert!(owner - .observe_finished_generation("download", &stale_generation) - .await - .is_none()); - assert!(owner.snapshot("download").is_some_and(|snapshot| { - owner - .generation_for_test("download") - .is_some_and(|current| current.matches(&successor)) - && snapshot.role == TaskRole::CancelFinalizer - })); - finish_sender.send(()).unwrap(); - } - - #[tokio::test] - async fn repeated_cancel_keeps_one_finalizer_owner() { - let owner = Arc::new(DownloadTaskOwner::new()); - let prepared = owner - .prepare("download".to_string(), TaskRole::Worker, |_| async { - std::future::pending::<()>().await; - }) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - let count = Arc::new(AtomicUsize::new(0)); - let count_in_finalizer = count.clone(); - let (finish_sender, finish_receiver) = oneshot::channel(); - let first = owner - .begin_cancel("download", move |_context, _| async move { - count_in_finalizer.fetch_add(1, Ordering::SeqCst); - let _ = finish_receiver.await; - }) - .unwrap(); - let CancelTransition::Started(finalizer) = first else { - panic!("first cancellation should install a finalizer"); - }; - finalizer.start(); - let first_generation = owner.generation_for_test("download").unwrap(); - let second = owner.begin_cancel("download", |_, _| async {}).unwrap(); - let CancelTransition::AlreadyRunning = second else { - panic!("repeat cancellation must not replace the finalizer"); - }; - let second_generation = owner.generation_for_test("download").unwrap(); - assert!(first_generation.matches(&second_generation)); - finish_sender.send(()).unwrap(); - tokio::task::yield_now().await; - assert_eq!(count.load(Ordering::SeqCst), 1); - } - - #[tokio::test] - async fn aborted_finalizer_retains_predecessor_drain_and_failure() { - for outcome in ["success", "error", "panic"] { - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered_sender, entered) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - let prepared = owner - .prepare( - "download".into(), - TaskRole::Worker, - move |context| async move { - let _ = context - .run_fallible_blocking_named( - "held cancellation predecessor", - move || -> Result<(), &'static str> { - entered_sender.send(()).unwrap(); - release.recv().unwrap(); - match outcome { - "success" => Ok(()), - "error" => Err("predecessor failure"), - _ => panic!("predecessor panic"), - } - }, - ) - .await; - }, - ) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - entered.await.unwrap(); - let finished = Arc::new(AtomicBool::new(false)); - let finished_in_finalizer = finished.clone(); - let CancelTransition::Started(finalizer) = owner - .begin_cancel("download", move |_, _| async move { - finished_in_finalizer.store(true, Ordering::Release); - }) - .unwrap() - else { - panic!("worker must receive a finalizer") - }; - let generation = finalizer.generation().clone(); - finalizer.start(); - owner.state.lock().unwrap().tasks["download"].outer.abort(); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner.outer_finished_for_test("download") { - tokio::task::yield_now().await; - } - }) - .await - .expect("finalizer outer must observe cancellation"); - let prematurely_finished = owner.snapshot("download").unwrap().finished; - let premature_observation = owner - .observe_finished_generation("download", &generation) - .await; - release_sender.send(()).unwrap(); - assert!(!prematurely_finished, "predecessor still held: {outcome}"); - assert!(premature_observation.is_none()); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .expect("predecessor observation must drain after release"); - let observation = owner - .observe_finished_generation("download", &generation) - .await - .unwrap(); - assert_eq!(observation.terminal, TaskTerminal::Cancelled); - assert_eq!( - observation.nested_failures, - usize::from(outcome != "success") - ); - assert!(!finished.load(Ordering::Acquire)); - } - } - - #[tokio::test] - async fn predecessor_failure_is_retained_without_failing_new_cleanup_effects() { - for abort_after_delivery in [false, true] { - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered_sender, entered) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - let prepared = owner - .prepare( - "download".into(), - TaskRole::Worker, - move |context| async move { - let _ = context - .run_fallible_blocking_named( - "failed predecessor", - move || -> Result<(), &'static str> { - entered_sender.send(()).unwrap(); - release.recv().unwrap(); - Err("predecessor failed") - }, - ) - .await; - }, - ) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - entered.await.unwrap(); - let (delivered_sender, delivered) = oneshot::channel(); - let (finish_sender, finish) = oneshot::channel(); - let CancelTransition::Started(finalizer) = owner - .begin_cancel("download", move |context, predecessor| async move { - let CancelPredecessor::Observed(observation) = predecessor else { - panic!("predecessor must be observed") - }; - assert_eq!(observation.nested_failures, 1); + }))); + let caller_owner = owner.clone(); + let caller = tokio::spawn(async move { + caller_owner + .run_invocation(move |context| async move { + let context = context.with_root_grant(phase_root).await?; context - .run_fallible_blocking_named("successful cleanup", || { - Ok::<_, &'static str>(()) - }) + .run_fallible_blocking_named( + "protected completed write", + || Ok::<_, ()>(()), + ) .await .unwrap() .unwrap(); - assert_eq!(context.drain_blocking().await, Ok(0)); - delivered_sender.send(()).unwrap(); - let _ = finish.await; + Ok(()) }) - .unwrap() - else { - panic!("worker must receive a finalizer") - }; - let generation = finalizer.generation().clone(); - finalizer.start(); - release_sender.send(()).unwrap(); - tokio::time::timeout(Duration::from_secs(1), delivered) .await - .expect("predecessor failure must not prevent cleanup drain") - .unwrap(); - if abort_after_delivery { - owner.state.lock().unwrap().tasks["download"].outer.abort(); - } else { - finish_sender.send(()).unwrap(); - } - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .expect("finalizer must become observable"); - let observation = owner - .observe_finished_generation("download", &generation) - .await - .unwrap(); - assert_eq!(observation.nested_failures, 1); - assert_eq!( - observation.terminal, - if abort_after_delivery { - TaskTerminal::Cancelled - } else { - TaskTerminal::Completed - } - ); - } - } - - #[tokio::test] - async fn failed_predecessor_observation_closes_receipt_only_after_effect_drain() { - let owner = Arc::new(DownloadTaskOwner::new()); - let (entered_sender, entered) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - let prepared = owner - .prepare( - "download".into(), - TaskRole::Worker, - move |context| async move { - let _ = context - .run_blocking(move || { - entered_sender.send(()).unwrap(); - release.recv().unwrap(); - }) - .await; - }, - ) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - entered.await.unwrap(); - let mut predecessor = owner - .state - .lock() - .unwrap() - .tasks - .remove("download") - .unwrap(); - let outer_abort = predecessor.outer.abort_handle(); - outer_abort.abort(); - tokio::time::timeout(Duration::from_secs(1), async { - while !outer_abort.is_finished() { - tokio::task::yield_now().await; - } - }) - .await - .expect("predecessor outer must finish before observing its nested work"); - let poisoned = Arc::new(ProjectionCell::new(false)); - assert!(std::panic::catch_unwind(AssertUnwindSafe(|| { - let _guard = poisoned.state.lock().unwrap(); - panic!("poison predecessor provenance"); - })) - .is_err()); - // Exercise the observer's own bookkeeping-failure path without placing - // a poisoned cell in a live successor's unrelated projection state. - predecessor.projection = Some(poisoned); - let completion = Arc::new(NestedCompletion { - finished: AtomicBool::new(false), - failed: AtomicBool::new(false), - notify: Notify::new(), }); - let (start, started) = oneshot::channel(); - let (receipt_sender, mut receipt) = oneshot::channel(); - let mut observer = Box::pin(observe_cancellation_predecessor( - Some(predecessor), - false, - started, - completion.clone(), - receipt_sender, - )); - start.send(()).unwrap(); - let observation_while_held = futures::poll!(&mut observer); - let finished_while_held = completion.finished.load(Ordering::Acquire); - let receipt_while_held = receipt.try_recv(); - release_sender.send(()).unwrap(); - assert!(observation_while_held.is_pending()); - tokio::time::timeout(Duration::from_secs(1), observer) + tokio::time::timeout(Duration::from_secs(3), ready) .await - .expect("failed observer must drain before completing"); - assert!(!finished_while_held); - assert!(matches!( - receipt_while_held, - Err(oneshot::error::TryRecvError::Empty) - )); - assert!(completion.finished.load(Ordering::Acquire)); - assert!(completion.failed.load(Ordering::Acquire)); - assert!( - receipt.await.is_err(), - "failed observation must not synthesize Absent" - ); - } - - #[tokio::test] - async fn blocking_panic_is_retained_when_outer_receiver_is_cancelled() { - let owner = Arc::new(DownloadTaskOwner::new()); - let (blocking_started_sender, blocking_started) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - let prepared = owner - .prepare( - "download".to_string(), - TaskRole::Worker, - move |context| async move { - let _ = context - .run_blocking(move || { - let _ = blocking_started_sender.send(()); - let _ = release.recv(); - panic!("nested sentinel panic"); - }) - .await; - }, - ) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - blocking_started.await.unwrap(); - - let (observed_sender, observed) = oneshot::channel(); - assert!(start_cancel( - owner - .begin_cancel("download", move |_context, predecessor| async move { - let _ = observed_sender.send(predecessor); - },) - .unwrap() - )); - release_sender.send(()).unwrap(); - let CancelPredecessor::Observed(observation) = observed.await.unwrap() else { - panic!("installed worker must be observed"); - }; - assert_eq!(observation.terminal, TaskTerminal::Cancelled); - assert_eq!(observation.nested_failures, 1); - } - - #[tokio::test] - async fn finished_unobserved_finalizer_is_replaced_and_observed_once() { - let owner = Arc::new(DownloadTaskOwner::new()); - let prepared = owner - .prepare("download".to_string(), TaskRole::Worker, |_| async {}) + .unwrap() .unwrap(); - owner.install_gated(prepared).unwrap().start(); - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - let count = Arc::new(AtomicUsize::new(0)); - let count_in_finalizer = count.clone(); - assert!(start_cancel( - owner - .begin_cancel("download", move |_, _| async move { - count_in_finalizer.fetch_add(1, Ordering::SeqCst); - },) - .unwrap() - )); - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - - let (predecessor_sender, predecessor) = oneshot::channel(); - let replacement = owner - .begin_cancel("download", move |_, predecessor| async move { - let _ = predecessor_sender.send(predecessor); - }) + caller.abort(); + let _ = caller.await; + let receipt = owner.request_shutdown(); + let mut shutdown = Box::pin(receipt.wait()); + let pending = futures::poll!(&mut shutdown).is_pending(); + let contention = root.try_acquire_execution_grant(); + release.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(3), shutdown) + .await + .unwrap() .unwrap(); - let CancelTransition::Started(replacement) = replacement else { - panic!("finished finalizer must be replaced by an observing finalizer"); - }; - replacement.start(); - let CancelPredecessor::Observed(observation) = predecessor.await.unwrap() else { - panic!("replacement must observe the finished finalizer"); - }; - assert_eq!(observation.role, TaskRole::CancelFinalizer); - assert_eq!(observation.terminal, TaskTerminal::Completed); - assert_eq!(count.load(Ordering::SeqCst), 1); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .unwrap(); - assert_eq!( - owner.observe_finished("download").await.unwrap().role, - TaskRole::CancelFinalizer - ); - assert!(owner.observe_finished("download").await.is_none()); + assert!(pending); + assert!(matches!( + contention, + Err(crate::PumasError::DownloadRootBusy) + )); + owner.set_blocking_result_observer(None); + root.try_acquire_execution_grant().unwrap(); } #[tokio::test] - async fn cancelling_an_outer_drain_keeps_nested_custody_with_the_finalizer() { + async fn scoped_root_grants_share_and_release_without_retiring_the_invocation() { + let temp = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(temp.path()).unwrap(); let owner = Arc::new(DownloadTaskOwner::new()); - let (drain_sender, drain_receiver) = oneshot::channel(); - let prepared = owner - .prepare( - "download".to_string(), - TaskRole::Worker, - move |context| async move { - let _ = drain_receiver.await; - let _ = context.drain_blocking().await; - }, - ) - .unwrap(); - let installed = owner.install_gated(prepared).unwrap(); - let generation = installed.generation().clone(); - installed.start(); - - let (blocking_started_sender, blocking_started) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - let result = owner - .register_blocking( - "download", - &generation, - "drain cancellation sentinel", - move || { - let _ = blocking_started_sender.send(()); - let _ = release.recv(); - }, - ) + owner + .run_invocation(move |context| async move { + let (first, second) = tokio::join!( + context.with_root_grant(root.clone()), + context.with_root_grant(root.clone()), + ); + let first = first?; + let second = second?; + assert!(Arc::ptr_eq( + first.root_grant.as_ref().unwrap(), + second.root_grant.as_ref().unwrap() + )); + assert!(matches!( + root.try_acquire_execution_grant(), + Err(crate::PumasError::DownloadRootBusy) + )); + drop(first); + drop(second); + context.drain_blocking().await.unwrap(); + // This invocation is still registered and running: only its mutation + // scope, not its registry membership, determines native custody. + root.try_acquire_execution_grant()?; + Ok(()) + }) + .await .unwrap(); - drop(result); - blocking_started.await.unwrap(); - let (drain_started_sender, drain_started) = oneshot::channel(); - let drain_started_sender = Arc::new(Mutex::new(Some(drain_started_sender))); - owner.set_drain_observer(Some(Arc::new(move || { - if let Some(sender) = drain_started_sender.lock().unwrap().take() { - let _ = sender.send(()); - } - }))); - drain_sender.send(()).unwrap(); - drain_started.await.unwrap(); - assert_eq!(owner.nested_count_for_test("download"), Some(1)); - - let terminal = Arc::new(AtomicBool::new(false)); - let terminal_in_finalizer = terminal.clone(); - assert!(start_cancel( - owner - .begin_cancel("download", move |_, _| async move { - terminal_in_finalizer.store(true, Ordering::SeqCst); - },) - .unwrap() - )); - tokio::task::yield_now().await; - let terminal_before_release = terminal.load(Ordering::SeqCst); - release_sender.send(()).unwrap(); - owner.set_drain_observer(None); - - assert!( - !terminal_before_release, - "cancelling a drain must not detach its registered blocking owner" - ); - tokio::time::timeout(Duration::from_secs(1), async { - while !terminal.load(Ordering::SeqCst) { - tokio::task::yield_now().await; - } - }) - .await - .expect("the same runtime must drive the finalizer after nested release"); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .expect("finalizer should reach an observable terminal outcome"); - let observation = owner.observe_finished("download").await.unwrap(); - assert_eq!(observation.role, TaskRole::CancelFinalizer); - assert_eq!(observation.terminal, TaskTerminal::Completed); - assert_eq!(observation.nested_failures, 0); + owner.shutdown().await.unwrap(); } #[tokio::test] - async fn finished_outer_is_not_observable_until_registered_blocking_work_finishes() { + async fn one_owner_acquires_distinct_physical_root_grants_concurrently() { + let model_root_dir = tempfile::TempDir::new().unwrap(); + let runtime_root_dir = tempfile::TempDir::new().unwrap(); + let model_root = DownloadDestinationRoot::open(model_root_dir.path()).unwrap(); + let reopened_model_root = DownloadDestinationRoot::open(model_root_dir.path()).unwrap(); + let runtime_root = DownloadDestinationRoot::open(runtime_root_dir.path()).unwrap(); let owner = Arc::new(DownloadTaskOwner::new()); - let (outer_release_sender, outer_release) = oneshot::channel(); - let prepared = owner - .prepare( - "download".to_string(), - TaskRole::Worker, - move |_| async move { - let _ = outer_release.await; - }, - ) - .unwrap(); - let installed = owner.install_gated(prepared).unwrap(); - let generation = installed.generation().clone(); - let (blocking_started_sender, blocking_started) = oneshot::channel(); - let (release_sender, release) = std::sync::mpsc::channel(); - // Register the real held blocking work synchronously through the - // owner/generation before allowing the outer future to complete. - let nested_result = owner - .register_blocking( - "download", - &generation, - "finished outer held operation", - move || { - let _ = blocking_started_sender.send(()); - let _ = release.recv(); - }, - ) - .unwrap(); - installed.start(); - tokio::time::timeout(Duration::from_secs(1), blocking_started) + owner + .run_invocation(move |context| async move { + let (model, reopened_model, runtime) = tokio::join!( + context.with_root_grant(model_root.clone()), + context.with_root_grant(reopened_model_root.clone()), + context.with_root_grant(runtime_root.clone()), + ); + let model = model?; + let reopened_model = reopened_model?; + let runtime = runtime?; + assert!(Arc::ptr_eq( + model.root_grant.as_ref().unwrap(), + reopened_model.root_grant.as_ref().unwrap() + )); + assert!(!Arc::ptr_eq( + model.root_grant.as_ref().unwrap(), + runtime.root_grant.as_ref().unwrap() + )); + assert!(matches!( + model_root.try_acquire_execution_grant(), + Err(crate::PumasError::DownloadRootBusy) + )); + assert!(matches!( + runtime_root.try_acquire_execution_grant(), + Err(crate::PumasError::DownloadRootBusy) + )); + drop(model); + drop(reopened_model); + drop(runtime); + context.drain_blocking().await.unwrap(); + model_root.try_acquire_execution_grant()?; + runtime_root.try_acquire_execution_grant()?; + Ok(()) + }) .await - .expect("registered blocking work should start") .unwrap(); - outer_release_sender.send(()).unwrap(); - tokio::task::yield_now().await; - - assert!(owner - .snapshot("download") - .is_some_and(|snapshot| { !snapshot.finished && snapshot.role == TaskRole::Worker })); - assert!(owner.observe_finished("download").await.is_none()); - assert!(owner.contains("download")); - - release_sender.send(()).unwrap(); - nested_result.await.unwrap().unwrap(); - tokio::time::timeout(Duration::from_secs(1), async { - while !owner - .snapshot("download") - .is_some_and(|snapshot| snapshot.finished) - { - tokio::task::yield_now().await; - } - }) - .await - .expect("nested completion should make the owner observable"); - let observation = owner - .observe_finished("download") - .await - .expect("finished nested work must be observable"); - assert_eq!(observation.role, TaskRole::Worker); - assert_eq!(observation.terminal, TaskTerminal::Completed); - assert_eq!(observation.nested_failures, 0); - assert!(!owner.contains("download")); + owner.shutdown().await.unwrap(); } #[tokio::test] - async fn completed_nested_work_is_reaped_without_losing_failure_evidence() { + async fn refused_root_grant_does_not_poison_shutdown() { + let temp = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(temp.path()).unwrap(); + let held = root.try_acquire_execution_grant().unwrap(); let owner = Arc::new(DownloadTaskOwner::new()); - let owner_in_task = owner.clone(); - let (metrics_sender, metrics) = oneshot::channel(); - let prepared = owner - .prepare( - "download".to_string(), - TaskRole::Worker, - move |context| async move { - let mut retained_max = 0; - for index in 0..512 { - let result = context - .run_blocking(move || { - if index == 0 { - panic!("archived nested failure sentinel"); - } - }) - .await; - if index == 0 { - assert!(matches!(result, Err(BlockingTaskError::Join(_)))); - } else { - result.unwrap(); - } - retained_max = retained_max.max( - owner_in_task - .nested_count_for_test("download") - .unwrap_or_default(), - ); - } - let drained_failures = context.drain_blocking().await.unwrap(); - let _ = metrics_sender.send((retained_max, drained_failures)); - std::future::pending::<()>().await; - }, - ) - .unwrap(); - owner.install_gated(prepared).unwrap().start(); - - let (retained_max, drained_failures) = metrics.await.unwrap(); - assert!( - retained_max <= 2, - "sequential blocking operations must retain only the current and terminalizing observer" - ); - assert_eq!(drained_failures, 1); - - let (predecessor_sender, predecessor) = oneshot::channel(); - assert!(start_cancel( - owner - .begin_cancel("download", move |_, predecessor| async move { - let _ = predecessor_sender.send(predecessor); - },) - .unwrap() - )); - let CancelPredecessor::Observed(observation) = predecessor.await.unwrap() else { - panic!("the worker remains owned until cancellation"); - }; - assert_eq!(observation.nested_failures, 1); + let outcome = owner + .run_invocation(move |context| async move { + context.with_root_grant(root).await.map(|_| ()) + }) + .await; + assert!(matches!(outcome, Err(crate::PumasError::DownloadRootBusy))); + owner.shutdown().await.unwrap(); + drop(held); } #[tokio::test] - async fn state_only_cancellation_does_not_fabricate_a_worker_observation() { - let owner = Arc::new(DownloadTaskOwner::new()); - let owner_in_finalizer = owner.clone(); - let (observation_sender, observation) = oneshot::channel(); - assert!(start_cancel( - owner - .begin_cancel("state-only", move |_, observation| async move { - assert!(owner_in_finalizer.contains("state-only")); - let _ = observation_sender.send(observation); - },) - .unwrap() - )); + async fn root_grant_retains_cancelled_blocking_and_async_effects_until_observed() { + for asynchronous in [false, true] { + for failure in 0..3 { + let temp = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(temp.path()).unwrap(); + let effect_root = root.clone(); + let owner = Arc::new(DownloadTaskOwner::new()); + let caller_owner = owner.clone(); + let (entered, ready) = oneshot::channel(); + let (release, released) = oneshot::channel(); + let caller = tokio::spawn(async move { + caller_owner + .run_invocation(move |context| async move { + let context = context.with_root_grant(effect_root).await?; + if asynchronous { + let _ = context + .run_fallible_async_named( + "held protected async effect", + move || async move { + let _ = entered.send(()); + released.await.unwrap(); + assert_ne!( + failure, 2, + "injected protected async panic" + ); + if failure == 1 { + Err("protected async failure") + } else { + Ok(()) + } + }, + ) + .await; + } else { + let _ = context + .run_fallible_blocking_named( + "held protected blocking effect", + move || { + let _ = entered.send(()); + released.blocking_recv().unwrap(); + assert_ne!( + failure, 2, + "injected protected blocking panic" + ); + if failure == 1 { + Err("protected blocking failure") + } else { + Ok(()) + } + }, + ) + .await; + } + Ok(()) + }) + .await + }); + tokio::time::timeout(Duration::from_secs(3), ready) + .await + .unwrap() + .unwrap(); + caller.abort(); + let _ = caller.await; + let receipt = owner.request_shutdown(); + let mut shutdown = Box::pin(receipt.wait()); + let pending = futures::poll!(&mut shutdown).is_pending(); + let contention = root.try_acquire_execution_grant(); + release.send(()).unwrap(); + let outcome = tokio::time::timeout(Duration::from_secs(3), shutdown) + .await + .unwrap(); + assert!(pending); + assert!(matches!( + contention, + Err(crate::PumasError::DownloadRootBusy) + )); + assert_eq!(outcome.is_err(), failure != 0); + root.try_acquire_execution_grant().unwrap(); + } + } + } - assert_eq!(observation.await.unwrap(), CancelPredecessor::Absent); + fn queue_destination() -> (tempfile::TempDir, DestinationIdentity) { + let root = tempfile::TempDir::new().unwrap(); + let authority = + crate::model_library::download_recovery::DownloadDestinationRoot::open(root.path()) + .unwrap(); + let destination = authority.resolve(Path::new("model")).unwrap().identity(); + (root, destination) } + #[test] + fn released_destination_claim_cannot_be_resurrected_from_stale_inventory() { + let owner = super::DestinationExecutionOwner::new(); + let (_root, path) = queue_destination(); + let first = super::TaskGeneration::new(); + let second = super::TaskGeneration::new(); + assert!(owner.reserve( + path.clone(), + "first".into(), + super::DestinationDomain::Ambient, + first.clone() + )); + assert!(owner.release(&path, "first", super::DestinationDomain::Ambient, &first)); + assert!(!owner.reserve_dormant( + path.clone(), + "first".into(), + super::DestinationDomain::Ambient + )); + assert!(!owner.reserve( + path.clone(), + "first".into(), + super::DestinationDomain::Ambient, + second + )); + assert_eq!(owner.claim_count(&path), 0); + } #[tokio::test] async fn destination_reservations_follow_admission_order_not_poll_order() { let owner = Arc::new(DestinationExecutionOwner::new()); diff --git a/rust/crates/pumas-core/src/model_library/hf/mod.rs b/rust/crates/pumas-core/src/model_library/hf/mod.rs index ae31bada..d11fd2a1 100644 --- a/rust/crates/pumas-core/src/model_library/hf/mod.rs +++ b/rust/crates/pumas-core/src/model_library/hf/mod.rs @@ -306,11 +306,15 @@ impl HuggingFaceClient { client, download_client, cache_dir, - downloads, + downloads: downloads.clone(), download_revision, download_updates, - download_publications, - download_tasks: Arc::new(DownloadTaskOwner::new()), + download_publications: download_publications.clone(), + download_tasks: Arc::new(DownloadTaskOwner::new_with_finalizer({ + let downloads = downloads.clone(); + let publications = download_publications.clone(); + move || download::project_download_shutdown(downloads, publications) + })), owns_lifecycle: true, destination_executions: Arc::new(DestinationExecutionOwner::new()), dest_locks: Arc::new(RwLock::new(HashMap::new())), @@ -493,11 +497,7 @@ impl HuggingFaceClient { impl Drop for HuggingFaceClient { fn drop(&mut self) { if self.owns_lifecycle { - let downloads = self.downloads.clone(); - let publications = self.download_publications.clone(); - self.download_tasks.request_shutdown(move || { - download::project_download_shutdown(downloads, publications) - }); + self.download_tasks.request_shutdown(); } } } From efc4d20bde6ae426c96f6a5fcb55ce0022788d09 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 19:26:45 -0700 Subject: [PATCH 10/20] docs(acquisition): record scoped custody candidate --- .../artifact-acquisition/execution-ledger.md | 12 +++++++++++ docs/plans/artifact-acquisition/plan.md | 4 ++-- .../reports/coding-standards-mcp-usability.md | 20 ++++++++++++++++++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index b8349767..f7db73b3 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -82,6 +82,18 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - These are local Rust tests, controlled/disposable filesystem cases, and static checks. No live Hugging Face import, shared llama.cpp acquisition/extraction, model generation, desktop workflow, retained live-root migration, real S3, or network-denied Torch install has been executed. The existing Ollama source/archive loop remains independent of acquisition. - The primary branch still requires a push and current hosted checks for this exact head; the prior hosted run belongs to an older SHA and skipped conditional jobs are not passes. AQ-HTTP remains not ready and no runtime slice may rely on it. +## 2026-09-30 — Q1 scoped task-custody extraction integrated + +- The worker change `584b5f45a17a80cd584b16ce77f7be5d9688a9b8` was reviewed and integrated without rewriting history into `work/acquisition-q1-http`; primary merge candidate `c3052583860baa76dc114edf6420805e9b06f327` is pushed and remains PR #7's draft head. The original baseline `a8359512a580aa25fb2f9c9e4cd7e0dd64fd970d` remains preserved. +- Exact source write set: `rust/crates/pumas-core/src/acquisition/task_custody.rs`, `acquisition/mod.rs`, `model_library/hf/lifecycle.rs`, `model_library/hf/mod.rs`, and `model_library/hf/download.rs`. One crate-private supervisor owns scope-keyed task populations, nested effects, generations, predecessor drainage, projections, and shutdown receipts. HF retains model admission, destination queues/root grants, existing persistence and final projection policy. No durable schema, migration, download marker/deletion rule, native transfer, generated interface, or lockfile changed. +- **Exact current candidate evidence:** On `c3052583860baa76dc114edf6420805e9b06f327`, the scoped custody test target passed 34 tests and the HF test target passed 219 tests using controlled local HTTP fixtures and disposable state. `git diff --check` passed. On source-identical worker commit `584b5f45…`, formatting, `pumas-library` all-target Clippy with `-D warnings`, and the no-default-features check passed. The reviewer executed no checks. This is custody/HF component evidence only, not a source-neutral owner, native consumer, retained migration/reopen, live source, desktop, S3, package, or runtime-admission claim. +- Independent read-only review of `584b5f45…` found no substantiated P0/P1/P2/P3 finding in the five-file scope. It verified scoped isolation, custody drainage and outcome retention, and preservation of HF policy. A generic scope still requires an explicit shutdown owner; HF currently has that owner. AQ-HTTP remains not ready. +- Hosted Actions run `36658935346` is for exact head `c3052583860baa76dc114edf6420805e9b06f327`; it was observed in progress, with Workflow/release contracts successful and frontend/desktop contracts, headless-without-inference, Rust quality and three Torch target jobs still in progress. Release/archive/RPC-E2E conditional jobs were skipped. No current-head full CI pass is claimed. +- The next Q1 implementation boundary is the canonical durable source-neutral acquisition owner/store and verified-file handoff, with the ordinary HF workflow and existing exact-tag llama.cpp installer as real consumers. Source inspection also confirmed `DownloadPersistence` schema 5 performs its v4-to-v5 publication during load and that `LibraryMutationAuthority` reads hidden admissions/quarantines to protect model deletion. The new owner/store and that mutation reader must change together; any migration must preserve those populations and must not claim exact consumer settlement from tag/path existence. Supported retained population and old-writer behavior remain to be established before any live-root mutation. +- **Coding-Standards MCP for this boundary:** snapshot `snapshot:v1:366ec2b5-a33e-48de-a36f-c2bde42d1d19`; the complete route selected 31 standards with zero unresolved categories. Focused reads covered Persistence, Architecture Replay, Concurrency, Contract Evolution, Schemas, Untrusted Execution, Independent Test Oracles and Platform Verification. The first route attempt requested `content.limit: 0` and the MCP rejected it (`minimum = 1`); omitting `content` returned a complete selection. An eight-policy read exceeded the visible output window, so the policies were re-read in two smaller batches. MCP routing/reads supplied obligations only; source, GitHub and tests supplied implementation evidence. Actual usability feedback is in [the separate report](reports/coding-standards-mcp-usability.md). +- A read-only architecture investigation of `c3052583` confirmed the supervisor extraction remains necessary preparation and does not create a shared durable owner. Its source trace identified the v4 migration-on-load, hidden custody and deletion-authority coupling, GitHub exact-tag/asset bridge, separate HF/native shutdown composition, and public constructors that need explicit lifecycle dispositions. The full write-set recommendation is retained in its handoff and will be narrowed to the exact production changes and tests before the next edits. +- User-owned untracked `docs/breif/future.md` remains untouched. No live retained store, installed environment, model, or release was opened or changed. + ### Coding-Standards route for the next shared-custody slice - Fresh route snapshot `snapshot:v1:2ea2f5e4-70d3-4a32-97c6-13a93989fb3f` selected 34 standards with zero unresolved fact categories after explicitly marking the unrelated framework fact known-absent. The first route omitted that fact and returned one required unresolved category; correcting the route preserved the same change scope. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index 92c4efad..7b28b090 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,8 +2,8 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** Q1 manifest/HTTP protocol and normal Hugging Face consumer slice implemented and under verification. The existing GitHub client has an additive exact-tag resolver that maps live publisher asset identity/digest metadata to a verified manifest while leaving the public/cache DTO unchanged. The HF lifecycle owner coalesces independently reopened handles for the same physical root while retaining distinct grants; this remains extraction preparation, not the shared acquisition cutover. Native installer shutdown custody is committed and integrated on the Q1 branch after repairs for metadata coordination, pending file-I/O settlement, fallible stage cleanup, and the public `OllamaVersionManager` wrapper's owned drain, with no remaining P0–P2 review findings in that custody scope. Its downloader remains independent. The next admitted slice moves the existing task/effect supervisor behind the acquisition boundary and scopes consumer operations while preserving HF model policy and retained state. The native installer does not yet consume the GitHub resolver or shared owner; durable handoff, retained-store evolution, desktop path, and required real-source qualification remain pending. -**Exactly one next slice:** Continue **Q1 — shared HTTP acquisition through the existing Hugging Face and native-runtime consumers.** +**Current phase:** Q1 manifest/HTTP protocol and the ordinary Hugging Face consumer are implemented. The exact-tag GitHub resolver maps publisher asset identity/digest evidence into a manifest without changing the public/cache DTO. HF physical-root grants coalesce independent opens of one root while retaining distinct grants. Native installation shutdown custody is committed, and the shared task/effect supervisor has been extracted with consumer-scoped admission, observation and drainage at `c3052583860baa76dc114edf6420805e9b06f327`. These are preparation/current-consumer slices: HF still owns transfer retry/effects, persistence and handoff; llama.cpp still uses its independent downloader. A durable source-neutral acquisition owner/store, exact-generation consumer reconciliation, real HF/native cutovers, desktop path and source/platform qualification remain pending. +**Exactly one next slice:** Continue **Q1 — one durable, source-neutral acquisition lifecycle and verified-file handoff consumed by both the normal Hugging Face workflow and the existing llama.cpp installer.** Keep consumer publication and model/runtime policy at their existing owners. Change the retained store only after supported schema/state and writer coordination are explicitly accounted for; do not use a live retained root as an implementation fixture. **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. **Operation:** `start` this exact plan on `work/acquisition-q1-http`. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index 5ed51d90..033eee53 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -146,6 +146,24 @@ This report records agent experience using the Coding-Standards MCP during Acqui ## Participation -The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, and shared-owner design reviewer used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. +### Primary integrator — current Q1 shared-store routing + +- **Useful calls:** A fresh `routing_facts` snapshot plus a complete route selected 31 standards with zero unresolved categories for the durable store/service boundary. Focused reads returned Persistence, Replay, Concurrency, Contract Evolution, Schemas, Untrusted Execution, Independent Test Oracles, and Platform Verification. Those standards make supported retained-state facts, one-writer semantics, effect cessation before release/publication, version scope, delegated authority and claim-matched evidence explicit. +- **Confusing or redundant steps:** An attempted selection-only call with `content.limit: 0` was rejected because the input contract requires a minimum of one; omitting `content` produced the desired compact route. A single eight-policy `read_many` returned duplicated content/metadata and exceeded the visible output window; the read was repeated in two focused batches. +- **Missing context:** The MCP did not expose the exact `c3052583` candidate, plan gate state, schema-5 hidden custody populations, v4-on-load mutation, model deletion reader, RPC drain order, or local CI state. Repository and GitHub inspection supplied those facts. +- **Where I left MCP:** Candidate identity, supported-state source inventory, tests, current Actions run, and the exact lifecycle call graph came from Git, repository source and GitHub. The MCP supplied obligations only. +- **Smallest sufficient workflow:** Capture facts, route without inline policy content, read two or three relevant policies at a time, inspect the exact source and evidence, then reroute when the implementation boundary changes. +- **Recommendations:** Make route selection-only the obvious default, reject or normalize a zero content limit with a direct hint, return policy bodies once without repeating them in another result representation, and support section-scoped reads with a visible already-read list. + +### Independent Q1 next-slice architecture investigator — GPT-6.1 Sol High + +- **Useful calls:** One `routing_facts`, one snapshot-bound `route`, and three focused `read_many` calls selected 28 standards with zero unresolved categories. Reads covered architecture/replay, persistence, contract evolution, concurrency, Rust async/API, security/trust, resilience, verification and planning. The route helped surface the one-writer, complete migration preconditions, consumer-specific proof, nested-effect drainage, and scoped-evidence obligations. +- **Confusing or redundant steps:** Broad discovery by “coding/standards/policy” returned unrelated tool descriptions. A route containing several whole policy bodies and an initial six-policy read exceeded the response window; the agent had to re-extract individual bodies. Relationship rationale and continuation metadata repeated around policy content. +- **Missing context:** The MCP did not know the source call graph, `DownloadPersistence` schema 5 and its v4 migration-on-load behavior, hidden admission/quarantine custody, the mutation-authority reader, library marker semantics, public constructors, or shutdown order. Those came from source, plan and contract inspection. +- **Where the agent left MCP:** Exact candidate/base identity, native asset selection semantics, state populations encoded by the source, and the proposed consumer/store composition were established with Git and repository inspection. The investigator edited nothing and ran no tests. +- **Smallest sufficient workflow:** Discover the namespace and fact vocabulary, route explicit facts without policy text, read a few selected standards in small batches, then inspect source and required evidence independently. +- **Recommendations:** Offer a read-only investigation entry point, compact route output with optional explanations, section-scoped policy reads, no duplicate body representations, and a clear indicator that routing does not inspect or certify application code. + +The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, shared-owner design reviewer, and Q1 next-slice architecture investigator used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. The architecture reviewer’s narrow follow-up inspected the repaired current source without making a new MCP call and reused the prior routed obligations; its findings confirm the two identified integrity paths are closed at source level, not that Q1 is accepted. The MCP `review` operation is an authoring workflow for changes to the standards corpus; it does not review application source. Application compliance was checked by final routing, standards reads, source/test inspection, and the independent code review. From 7129d51631d7257742573d4e4873009f20158bec Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 19:33:42 -0700 Subject: [PATCH 11/20] docs(acquisition): admit shared store implementation slice --- .../artifact-acquisition/execution-ledger.md | 8 ++++++++ .../artifact-acquisition/reports/write-sets.md | 16 ++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index f7db73b3..cba96002 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -94,6 +94,14 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - A read-only architecture investigation of `c3052583` confirmed the supervisor extraction remains necessary preparation and does not create a shared durable owner. Its source trace identified the v4 migration-on-load, hidden custody and deletion-authority coupling, GitHub exact-tag/asset bridge, separate HF/native shutdown composition, and public constructors that need explicit lifecycle dispositions. The full write-set recommendation is retained in its handoff and will be narrowed to the exact production changes and tests before the next edits. - User-owned untracked `docs/breif/future.md` remains untouched. No live retained store, installed environment, model, or release was opened or changed. +### Exact next worker admission — HF consumer of the durable owner + +- Parent milestone: `work/acquisition-q1-http` / draft PR #7, targeting `main`; exact worker base: `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`. +- Worker role: one GPT-6.1 Sol High implementation writer for the canonical acquisition lifecycle/store/workspace and the existing HF consumer cutover. It is a Q1 sub-slice; it does not open AQ-HTTP. +- Exact primary paths, test requirements and exclusions are recorded in [the Q1 write-set admission](reports/write-sets.md#exact-next-worker-admission--durable-acquisition-owner-and-hugging-face-cutover). The shared semantic contract and gate/plan records remain integrator-owned; no other writer may change the selected persistence schema or migration while this proposal is active. +- Required proof is the actual normal HF workflow through the acquisition owner, versioned store migration/reopen against disposable supported fixtures, all hidden custody/Pending refusal, lease retention through importer cleanup, concurrent writer isolation, cancellation/restart and shutdown evidence. A helper-only service, mock consumer, or passing unit test is not sufficient for this handoff. +- After this handoff is reviewed and integrated, the next Q1 sub-slice is the existing exact-tag llama.cpp installer using that same service and verified-input lease, with shared RPC composition and consumer-before-owner shutdown. AQ-HTTP remains blocked until both consumers and the plan acceptance matrix are satisfied. + ### Coding-Standards route for the next shared-custody slice - Fresh route snapshot `snapshot:v1:2ea2f5e4-70d3-4a32-97c6-13a93989fb3f` selected 34 standards with zero unresolved fact categories after explicitly marking the unrelated framework fact known-absent. The first route omitted that fact and returned one required unresolved category; correcting the route preserved the same change scope. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index 88ae9740..4f95e7f8 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -23,6 +23,22 @@ The isolated native-custody worker proposal is complete and integrated from `wor **Serial adjacent writes:** `rust/crates/pumas-rpc/src/contract.rs`, `contract/export.rs`, affected HF/version/status handlers, `electron/src/{preload.ts,rpc-method-registry.ts,ipc-validation.ts}`, actual corresponding frontend download/install/source views and generated DTOs. Enumerate outputs from the actual exporter rather than guess or edit generated files manually. Reuse the existing model/native UI; this is not a dashboard redesign. +### Exact next worker admission — durable acquisition owner and Hugging Face cutover + +Parent milestone: `work/acquisition-q1-http`, draft PR #7, target `main`. The worker starts from the exact primary documentation head `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`, in a task-owned branch/worktree. The primary integrator owns PR history, the shared semantic contract, gate state, later native composition, review and final candidate evidence. + +Primary worker write set: + +- `rust/crates/pumas-core/src/acquisition/{mod.rs,http.rs,task_custody.rs,service.rs,store.rs,workspace.rs}`; the last three are new canonical acquisition modules. +- `rust/crates/pumas-core/src/model_library/{download_store.rs,download_recovery.rs,mutation_authority.rs,library.rs,mod.rs}`. +- `rust/crates/pumas-core/src/model_library/hf/{mod.rs,download.rs,lifecycle.rs,acquisition_source.rs,types.rs}`. +- `rust/crates/pumas-core/src/{lib.rs,api/builder.rs,api/state.rs,api/hf.rs}`. +- Co-located regressions in those files and new `rust/crates/pumas-core/tests/artifact_acquisition.rs` when a public composition/reopen path is required. + +The worker implements one durable neutral acquisition owner/store and makes the ordinary HF path consume its verified-file handoff through awaited import/finalization. It moves the v4/v5 compatibility boundary under that same writer and preserves all current model snapshots, hidden admission attempts, revocations, queues, release proofs and quarantines. The model mutation authority must read the canonical store's custody view. One store file/transaction authority remains; no second transfer writer, model marker initializer in neutral workspace code, inferred v4/v5 compatibility, cross-store exactly-once claim, or new public wire representation is allowed. The worker does not modify `docs/contracts/artifact-acquisition.md`, plan/gate/ledger files, RPC/generated/frontend outputs, `pumas-app-manager`, Cargo manifests/lockfiles, or package/S3/runtime/adapter state. Report any discovered contract or scope change before editing outside this set. + +Required handoff evidence: exact branch/base/head and path list; schema 4/5 representative fixture inventory and migration/reopen proof without live roots; single-writer and concurrent-instance tests; preservation/refusal of every hidden custody and Pending-cleanup state; controlled HTTP acquisition through the actual HF workflow; cancellation, dropped observer, shutdown and lease-through-import cleanup; consumer commit/reopen settlement behavior; focused store/recovery/HF/acquisition tests; `pumas-library` Clippy, no-default-features and formatting; and limitations separate from mocks/local fixtures. This HF-only vertical cutover is not AQ-HTTP acceptance; native llama.cpp must later consume the same service on the same PR before that gate can be considered. + **Docs:** the canonical shared contract, acquisition plan records, bounded handoffs in the existing HF/Rust remediation and upstream-runtime plans, and current architecture/development documentation where behavior has landed. Preserve the `docs/breif` path spelling; do not rename unrelated source-intent files. **Forbidden:** runtime installation-ID/profile migration, new model-adapter registry, package dependency reinterpretation, arbitrary source/plugin execution, consumer data deletion, claims of completed Pending replay, and concurrent second writers to the same transfer state. From 5ef3e0530a3b2c050d35cc302a22b9f783ccc777 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 19:42:43 -0700 Subject: [PATCH 12/20] docs(acquisition): admit explicit schema migration boundary --- docs/contracts/artifact-acquisition.md | 4 +++- docs/plans/artifact-acquisition/execution-ledger.md | 6 ++++-- docs/plans/artifact-acquisition/plan.md | 2 +- .../reports/coding-standards-mcp-usability.md | 9 +++++++++ docs/plans/artifact-acquisition/reports/write-sets.md | 4 ++-- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/contracts/artifact-acquisition.md b/docs/contracts/artifact-acquisition.md index 8870fc36..e335648f 100644 --- a/docs/contracts/artifact-acquisition.md +++ b/docs/contracts/artifact-acquisition.md @@ -101,6 +101,8 @@ An installer holds the handoff until package/extraction workers and their cleanu Persist the selected manifest/specification identity, current demand/attempt, source-scoped nonsecret revision evidence, workspace identity, validated resume progress and lifecycle dispositions. Reuse the owned atomic store/publication machinery after separating model-specific records. Runtime artifacts must not require a fake repo/model/library UUID. One shared acquisition implementation can retain a supported legacy decoding boundary, but two stores must not both authorize the same transfer. +The current model-download document is schema 5, with schema 4 as its supported legacy input. Q1 adds neutral acquisition facts to that same physical store under schema 6; it does not create a second transfer store or reinterpret old model records as neutral manifests. Schema 6 retains the complete legacy model-download, recovery, queue, hidden-admission, release-proof and quarantine partitions, including unresolved dispositions, alongside neutral acquisition state. New empty roots use schema 6. An ordinary open of schema 4 or 5 reports that explicit migration is required and does not publish or authorize neutral work. A separate, explicit offline migration operation may convert only supported schema 4 or 5 state to schema 6, preserving legacy facts and publishing the new document atomically under the existing store coordination. Its operational precondition is that every process capable of reading or writing the old store has been stopped; the file lock serializes store operations but cannot prove that an older process will not resume with cached state. Unsupported, corrupt, or unknown state fails closed without a guessed conversion. Pending cleanup remains non-authorizing and is never replayed by migration. No automatic downgrade or old-binary rollback is supported after schema 6 publication; older strict readers must reject schema 6 without rewriting it. + Persisting progress is not the same as making the file bytes durable. After a crash, validate source identity and actual partial files under reopened authority; do not trust a saved byte counter as proof. Complete-file verification must re-establish readiness when prior verification cannot safely be reused. Unsupported recovery remains non-authorizing. Acquisition FilesReady and model/runtime publication are distinct commits. Consumers record their own idempotent finalization linked to the exact request/generation. If a crash occurs after consumer commit but before acquisition settlement, retain inputs conservatively and reconcile through the consumer's authoritative result; do not automatically repeat installation/import or delete final output. There is no assumed cross-store atomic transaction or exactly-once remote-I/O guarantee. @@ -140,4 +142,4 @@ The decisive test denies network during the final installation leg and rejects h Internal coordinated DTOs, public Rust/IPC APIs, persisted formats, optional source implementations and consumer install/model records have different evolution obligations. Update actual generated consumers with their producer. New source support within this contract does not change model-adapter registration or runtime installation identity. Unknown schema/protocol versions are explicitly rejected, not decoded into weaker defaults. -Before independently deploying a breaking contract, disposition every supported public client and retained data state. Gate status is owned by the acquisition plan, not by a version number in this document. This document remains proposed until the implemented producer/consumer evidence exists. +Before independently deploying a breaking contract, disposition every supported public client and retained data state. Schema 6 is a breaking retained-store transition: migration is opt-in and offline, schema 4 and 5 remain untouched until that operation is invoked, and old writers must be stopped before publication. The actual deployed root population, retirement of old writers, and operational rollback policy are not established by disposable fixtures; those facts remain required for live-root qualification. Gate status is owned by the acquisition plan, not by a version number in this document. This document remains proposed until the implemented producer/consumer evidence exists. diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index cba96002..789dfb5f 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -98,9 +98,11 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - Parent milestone: `work/acquisition-q1-http` / draft PR #7, targeting `main`; exact worker base: `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`. - Worker role: one GPT-6.1 Sol High implementation writer for the canonical acquisition lifecycle/store/workspace and the existing HF consumer cutover. It is a Q1 sub-slice; it does not open AQ-HTTP. -- Exact primary paths, test requirements and exclusions are recorded in [the Q1 write-set admission](reports/write-sets.md#exact-next-worker-admission--durable-acquisition-owner-and-hugging-face-cutover). The shared semantic contract and gate/plan records remain integrator-owned; no other writer may change the selected persistence schema or migration while this proposal is active. -- Required proof is the actual normal HF workflow through the acquisition owner, versioned store migration/reopen against disposable supported fixtures, all hidden custody/Pending refusal, lease retention through importer cleanup, concurrent writer isolation, cancellation/restart and shutdown evidence. A helper-only service, mock consumer, or passing unit test is not sufficient for this handoff. +- Exact primary paths, test requirements and exclusions are recorded in [the Q1 write-set admission](reports/write-sets.md#exact-next-worker-admission--durable-acquisition-owner-and-hugging-face-cutover). The shared semantic contract and gate/plan records remain integrator-owned. The admitted durable format is schema 6 in the existing canonical store; fresh roots use it, normal open does not migrate schema 4/5, and a separate explicit offline migration operation is required. Stop every old reader/writer before migration. No live retained root is authorized as a fixture, and deployed population/rollback qualification remains open. +- Required proof is the actual normal HF workflow through the acquisition owner, schema-6 migration/reopen against disposable supported schema-4 and schema-5 fixtures, all hidden custody/Pending refusal, lease retention through importer cleanup, concurrent writer isolation, cancellation/restart and shutdown evidence. Normal open must not mutate retained schema-4/5 stores; migration is a separate explicit offline operation, and older writers/readers must be stopped before it is invoked. Fresh schema-6 roots and reopened schema-6 state must be covered. A helper-only service, mock consumer, or passing unit test is not sufficient for this handoff. Disposable fixtures do not qualify the unknown deployed retained-state population or rollback to old binaries. - After this handoff is reviewed and integrated, the next Q1 sub-slice is the existing exact-tag llama.cpp installer using that same service and verified-input lease, with shared RPC composition and consumer-before-owner shutdown. AQ-HTTP remains blocked until both consumers and the plan acceptance matrix are satisfied. +- The completed custody worker worktree `/tmp/pumas-q1-hf-custody` was clean at `584b5f45a17a80cd584b16ce77f7be5d9688a9b8`, which remains reachable from primary `work/acquisition-q1-http` (`7129d51631d7257742573d4e4873009f20158bec`) through the preserved merge. Its task-owned worktree and redundant local branch were removed after the reachability check (`removed-reachable`). Other task-owned and user-owned worktrees remain untouched. +- The current worker worktree is `/tmp/pumas-q1-hf-acquisition`, branch `agent/q1-hf-acquisition`, based exactly on `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`. The primary advanced only by plan/write-set documentation after that base was created; production source is identical. Integrate with the history-preserving mechanism selected after reviewing its actual commit graph, then record its terminal disposition separately. ### Coding-Standards route for the next shared-custody slice diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index 7b28b090..1c1ed3dd 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -98,7 +98,7 @@ Runtime R2 adds arbitrary registered adapters later using that same accepted fil ### AD8 — own evolution before mutation -The existing download schema, recovery admissions, root markers, hidden history and cleanup custody have retained consumers. Q1 must reconcile the active Rust recovery owner and enumerate supported source states before migration. Preserve legacy IDs through explicit mapping where needed for existing UI/consumer operations. Unsupported/corrupt or custody-unresolved states remain visible and non-authorizing. +The existing download schema, recovery admissions, root markers, hidden history and cleanup custody have retained consumers. Q1 must reconcile the active Rust recovery owner and enumerate supported source states before migration. Preserve legacy IDs through explicit mapping where needed for existing UI/consumer operations. Unsupported/corrupt or custody-unresolved states remain visible and non-authorizing. The source-neutral fields extend the existing canonical store as schema 6; they do not reuse schema 5 with an expanded shape or add a second writer. Fresh roots use schema 6. Existing schema 4/5 roots require a separate explicit offline migration operation, while normal open/admission fails closed without modifying them. Every old reader/writer must be stopped before that operation; disposable fixture migration is implementation evidence, not proof that the deployed root population or rollback policy is qualified. Reopen after each interrupted publication boundary to source, destination or an explicit uncertainty state. Do not mark completion, delete partials, auto-retry imports/installations, or replay Pending cleanup from guessed facts. New readers reject unknown future formats. Older writers must be retired or isolated; a new format number cannot constrain an old binary. Rollback requires evidence about both stores and subsequent authored changes, not just a backup file. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index 033eee53..f8095931 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -164,6 +164,15 @@ This report records agent experience using the Coding-Standards MCP during Acqui - **Smallest sufficient workflow:** Discover the namespace and fact vocabulary, route explicit facts without policy text, read a few selected standards in small batches, then inspect source and required evidence independently. - **Recommendations:** Offer a read-only investigation entry point, compact route output with optional explanations, section-scoped policy reads, no duplicate body representations, and a clear indicator that routing does not inspect or certify application code. +### Q1 durable-store implementation admission follow-up — GPT-6.1 Sol High + +- **Useful calls:** The worker refreshed routing facts, completed an implementation/verification/commit route with 26 selected policies and no unresolved categories, then read Persistence, Evolution, Replay, Commit, Concurrency, Rust Async, Security and Verification. That supplied the applicable obligations while source inspection located the migration-on-load and strict-schema behavior. +- **Confusing or redundant steps:** Whole-policy batches still exceeded a practical output window. The worker also reached an implementation boundary the MCP could not resolve: it routed migration obligations but did not select the exact operation/API or default builder behavior needed to satisfy them. The integrator had to make and record that product-contract decision before source changes could begin. +- **Missing context:** The MCP did not know the exact accepted schema versions, that v4 migration currently publishes during load, whether older processes might retain cached state, or that live retained roots are explicitly out of bounds. The worker established those facts from source and the plan, then paused for contract ownership. +- **Where the worker left MCP:** The exact schema version, migration authorization, API behavior, and production evidence scope were settled in the shared contract and plan by the integrator. The worker made no source edits and ran no tests before that decision. +- **Smallest sufficient workflow:** One current routing-facts snapshot, one complete route without inline content, focused reads of persistence/evolution/concurrency/async/security/verification/commit policies, then inspect the source and plan. A concrete source migration proposal still requires an owning human/agent to choose the API and rollout policy. +- **Recommendations:** Keep policy batches bounded; add a prompt that distinguishes normative obligations from design choices the caller must resolve; include a direct migration-cutover checklist for old readers/writers, default-open behavior, explicit authorization, and fixture versus deployed-state evidence. Continue to state that routing does not decide product semantics or certify code. + The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, shared-owner design reviewer, and Q1 next-slice architecture investigator used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. The architecture reviewer’s narrow follow-up inspected the repaired current source without making a new MCP call and reused the prior routed obligations; its findings confirm the two identified integrity paths are closed at source level, not that Q1 is accepted. The MCP `review` operation is an authoring workflow for changes to the standards corpus; it does not review application source. Application compliance was checked by final routing, standards reads, source/test inspection, and the independent code review. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index 4f95e7f8..ae9f3a7d 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -35,9 +35,9 @@ Primary worker write set: - `rust/crates/pumas-core/src/{lib.rs,api/builder.rs,api/state.rs,api/hf.rs}`. - Co-located regressions in those files and new `rust/crates/pumas-core/tests/artifact_acquisition.rs` when a public composition/reopen path is required. -The worker implements one durable neutral acquisition owner/store and makes the ordinary HF path consume its verified-file handoff through awaited import/finalization. It moves the v4/v5 compatibility boundary under that same writer and preserves all current model snapshots, hidden admission attempts, revocations, queues, release proofs and quarantines. The model mutation authority must read the canonical store's custody view. One store file/transaction authority remains; no second transfer writer, model marker initializer in neutral workspace code, inferred v4/v5 compatibility, cross-store exactly-once claim, or new public wire representation is allowed. The worker does not modify `docs/contracts/artifact-acquisition.md`, plan/gate/ledger files, RPC/generated/frontend outputs, `pumas-app-manager`, Cargo manifests/lockfiles, or package/S3/runtime/adapter state. Report any discovered contract or scope change before editing outside this set. +The worker implements one durable neutral acquisition owner/store and makes the ordinary HF path consume its verified-file handoff through awaited import/finalization. Schema 6 extends the same canonical file and preserves all current model snapshots, hidden admission attempts, revocations, queues, release proofs and quarantines. Fresh roots use schema 6; ordinary open/admission does not migrate schema 4/5. A separate explicit offline migration operation is required, exposed independently from normal API construction; its caller must ensure all old readers and writers are stopped. Without migration, acquisition admission reports a typed migration-required result and performs no transfer effects. The model mutation authority must read the canonical store's custody view. One store file/transaction authority remains; no second transfer writer, model marker initializer in neutral workspace code, inferred v4/v5 compatibility, cross-store exactly-once claim, or new public wire representation is allowed. The worker does not modify `docs/contracts/artifact-acquisition.md`, plan/gate/ledger files, RPC/generated/frontend outputs, `pumas-app-manager`, Cargo manifests/lockfiles, or package/S3/runtime/adapter state. Report any discovered contract or scope change before editing outside this set. -Required handoff evidence: exact branch/base/head and path list; schema 4/5 representative fixture inventory and migration/reopen proof without live roots; single-writer and concurrent-instance tests; preservation/refusal of every hidden custody and Pending-cleanup state; controlled HTTP acquisition through the actual HF workflow; cancellation, dropped observer, shutdown and lease-through-import cleanup; consumer commit/reopen settlement behavior; focused store/recovery/HF/acquisition tests; `pumas-library` Clippy, no-default-features and formatting; and limitations separate from mocks/local fixtures. This HF-only vertical cutover is not AQ-HTTP acceptance; native llama.cpp must later consume the same service on the same PR before that gate can be considered. +Required handoff evidence: exact branch/base/head and path list; representative schema 4/5 fixture inventory; prove ordinary open/admission leaves legacy fixture bytes unchanged and returns migration-required; invoke the separate explicit migration operation and prove v6 reopen without live roots; single-writer and concurrent-instance tests; preservation/refusal of every hidden custody and Pending-cleanup state; controlled HTTP acquisition through the actual HF workflow; cancellation, dropped observer, shutdown and lease-through-import cleanup; consumer commit/reopen settlement behavior; focused store/recovery/HF/acquisition tests; `pumas-library` Clippy, no-default-features and formatting; and limitations separate from mocks/local fixtures. This HF-only vertical cutover is not AQ-HTTP acceptance; native llama.cpp must later consume the same service on the same PR before that gate can be considered. **Docs:** the canonical shared contract, acquisition plan records, bounded handoffs in the existing HF/Rust remediation and upstream-runtime plans, and current architecture/development documentation where behavior has landed. Preserve the `docs/breif` path spelling; do not rename unrelated source-intent files. From 65274dffd90e9c5272a89ec5a3e9c1ae564bfa0c Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 21:11:11 -0700 Subject: [PATCH 13/20] feat(acquisition): route HF through durable shared custody Own the schema 6 download transaction, neutral workspace and verified-file handoff in acquisition while retaining typed model custody in its facade. Keep legacy v4/v5 projection read-only and expose explicit offline migration. Retain unresolved consumer use on reopen and settle only exact observed operations. --- rust/crates/pumas-core/src/acquisition/mod.rs | 15 +- .../pumas-core/src/acquisition/service.rs | 1086 +++++++++++++++++ .../pumas-core/src/acquisition/store.rs | 326 +++++ .../pumas-core/src/acquisition/workspace.rs | 425 +++++++ rust/crates/pumas-core/src/api/builder.rs | 5 + rust/crates/pumas-core/src/api/hf.rs | 1 + rust/crates/pumas-core/src/api/state.rs | 1 + rust/crates/pumas-core/src/lib.rs | 10 + .../src/model_library/download_recovery.rs | 114 +- .../src/model_library/download_store.rs | 283 +++-- .../src/model_library/hf/download.rs | 1046 ++++++---------- .../src/model_library/hf/lifecycle.rs | 10 + .../pumas-core/src/model_library/hf/mod.rs | 36 +- .../src/model_library/mutation_authority.rs | 92 +- .../pumas-core/tests/artifact_acquisition.rs | 43 + 15 files changed, 2674 insertions(+), 819 deletions(-) create mode 100644 rust/crates/pumas-core/src/acquisition/service.rs create mode 100644 rust/crates/pumas-core/src/acquisition/store.rs create mode 100644 rust/crates/pumas-core/src/acquisition/workspace.rs create mode 100644 rust/crates/pumas-core/tests/artifact_acquisition.rs diff --git a/rust/crates/pumas-core/src/acquisition/mod.rs b/rust/crates/pumas-core/src/acquisition/mod.rs index cbdbe197..3ddf0a81 100644 --- a/rust/crates/pumas-core/src/acquisition/mod.rs +++ b/rust/crates/pumas-core/src/acquisition/mod.rs @@ -2,16 +2,18 @@ //! //! This module owns validated selection data and the shared HTTP response and //! body-streaming protocol plus consumer-scoped task/effect supervision. Durable -//! acquisition state and consumer publication remain with their current owners. +//! acquisition custody and the canonical store are shared; consumer publication +//! remains with its consumer. mod github_release; mod http; mod manifest; +mod service; +pub(crate) mod store; pub(crate) mod task_custody; +mod workspace; -pub(crate) use http::{ - open_http_artifact, stream_http_artifact, HttpArtifactSink, HttpAttemptHost, HttpBodyOutcome, -}; +pub(crate) use http::HttpAttemptHost; pub(crate) use github_release::{select_github_release_asset, GitHubReleaseAssetMetadata}; pub use github_release::{ @@ -22,3 +24,8 @@ pub use manifest::{ FileVerificationRequirement, ManifestValidationError, RevisionStrength, Sha256Evidence, CURRENT_MANIFEST_VERSION, }; + +pub use service::{AcquisitionDemand, AcquisitionPhase, AcquisitionRecord, AcquisitionService}; +pub(crate) use service::{AcquisitionHost, AcquisitionRetryPolicy}; +pub use store::AcquisitionStore; +pub use workspace::{AcquisitionWorkspace, VerifiedFile, WorkspaceIdentity}; diff --git a/rust/crates/pumas-core/src/acquisition/service.rs b/rust/crates/pumas-core/src/acquisition/service.rs new file mode 100644 index 00000000..20b18663 --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/service.rs @@ -0,0 +1,1086 @@ +//! Durable acquisition custody shared by source adapters and consumers. +use super::http::{ + open_http_artifact, stream_http_artifact, HttpArtifactSink, HttpAttemptHost, HttpBodyOutcome, +}; +use super::store::AcquisitionStore; +use super::task_custody::{TaskContext, TaskCustodyOwner}; +use super::workspace::{write_chunk, AcquisitionWorkspace, VerifiedFile, WorkspaceIdentity}; +use super::ArtifactManifest; +use crate::{PumasError, Result}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use uuid::Uuid; + +/// Consumer identity and exact demand operation; neither authorizes file access. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AcquisitionDemand { + pub consumer: String, + pub operation: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum AcquisitionPhase { + Transferring, + FilesReady, + Using { + #[serde(with = "uuid_wire")] + lease: Uuid, + }, + Adopted { + #[serde(with = "uuid_wire")] + lease: Uuid, + }, + Withdrawn, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AcquisitionRecord { + #[serde(with = "uuid_wire")] + pub id: Uuid, + pub demand: AcquisitionDemand, + pub manifest: ArtifactManifest, + pub workspace: WorkspaceIdentity, + pub phase: AcquisitionPhase, + pub files: Vec, +} + +fn invalid(message: &str) -> PumasError { + PumasError::Validation { + field: "acquisition.custody".into(), + message: message.into(), + } +} + +impl AcquisitionRecord { + pub(crate) fn validate(&self, id: Uuid) -> Result<()> { + if id != self.id + || id.is_nil() + || self.demand.consumer.is_empty() + || self.demand.operation.is_empty() + || self.workspace.root_identity.is_empty() + || self.workspace.relative_target.is_empty() + { + return Err(invalid( + "Acquisition identity is incomplete or inconsistent", + )); + } + if matches!( + self.phase, + AcquisitionPhase::FilesReady + | AcquisitionPhase::Using { .. } + | AcquisitionPhase::Adopted { .. } + ) { + if self.files.len() != self.manifest.files().len() { + return Err(invalid("Verified-file set is incomplete")); + } + for (receipt, selected) in self.files.iter().zip(self.manifest.files()) { + if receipt.path != selected.logical_path() + || selected + .expected_size() + .is_some_and(|size| size != receipt.bytes) + || receipt.sha256.len() != 64 + || !receipt + .sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + || selected + .expected_sha256() + .is_some_and(|digest| digest.value() != receipt.sha256) + { + return Err(invalid("Verified-file receipt contradicts selection")); + } + } + } else if !self.files.is_empty() { + return Err(invalid( + "Unverified custody cannot contain a verified-file set", + )); + } + if matches!(self.phase, AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } if lease.is_nil()) + { + return Err(invalid("Consumer lease identity is invalid")); + } + Ok(()) + } +} + +/// Exact operation handle; it is not a filesystem capability. +#[derive(Clone)] +pub(crate) struct AcquisitionOperation { + record: AcquisitionRecord, +} + +impl AcquisitionOperation { + pub(crate) fn is_adopted(&self) -> bool { + matches!(self.record.phase, AcquisitionPhase::Adopted { .. }) + } +} + +#[derive(Clone)] +pub(crate) enum AcquisitionReconciliation { + Using, + Adopted(Uuid), +} + +/// Pins the held workspace while a consumer uses the verified set. Dropping an +/// unacknowledged lease leaves durable `Using` custody for explicit reconciliation. +pub(crate) struct AcquisitionUseLease { + operation: AcquisitionOperation, + workspace: AcquisitionWorkspace, + lease: Uuid, +} + +#[derive(Clone)] +pub(crate) struct AcquisitionRetryPolicy { + pub(crate) attempts: Option, + pub(crate) elapsed: Duration, + pub(crate) backoff: crate::network::RetryConfig, +} + +#[async_trait::async_trait] +pub(crate) trait AcquisitionHost: HttpAttemptHost { + async fn retry( + &mut self, + attempt: u32, + delay: Option, + error: Option<&str>, + ) -> Result<()>; +} + +/// One durable lifecycle/store owner and the existing shared task supervisor. +/// Source access is ephemeral and is never written to the durable manifest. +pub struct AcquisitionService { + store: Arc, + supervisor: Arc, +} + +impl AcquisitionService { + pub fn new(store: Arc) -> Self { + Self { + store, + supervisor: Arc::new(TaskCustodyOwner::new()), + } + } + + pub(crate) fn with_store(&self, store: Arc) -> Self { + Self { + store, + supervisor: self.supervisor.clone(), + } + } + + pub fn store(&self) -> &Arc { + &self.store + } + pub(crate) fn supervisor(&self) -> Arc { + self.supervisor.clone() + } + + /// Global closure after every consumer has stopped admitting work. Narrow + /// consumer shutdown must close that consumer's scope first. + pub async fn shutdown(self: &Arc) -> Result<()> { + self.supervisor.request_shutdown().wait().await + } + + pub(crate) async fn require_schema(&self, context: &TaskContext) -> Result<()> { + let store = self.store.clone(); + owned( + context, + "validate acquisition schema eligibility", + move || store.require_acquisition_schema(), + ) + .await + } + + /// The owner calls this only after observing exact-operation cleanup and + /// its registered effects. Unknown/Pending cleanup cannot reach this seam. + pub(crate) async fn withdraw( + &self, + context: &TaskContext, + demand: AcquisitionDemand, + workspace: WorkspaceIdentity, + ) -> Result<()> { + let store = self.store.clone(); + owned(context, "withdraw cleaned acquisition demand", move || { + store.update_acquisitions_if_changed(|records| { + if let Some(record) = records.values_mut().find(|record| record.demand == demand) { + if record.workspace != workspace { + return Err(invalid("Withdrawal workspace does not match exact demand")); + } + if !matches!( + record.phase, + AcquisitionPhase::Adopted { .. } | AcquisitionPhase::Withdrawn + ) { + record.phase = AcquisitionPhase::Withdrawn; + record.files.clear(); + } + } + Ok(()) + }) + }) + .await + } + + pub(crate) async fn begin( + &self, + context: &TaskContext, + demand: AcquisitionDemand, + manifest: ArtifactManifest, + workspace: WorkspaceIdentity, + reconciliation: Option, + ) -> Result { + let store = self.store.clone(); + owned(context, "admit durable acquisition", move || { + store.update_acquisitions(|records| { + if let Some(existing) = records.values().find(|record| record.demand == demand) { + if existing.manifest != manifest || existing.workspace != workspace { return Err(invalid("Exact acquisition demand changed selection or workspace")); } + if matches!(existing.phase, AcquisitionPhase::Using { .. }) { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Retained consumer use requires its authoritative exact result; input custody alone cannot authorize repeating import".into(), + }); + } + let reconciling = matches!((&existing.phase, &reconciliation), + (AcquisitionPhase::Adopted { lease }, Some(AcquisitionReconciliation::Adopted(expected))) if lease == expected); + if !matches!(existing.phase, AcquisitionPhase::Transferring | AcquisitionPhase::FilesReady) && !reconciling { + return Err(invalid("Acquisition demand has unresolved or terminal consumer custody")); + } + return Ok(AcquisitionOperation { record: existing.clone() }); + } + if records.values().any(|record| record.workspace == workspace && !matches!(record.phase, AcquisitionPhase::Adopted { .. } | AcquisitionPhase::Withdrawn)) { + return Err(PumasError::DownloadRootBusy); + } + let record = AcquisitionRecord { id: Uuid::new_v4(), demand, manifest, workspace, phase: AcquisitionPhase::Transferring, files: Vec::new() }; + record.validate(record.id)?; + records.insert(record.id, record.clone()); + Ok(AcquisitionOperation { record }) + }) + }).await + } + + pub(crate) async fn verified_existing_file( + &self, + context: &TaskContext, + operation: &AcquisitionOperation, + workspace: &AcquisitionWorkspace, + file_index: usize, + ) -> Result> { + let selected = operation + .record + .manifest + .files() + .get(file_index) + .ok_or_else(|| invalid("Selected file is unavailable"))? + .clone(); + let grant = workspace.clone(); + owned(context, "inspect reusable acquisition file", move || { + grant.prepare_file(&selected)?; + let Some(size) = grant.file_len(selected.logical_path(), false)? else { + return Ok(None); + }; + if selected + .expected_size() + .is_some_and(|expected| expected != size) + { + return Err(invalid( + "Existing selected file size conflicts with selection", + )); + } + if selected.expected_sha256().is_none() { + return Ok(None); + } + let verified = grant.verify_file(&selected, false)?; + grant.remove_part(selected.logical_path())?; + Ok(Some(verified.bytes)) + }) + .await + } + + /// The only byte attempt/retry/file-promotion path. The host projects + /// progress and cancellation; it never writes artifact bytes or retries. + #[allow(clippy::too_many_arguments)] + pub(crate) async fn acquire_file( + &self, + context: &TaskContext, + operation: &AcquisitionOperation, + workspace: &AcquisitionWorkspace, + file_index: usize, + client: &reqwest::Client, + url: &str, + authorization: Option<&str>, + retry: &AcquisitionRetryPolicy, + host: &mut dyn AcquisitionHost, + ) -> Result { + if &operation.record.workspace != workspace.identity() { + return Err(invalid("Workspace grant does not match acquisition")); + } + let file = operation + .record + .manifest + .files() + .get(file_index) + .ok_or_else(|| invalid("Selected file is unavailable"))? + .clone(); + let prepare = workspace.clone(); + let selected = file.clone(); + owned(context, "prepare acquisition file parent", move || { + prepare.prepare_file(&selected) + }) + .await?; + if matches!( + operation.record.phase, + AcquisitionPhase::FilesReady + | AcquisitionPhase::Using { .. } + | AcquisitionPhase::Adopted { .. } + ) { + let verify = workspace.clone(); + let manifest = operation.record.manifest.clone(); + let receipts = operation.record.files.clone(); + owned(context, "verify reopened acquisition receipts", move || { + verify.verify_receipts(&manifest, &receipts) + }) + .await?; + return Ok(operation.record.files[file_index].bytes); + } + let inspect = workspace.clone(); + let path = file.logical_path().to_owned(); + let existing = owned(context, "inspect acquisition final file", move || { + inspect.file_len(&path, false) + }) + .await?; + let compare_existing = match existing { + Some(size) => { + if file + .expected_size() + .is_some_and(|expected| expected != size) + { + return Err(invalid( + "Existing selected file size conflicts with selection", + )); + } + if file.expected_sha256().is_some() { + let verify = workspace.clone(); + let selected = file.clone(); + let receipt = owned(context, "verify existing acquisition file", move || { + verify.verify_file(&selected, false) + }) + .await?; + return Ok(receipt.bytes); + } + if operation.record.manifest.source().revision().strength() + != super::RevisionStrength::Immutable + { + return Err(invalid( + "Existing file has no digest or custody receipt for its mutable source", + )); + } + true + } + None => false, + }; + let started = Instant::now(); + let mut attempt = 0_u32; + loop { + attempt = attempt + .checked_add(1) + .ok_or_else(|| invalid("Retry counter exhausted"))?; + host.retry(attempt, None, None).await?; + if host.cancel_requested() { + return Err(PumasError::DownloadCancelled); + } + if host.pause_requested_now() { + return Err(PumasError::DownloadPaused); + } + let inspect = workspace.clone(); + let path = file.logical_path().to_owned(); + let mut resume = owned(context, "inspect acquisition partial file", move || { + inspect.file_len(&path, true) + }) + .await? + .unwrap_or(0); + if resume > 0 + && ((!operation.record.manifest.permits_resume(file_index)) + || (compare_existing && attempt == 1)) + { + let remove = workspace.clone(); + let path = file.logical_path().to_owned(); + owned(context, "discard unbound acquisition partial", move || { + remove.remove_part(&path) + }) + .await?; + resume = 0; + } + if !compare_existing + && file.expected_size() == Some(resume) + && resume > 0 + && operation.record.manifest.permits_resume(file_index) + { + let publish = workspace.clone(); + let selected = file.clone(); + return owned(context, "publish complete acquisition partial", move || { + publish + .publish_part(&selected, false) + .map(|receipt| receipt.bytes) + }) + .await; + } + let response = tokio::select! { + biased; + _ = host.pause_requested() => return Err(PumasError::DownloadPaused), + response = open_http_artifact(client, url, &operation.record.manifest, file_index, resume, authorization) => response, + }; + let outcome = match response { + Ok(response) => { + let open = workspace.clone(); + let path = file.logical_path().to_owned(); + let append = response.resumed; + let file = owned(context, "open acquisition partial file", move || { + open.open_part(&path, append) + }) + .await?; + let mut sink = AcquisitionSink { + file: Some(file), + context, + }; + stream_http_artifact(response, resume, &mut sink, host).await + } + Err(error) => Err(error), + }; + match outcome { + Ok(HttpBodyOutcome::Complete { .. }) => { + if host.cancel_requested() { + return Err(PumasError::DownloadCancelled); + } + if host.pause_requested_now() { + return Err(PumasError::DownloadPaused); + } + let publish = workspace.clone(); + let selected = file.clone(); + return owned(context, "publish verified acquisition file", move || { + publish + .publish_part(&selected, compare_existing) + .map(|receipt| receipt.bytes) + }) + .await; + } + Ok(HttpBodyOutcome::Paused) => return Err(PumasError::DownloadPaused), + Ok(HttpBodyOutcome::Cancelled) => return Err(PumasError::DownloadCancelled), + Err(error) => { + if !error.is_retryable() || host.cancel_requested() { + return Err(error); + } + if retry.attempts.is_some_and(|limit| attempt >= limit) + || (retry.elapsed > Duration::ZERO && started.elapsed() >= retry.elapsed) + { + return Err(PumasError::DownloadFailed { url: "artifact source".into(), message: format!("Acquisition retry budget exhausted after {attempt} attempts: {error}") }); + } + let delay = retry.backoff.calculate_delay(attempt.saturating_sub(1)); + host.retry(attempt, Some(delay), Some(&error.to_string())) + .await?; + tokio::select! { + biased; + _ = host.pause_requested() => return Err(PumasError::DownloadPaused), + _ = tokio::time::sleep(delay) => {}, + } + } + } + } + } + + /// Reopening `Using` requires an exact consumer reconciliation lease. The + /// consumer must first revalidate its durable admission under root custody; + /// the store token by itself never authorizes replay of consumer effects. + pub(crate) async fn reconciliation_lease( + &self, + context: &TaskContext, + demand: &AcquisitionDemand, + ) -> Result> { + let store = self.store.clone(); + let demand = demand.clone(); + owned( + context, + "observe acquisition reconciliation custody", + move || { + Ok(store + .acquisitions()? + .values() + .find(|record| record.demand == demand) + .and_then(|record| match record.phase { + AcquisitionPhase::Using { .. } => Some(AcquisitionReconciliation::Using), + AcquisitionPhase::Adopted { lease } => { + Some(AcquisitionReconciliation::Adopted(lease)) + } + _ => None, + })) + }, + ) + .await + } + + pub(crate) async fn files_ready( + &self, + context: &TaskContext, + operation: AcquisitionOperation, + workspace: AcquisitionWorkspace, + ) -> Result { + let seal = workspace.clone(); + let manifest = operation.record.manifest.clone(); + let files = owned(context, "seal verified acquisition file set", move || { + seal.seal(&manifest) + }) + .await?; + let store = self.store.clone(); + let mut expected = operation.record.clone(); + if matches!(expected.phase, AcquisitionPhase::Transferring) { + let ready = expected.clone(); + let receipts = files.clone(); + owned(context, "persist acquisition files ready", move || { + store.update_acquisitions(|records| { + let record = records + .get_mut(&ready.id) + .ok_or_else(|| invalid("Acquisition custody disappeared"))?; + if record != &ready { + return Err(invalid("Acquisition readiness is stale")); + } + record.files = receipts; + record.phase = AcquisitionPhase::FilesReady; + Ok(()) + }) + }) + .await?; + expected.files = files.clone(); + expected.phase = AcquisitionPhase::FilesReady; + } + if files != expected.files { + return Err(invalid("Reopened verified-file receipts changed")); + } + let lease = match expected.phase { + AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } => lease, + AcquisitionPhase::FilesReady => { + let store = self.store.clone(); + let lease = Uuid::new_v4(); + owned( + context, + "handoff durable verified acquisition files", + move || { + store.update_acquisitions(|records| { + let record = records + .get_mut(&expected.id) + .ok_or_else(|| invalid("Acquisition custody disappeared"))?; + if record != &expected { + return Err(invalid("Acquisition handoff is stale")); + } + record.phase = AcquisitionPhase::Using { lease }; + Ok(()) + }) + }, + ) + .await?; + lease + } + _ => return Err(invalid("Acquisition is not ready for consumer use")), + }; + Ok(AcquisitionUseLease { + operation, + workspace, + lease, + }) + } + + /// Called only after the exact consumer operation and every nested effect + /// have been positively observed. A path/tag probe cannot acknowledge it. + pub(crate) async fn acknowledge( + &self, + context: &TaskContext, + lease: AcquisitionUseLease, + ) -> Result<()> { + let verify = lease.workspace.clone(); + let store = self.store.clone(); + let id = lease.operation.record.id; + let token = lease.lease; + let manifest = lease.operation.record.manifest.clone(); + let read_store = self.store.clone(); + let record = owned( + context, + "observe exact acquisition consumer custody", + move || { + read_store + .acquisitions()? + .remove(&id) + .ok_or_else(|| invalid("Consumer custody disappeared")) + }, + ) + .await?; + let expected = record.clone(); + owned(context, "verify acquisition consumer receipts", move || { + verify.verify_receipts(&manifest, &record.files) + }) + .await?; + owned(context, "acknowledge adopted acquisition files", move || { + store.update_acquisitions(|records| { + let record = records.get_mut(&id).ok_or_else(|| invalid("Consumer custody disappeared"))?; + if record != &expected || !matches!(record.phase, AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } if lease == token) { + return Err(invalid("Consumer acknowledgment is stale")); + } + record.phase = AcquisitionPhase::Adopted { lease: token }; + Ok(()) + }) + }).await + } +} + +async fn owned( + context: &TaskContext, + name: &'static str, + work: impl FnOnce() -> Result + Send + 'static, +) -> Result { + context + .run_fallible_blocking_named(name, move || match work() { + Err( + error @ (PumasError::Validation { .. } + | PumasError::HashMismatch { .. } + | PumasError::DownloadRootBusy), + ) => Ok(Err(error)), + Err(error) => Err(error), + Ok(value) => Ok(Ok(value)), + }) + .await + .map_err(|error| { + PumasError::Other(format!("Acquisition effect observation failed: {error}")) + })? + .and_then(|result| result) +} + +struct AcquisitionSink<'a> { + file: Option, + context: &'a TaskContext, +} + +#[async_trait::async_trait] +impl HttpArtifactSink for AcquisitionSink<'_> { + async fn write_all(&mut self, bytes: &[u8]) -> Result<()> { + let mut file = self + .file + .take() + .ok_or_else(|| invalid("Partial file effect is unfinished"))?; + let bytes = bytes.to_owned(); + self.file = Some( + owned(self.context, "write acquisition partial file", move || { + write_chunk(&mut file, &bytes)?; + Ok(file) + }) + .await?, + ); + Ok(()) + } + async fn flush(&mut self) -> Result<()> { + let file = self + .file + .take() + .ok_or_else(|| invalid("Partial file effect is unfinished"))?; + self.file = Some( + owned(self.context, "sync acquisition partial file", move || { + file.sync_all()?; + Ok(file) + }) + .await?, + ); + Ok(()) + } +} + +mod uuid_wire { + use serde::{Deserialize, Deserializer, Serializer}; + use uuid::Uuid; + pub(super) fn serialize(value: &Uuid, serializer: S) -> Result { + serializer.serialize_str(&value.to_string()) + } + pub(super) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result { + Uuid::parse_str(&String::deserialize(deserializer)?).map_err(serde::de::Error::custom) + } +} + +pub(super) mod uuid_map { + use super::AcquisitionRecord; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use std::collections::BTreeMap; + use uuid::Uuid; + pub(crate) fn serialize( + values: &BTreeMap, + serializer: S, + ) -> Result { + values + .iter() + .map(|(id, value)| (id.to_string(), value)) + .collect::>() + .serialize(serializer) + } + pub(crate) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + BTreeMap::::deserialize(deserializer)? + .into_iter() + .map(|(id, record)| { + Uuid::parse_str(&id) + .map(|id| (id, record)) + .map_err(serde::de::Error::custom) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::acquisition::{ + ArtifactFile, ArtifactRevisionEvidence, ArtifactSourceIdentity, + FileVerificationRequirement, RevisionStrength, + }; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + fn manifest(path: &str) -> ArtifactManifest { + ArtifactManifest::new( + ArtifactSourceIdentity::new( + "fixture", + "selected-object", + ArtifactRevisionEvidence::new( + "fixture.revision", + "immutable-v1", + RevisionStrength::Immutable, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + path, + "payload", + Some(4), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap()], + ) + .unwrap() + } + + fn workspace(path: &std::path::Path) -> AcquisitionWorkspace { + let root = crate::platform::capability_fs::open_directory(path).unwrap(); + let check = root.try_clone().unwrap(); + let expected = std::fs::canonicalize(path).unwrap(); + let source = path.to_path_buf(); + AcquisitionWorkspace::from_capability( + root, + WorkspaceIdentity { + root_identity: "fixture-physical-root".into(), + relative_target: "staging".into(), + }, + Arc::new(()), + move || { + if std::fs::canonicalize(&source)? != expected || !check.dir_metadata()?.is_dir() { + return Err(invalid("Fixture grant changed")); + } + Ok(()) + }, + ) + .unwrap() + } + + struct Host; + #[async_trait::async_trait] + impl HttpAttemptHost for Host { + async fn pause_requested(&self) { + std::future::pending::<()>().await; + } + fn pause_requested_now(&self) -> bool { + false + } + fn cancel_requested(&self) -> bool { + false + } + async fn record_progress(&mut self, _bytes: u64) -> Result<()> { + Ok(()) + } + } + #[async_trait::async_trait] + impl AcquisitionHost for Host { + async fn retry( + &mut self, + _attempt: u32, + _delay: Option, + _error: Option<&str>, + ) -> Result<()> { + Ok(()) + } + } + + fn retry() -> AcquisitionRetryPolicy { + AcquisitionRetryPolicy { + attempts: Some(1), + elapsed: Duration::from_secs(5), + backoff: crate::network::RetryConfig::new(), + } + } + + async fn serve(body: &'static [u8]) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let url = format!("http://{}/fixture", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0_u8; 2048]; + let received = socket.read(&mut request).await.unwrap(); + assert!(received > 0); + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + socket.write_all(body).await.unwrap(); + }); + (url, server) + } + + #[tokio::test] + async fn promoted_digestless_file_reopens_by_fresh_http_comparison_and_retains_using() { + let temp = tempfile::TempDir::new().unwrap(); + let stage = temp.path().join("stage"); + std::fs::create_dir(&stage).unwrap(); + let store = Arc::new(AcquisitionStore::new(temp.path())); + let service = Arc::new(AcquisitionService::new(store)); + let scope = service + .supervisor() + .open_scope(|| async { Ok(()) }) + .unwrap(); + let demand = AcquisitionDemand { + consumer: "fixture".into(), + operation: "exact-demand".into(), + }; + let first_service = service.clone(); + let first_stage = stage.clone(); + let first_demand = demand.clone(); + let (url, server) = serve(b"DATA").await; + scope + .run_invocation(move |context| async move { + let grant = workspace(&first_stage); + let operation = first_service + .begin( + &context, + first_demand, + manifest("payload.bin"), + grant.identity().clone(), + None, + ) + .await?; + first_service + .acquire_file( + &context, + &operation, + &grant, + 0, + &reqwest::Client::new(), + &url, + None, + &retry(), + &mut Host, + ) + .await?; + Ok(()) + }) + .await + .unwrap(); + server.await.unwrap(); + scope.shutdown().await.unwrap(); + assert!(matches!( + service + .store + .acquisitions() + .unwrap() + .values() + .next() + .unwrap() + .phase, + AcquisitionPhase::Transferring + )); + drop(service); + + // A promoted file has no durable receipt yet: reopen must obtain a fresh + // immutable-source representation instead of trusting path/size/tag. + let reopened = Arc::new(AcquisitionService::new(Arc::new(AcquisitionStore::new( + temp.path(), + )))); + let scope = reopened + .supervisor() + .open_scope(|| async { Ok(()) }) + .unwrap(); + let owner = reopened.clone(); + let use_stage = stage.clone(); + let use_demand = demand.clone(); + let (url, server) = serve(b"DATA").await; + scope + .run_invocation(move |context| async move { + let grant = workspace(&use_stage); + let operation = owner + .begin( + &context, + use_demand, + manifest("payload.bin"), + grant.identity().clone(), + None, + ) + .await?; + owner + .acquire_file( + &context, + &operation, + &grant, + 0, + &reqwest::Client::new(), + &url, + None, + &retry(), + &mut Host, + ) + .await?; + let lease = owner.files_ready(&context, operation, grant).await?; + drop(lease); // Simulate loss before the consumer can acknowledge. + Ok(()) + }) + .await + .unwrap(); + server.await.unwrap(); + scope.shutdown().await.unwrap(); + drop(reopened); + let owner = Arc::new(AcquisitionService::new(Arc::new(AcquisitionStore::new( + temp.path(), + )))); + let scope = owner.supervisor().open_scope(|| async { Ok(()) }).unwrap(); + let current = owner.clone(); + let before = std::fs::read(temp.path().join("downloads.json")).unwrap(); + let result = scope + .run_invocation(move |context| async move { + let grant = workspace(&stage); + let evidence = current.reconciliation_lease(&context, &demand).await?; + current + .begin( + &context, + demand, + manifest("payload.bin"), + grant.identity().clone(), + evidence, + ) + .await + .map(|_| ()) + }) + .await; + assert!( + matches!(result, Err(PumasError::Validation { field, .. }) if field == "acquisition.consumer_recovery_required") + ); + assert_eq!( + std::fs::read(temp.path().join("downloads.json")).unwrap(), + before + ); + scope.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn adopted_proof_is_terminal_and_exact_withdrawal_does_not_release_a_successor() { + let temp = tempfile::TempDir::new().unwrap(); + let stage = temp.path().join("stage"); + std::fs::create_dir(&stage).unwrap(); + std::fs::write(stage.join("payload.bin"), b"DATA").unwrap(); + let owner = Arc::new(AcquisitionService::new(Arc::new(AcquisitionStore::new( + temp.path(), + )))); + let scope = owner.supervisor().open_scope(|| async { Ok(()) }).unwrap(); + let service = owner.clone(); + scope + .run_invocation(move |context| async move { + let grant = workspace(&stage); + let demand = AcquisitionDemand { + consumer: "fixture".into(), + operation: "first".into(), + }; + let operation = service + .begin( + &context, + demand.clone(), + manifest("payload.bin"), + grant.identity().clone(), + None, + ) + .await?; + let lease = service + .files_ready(&context, operation, grant.clone()) + .await?; + service.acknowledge(&context, lease).await?; + let evidence = service.reconciliation_lease(&context, &demand).await?; + let terminal = service + .begin( + &context, + demand.clone(), + manifest("payload.bin"), + grant.identity().clone(), + evidence, + ) + .await?; + assert!(terminal.is_adopted()); + let successor = AcquisitionDemand { + consumer: "fixture".into(), + operation: "successor".into(), + }; + service + .begin( + &context, + successor.clone(), + manifest("payload.bin"), + grant.identity().clone(), + None, + ) + .await?; + service + .withdraw(&context, demand, grant.identity().clone()) + .await?; + assert!(matches!( + service + .store + .acquisitions()? + .values() + .find(|record| record.demand == successor) + .unwrap() + .phase, + AcquisitionPhase::Transferring + )); + service + .withdraw(&context, successor, grant.identity().clone()) + .await?; + Ok(()) + }) + .await + .unwrap(); + scope.shutdown().await.unwrap(); + let records = owner.store.acquisitions().unwrap(); + assert_eq!( + records + .values() + .filter(|record| matches!(record.phase, AcquisitionPhase::Adopted { .. })) + .count(), + 1 + ); + assert_eq!( + records + .values() + .filter(|record| matches!(record.phase, AcquisitionPhase::Withdrawn)) + .count(), + 1 + ); + } +} diff --git a/rust/crates/pumas-core/src/acquisition/store.rs b/rust/crates/pumas-core/src/acquisition/store.rs new file mode 100644 index 00000000..6c8a6c3b --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/store.rs @@ -0,0 +1,326 @@ +//! The single transaction and atomic-publication authority for downloads.json. +//! +//! Model custody is an opaque partition decoded by its model-owned facade. +//! This module never interprets model requests, statuses, or recovery policy. +use super::service::AcquisitionRecord; +use crate::metadata::{ + AtomicJsonTarget, AtomicPublication, AtomicPublishFailure, AtomicPublishFailureKind, + AtomicPublishResult, AtomicPublishStage, StagingCleanup, +}; +use crate::{PumasError, Result}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, MutexGuard}; +use uuid::Uuid; + +const SCHEMA_VERSION: u32 = 6; +const LOCK_FILE: &str = ".downloads.lock"; + +#[derive(Clone, Serialize, Deserialize)] +pub(crate) struct AcquisitionDocument { + schema_version: u32, + #[serde(with = "super::service::uuid_map")] + pub(crate) acquisitions: BTreeMap, + /// Existing model partitions retain their exact serialized custody shape. + /// Only the trusted model facade may decode or replace these values. + #[serde(flatten)] + legacy: BTreeMap, +} + +impl AcquisitionDocument { + fn empty() -> Self { + Self { + schema_version: SCHEMA_VERSION, + acquisitions: BTreeMap::new(), + legacy: BTreeMap::new(), + } + } + fn validate(&self) -> Result<()> { + if self.schema_version != SCHEMA_VERSION { + return Err(invalid_schema()); + } + for (id, record) in &self.acquisitions { + record.validate(*id)?; + } + Ok(()) + } +} + +/// Source-neutral physical store authority. Construction has no I/O effects. +pub struct AcquisitionStore { + path: PathBuf, + mutation: Mutex<()>, +} + +pub(crate) struct AcquisitionTransaction<'a> { + _instance_guard: MutexGuard<'a, ()>, + target: AtomicJsonTarget, + _os_lock: Option, + legacy_read_only: bool, + consumer_settlement: Option<(String, String)>, +} + +fn schema(value: &Value) -> Option { + value.get("schema_version").and_then(Value::as_u64) +} + +pub(crate) fn migration_required() -> PumasError { + PumasError::Validation { + field: "acquisition.migration_required".into(), + message: "Acquisition requires schema 6; stop all old readers/writers and explicitly migrate the supported v4/v5 store offline".into(), + } +} + +fn invalid_schema() -> PumasError { + PumasError::Validation { field: "downloads.schema_version".into(), message: "Only complete supported schema 4/5 projection and schema 6 acquisition documents are accepted".into() } +} + +impl AcquisitionStore { + pub fn new(data_dir: &Path) -> Self { + Self { + path: data_dir.join("downloads.json"), + mutation: Mutex::new(()), + } + } + + /// Eligibility check; does not publish or obtain workspace authority. + pub fn require_acquisition_schema(&self) -> Result<()> { + let target = AtomicJsonTarget::open(&self.path)?; + if let Some(value) = target.read_json::()? { + if matches!(schema(&value), Some(4 | 5)) { + return Err(migration_required()); + } + let document: AcquisitionDocument = serde_json::from_value(value)?; + document.validate()?; + } + Ok(()) + } + + pub(crate) fn transaction(&self, read_only: bool) -> Result> { + self.transaction_observed(read_only, || {}, || {}) + } + + pub(crate) fn transaction_observed( + &self, + read_only: bool, + attempting: impl FnOnce(), + acquired: impl FnOnce(), + ) -> Result> { + let guard = self + .mutation + .lock() + .map_err(|_| PumasError::Other("Acquisition store lock is poisoned".into()))?; + let target = AtomicJsonTarget::open(&self.path)?; + let legacy = target + .read_json::()? + .is_some_and(|value| matches!(schema(&value), Some(4 | 5))); + if legacy { + if !read_only { + return Err(migration_required()); + } + return Ok(AcquisitionTransaction { + _instance_guard: guard, + target, + _os_lock: None, + legacy_read_only: true, + consumer_settlement: None, + }); + } + let os_lock = target.open_lock_file(LOCK_FILE)?; + attempting(); + os_lock.lock()?; + acquired(); + // Repeat the schema check after taking the shared lock. + if !read_only + && target + .read_json::()? + .is_some_and(|value| matches!(schema(&value), Some(4 | 5))) + { + return Err(migration_required()); + } + Ok(AcquisitionTransaction { + _instance_guard: guard, + target, + _os_lock: Some(os_lock), + legacy_read_only: false, + consumer_settlement: None, + }) + } + + /// Private trusted conversion hook; only the model facade supplies it. + /// The caller has stopped all old readers/writers. The converter sees the + /// exact locked source and validates every supported legacy custody field. + pub(crate) fn migrate_legacy_offline( + &self, + convert: impl FnOnce(Value) -> Result, + ) -> Result<()> { + let guard = self + .mutation + .lock() + .map_err(|_| PumasError::Other("Acquisition store lock is poisoned".into()))?; + let target = AtomicJsonTarget::open(&self.path)?; + let lock = target.open_lock_file(LOCK_FILE)?; + lock.lock()?; + let transaction = AcquisitionTransaction { + _instance_guard: guard, + target, + _os_lock: Some(lock), + legacy_read_only: false, + consumer_settlement: None, + }; + let value = transaction + .target + .read_json::()? + .ok_or_else(invalid_schema)?; + if !matches!(schema(&value), Some(4 | 5)) { + return Err(invalid_schema()); + } + let legacy = convert(value)?; + let document = document_with_partition(AcquisitionDocument::empty(), legacy)?; + require_durable(transaction.publish_document(&document)) + } + + pub(crate) fn update_acquisitions( + &self, + update: impl FnOnce(&mut BTreeMap) -> Result, + ) -> Result { + self.update_acquisition_records(update, true) + } + + pub(crate) fn update_acquisitions_if_changed( + &self, + update: impl FnOnce(&mut BTreeMap) -> Result, + ) -> Result { + self.update_acquisition_records(update, false) + } + + fn update_acquisition_records( + &self, + update: impl FnOnce(&mut BTreeMap) -> Result, + publish_unchanged: bool, + ) -> Result { + let transaction = self.transaction(false)?; + let mut document = transaction.document()?; + let before = document.acquisitions.clone(); + let result = update(&mut document.acquisitions)?; + if !publish_unchanged && document.acquisitions == before { + return Ok(result); + } + document.validate()?; + require_durable(transaction.publish_document(&document))?; + Ok(result) + } + + pub(crate) fn acquisitions(&self) -> Result> { + let transaction = self.transaction(true)?; + if transaction.legacy_read_only { + return Ok(BTreeMap::new()); + } + Ok(transaction.document()?.acquisitions) + } +} + +fn document_with_partition( + mut document: AcquisitionDocument, + value: Value, +) -> Result { + let mut legacy = value.as_object().cloned().ok_or_else(invalid_schema)?; + if schema(&value) != Some(5) || legacy.contains_key("acquisitions") { + return Err(invalid_schema()); + } + legacy.remove("schema_version"); + document.legacy = legacy.into_iter().collect(); + document.validate()?; + Ok(document) +} + +impl AcquisitionTransaction<'_> { + fn document(&self) -> Result { + let Some(value) = self.target.read_json::()? else { + return Ok(AcquisitionDocument::empty()); + }; + if schema(&value) != Some(6) { + return Err(invalid_schema()); + } + let document: AcquisitionDocument = serde_json::from_value(value)?; + document.validate()?; + Ok(document) + } + + pub(crate) fn model_partition(&self) -> Result> { + let Some(value) = self.target.read_json::()? else { + return Ok(None); + }; + if matches!(schema(&value), Some(4 | 5)) && self.legacy_read_only { + return Ok(Some(value)); + } + let document = self.document()?; + if document.legacy.is_empty() { + return Ok(None); + } + let mut value: serde_json::Map = document.legacy.into_iter().collect(); + value.insert("schema_version".into(), 5.into()); + Ok(Some(Value::Object(value))) + } + + pub(crate) fn publish_model_partition(&self, data: &T) -> AtomicPublishResult { + let document = (|| -> Result<_> { + if self.legacy_read_only { + return Err(migration_required()); + } + let mut document = + document_with_partition(self.document()?, serde_json::to_value(data)?)?; + if let Some((consumer, operation)) = &self.consumer_settlement { + for record in document.acquisitions.values_mut().filter(|record| { + &record.demand.consumer == consumer && &record.demand.operation == operation + }) { + if !matches!( + record.phase, + super::service::AcquisitionPhase::Using { .. } + | super::service::AcquisitionPhase::Adopted { .. } + ) { + record.phase = super::service::AcquisitionPhase::Withdrawn; + record.files.clear(); + } + } + } + document.validate()?; + Ok(document) + })(); + match document { + Ok(document) => self.publish_document(&document), + Err(error) => Err(Box::new(AtomicPublishFailure { + stage: AtomicPublishStage::Serialization, + kind: AtomicPublishFailureKind::InvalidData, + error, + cleanup: StagingCleanup::NotRequired, + })), + } + } + + pub(crate) fn stage_consumer_settlement(&mut self, consumer: &str, operation: &str) { + self.consumer_settlement = Some((consumer.into(), operation.into())); + } + + fn publish_document(&self, document: &AcquisitionDocument) -> AtomicPublishResult { + self.target.publish_json(document) + } +} + +fn require_durable(publication: AtomicPublishResult) -> Result<()> { + match publication { + Ok(AtomicPublication::Durable) => Ok(()), + Ok(AtomicPublication::PublishedDurabilityUnknown { error }) => Err(error), + Ok(AtomicPublication::VisibilityUnknown { error, cleanup }) => Err(AtomicPublishFailure { + stage: AtomicPublishStage::Rename, + kind: AtomicPublishFailureKind::Filesystem, + error, + cleanup, + } + .into_error()), + Err(failure) => Err(failure.into_error()), + } +} diff --git a/rust/crates/pumas-core/src/acquisition/workspace.rs b/rust/crates/pumas-core/src/acquisition/workspace.rs new file mode 100644 index 00000000..1df4a99d --- /dev/null +++ b/rust/crates/pumas-core/src/acquisition/workspace.rs @@ -0,0 +1,425 @@ +//! Held, source-neutral filesystem authority. Persisted identity never opens it. +#![deny(unsafe_code)] + +use super::{ArtifactFile, ArtifactManifest}; +use crate::platform::capability_fs::sync_directory; +use crate::{PumasError, Result}; +#[cfg(unix)] +use cap_std::fs::OpenOptionsExt; +use cap_std::fs::{Dir, Metadata, OpenOptions}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::io::{Read, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +/// Equality-only workspace locator. It conveys no filesystem permission. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceIdentity { + pub root_identity: String, + pub relative_target: String, +} + +/// Bounded verification receipt tied to a held regular file. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct VerifiedFile { + pub path: String, + pub bytes: u64, + pub sha256: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Identity(u64, u64); + +fn identity(metadata: &Metadata) -> Result { + #[cfg(unix)] + { + use cap_std::fs::MetadataExt; + Ok(Identity(metadata.dev(), metadata.ino())) + } + #[cfg(windows)] + { + use cap_primitives::fs::_WindowsByHandle; + Ok(Identity( + u64::from(metadata.volume_serial_number().ok_or_else(changed)?), + metadata.file_index().ok_or_else(changed)?, + )) + } + #[cfg(not(any(unix, windows)))] + { + let _ = metadata; + Err(changed()) + } +} + +fn changed() -> PumasError { + PumasError::Validation { + field: "acquisition.workspace".into(), + message: "Held acquisition workspace or file binding changed".into(), + } +} + +fn options() -> OpenOptions { + let mut options = OpenOptions::new(); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + #[cfg(windows)] + { + use cap_std::fs::OpenOptionsExt; + options.custom_flags(windows_sys::Win32::Storage::FileSystem::FILE_FLAG_OPEN_REPARSE_POINT); + } + options +} + +struct HeldParent { + directory: Dir, + binding: Identity, +} + +struct HeldWorkspace { + directory: Dir, + binding: Identity, + validate: Box Result<()> + Send + Sync>, + _execution_lease: Arc, + parents: Mutex>>, + sealed: Mutex, +} + +/// A runtime grant captured from an already-held, exclusively reserved directory. +/// Its validator must retain and revalidate the caller's root/execution grant. +/// It never opens a directory from its persisted or displayed locator. +#[derive(Clone)] +pub struct AcquisitionWorkspace { + held: Arc, + locator: WorkspaceIdentity, +} + +impl AcquisitionWorkspace { + pub(crate) fn from_capability( + directory: Dir, + locator: WorkspaceIdentity, + execution_lease: Arc, + validate: impl Fn() -> Result<()> + Send + Sync + 'static, + ) -> Result { + validate()?; + let binding = identity(&directory.dir_metadata()?)?; + Ok(Self { + held: Arc::new(HeldWorkspace { + directory, + binding, + validate: Box::new(validate), + _execution_lease: execution_lease, + parents: Mutex::new(BTreeMap::new()), + sealed: Mutex::new(false), + }), + locator, + }) + } + + pub fn identity(&self) -> &WorkspaceIdentity { + &self.locator + } + + fn validate(&self) -> Result<()> { + (self.held.validate)()?; + if identity(&self.held.directory.dir_metadata()?)? != self.held.binding { + return Err(changed()); + } + Ok(()) + } + + fn parent(&self, path: &str, create: bool) -> Result<(Dir, String)> { + self.validate()?; + let path = Path::new(path); + if path + .components() + .any(|part| !matches!(part, Component::Normal(_))) + { + return Err(changed()); + } + let name = path + .file_name() + .and_then(|name| name.to_str()) + .ok_or_else(changed)?; + let mut directory = self.held.directory.try_clone()?; + let mut relative = PathBuf::new(); + for component in path.parent().unwrap_or(Path::new("")).components() { + let Component::Normal(component) = component else { + return Err(changed()); + }; + relative.push(component); + let known = self + .held + .parents + .lock() + .map_err(|_| changed())? + .get(&relative) + .cloned(); + if create && known.is_none() { + match directory.create_dir(component) { + Ok(()) => sync_directory(&directory)?, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => return Err(error.into()), + } + } + let metadata = directory.symlink_metadata(component)?; + if !metadata.is_dir() || metadata.is_symlink() { + return Err(changed()); + } + let next = directory.open_dir(component)?; + let binding = identity(&next.dir_metadata()?)?; + if identity(&metadata)? != binding + || known.as_ref().is_some_and(|held| held.binding != binding) + { + return Err(changed()); + } + let candidate = Arc::new(HeldParent { + directory: next, + binding, + }); + let held = { + let mut parents = self.held.parents.lock().map_err(|_| changed())?; + parents.entry(relative.clone()).or_insert(candidate).clone() + }; + if held.binding != binding { + return Err(changed()); + } + directory = held.directory.try_clone()?; + } + self.validate()?; + Ok((directory, name.to_owned())) + } + + fn require_writable(&self) -> Result<()> { + if *self.held.sealed.lock().map_err(|_| changed())? { + return Err(changed()); + } + self.validate() + } + + pub(crate) fn prepare_file(&self, file: &ArtifactFile) -> Result<()> { + self.require_writable()?; + self.parent(file.logical_path(), true).map(|_| ()) + } + + pub(crate) fn file_len(&self, path: &str, partial: bool) -> Result> { + let (parent, name) = self.parent(path, false)?; + let name = if partial { + format!("{name}.part") + } else { + name + }; + match parent.symlink_metadata(name) { + Ok(metadata) if metadata.is_file() && !metadata.is_symlink() => { + Ok(Some(metadata.len())) + } + Ok(_) => Err(changed()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(error) => Err(error.into()), + } + } + + pub(crate) fn open_part(&self, path: &str, append: bool) -> Result { + self.require_writable()?; + let (parent, name) = self.parent(path, true)?; + let mut options = options(); + if append { + options.append(true); + } else { + options.write(true).create(true); + } + let file = parent + .open_with(format!("{name}.part"), &options)? + .into_std(); + if !file.metadata()?.is_file() { + return Err(changed()); + } + if !append { + file.set_len(0)?; + } + self.validate()?; + Ok(file) + } + + pub(crate) fn remove_part(&self, path: &str) -> Result<()> { + self.require_writable()?; + let (parent, name) = self.parent(path, false)?; + match parent.remove_file(format!("{name}.part")) { + Ok(()) => sync_directory(&parent)?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error.into()), + } + self.validate() + } + + pub(crate) fn verify_file(&self, file: &ArtifactFile, partial: bool) -> Result { + let (parent, name) = self.parent(file.logical_path(), false)?; + let name = if partial { + format!("{name}.part") + } else { + name + }; + let mut options = options(); + options.read(true); + let mut handle = parent.open_with(&name, &options)?.into_std(); + let before = Metadata::from_file(&handle)?; + if !before.is_file() + || file + .expected_size() + .is_some_and(|size| size != before.len()) + { + return Err(changed()); + } + let binding = identity(&before)?; + let mut hash = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let bytes = handle.read(&mut buffer)?; + if bytes == 0 { + break; + } + hash.update(&buffer[..bytes]); + } + let digest = hex::encode(hash.finalize()); + let after = Metadata::from_file(&handle)?; + let (current_parent, _) = self.parent(file.logical_path(), false)?; + let current = Metadata::from_file(¤t_parent.open_with(&name, &options)?.into_std())?; + if identity(&after)? != binding + || identity(¤t)? != binding + || before.len() != after.len() + || before.len() != current.len() + || before.modified()? != after.modified()? + || before.modified()? != current.modified()? + { + return Err(changed()); + } + if let Some(expected) = file.expected_sha256() { + if expected.value() != digest { + return Err(PumasError::HashMismatch { + expected: expected.value().into(), + actual: digest, + }); + } + } + handle.sync_all()?; + sync_directory(&parent)?; + self.validate()?; + Ok(VerifiedFile { + path: file.logical_path().into(), + bytes: before.len(), + sha256: digest, + }) + } + + pub(crate) fn publish_part( + &self, + file: &ArtifactFile, + compare_existing: bool, + ) -> Result { + self.require_writable()?; + let staged = self.verify_file(file, true)?; + let (parent, name) = self.parent(file.logical_path(), false)?; + if compare_existing { + let current = self.verify_file(file, false)?; + if current != staged { + return Err(PumasError::Validation { + field: "download.integrity".into(), + message: format!("Existing selected file {} differs from the immutable source artifact; preserving both files for reconciliation", file.logical_path()), + }); + } + self.remove_part(file.logical_path())?; + } else { + if self.file_len(file.logical_path(), false)?.is_some() { + return Err(changed()); + } + parent.rename(format!("{name}.part"), &parent, name)?; + sync_directory(&parent)?; + } + self.verify_file(file, false) + } + + pub(crate) fn seal(&self, manifest: &ArtifactManifest) -> Result> { + self.validate()?; + *self.held.sealed.lock().map_err(|_| changed())? = true; + manifest + .files() + .iter() + .map(|file| self.verify_file(file, false)) + .collect() + } + + pub(crate) fn verify_receipts( + &self, + manifest: &ArtifactManifest, + receipts: &[VerifiedFile], + ) -> Result<()> { + let observed: Vec<_> = manifest + .files() + .iter() + .map(|file| self.verify_file(file, false)) + .collect::>()?; + if observed != receipts { + return Err(changed()); + } + Ok(()) + } +} + +pub(crate) fn write_chunk(file: &mut std::fs::File, bytes: &[u8]) -> Result<()> { + file.write_all(bytes)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::acquisition::FileVerificationRequirement; + + #[test] + fn concurrent_parent_creation_is_coalesced_and_replacement_is_refused() { + let temp = tempfile::TempDir::new().unwrap(); + let root = crate::platform::capability_fs::open_directory(temp.path()).unwrap(); + let workspace = AcquisitionWorkspace::from_capability( + root, + WorkspaceIdentity { + root_identity: "physical-fixture-root".into(), + relative_target: "stage".into(), + }, + Arc::new(()), + || Ok(()), + ) + .unwrap(); + let file = ArtifactFile::new( + "nested/payload.bin", + "payload", + Some(4), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap(); + let barrier = Arc::new(std::sync::Barrier::new(3)); + let handles: Vec<_> = (0..2) + .map(|_| { + let grant = workspace.clone(); + let selected = file.clone(); + let ready = barrier.clone(); + std::thread::spawn(move || { + ready.wait(); + grant.prepare_file(&selected) + }) + }) + .collect(); + barrier.wait(); + for handle in handles { + handle.join().unwrap().unwrap(); + } + std::fs::rename(temp.path().join("nested"), temp.path().join("retired")).unwrap(); + std::fs::create_dir(temp.path().join("nested")).unwrap(); + assert!(workspace.open_part("nested/payload.bin", false).is_err()); + assert!(!temp.path().join("nested/payload.bin.part").exists()); + assert!(!temp.path().join("retired/payload.bin.part").exists()); + } +} diff --git a/rust/crates/pumas-core/src/api/builder.rs b/rust/crates/pumas-core/src/api/builder.rs index 9f3039bd..da0e1cd0 100644 --- a/rust/crates/pumas-core/src/api/builder.rs +++ b/rust/crates/pumas-core/src/api/builder.rs @@ -407,6 +407,9 @@ impl PumasApiBuilder { let download_persistence = Arc::new(model_library::DownloadPersistence::new( &self.launcher_root.join("launcher-data"), )); + let acquisition = Arc::new(crate::acquisition::AcquisitionService::new( + download_persistence.acquisition_store(), + )); let mutation_root = model_library::DownloadDestinationRoot::open(&model_library_dir)?; model_library.install_mutation_authority( runtime_tasks.clone(), @@ -493,6 +496,7 @@ impl PumasApiBuilder { // Import mutation belongs to the download lifecycle, not an external // notification callback. Configure it before restoring completed bytes. if let Some(ref mut client) = hf_client { + client.set_acquisition_service(acquisition.clone())?; client.set_download_importer(Arc::new(model_importer.clone())); client.restore_persisted_downloads().await?; } @@ -549,6 +553,7 @@ impl PumasApiBuilder { system_utils, model_library, hf_client, + acquisition, intent_service, model_importer, conversion_manager, diff --git a/rust/crates/pumas-core/src/api/hf.rs b/rust/crates/pumas-core/src/api/hf.rs index 2b9ec730..16224425 100644 --- a/rust/crates/pumas-core/src/api/hf.rs +++ b/rust/crates/pumas-core/src/api/hf.rs @@ -2126,6 +2126,7 @@ pub(super) mod tests { provider_registry, )), model_library: library, + acquisition: client.acquisition.clone(), hf_client: Some(client), intent_service, runtime_tasks: tasks.clone(), diff --git a/rust/crates/pumas-core/src/api/state.rs b/rust/crates/pumas-core/src/api/state.rs index 0fe45507..a4351872 100644 --- a/rust/crates/pumas-core/src/api/state.rs +++ b/rust/crates/pumas-core/src/api/state.rs @@ -229,6 +229,7 @@ pub(crate) struct PrimaryState { pub(crate) system_utils: Arc, pub(crate) model_library: Arc, pub(crate) hf_client: Option>, + pub(crate) acquisition: Arc, pub(crate) intent_service: Arc, pub(crate) model_importer: model_library::ModelImporter, pub(crate) conversion_manager: Arc, diff --git a/rust/crates/pumas-core/src/lib.rs b/rust/crates/pumas-core/src/lib.rs index 909561ec..fdeb9d38 100644 --- a/rust/crates/pumas-core/src/lib.rs +++ b/rust/crates/pumas-core/src/lib.rs @@ -145,6 +145,16 @@ impl PumasApi { intent::IntentApi::new(&self.primary().intent_service) } + /// Shared source-neutral artifact acquisition owner used by local consumers. + pub fn acquisition(&self) -> &Arc { + &self.primary().acquisition + } + + /// Close the shared acquisition supervisor after consumer-specific shutdown. + pub async fn shutdown_acquisition(&self) -> Result<()> { + self.primary().acquisition.shutdown().await + } + /// Close local intent admission, drain admitted local effects, then drain downloads. /// /// Dropping this waiter does not cancel admitted work. Repeated calls observe diff --git a/rust/crates/pumas-core/src/model_library/download_recovery.rs b/rust/crates/pumas-core/src/model_library/download_recovery.rs index f144eae2..2a76d3b6 100644 --- a/rust/crates/pumas-core/src/model_library/download_recovery.rs +++ b/rust/crates/pumas-core/src/model_library/download_recovery.rs @@ -652,6 +652,32 @@ impl DownloadRecoveryDestination { }) } + /// Transfer already-held directory authority into the neutral workspace. + /// The model root and destination chain remain revalidated on every effect. + pub(crate) fn acquisition_workspace( + &self, + execution_lease: Arc, + ) -> Result { + let directory = self.directory(false)?; + let destination = self.clone(); + let expected = directory_identity(&directory)?; + let locator = self.persisted_identity()?; + crate::acquisition::AcquisitionWorkspace::from_capability( + directory, + crate::acquisition::WorkspaceIdentity { + root_identity: locator.library_root, + relative_target: locator.relative_target, + }, + execution_lease, + move || { + if directory_identity(&destination.directory(false)?)? != expected { + return Err(invalid_capability_path().into()); + } + Ok(()) + }, + ) + } + pub(crate) fn identity(&self) -> DestinationIdentity { DestinationIdentity { root: self.authority.root_identity, @@ -1060,6 +1086,7 @@ impl DownloadRecoveryDestination { Ok(()) } + #[cfg(test)] pub(crate) fn create_parent(&self, file: &str) -> io::Result<()> { self.file_parent(file, true).map(|_| ()) } @@ -1151,71 +1178,7 @@ impl DownloadRecoveryDestination { Ok(()) } - /// Compare an immutable-source response staged in `.part` with an - /// existing final file without replacing either file. Identity and - /// metadata checks detect replacement and ordinary concurrent changes; - /// this is not isolation from an uncooperative writer that can modify a - /// file in place while restoring its metadata. - pub(crate) fn download_part_matches_final(&self, filename: &str) -> Result { - let (parent, name) = self.file_parent(filename, false)?; - let part_name = format!( - "{name}{}", - crate::config::NetworkConfig::DOWNLOAD_TEMP_SUFFIX - ); - let mut final_options = OpenOptions::new(); - final_options.read(true); - nofollow_options(&mut final_options); - let mut part_options = OpenOptions::new(); - part_options.read(true); - nofollow_options(&mut part_options); - let mut final_file = parent.open_with(&name, &final_options)?.into_std(); - let mut part_file = parent.open_with(&part_name, &part_options)?.into_std(); - let final_before = Metadata::from_file(&final_file)?; - let part_before = Metadata::from_file(&part_file)?; - if !final_before.is_file() || !part_before.is_file() { - return Err(invalid_capability_path().into()); - } - let final_identity = filesystem_identity(&final_before).ok_or_else(|| { - invalid_download_integrity("Platform cannot bind the existing file during comparison") - })?; - let part_identity = filesystem_identity(&part_before).ok_or_else(|| { - invalid_download_integrity("Platform cannot bind the staged file during comparison") - })?; - - let comparison = if final_before.len() == part_before.len() { - compare_file_contents(&mut final_file, &mut part_file, final_before.len()) - } else { - Ok(false) - }; - - let final_after = Metadata::from_file(&final_file)?; - let part_after = Metadata::from_file(&part_file)?; - let current_final = parent.symlink_metadata(&name)?; - let current_part = parent.symlink_metadata(&part_name)?; - let unchanged = current_final.is_file() - && current_part.is_file() - && filesystem_identity(&final_before) == Some(final_identity) - && filesystem_identity(&final_after) == Some(final_identity) - && filesystem_identity(¤t_final) == Some(final_identity) - && filesystem_identity(&part_before) == Some(part_identity) - && filesystem_identity(&part_after) == Some(part_identity) - && filesystem_identity(¤t_part) == Some(part_identity) - && final_before.len() == final_after.len() - && final_before.len() == current_final.len() - && part_before.len() == part_after.len() - && part_before.len() == current_part.len() - && final_before.modified()? == final_after.modified()? - && final_before.modified()? == current_final.modified()? - && part_before.modified()? == part_after.modified()? - && part_before.modified()? == current_part.modified()?; - if !unchanged { - return Err(invalid_download_integrity( - "Downloaded files changed during source comparison", - )); - } - Ok(comparison?) - } - + #[cfg(test)] pub(crate) fn open_part(&self, file: &str, append: bool) -> io::Result { let (parent, name) = self.file_parent(file, true)?; let name = format!( @@ -1252,6 +1215,7 @@ impl DownloadRecoveryDestination { self.remove_file_durable(&parent, &name) } + #[cfg(test)] pub(crate) fn rename_part_to_file(&self, file: &str) -> io::Result<()> { let (parent, name) = self.file_parent(file, false)?; let part = format!( @@ -1333,26 +1297,6 @@ impl DownloadRecoveryDestination { } } -fn compare_file_contents( - left: &mut std::fs::File, - right: &mut std::fs::File, - length: u64, -) -> io::Result { - let mut left_buffer = [0_u8; 64 * 1024]; - let mut right_buffer = [0_u8; 64 * 1024]; - let mut remaining = length; - while remaining > 0 { - let chunk_length = remaining.min(left_buffer.len() as u64) as usize; - left.read_exact(&mut left_buffer[..chunk_length])?; - right.read_exact(&mut right_buffer[..chunk_length])?; - if left_buffer[..chunk_length] != right_buffer[..chunk_length] { - return Ok(false); - } - remaining -= chunk_length as u64; - } - Ok(true) -} - fn directory_identity(directory: &Dir) -> io::Result { filesystem_identity(&directory.dir_metadata()?).ok_or_else(invalid_capability_path) } diff --git a/rust/crates/pumas-core/src/model_library/download_store.rs b/rust/crates/pumas-core/src/model_library/download_store.rs index 87e970fb..2c10d67b 100644 --- a/rust/crates/pumas-core/src/model_library/download_store.rs +++ b/rust/crates/pumas-core/src/model_library/download_store.rs @@ -5,10 +5,11 @@ //! Strict inventory hides unresolved ownership transitions; //! durable terminal proofs may outlive the resumable snapshot they protect. +use crate::acquisition::store::{AcquisitionStore, AcquisitionTransaction}; use crate::error::Result; use crate::metadata::{ - AtomicJsonTarget, AtomicPublication, AtomicPublishFailure, AtomicPublishFailureKind, - AtomicPublishResult, AtomicPublishStage, StagingCleanup, + AtomicPublication, AtomicPublishFailure, AtomicPublishFailureKind, AtomicPublishResult, + AtomicPublishStage, StagingCleanup, }; use crate::model_library::artifact_identity::DownloadRevision; use crate::model_library::types::DownloadRequest; @@ -16,15 +17,13 @@ use crate::models::DownloadStatus; use crate::models::HuggingFaceEvidence; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::fs::File; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex}; use tracing::{debug, warn}; use uuid::Uuid; const DOWNLOAD_STORE_SCHEMA_VERSION: u32 = 5; const LEGACY_DOWNLOAD_STORE_SCHEMA_VERSION: u32 = 4; -const DOWNLOAD_STORE_LOCK_FILE: &str = ".downloads.lock"; /// A single persisted download entry. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -342,7 +341,7 @@ enum PersistedRevocationDisposition { #[derive(Clone)] pub struct DownloadPersistence { path: PathBuf, - mutation: Arc>, + store: Arc, confirmed_admissions: Arc>>, confirmed_cleanups: Arc>>, publisher: Arc, @@ -359,14 +358,22 @@ enum StoreOperation { } trait DownloadStorePublisher: Send + Sync { - fn publish(&self, target: &AtomicJsonTarget, data: &DownloadStoreData) -> AtomicPublishResult; + fn publish( + &self, + target: &AcquisitionTransaction<'_>, + data: &DownloadStoreData, + ) -> AtomicPublishResult; } struct AtomicDownloadStorePublisher; impl DownloadStorePublisher for AtomicDownloadStorePublisher { - fn publish(&self, target: &AtomicJsonTarget, data: &DownloadStoreData) -> AtomicPublishResult { - target.publish_json(data) + fn publish( + &self, + target: &AcquisitionTransaction<'_>, + data: &DownloadStoreData, + ) -> AtomicPublishResult { + target.publish_model_partition(data) } } @@ -381,9 +388,7 @@ struct NoopStoreTransactionObserver; impl StoreTransactionObserver for NoopStoreTransactionObserver {} struct StoreTransaction<'a> { - _instance_guard: MutexGuard<'a, ()>, - target: AtomicJsonTarget, - _os_lock: File, + target: AcquisitionTransaction<'a>, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -476,7 +481,7 @@ impl DownloadPersistence { pub fn new(data_dir: &Path) -> Self { Self { path: data_dir.join("downloads.json"), - mutation: Arc::new(Mutex::new(())), + store: Arc::new(AcquisitionStore::new(data_dir)), confirmed_admissions: Arc::new(Mutex::new(HashSet::new())), confirmed_cleanups: Arc::new(Mutex::new(HashSet::new())), publisher: Arc::new(AtomicDownloadStorePublisher), @@ -663,7 +668,7 @@ impl DownloadPersistence { download_id: &str, attempt_id: &str, ) -> Result { - let transaction = self.transaction(StoreOperation::Remove)?; + let mut transaction = self.transaction(StoreOperation::Remove)?; let mut data = self.load_data_strict(&transaction)?; let Some(admission) = data .queue_admissions @@ -696,6 +701,9 @@ impl DownloadPersistence { data.lifecycle_quarantines .retain(|id, quarantine| id != download_id || quarantine.sticky_failure); } + transaction + .target + .stage_consumer_settlement("hf.model", attempt_id); self.write_data(&transaction, &mut data)?; Ok(true) } @@ -1482,50 +1490,36 @@ impl DownloadPersistence { Ok(true) } - fn transaction(&self, operation: StoreOperation) -> Result> { - let instance_guard = self.mutation.lock().map_err(|_| { - crate::PumasError::Other("Download persistence lock is poisoned".to_string()) - })?; - let target = AtomicJsonTarget::open(&self.path)?; - let os_lock = target.open_lock_file(DOWNLOAD_STORE_LOCK_FILE)?; - self.observer.attempting(operation); - os_lock.lock().map_err(|source| crate::PumasError::Io { - message: format!("Failed to lock download store {}", self.path.display()), - path: Some(self.path.clone()), - source: Some(source), - })?; - self.observer.acquired(operation); - Ok(StoreTransaction { - _instance_guard: instance_guard, - target, - _os_lock: os_lock, + /// Explicit one-shot offline migration of the complete supported v4/v5 + /// model custody representation into schema 6. The caller must stop all + /// old readers and writers first; the advisory lock cannot prove that. + /// Normal construction/open never invokes this operation. + pub fn migrate_legacy_offline(data_dir: impl AsRef) -> Result<()> { + let store = Self::new(data_dir.as_ref()); + store.store.migrate_legacy_offline(|value| { + let data = normalize_legacy_store(value, &store.path)?; + serde_json::to_value(data).map_err(Into::into) }) } + pub(crate) fn acquisition_store(&self) -> Arc { + self.store.clone() + } + + fn transaction(&self, operation: StoreOperation) -> Result> { + let target = self.store.transaction_observed( + operation == StoreOperation::Load, + || self.observer.attempting(operation), + || self.observer.acquired(operation), + )?; + Ok(StoreTransaction { target }) + } + fn load_data_strict(&self, transaction: &StoreTransaction<'_>) -> Result { - let Some(value) = transaction.target.read_json::()? else { + let Some(value) = transaction.target.model_partition()? else { return Ok(DownloadStoreData::empty()); }; - let schema_version = value - .get("schema_version") - .and_then(serde_json::Value::as_u64); - let value = match schema_version { - Some(version) if version == u64::from(DOWNLOAD_STORE_SCHEMA_VERSION) => value, - Some(version) if version == u64::from(LEGACY_DOWNLOAD_STORE_SCHEMA_VERSION) => { - let mut data = migrate_v4_to_v5(value, &self.path)?; - self.write_data(transaction, &mut data)?; - return Ok(data); - } - _ => { - return Err(crate::PumasError::Validation { - field: "downloads.schema_version".into(), - message: format!( - "Download store requires schema version {DOWNLOAD_STORE_SCHEMA_VERSION}; only schema version {LEGACY_DOWNLOAD_STORE_SCHEMA_VERSION} can be upgraded automatically" - ), - }); - } - }; - parse_current_store(value, &self.path) + normalize_legacy_store(value, &self.path) } /// Replace the complete versioned store document and require `Durable`. @@ -1571,11 +1565,22 @@ fn parse_current_store(value: serde_json::Value, path: &Path) -> Result Result { +fn normalize_legacy_store(mut value: serde_json::Value, path: &Path) -> Result { let root = value .as_object_mut() .ok_or_else(|| invalid_store_migration("Download store document must be an object"))?; + let version = root + .get("schema_version") + .and_then(serde_json::Value::as_u64); + if version == Some(5) { + return parse_current_store(value, path); + } + if version != Some(4) { + return Err(invalid_store_migration( + "Only complete supported v4/v5 model custody is accepted", + )); + } add_legacy_revision_to_snapshot_array(root.get_mut("downloads"), "downloads")?; add_legacy_revision_to_nested_snapshots( root.get_mut("lifecycle_quarantines"), @@ -2650,7 +2655,7 @@ mod tests { impl DownloadStorePublisher for ScriptedPublisher { fn publish( &self, - target: &AtomicJsonTarget, + target: &AcquisitionTransaction<'_>, data: &DownloadStoreData, ) -> AtomicPublishResult { self.calls.fetch_add(1, Ordering::SeqCst); @@ -2661,7 +2666,7 @@ mod tests { .pop_front() .unwrap_or(ScriptedPublication::Durable) { - ScriptedPublication::Durable => target.publish_json(data), + ScriptedPublication::Durable => target.publish_model_partition(data), ScriptedPublication::NotPublished => Err(Box::new(AtomicPublishFailure { stage: AtomicPublishStage::Staging, kind: AtomicPublishFailureKind::Filesystem, @@ -2670,7 +2675,7 @@ mod tests { })), ScriptedPublication::PublishedDurabilityUnknown => { assert!(matches!( - target.publish_json(data).unwrap(), + target.publish_model_partition(data).unwrap(), AtomicPublication::Durable )); Ok(AtomicPublication::PublishedDurabilityUnknown { @@ -2689,7 +2694,7 @@ mod tests { } ScriptedPublication::VisibilityUnknownAfterEffect => { assert!(matches!( - target.publish_json(data).unwrap(), + target.publish_model_partition(data).unwrap(), AtomicPublication::Durable )); Ok(AtomicPublication::VisibilityUnknown { @@ -3182,10 +3187,10 @@ mod tests { impl DownloadStorePublisher for ExitAfterPublisher { fn publish( &self, - target: &AtomicJsonTarget, + target: &AcquisitionTransaction<'_>, data: &DownloadStoreData, ) -> AtomicPublishResult { - let outcome = target.publish_json(data); + let outcome = target.publish_model_partition(data); let call = self.calls.fetch_add(1, Ordering::SeqCst) + 1; if call == self.exit_after { std::process::exit(80 + i32::try_from(call).expect("test call count fits i32")); @@ -3337,7 +3342,7 @@ mod tests { assert_eq!(reopened.downloads[0].revision.as_deref(), Some(commit)); let document = std::fs::read_to_string(&store.path).unwrap(); assert!(document.contains(&format!("\"revision\": \"{commit}\""))); - assert!(document.contains("\"schema_version\": 5")); + assert!(document.contains("\"schema_version\": 6")); } #[test] @@ -3358,6 +3363,114 @@ mod tests { } } + #[test] + fn offline_v4_v5_migration_preserves_all_custody_partitions_without_authorizing_pending() { + for version in [4, 5] { + let tmp = TempDir::new().unwrap(); + let store = DownloadPersistence::new(tmp.path()); + let mut active = admission_request("active"); + active.destination.relative_target = "active".into(); + store + .admit_download(&Uuid::new_v4().to_string(), &active) + .unwrap() + .into_result() + .unwrap(); + let mut pending = admission_request("pending"); + pending.destination.relative_target = "pending".into(); + let pending_attempt = Uuid::new_v4().to_string(); + store + .admit_download(&pending_attempt, &pending) + .unwrap() + .into_result() + .unwrap(); + store + .begin_lifecycle_quarantine( + &pending.snapshot, + LifecycleQuarantineDomain::Ambient, + true, + Some(&pending_attempt), + ) + .unwrap(); + let mut released = admission_request("released"); + released.destination.relative_target = "released".into(); + let released_attempt = Uuid::new_v4().to_string(); + store + .admit_download(&released_attempt, &released) + .unwrap() + .into_result() + .unwrap(); + store + .settle_queue_admission("released", &released_attempt) + .unwrap(); + let mut recovery = admission_request("recovery"); + recovery.destination.relative_target = "recovery".into(); + let recovery_attempt = Uuid::new_v4().to_string(); + store + .admit_download(&recovery_attempt, &recovery) + .unwrap() + .into_result() + .unwrap(); + store + .revoke_admitted_for_recovery("recovery", &recovery_attempt, &recovery.snapshot) + .unwrap(); + let mut hidden = admission_request("hidden"); + hidden.destination.relative_target = "hidden".into(); + let hidden_store = store + .clone() + .with_test_publisher(Arc::new(ScriptedPublisher::new([ + ScriptedPublication::Durable, + ScriptedPublication::NotPublished, + ]))); + assert!(matches!( + hidden_store + .admit_download(&Uuid::new_v4().to_string(), &hidden) + .unwrap(), + DownloadAdmissionTransition::NotPublished { .. } + )); + let mut expected: serde_json::Value = + serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); + let mut legacy = expected.clone(); + legacy.as_object_mut().unwrap().remove("acquisitions"); + legacy["schema_version"] = 5.into(); + std::fs::write(&store.path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + if version == 4 { + rewrite_current_store_as_v4(&store); + } + let original = std::fs::read(&store.path).unwrap(); + let fresh = DownloadPersistence::new(tmp.path()); + let inventory = fresh.load_lifecycle_inventory_strict().unwrap(); + assert!(inventory.hidden_admissions.contains_key("hidden")); + assert_eq!( + inventory.quarantines["pending"].disposition, + LifecycleCleanupDisposition::Pending + ); + assert!(fresh.reconcile_lifecycle_inventory_strict().is_err()); + assert_eq!(std::fs::read(&store.path).unwrap(), original); + DownloadPersistence::migrate_legacy_offline(tmp.path()).unwrap(); + expected["schema_version"] = 6.into(); + let migrated_bytes = std::fs::read(&store.path).unwrap(); + let actual: serde_json::Value = serde_json::from_slice(&migrated_bytes).unwrap(); + assert_eq!( + actual, expected, + "only the envelope may change during v{version} migration" + ); + // An old strict decoder rejects the new envelope, never downgrades it. + assert!(serde_json::from_value::(actual.clone()).is_err()); + let reopened = DownloadPersistence::new(tmp.path()); + let inventory = reopened.load_lifecycle_inventory_strict().unwrap(); + assert_eq!( + inventory.quarantines["pending"].disposition, + LifecycleCleanupDisposition::Pending + ); + assert!(inventory.hidden_admissions.contains_key("hidden")); + assert!(reopened + .settle_queue_admission("pending", &pending_attempt) + .is_err()); + assert_eq!(std::fs::read(&store.path).unwrap(), migrated_bytes); + assert!(DownloadPersistence::migrate_legacy_offline(tmp.path()).is_err()); + } + } + #[test] fn schema_v4_upgrade_persists_legacy_main_revision() { let tmp = TempDir::new().unwrap(); @@ -3369,22 +3482,26 @@ mod tests { .map(|contents| serde_json::from_str::(&contents).unwrap()) .unwrap(); legacy["schema_version"] = serde_json::Value::from(4); + legacy.as_object_mut().unwrap().remove("acquisitions"); legacy["downloads"][0] .as_object_mut() .unwrap() .remove("revision"); std::fs::write(&store.path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); + let original = std::fs::read(&store.path).unwrap(); let reopened = DownloadPersistence::new(tmp.path()); let transaction = reopened.transaction(StoreOperation::Load).unwrap(); let upgraded = reopened.load_data_strict(&transaction).unwrap(); assert_eq!(upgraded.schema_version, 5); assert_eq!(upgraded.downloads[0].revision, None); drop(transaction); + assert_eq!(std::fs::read(&store.path).unwrap(), original); + DownloadPersistence::migrate_legacy_offline(tmp.path()).unwrap(); let durable: serde_json::Value = serde_json::from_slice(&std::fs::read(&reopened.path).unwrap()).unwrap(); - assert_eq!(durable["schema_version"], 5); + assert_eq!(durable["schema_version"], 6); assert!(durable["downloads"][0]["revision"].is_null()); } @@ -3495,6 +3612,7 @@ mod tests { let mut document: serde_json::Value = serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); document["schema_version"] = serde_json::Value::from(4); + document.as_object_mut().unwrap().remove("acquisitions"); for snapshot in document["downloads"].as_array_mut().unwrap() { snapshot.as_object_mut().unwrap().remove("revision"); } @@ -3556,6 +3674,7 @@ mod tests { let mut legacy: serde_json::Value = serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); legacy["schema_version"] = serde_json::Value::from(4); + legacy.as_object_mut().unwrap().remove("acquisitions"); let original = serde_json::to_vec_pretty(&legacy).unwrap(); std::fs::write(&store.path, &original).unwrap(); @@ -3578,6 +3697,7 @@ mod tests { let mut legacy: serde_json::Value = serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); legacy["schema_version"] = serde_json::Value::from(4); + legacy.as_object_mut().unwrap().remove("acquisitions"); legacy["downloads"][0] .as_object_mut() .unwrap() @@ -3603,39 +3723,32 @@ mod tests { ScriptedPublisher::new([ScriptedPublication::NotPublished]), )); - assert!(reopened.load_all_strict().is_err()); + assert!(reopened.load_all_strict().is_ok()); + assert!( + matches!(reopened.reconcile_lifecycle_inventory_strict(), Err(crate::PumasError::Validation { ref field, .. }) if field == "acquisition.migration_required") + ); assert_eq!(std::fs::read(&reopened.path).unwrap(), original); } #[test] - fn schema_v4_upgrade_unknown_durability_returns_error_and_retains_valid_v5_custody() { + fn schema_v4_explicit_migration_preserves_reopen_custody() { let tmp = TempDir::new().unwrap(); let store = DownloadPersistence::new(tmp.path()); store - .admit_test_download(&persisted("dl-v4-unknown-durability")) + .admit_test_download(&persisted("dl-v4-migration")) .unwrap(); - rewrite_current_store_as_v4(&store); - let reopened = DownloadPersistence::new(tmp.path()).with_test_publisher(Arc::new( - ScriptedPublisher::new([ScriptedPublication::PublishedDurabilityUnknown]), - )); - - assert!(matches!( - reopened.load_all_strict(), - Err(crate::PumasError::Other(ref message)) - if message == "injected parent-sync uncertainty" - )); - let durable: serde_json::Value = - serde_json::from_slice(&std::fs::read(&reopened.path).unwrap()).unwrap(); - assert_eq!(durable["schema_version"], 5); - assert!(durable["downloads"][0]["revision"].is_null()); - + let original = rewrite_current_store_as_v4(&store); + let reopened = DownloadPersistence::new(tmp.path()); + assert!(reopened.load_all_strict().is_ok()); + assert_eq!(std::fs::read(&store.path).unwrap(), original); + DownloadPersistence::migrate_legacy_offline(tmp.path()).unwrap(); let fresh = DownloadPersistence::new(tmp.path()); - let transaction = fresh.transaction(StoreOperation::Load).unwrap(); - let data = fresh.load_data_strict(&transaction).unwrap(); - assert_eq!(data.downloads[0].download_id, "dl-v4-unknown-durability"); - assert!(data - .queue_admissions - .contains_key("dl-v4-unknown-durability")); + fresh.reconcile_lifecycle_inventory_strict().unwrap(); + assert_eq!( + fresh.load_all_strict().unwrap()[0].download_id, + "dl-v4-migration" + ); + assert!(DownloadPersistence::migrate_legacy_offline(tmp.path()).is_err()); } #[test] diff --git a/rust/crates/pumas-core/src/model_library/hf/download.rs b/rust/crates/pumas-core/src/model_library/hf/download.rs index f892e595..81dfb51b 100644 --- a/rust/crates/pumas-core/src/model_library/hf/download.rs +++ b/rust/crates/pumas-core/src/model_library/hf/download.rs @@ -33,7 +33,7 @@ use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::{broadcast, Mutex as TokioMutex, OwnedMutexGuard, RwLock}; -use tracing::{debug, error, info, warn}; +use tracing::{error, info, warn}; /// Regular (non-LFS) filenames that should be automatically fetched alongside /// weight files. These are config/tokenizer files needed by inference engines. @@ -65,6 +65,7 @@ struct PendingDownloadPublication { completed: tokio::sync::oneshot::Sender<()>, } +#[cfg(test)] fn existing_artifact_requires_source_comparison( file: &crate::acquisition::ArtifactFile, observed_size: u64, @@ -326,10 +327,8 @@ fn resolve_exact_recovery_files( .collect() } -enum DownloadFile { - Ambient(Arc>), - Recovery(Arc>), -} +#[cfg(test)] +struct DownloadFile(Option); struct DownloadStartSetup { marker_contents: String, @@ -566,6 +565,7 @@ struct PreparedDownloadTask { #[cfg(test)] download_base_url: Option, client: reqwest::Client, + acquisition: Arc, metadata_client: HuggingFaceClient, downloads: Arc>>, download_publications: Arc, @@ -852,7 +852,11 @@ impl PreparedDownloadTask { Ok(()) } - async fn finalize_pinned_restored_files(&self, context: &TaskContext) -> Result { + async fn finalize_pinned_restored_files( + &self, + context: &TaskContext, + workspace: &crate::acquisition::AcquisitionWorkspace, + ) -> Result { // A restored file with no admitted whole-file digest cannot be tied to // the selected source from its pathname, size, or immutable revision // alone. Leave it under its existing recovery custody for an ordinary @@ -891,15 +895,37 @@ impl PreparedDownloadTask { return Ok(false); } self.destination.verify_file(context, file, true).await?; - self.destination - .rename_part_to_file(context, &file.filename) - .await?; + let manifest = super::acquisition_source::manifest_for_download( + &self.repo_id, + &self.revision, + &self.files, + )?; + let selected = manifest + .files() + .iter() + .find(|selected| selected.logical_path() == file.filename) + .ok_or_else(|| PumasError::Other("Restored selected file is unavailable".into()))? + .clone(); + let grant = workspace.clone(); + context + .run_fallible_blocking_named( + "publish verified retained acquisition file", + move || grant.publish_part(&selected, false), + ) + .await + .map_err(|error| { + PumasError::Other(format!("Restore publication observation failed: {error}")) + })??; } self.verify_pinned_final_files(context).await?; Ok(true) } - async fn finalize_digest_verified_restored_files(&self, context: &TaskContext) -> Result { + async fn finalize_digest_verified_restored_files( + &self, + context: &TaskContext, + workspace: &crate::acquisition::AcquisitionWorkspace, + ) -> Result { for file in &self.files { // Legacy snapshots can lack enough identity to prove a completed // file. Keep them resumable/reconcilable instead of promoting or @@ -942,9 +968,27 @@ impl PreparedDownloadTask { } return Err(error); } - self.destination - .rename_part_to_file(context, &file.filename) - .await?; + let manifest = super::acquisition_source::manifest_for_download( + &self.repo_id, + &self.revision, + &self.files, + )?; + let selected = manifest + .files() + .iter() + .find(|selected| selected.logical_path() == file.filename) + .ok_or_else(|| PumasError::Other("Restored selected file is unavailable".into()))? + .clone(); + let grant = workspace.clone(); + context + .run_fallible_blocking_named( + "publish verified retained acquisition file", + move || grant.publish_part(&selected, false), + ) + .await + .map_err(|error| { + PumasError::Other(format!("Restore publication observation failed: {error}")) + })??; } Ok(true) } @@ -992,10 +1036,46 @@ impl PreparedDownloadTask { "Download provenance observation failed: {error}" ))) })??; + self.acquisition.require_schema(context).await?; + let destination = self.destination.capability().clone(); + let execution_lease = context.held_execution_lease()?; + let workspace = context + .run_fallible_blocking_named("capture restored acquisition workspace", move || { + destination.acquisition_workspace(execution_lease) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Restored workspace observation failed: {error}")) + })??; + let demand = crate::acquisition::AcquisitionDemand { + consumer: "hf.model".into(), + operation: attempt.clone(), + }; + let reconciliation = self + .acquisition + .reconciliation_lease(context, &demand) + .await?; + let manifest = super::acquisition_source::manifest_for_download( + &self.repo_id, + &self.revision, + &self.files, + )?; + let operation = self + .acquisition + .begin( + context, + demand, + manifest, + workspace.identity().clone(), + reconciliation, + ) + .await?; + let already_adopted = operation.is_adopted(); let complete = if self.revision.as_persisted().is_some() { - self.finalize_pinned_restored_files(context).await? + self.finalize_pinned_restored_files(context, &workspace) + .await? } else { - self.finalize_digest_verified_restored_files(context) + self.finalize_digest_verified_restored_files(context, &workspace) .await? }; if !matches!(context.drain_blocking().await, Ok(0)) { @@ -1021,6 +1101,10 @@ impl PreparedDownloadTask { state.files_completed = state.files.len(); } self.verify_pinned_final_files(context).await?; + let lease = self + .acquisition + .files_ready(context, operation, workspace) + .await?; self.destination.remove_marker(context).await?; let info = self .downloads @@ -1029,14 +1113,16 @@ impl PreparedDownloadTask { .get(&self.download_id) .and_then(download_completion_info); drop(destination_guard.take()); - import_completed_download( - &self.download_importer, - context, - info, - self.revision.clone(), - ) - .await - .map_err(RestoredFinalizationError::Import)?; + if !already_adopted { + import_completed_download( + &self.download_importer, + context, + info, + self.revision.clone(), + ) + .await + .map_err(RestoredFinalizationError::Import)?; + } destination_guard = Some(self.destination_lock.clone().lock_owned().await); { let mut states = self.downloads.write().await; @@ -1047,6 +1133,10 @@ impl PreparedDownloadTask { &[DownloadStatus::Downloading], )?; } + if !matches!(context.drain_blocking().await, Ok(0)) { + return Err(PumasError::Other("Restored consumer effects did not drain".into()).into()); + } + self.acquisition.acknowledge(context, lease).await?; let persistence = self.persistence.clone().ok_or_else(|| PumasError::Config { message: "Restore finalization persistence is unavailable".into(), })?; @@ -1212,6 +1302,7 @@ impl PreparedDownloadTask { } else if acquired == Some(true) { AssertUnwindSafe(HuggingFaceClient::run_download( self.client, + self.acquisition.clone(), self.downloads.clone(), self.download_publications.clone(), &self.download_id, @@ -1341,6 +1432,47 @@ impl PreparedDownloadTask { true }; let nested_failures = task_context.drain_blocking().await.unwrap_or(1); + let has_admission = self + .downloads + .read() + .await + .get(&self.download_id) + .is_some_and(|state| state.admission.is_some()); + if nested_failures == 0 + && !has_admission + && result.as_ref().is_err_and(|error| { + !matches!( + error, + PumasError::DownloadPaused | PumasError::DownloadCancelled + ) + }) + { + let mut cleanup_ok = true; + for file in &self.files { + cleanup_ok &= self + .destination + .remove_part(&task_context, &file.filename) + .await + .is_ok(); + } + cleanup_ok &= self.destination.remove_marker(&task_context).await.is_ok(); + if cleanup_ok && matches!(task_context.drain_blocking().await, Ok(0)) { + let identity = self.destination.persisted_identity()?; + self.acquisition + .withdraw( + &task_context, + crate::acquisition::AcquisitionDemand { + consumer: "hf.model".into(), + operation: self.download_id.clone(), + }, + crate::acquisition::WorkspaceIdentity { + root_identity: identity.library_root, + relative_target: identity.relative_target, + }, + ) + .await?; + } + } let mut error_projected = false; if let Err(error) = &result { @@ -1472,6 +1604,7 @@ impl DownloadDestination { } } + #[cfg(test)] async fn prepare_file(&self, task_context: &TaskContext, filename: &str) -> Result<()> { match self { Self::Recovery(destination) | Self::Managed(destination) => { @@ -1544,25 +1677,6 @@ impl DownloadDestination { } } - async fn part_matches_final(&self, task_context: &TaskContext, filename: &str) -> Result { - match self { - Self::Recovery(destination) | Self::Managed(destination) => { - let destination = destination.clone(); - let filename = filename.to_string(); - task_context - .run_blocking_named("compare staged and existing artifact files", move || { - destination.download_part_matches_final(&filename) - }) - .await - .map_err(|error| { - PumasError::Other(format!( - "download source comparison task failed: {error}" - )) - })? - } - } - } - async fn remove_part(&self, task_context: &TaskContext, filename: &str) -> Result<()> { let operation = if self.is_recovery() { "remove partial download file" @@ -1589,35 +1703,7 @@ impl DownloadDestination { } } - async fn finalize_complete_part_file( - &self, - task_context: &TaskContext, - file: &FileToDownload, - verify_integrity: bool, - ) -> Result { - let filename = &file.filename; - let expected_size = file.size; - let Some(expected_size) = expected_size else { - return Ok(false); - }; - let observed_size = self.part_len(task_context, filename).await?; - if observed_size != Some(expected_size) { - if verify_integrity && observed_size.is_some_and(|observed| observed > expected_size) { - self.verify_file(task_context, file, true).await?; - } - return Ok(false); - } - if verify_integrity { - self.verify_file(task_context, file, true).await?; - } - self.rename_part_to_file(task_context, filename).await?; - info!( - "Finalized fully downloaded partial file {} ({} bytes)", - filename, expected_size - ); - Ok(true) - } - + #[cfg(test)] async fn rename_part_to_file(&self, task_context: &TaskContext, filename: &str) -> Result<()> { match self { Self::Recovery(destination) | Self::Managed(destination) => { @@ -1633,6 +1719,7 @@ impl DownloadDestination { } } + #[cfg(test)] async fn open_part( &self, task_context: &TaskContext, @@ -1649,12 +1736,7 @@ impl DownloadDestination { move || destination.open_part(&filename, append), ) .await?; - let file = Arc::new(std::sync::Mutex::new(file)); - Ok(if self.is_recovery() { - DownloadFile::Recovery(file) - } else { - DownloadFile::Ambient(file) - }) + Ok(DownloadFile(Some(file))) } } } @@ -1713,89 +1795,38 @@ impl DownloadDestination { } } +#[cfg(test)] impl DownloadFile { - async fn write_all(&mut self, task_context: &TaskContext, bytes: &[u8]) -> Result<()> { - match self { - Self::Ambient(file) => { - let file = file.clone(); - let bytes = bytes.to_vec(); - recovery_filesystem_operation( - task_context, - "write ambient partial download file", - move || { - let mut file = file - .lock() - .map_err(|_| std::io::Error::other("ambient file lock was poisoned"))?; - std::io::Write::write_all(&mut *file, &bytes) - }, - ) - .await - } - Self::Recovery(file) => { - let file = file.clone(); - let bytes = bytes.to_vec(); - recovery_filesystem_operation( - task_context, - "write partial download file", - move || { - let mut file = file.lock().map_err(|_| { - std::io::Error::other("recovery file lock was poisoned") - })?; - std::io::Write::write_all(&mut *file, &bytes) - }, - ) + async fn write_all(&mut self, context: &TaskContext, bytes: &[u8]) -> Result<()> { + let mut file = self.0.take().expect("one owned fixture file effect"); + let bytes = bytes.to_vec(); + self.0 = Some( + context + .run_fallible_blocking_named("write partial download file", move || { + std::io::Write::write_all(&mut file, &bytes)?; + Ok::<_, std::io::Error>(file) + }) .await - } - } + .map_err(|error| { + PumasError::Other(format!("Fixture write observation failed: {error}")) + })??, + ); + Ok(()) } - - async fn flush(&mut self, task_context: &TaskContext) -> Result<()> { - match self { - Self::Ambient(file) => { - let file = file.clone(); - recovery_filesystem_operation( - task_context, - "flush ambient partial download file", - move || { - let mut file = file - .lock() - .map_err(|_| std::io::Error::other("ambient file lock was poisoned"))?; - std::io::Write::flush(&mut *file) - }, - ) - .await - } - Self::Recovery(file) => { - let file = file.clone(); - recovery_filesystem_operation( - task_context, - "flush partial download file", - move || { - let mut file = file.lock().map_err(|_| { - std::io::Error::other("recovery file lock was poisoned") - })?; - std::io::Write::flush(&mut *file) - }, - ) + async fn flush(&mut self, context: &TaskContext) -> Result<()> { + let file = self.0.take().expect("one owned fixture file effect"); + self.0 = Some( + context + .run_fallible_blocking_named("flush partial download file", move || { + file.sync_all()?; + Ok::<_, std::io::Error>(file) + }) .await - } - } - } -} - -struct HuggingFaceHttpSink<'a> { - file: DownloadFile, - task_context: &'a TaskContext, -} - -#[async_trait::async_trait] -impl crate::acquisition::HttpArtifactSink for HuggingFaceHttpSink<'_> { - async fn write_all(&mut self, bytes: &[u8]) -> Result<()> { - self.file.write_all(self.task_context, bytes).await - } - - async fn flush(&mut self) -> Result<()> { - self.file.flush(self.task_context).await + .map_err(|error| { + PumasError::Other(format!("Fixture sync observation failed: {error}")) + })??, + ); + Ok(()) } } @@ -1812,6 +1843,7 @@ struct HuggingFaceHttpAttemptHost<'a> { bytes_offset: u64, started_at: Instant, last_publish: Instant, + retry_limit: Option, } #[async_trait::async_trait] @@ -1871,6 +1903,59 @@ impl crate::acquisition::HttpAttemptHost for HuggingFaceHttpAttemptHost<'_> { } } +#[async_trait::async_trait] +impl crate::acquisition::AcquisitionHost for HuggingFaceHttpAttemptHost<'_> { + async fn retry( + &mut self, + attempt: u32, + delay: Option, + error: Option<&str>, + ) -> Result<()> { + if delay.is_none() && attempt > 1 { + project_worker_retry_reset( + self.downloads, + self.download_id, + self.task_context, + attempt, + self.retry_limit, + ) + .await?; + } + { + let mut downloads = self.downloads.write().await; + let state = current_worker_state( + &mut downloads, + self.download_id, + self.task_context, + &[DownloadStatus::Downloading], + )?; + state.retry_attempt = attempt; + state.retry_limit = self.retry_limit; + state.retrying = delay.is_some(); + state.next_retry_delay_seconds = delay.map(|delay| delay.as_secs_f64()); + state.error = delay.zip(error).map(|(delay, error)| { + format!( + "Transient network error, retrying attempt {} in {:.1}s: {}", + attempt + 1, + delay.as_secs_f64(), + error + ) + }); + } + publish_worker_snapshot_and_revalidate( + self.publications, + self.downloads, + self.download_id, + self.task_context, + self.destination, + self.destination_lock, + self.destination_guard, + &[DownloadStatus::Downloading], + ) + .await + } +} + async fn recovery_filesystem_operation( task_context: &TaskContext, operation: &'static str, @@ -2075,13 +2160,7 @@ fn retry_limit(max_attempts: u32) -> Option { } } -fn retry_limit_display(limit: Option) -> String { - match limit { - Some(limit) => limit.to_string(), - None => "unlimited".to_string(), - } -} - +#[cfg(test)] fn retry_exhausted( attempt: u32, limit: Option, @@ -2093,24 +2172,6 @@ fn retry_exhausted( attempts_exhausted || elapsed_exhausted } -fn retry_exhausted_message( - attempt: u32, - limit: Option, - elapsed: Duration, - last_error: &str, -) -> String { - let limit_text = limit - .map(|value| value.to_string()) - .unwrap_or_else(|| "unlimited".to_string()); - format!( - "Retry budget exhausted after {} attempt(s) (limit {}, elapsed {:.1}s). Last error: {}", - attempt, - limit_text, - elapsed.as_secs_f64(), - last_error - ) -} - fn selected_artifact_id_for_state(state: &DownloadState) -> Option { let request = state.download_request.as_ref()?; let request_has_explicit_file_scope = @@ -3563,6 +3624,7 @@ impl HuggingFaceClient { #[cfg(test)] download_base_url: self.download_base_url.clone(), client: self.download_client.clone(), + acquisition: self.acquisition.clone(), metadata_client: self.clone_for_invocation(), downloads: self.downloads.clone(), download_publications: self.download_publications.clone(), @@ -3817,6 +3879,7 @@ impl HuggingFaceClient { #[cfg(test)] download_base_url: self.download_base_url.clone(), client: self.download_client.clone(), + acquisition: self.acquisition.clone(), metadata_client: self.clone_for_invocation(), downloads: self.downloads.clone(), download_publications: self.download_publications.clone(), @@ -4038,6 +4101,7 @@ impl HuggingFaceClient { #[allow(clippy::too_many_arguments)] async fn run_download( client: reqwest::Client, + acquisition: Arc, downloads: Arc>>, download_publications: Arc, download_id: &str, @@ -4061,6 +4125,7 @@ impl HuggingFaceClient { use crate::config::NetworkConfig; use crate::network::RetryConfig; + acquisition.require_schema(&task_context).await?; let artifact_manifest = super::acquisition_source::manifest_for_download(repo_id, revision, files)?; @@ -4131,6 +4196,43 @@ impl HuggingFaceClient { .await?; destination.prepare(&task_context).await?; + let granted = destination.capability().clone(); + let execution_lease = task_context.held_execution_lease()?; + let workspace = task_context + .run_fallible_blocking_named("capture acquisition workspace", move || { + granted.acquisition_workspace(execution_lease) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Workspace observation failed: {error}")) + })??; + let operation_receipt = downloads + .read() + .await + .get(download_id) + .and_then(|state| { + state + .admission + .as_ref() + .map(|admission| admission.attempt_id.clone()) + }) + .unwrap_or_else(|| download_id.to_string()); + let demand = crate::acquisition::AcquisitionDemand { + consumer: "hf.model".into(), + operation: operation_receipt, + }; + let reconcile_lease = acquisition + .reconciliation_lease(&task_context, &demand) + .await?; + let operation = acquisition + .begin( + &task_context, + demand, + artifact_manifest.clone(), + workspace.identity().clone(), + reconcile_lease, + ) + .await?; if let Some(start_setup) = start_setup { destination @@ -4247,9 +4349,13 @@ impl HuggingFaceClient { let max_attempts = NetworkConfig::hf_download_max_retries(); let retry_limit = retry_limit(max_attempts); let max_retry_elapsed = NetworkConfig::hf_download_max_retry_elapsed(); - let retry_config = RetryConfig::new() - .with_max_attempts(max_attempts.max(1)) - .with_base_delay(NetworkConfig::HF_DOWNLOAD_RETRY_BASE_DELAY); + let retry_policy = crate::acquisition::AcquisitionRetryPolicy { + attempts: retry_limit, + elapsed: max_retry_elapsed, + backoff: RetryConfig::new() + .with_max_attempts(max_attempts.max(1)) + .with_base_delay(NetworkConfig::HF_DOWNLOAD_RETRY_BASE_DELAY), + }; // Download each file sequentially let mut bytes_offset: u64 = 0; @@ -4265,78 +4371,40 @@ impl HuggingFaceClient { message: "Resolved Hugging Face artifact manifest lost a selected file" .into(), })?; - let verify_selected_file = artifact_manifest.requires_file_verification(file_idx); - // Ensure parent directory exists (needed for subdirectory files - // like transformer/model.safetensors in diffusion repos) - destination.prepare_file(&task_context, filename).await?; - - let mut compare_existing_source = false; - // Digest-backed local files can be verified directly. Without a - // digest, only an immutable source can prove an existing file by - // fetching the selected bytes and comparing them under custody. - if let Some(existing_size) = destination.file_len(&task_context, filename).await? { - compare_existing_source = existing_artifact_requires_source_comparison( - artifact_file, - existing_size, - artifact_manifest.source().revision().strength() - == crate::acquisition::RevisionStrength::Immutable, - )?; - if !compare_existing_source { - destination - .verify_file(&task_context, file_info, false) - .await?; - if destination - .part_len(&task_context, filename) - .await? - .is_some() - { - if let Err(error) = destination.remove_part(&task_context, filename).await { - warn!( - "Failed to remove stale partial file for {}/{}: {}", - repo_id, filename, error - ); - } - } - bytes_offset += existing_size; - info!( - "Skipping already-downloaded file {}/{} ({} bytes)", - repo_id, filename, existing_size - ); - - // Update state - #[cfg(test)] - task_context.observe_worker_projection("before-existing-file-projection"); - { - let mut downloads = downloads.write().await; - let state = current_worker_state( - &mut downloads, - download_id, - &task_context, - &[DownloadStatus::Downloading], - )?; - state.files_completed = file_idx + 1; - state.downloaded_bytes = bytes_offset; - if let Some(total) = state.total_bytes { - state.progress = bytes_offset as f32 / total as f32; - } - } - #[cfg(test)] - if file_idx + 1 == files.len() { - task_context.observe_worker_projection("terminal-cleanup-committed"); - } - publish_worker_snapshot_and_revalidate( - &download_publications, - &downloads, + if let Some(size) = acquisition + .verified_existing_file(&task_context, &operation, &workspace, file_idx) + .await? + { + #[cfg(test)] + task_context.observe_worker_projection("before-existing-file-projection"); + bytes_offset += size; + { + let mut states = downloads.write().await; + let state = current_worker_state( + &mut states, download_id, &task_context, - destination, - &destination_lock, - &mut destination_guard, &[DownloadStatus::Downloading], - ) - .await?; - continue; + )?; + state.files_completed = file_idx + 1; + state.downloaded_bytes = bytes_offset; + } + #[cfg(test)] + if file_idx + 1 == files.len() { + task_context.observe_worker_projection("terminal-cleanup-committed"); } + publish_worker_snapshot_and_revalidate( + &download_publications, + &downloads, + download_id, + &task_context, + destination, + &destination_lock, + &mut destination_guard, + &[DownloadStatus::Downloading], + ) + .await?; + continue; } // Fire aux-complete callback at the boundary between auxiliary and weight files. @@ -4475,43 +4543,39 @@ impl HuggingFaceClient { )? .to_string(); - let mut last_error: Option = None; - - let mut file_completed = false; - let mut attempt: u32 = 0; - let retry_started = Instant::now(); - loop { - attempt += 1; - { - let mut downloads = downloads.write().await; - let state = current_worker_state( - &mut downloads, - download_id, + let acquired_size = { + let mut host = HuggingFaceHttpAttemptHost { + downloads: &downloads, + publications: &download_publications, + destination_lock: &destination_lock, + destination_guard: &mut destination_guard, + download_id, + destination, + cancel_flag: &cancel_flag, + pause_flag: &pause_flag, + task_context: &task_context, + bytes_offset, + started_at: Instant::now(), + last_publish: Instant::now(), + retry_limit, + }; + acquisition + .acquire_file( &task_context, - &[DownloadStatus::Downloading], - )?; - state.retry_attempt = attempt; - state.retry_limit = retry_limit; - state.retrying = false; - state.next_retry_delay_seconds = None; - } - - // Check cancellation before each attempt - #[cfg(test)] - task_context.observe_cancellation_check(); - if cancel_flag.load(Ordering::Relaxed) { - let _ = destination.remove_part(&task_context, filename).await; - // `cancel_download` has already generation-replaced this - // worker. Its caller-independent finalizer exclusively - // owns terminal state, persistence cleanup, and recovery - // capability release after observing this worker. - return Err(PumasError::DownloadCancelled); - } - - // Check pause before each attempt - if pause_flag.load(Ordering::Relaxed) { - #[cfg(test)] - task_context.observe_worker_projection("pause-before-attempt"); + &operation, + &workspace, + file_idx, + &client, + &url, + auth_header.as_deref(), + &retry_policy, + &mut host, + ) + .await + }; + let actual_size = match acquired_size { + Ok(size) => size, + Err(PumasError::DownloadPaused) => { return Self::settle_worker_pause( &downloads, &download_publications, @@ -4522,240 +4586,10 @@ impl HuggingFaceClient { ) .await; } - - // Determine resume offset from existing .part file - let mut resume_from_byte = destination - .part_len(&task_context, filename) - .await? - .unwrap_or(0); - - if compare_existing_source && attempt == 1 && resume_from_byte > 0 { - // A retained partial has no durable evidence tying its - // prefix to this source. Start this comparison from a - // fresh complete representation; retries in this same - // supervised worker may resume bytes it just received. - destination.remove_part(&task_context, filename).await?; - resume_from_byte = 0; - } - - if resume_from_byte > 0 && !artifact_manifest.permits_resume(file_idx) { - info!( - "Discarding partial file {}/{} because its selected source has no resume identity", - repo_id, filename - ); - destination.remove_part(&task_context, filename).await?; - resume_from_byte = 0; - } - - if !compare_existing_source - && destination - .finalize_complete_part_file(&task_context, file_info, verify_selected_file) - .await? - { - file_completed = true; - break; - } - - if attempt > 1 { - warn!( - "Retry {}/{} for {}/{} (resuming from byte {})", - attempt, - retry_limit_display(retry_limit), - repo_id, - filename, - resume_from_byte - ); - - // Reset status to Downloading for the retry - project_worker_retry_reset( - &downloads, - download_id, - &task_context, - attempt, - retry_limit, - ) - .await?; - publish_worker_snapshot_and_revalidate( - &download_publications, - &downloads, - download_id, - &task_context, - destination, - &destination_lock, - &mut destination_guard, - &[DownloadStatus::Downloading], - ) - .await?; - } - - match Self::download_attempt( - &client, - &downloads, - &download_publications, - &destination_lock, - &mut destination_guard, - download_id, - &url, - destination, - filename, - &artifact_manifest, - file_idx, - resume_from_byte, - bytes_offset, - &cancel_flag, - &pause_flag, - persistence.as_ref(), - auth_header.as_deref(), - &task_context, - ) - .await - { - Ok(_) => { - #[cfg(test)] - task_context.observe_worker_projection("before-rename-pause-check"); - if pause_flag.load(Ordering::Relaxed) { - #[cfg(test)] - task_context.observe_worker_projection("pause-before-rename"); - return Self::settle_worker_pause( - &downloads, - &download_publications, - download_id, - &task_context, - persistence.as_ref(), - &mut destination_guard, - ) - .await; - } - if verify_selected_file { - destination - .verify_file(&task_context, file_info, true) - .await?; - } - if compare_existing_source { - if !destination - .part_matches_final(&task_context, filename) - .await? - { - return Err(PumasError::Validation { - field: "download.integrity".into(), - message: format!( - "Existing selected file {filename} differs from the immutable source artifact; preserving both files for reconciliation" - ), - }); - } - destination.remove_part(&task_context, filename).await?; - } else { - // Rename .part to final path atomically. - destination - .rename_part_to_file(&task_context, filename) - .await - .map_err(|e| PumasError::DownloadFailed { - url: "Hugging Face artifact".into(), - message: format!("Failed to rename temp file: {}", e), - })?; - } - - file_completed = true; - break; - } - Err(e) => { - // Paused -- .part preserved, not a real error - if matches!(e, PumasError::DownloadPaused) { - return Err(e); - } - - if !e.is_retryable() || cancel_flag.load(Ordering::Relaxed) { - if cancel_flag.load(Ordering::Relaxed) { - let _ = destination.remove_part(&task_context, filename).await; - } - return Err(e); - } - - warn!( - "Download attempt {}/{} failed for {}/{}: {}", - attempt, - retry_limit_display(retry_limit), - repo_id, - filename, - e - ); - let error_text = e.to_string(); - last_error = Some(e); - - let elapsed = retry_started.elapsed(); - if retry_exhausted(attempt, retry_limit, elapsed, max_retry_elapsed) { - break; - } - - let delay = retry_config.calculate_delay(attempt.saturating_sub(1)); - let limit_text = retry_limit_display(retry_limit); - let next_attempt = attempt + 1; - { - let mut downloads = downloads.write().await; - let state = current_worker_state( - &mut downloads, - download_id, - &task_context, - &[DownloadStatus::Downloading], - )?; - state.retry_attempt = attempt; - state.retry_limit = retry_limit; - state.retrying = true; - state.next_retry_delay_seconds = Some(delay.as_secs_f64()); - state.error = Some(format!( - "Transient network error, retrying attempt {}/{} in {:.1}s: {}", - next_attempt, - limit_text, - delay.as_secs_f64(), - error_text - )); - } - publish_worker_snapshot_and_revalidate( - &download_publications, - &downloads, - download_id, - &task_context, - destination, - &destination_lock, - &mut destination_guard, - &[DownloadStatus::Downloading], - ) - .await?; - debug!("Waiting {:?} before retry", delay); - tokio::select! { - biased; - _ = task_context.pause_requested(&pause_flag) => return Err(PumasError::DownloadPaused), - _ = tokio::time::sleep(delay) => {} - } - } - } - } - - if !file_completed { - let elapsed = retry_started.elapsed(); - if let Some(last_error) = last_error { - let detail = retry_exhausted_message( - attempt, - retry_limit, - elapsed, - &last_error.to_string(), - ); - return Err(PumasError::DownloadFailed { - url, - message: detail, - }); - } - return Err(PumasError::DownloadFailed { - url, - message: "Download stopped before completion".to_string(), - }); - } + Err(error) => return Err(error), + }; // File completed -- use actual file size for accurate offset - let actual_size = destination - .file_len(&task_context, filename) - .await? - .unwrap_or(file_info.size.unwrap_or(0)); bytes_offset += actual_size; { let mut downloads = downloads.write().await; @@ -4797,11 +4631,10 @@ impl HuggingFaceClient { ); } - for (file_idx, file) in files.iter().enumerate() { - if artifact_manifest.requires_file_verification(file_idx) { - destination.verify_file(&task_context, file, false).await?; - } - } + let already_adopted = operation.is_adopted(); + let use_lease = acquisition + .files_ready(&task_context, operation, workspace) + .await?; // Remove the marker through the same destination authority before // releasing a recovery capability from state. If this fails, the @@ -4814,13 +4647,15 @@ impl HuggingFaceClient { .get(download_id) .and_then(download_completion_info); drop(destination_guard.take()); - import_completed_download( - &download_importer, - &task_context, - completion_info, - revision.clone(), - ) - .await?; + if !already_adopted { + import_completed_download( + &download_importer, + &task_context, + completion_info, + revision.clone(), + ) + .await?; + } destination_guard = Some(destination_lock.clone().lock_owned().await); { let mut states = downloads.write().await; @@ -4848,6 +4683,13 @@ impl HuggingFaceClient { } } + acquisition.acknowledge(&task_context, use_lease).await?; + if !matches!(task_context.drain_blocking().await, Ok(0)) { + return Err(PumasError::Other( + "Acquisition adoption effects did not drain".into(), + )); + } + // Persistence cleanup is part of successful completion. It is // registered with the same task owner and must finish before the final // drain, Completed projection, or recovery-capability release. @@ -4925,110 +4767,6 @@ impl HuggingFaceClient { Ok(()) } - /// Execute a single download attempt, optionally resuming from a byte offset. - /// - /// `artifact_manifest` is the validated source-neutral selection for this - /// download, and `file_index` identifies the selected file for resume policy. - /// `bytes_offset` is bytes already downloaded from previous files in a multi-file download. - /// Overall progress is calculated as `(bytes_offset + file_downloaded) / overall_total`. - #[allow(clippy::too_many_arguments)] - async fn download_attempt( - client: &reqwest::Client, - downloads: &Arc>>, - download_publications: &Arc, - destination_lock: &Arc>, - destination_guard: &mut Option>, - download_id: &str, - url: &str, - destination: &DownloadDestination, - filename: &str, - artifact_manifest: &crate::acquisition::ArtifactManifest, - file_index: usize, - resume_from_byte: u64, - bytes_offset: u64, - cancel_flag: &Arc, - pause_flag: &Arc, - persistence: Option<&Arc>, - auth_header: Option<&str>, - task_context: &TaskContext, - ) -> Result<()> { - if resume_from_byte > 0 { - info!("Resuming download from byte {}", resume_from_byte); - } - - let response = tokio::select! { - biased; - _ = task_context.pause_requested(pause_flag) => return Err(PumasError::DownloadPaused), - response = crate::acquisition::open_http_artifact( - client, - url, - artifact_manifest, - file_index, - resume_from_byte, - auth_header, - ) => response, - }?; - - let is_resuming = response.resumed; - if resume_from_byte > 0 && !is_resuming { - warn!("Server ignored Range; replacing the partial file from byte zero"); - } - - // Open file: append for resume, create for fresh start - let file = destination - .open_part(task_context, filename, is_resuming) - .await?; - let (sink, outcome) = { - let mut sink = HuggingFaceHttpSink { file, task_context }; - let mut host = HuggingFaceHttpAttemptHost { - downloads, - publications: download_publications, - destination_lock, - destination_guard, - download_id, - destination, - cancel_flag, - pause_flag, - task_context, - bytes_offset, - started_at: Instant::now(), - last_publish: Instant::now(), - }; - let outcome = crate::acquisition::stream_http_artifact( - response, - resume_from_byte, - &mut sink, - &mut host, - ) - .await?; - (sink, outcome) - }; - - drop(sink); - match outcome { - crate::acquisition::HttpBodyOutcome::Complete { downloaded: _ } => Ok(()), - crate::acquisition::HttpBodyOutcome::Paused => { - #[cfg(test)] - task_context.observe_worker_projection("pause-during-stream"); - Self::settle_worker_pause( - downloads, - download_publications, - download_id, - task_context, - persistence, - destination_guard, - ) - .await - } - crate::acquisition::HttpBodyOutcome::Cancelled => { - let _ = destination.remove_part(task_context, filename).await; - // Terminal cancellation belongs to the generation-replacing - // finalizer, which observes this worker and its nested work. - Err(PumasError::DownloadCancelled) - } - } - } - async fn persist_status_update_owned( task_context: &TaskContext, persistence: Arc, @@ -5148,6 +4886,7 @@ impl HuggingFaceClient { let destination_executions = self.destination_executions.clone(); let persistence = self.persistence.clone(); let finalizer_id = download_id.to_string(); + let acquisition = self.acquisition.clone(); let cancellation_persistence = persistence.map(|store| CancellationPersistence { store, download_id: finalizer_id.clone(), @@ -5283,6 +5022,18 @@ impl HuggingFaceClient { let _ = task_context.run_fallible_blocking_named("mark cancelled download cleanup failure", move || persistence.mark_failed()).await; } } + if !quarantine_failed && !filesystem_cleanup_failed && !effect_drain_failed && !persistence_cleanup_failed { + let capability = cleanup_destination.capability().clone(); + let identity = task_context.run_fallible_blocking_named("observe cancelled acquisition workspace", move || capability.persisted_identity()).await; + if let Ok(Ok(identity)) = identity { + let receipt = cancellation_persistence.as_ref().and_then(|persistence| persistence.admission_attempt.clone()).unwrap_or_else(|| finalizer_id.clone()); + if acquisition.withdraw(&task_context, + crate::acquisition::AcquisitionDemand { consumer: "hf.model".into(), operation: receipt }, + crate::acquisition::WorkspaceIdentity { root_identity: identity.library_root, relative_target: identity.relative_target }).await.is_err() { + filesystem_cleanup_failed = true; + } + } else { filesystem_cleanup_failed = true; } + } // Join registered finalizer blocking work before exposing // Cancelled or releasing the recovery capability. let finalizer_drain = task_context.drain_blocking().await; @@ -10879,7 +10630,12 @@ mod tests { assert!(!completion_called.load(Ordering::SeqCst)); assert!(!aux_called.load(Ordering::SeqCst)); assert!(persistence.load_all().is_empty()); - assert!(!temp.path().join("downloads.json").exists()); + let records = persistence.acquisition_store().acquisitions().unwrap(); + assert_eq!(records.len(), 1); + assert!(matches!( + records.values().next().unwrap().phase, + crate::acquisition::AcquisitionPhase::Adopted { .. } + )); assert_eq!( std::fs::read(verified.destination.display_path().join("weights.gguf")).unwrap(), b"done" @@ -13731,7 +13487,7 @@ mod tests { let started_sender = started_sender.clone(); let release = release.clone(); move |operation| { - if operation == "write ambient partial download file" { + if operation == "write partial download file" { if let Some(sender) = started_sender.lock().unwrap().take() { let _ = sender.send(()); let _ = release.lock().unwrap().recv(); @@ -15604,7 +15360,7 @@ mod tests { let result_sender = result_sender.clone(); let release = release.clone(); move |operation| { - if operation == "create file parent" { + if operation == "inspect reusable acquisition file" { if let Some(sender) = result_sender.lock().unwrap().take() { let _ = sender.send(()); let _ = release.lock().unwrap().recv(); @@ -19166,6 +18922,18 @@ mod tests { .unwrap() .is_some()); assert!(reopened_persistence.load_all().is_empty()); + let neutral = reopened.acquisition.store().acquisitions().unwrap(); + let adopted = neutral + .values() + .find(|record| record.demand.consumer == "hf.model") + .unwrap(); + assert!(matches!( + adopted.phase, + crate::acquisition::AcquisitionPhase::Adopted { .. } + )); + assert_eq!(adopted.manifest.source().revision().value(), COMMIT); + assert_eq!(adopted.files.len(), 2); + assert_eq!(adopted.files[1].bytes, PAYLOAD.len() as u64); } async fn await_pinned_integrity_error(client: &HuggingFaceClient, download_id: &str) -> String { diff --git a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs index e86a3f04..eca6f028 100644 --- a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs +++ b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs @@ -408,6 +408,15 @@ impl TaskContext { Ok(scoped) } + pub(crate) fn held_execution_lease(&self) -> crate::Result> { + self.root_grant + .clone() + .map(|grant| grant as Arc) + .ok_or_else(|| crate::PumasError::Config { + message: "Verified-file handoff requires a held execution grant".into(), + }) + } + pub(crate) fn without_root_grant(&self) -> Self { let mut context = self.clone(); context.inner = context.inner.with_effect_lease(None); @@ -439,6 +448,7 @@ impl DownloadTaskOwner { Self::new_with_finalizer(|| async { Ok(()) }) } + #[cfg(test)] pub(super) fn new_with_finalizer(finalizer: F) -> Self where F: FnOnce() -> Fut + Send + 'static, diff --git a/rust/crates/pumas-core/src/model_library/hf/mod.rs b/rust/crates/pumas-core/src/model_library/hf/mod.rs index d11fd2a1..3e7ea7c0 100644 --- a/rust/crates/pumas-core/src/model_library/hf/mod.rs +++ b/rust/crates/pumas-core/src/model_library/hf/mod.rs @@ -87,6 +87,7 @@ pub struct HuggingFaceClient { pub(super) download_publications: Arc, /// Owner of background download tasks and their blocking filesystem work. download_tasks: Arc, + pub(crate) acquisition: Arc, /// Only the public client requests closure on Drop; invocation snapshots /// borrow its configuration while their work belongs to `download_tasks`. owns_lifecycle: bool, @@ -196,6 +197,7 @@ impl HuggingFaceClient { download_updates: self.download_updates.clone(), download_publications: self.download_publications.clone(), download_tasks: self.download_tasks.clone(), + acquisition: self.acquisition.clone(), owns_lifecycle: false, destination_executions: self.destination_executions.clone(), dest_locks: self.dest_locks.clone(), @@ -301,6 +303,9 @@ impl HuggingFaceClient { download_updates.clone(), )); + let acquisition = Arc::new(crate::acquisition::AcquisitionService::new(Arc::new( + crate::acquisition::AcquisitionStore::new(&cache_dir), + ))); Ok(Self { destination_root: None, client, @@ -310,11 +315,15 @@ impl HuggingFaceClient { download_revision, download_updates, download_publications: download_publications.clone(), - download_tasks: Arc::new(DownloadTaskOwner::new_with_finalizer({ - let downloads = downloads.clone(); - let publications = download_publications.clone(); - move || download::project_download_shutdown(downloads, publications) - })), + acquisition: acquisition.clone(), + download_tasks: Arc::new(DownloadTaskOwner::with_supervisor( + acquisition.supervisor(), + { + let downloads = downloads.clone(); + let publications = download_publications.clone(); + move || download::project_download_shutdown(downloads, publications) + }, + )?), owns_lifecycle: true, destination_executions: Arc::new(DestinationExecutionOwner::new()), dest_locks: Arc::new(RwLock::new(HashMap::new())), @@ -356,9 +365,26 @@ impl HuggingFaceClient { /// Set the download persistence store. pub fn set_persistence(&mut self, persistence: Arc) { + self.acquisition = Arc::new(self.acquisition.with_store(persistence.acquisition_store())); self.persistence = Some(persistence); } + pub(crate) fn set_acquisition_service( + &mut self, + acquisition: Arc, + ) -> Result<()> { + self.download_tasks = Arc::new(DownloadTaskOwner::with_supervisor( + acquisition.supervisor(), + { + let downloads = self.downloads.clone(); + let publications = self.download_publications.clone(); + move || download::project_download_shutdown(downloads, publications) + }, + )?); + self.acquisition = acquisition; + Ok(()) + } + pub(crate) fn set_download_importer(&mut self, importer: Arc) { self.download_importer = Some(importer); } diff --git a/rust/crates/pumas-core/src/model_library/mutation_authority.rs b/rust/crates/pumas-core/src/model_library/mutation_authority.rs index b298d546..dc857ad4 100644 --- a/rust/crates/pumas-core/src/model_library/mutation_authority.rs +++ b/rust/crates/pumas-core/src/model_library/mutation_authority.rs @@ -161,10 +161,21 @@ impl LibraryMutationAuthority { .hidden_admissions .values() .any(|admission| targets.contains(&admission.request.destination)); + let acquisitions = self.downloads.acquisition_store().acquisitions()?; + let acquiring = acquisitions.values().any(|record| { + !matches!( + record.phase, + crate::acquisition::AcquisitionPhase::Adopted { .. } + | crate::acquisition::AcquisitionPhase::Withdrawn + ) && targets.iter().any(|target| { + target.library_root == record.workspace.root_identity + && target.relative_target == record.workspace.relative_target + }) + }); // Settled quarantine records do not retain a trustworthy destination // identity. Preserve all model bytes until explicit recovery resolves // that custody. - if queued || hidden || !inventory.quarantines.is_empty() { + if queued || hidden || acquiring || !inventory.quarantines.is_empty() { return Err(PumasError::DownloadRootBusy); } Ok(()) @@ -269,4 +280,83 @@ mod tests { PumasError::Validation { ref field, .. } if field == "model_library.mutation" )); } + + #[tokio::test] + async fn canonical_acquisition_custody_blocks_model_mutation_until_withdrawal() { + use crate::acquisition::{ + AcquisitionDemand, AcquisitionPhase, AcquisitionRecord, ArtifactFile, ArtifactManifest, + ArtifactRevisionEvidence, ArtifactSourceIdentity, FileVerificationRequirement, + RevisionStrength, WorkspaceIdentity, + }; + let temp = tempfile::TempDir::new().unwrap(); + let library_root = temp.path().join("library"); + let model = library_root.join("llm/family/model"); + std::fs::create_dir_all(&model).unwrap(); + let downloads = Arc::new(DownloadPersistence::new(temp.path())); + let authority = LibraryMutationAuthority::new( + &library_root, + RuntimeTasks::new(), + DownloadDestinationRoot::open(&library_root).unwrap(), + downloads.clone(), + ) + .unwrap(); + let targets = authority + .validate_targets(&[("llm/family/model".into(), model)]) + .unwrap(); + let operation = Uuid::new_v4(); + let record = AcquisitionRecord { + id: operation, + demand: AcquisitionDemand { + consumer: "fixture.consumer".into(), + operation: "retained-demand".into(), + }, + manifest: ArtifactManifest::new( + ArtifactSourceIdentity::new( + "fixture", + "object", + ArtifactRevisionEvidence::new( + "fixture.revision", + "v1", + RevisionStrength::Immutable, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + "weights.gguf", + "weights", + Some(4), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap()], + ) + .unwrap(), + workspace: WorkspaceIdentity { + root_identity: targets[0].library_root.clone(), + relative_target: targets[0].relative_target.clone(), + }, + phase: AcquisitionPhase::Transferring, + files: Vec::new(), + }; + downloads + .acquisition_store() + .update_acquisitions(|records| { + records.insert(operation, record); + Ok(()) + }) + .unwrap(); + assert!(matches!( + authority.require_no_download_custody(&targets), + Err(PumasError::DownloadRootBusy) + )); + downloads + .acquisition_store() + .update_acquisitions(|records| { + records.get_mut(&operation).unwrap().phase = AcquisitionPhase::Withdrawn; + Ok(()) + }) + .unwrap(); + authority.require_no_download_custody(&targets).unwrap(); + } } diff --git a/rust/crates/pumas-core/tests/artifact_acquisition.rs b/rust/crates/pumas-core/tests/artifact_acquisition.rs new file mode 100644 index 00000000..2e496eb6 --- /dev/null +++ b/rust/crates/pumas-core/tests/artifact_acquisition.rs @@ -0,0 +1,43 @@ +//! Public construction/migration regression; all roots are local temporary fixtures. +use pumas_library::{model_library::DownloadPersistence, PumasApi, PumasError}; + +#[tokio::test] +async fn ordinary_builder_leaves_legacy_state_read_only_until_explicit_offline_migration() { + let root = tempfile::TempDir::new().unwrap(); + let data = root.path().join("launcher-data"); + std::fs::create_dir(&data).unwrap(); + let path = data.join("downloads.json"); + let legacy = serde_json::to_vec_pretty(&serde_json::json!({ + "schema_version": 5, + "downloads": [], + "recovery_revocations": {}, + "lifecycle_quarantines": {}, + "admission_attempts": {}, + "queue_admissions": {}, + "released_queue_admissions": {} + })) + .unwrap(); + std::fs::write(&path, &legacy).unwrap(); + let api = PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() + .await + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), legacy); + assert!( + matches!(api.acquisition().store().require_acquisition_schema(), + Err(PumasError::Validation { field, .. }) if field == "acquisition.migration_required") + ); + api.shutdown_intent().await.unwrap(); + api.shutdown_acquisition().await.unwrap(); + drop(api); + // This fixture has no old readers/writers. Migration is a separate operator action. + DownloadPersistence::migrate_legacy_offline(&data).unwrap(); + let current: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(current["schema_version"], 6); + assert_eq!(current["acquisitions"], serde_json::json!({})); + assert!(DownloadPersistence::new(&data).load_all().is_empty()); + assert!(DownloadPersistence::migrate_legacy_offline(&data).is_err()); +} From d0b71b51075815cf307cea95c9364be0ed25472d Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 21:25:26 -0700 Subject: [PATCH 14/20] fix(acquisition): reject conflicting durable custody --- .../pumas-core/src/acquisition/store.rs | 192 +++++++++++++++++- 1 file changed, 191 insertions(+), 1 deletion(-) diff --git a/rust/crates/pumas-core/src/acquisition/store.rs b/rust/crates/pumas-core/src/acquisition/store.rs index 6c8a6c3b..6ac08da5 100644 --- a/rust/crates/pumas-core/src/acquisition/store.rs +++ b/rust/crates/pumas-core/src/acquisition/store.rs @@ -10,7 +10,7 @@ use crate::metadata::{ use crate::{PumasError, Result}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::path::{Path, PathBuf}; use std::sync::{Mutex, MutexGuard}; @@ -42,8 +42,33 @@ impl AcquisitionDocument { if self.schema_version != SCHEMA_VERSION { return Err(invalid_schema()); } + let mut demands = BTreeSet::new(); + let mut active_workspaces = BTreeSet::new(); for (id, record) in &self.acquisitions { record.validate(*id)?; + if !demands.insert(( + record.demand.consumer.clone(), + record.demand.operation.clone(), + )) { + return Err(PumasError::Validation { + field: "acquisition.custody".into(), + message: "Acquisition demand is duplicated in the durable document".into(), + }); + } + if matches!( + record.phase, + super::service::AcquisitionPhase::Transferring + | super::service::AcquisitionPhase::FilesReady + | super::service::AcquisitionPhase::Using { .. } + ) && !active_workspaces.insert(( + record.workspace.root_identity.clone(), + record.workspace.relative_target.clone(), + )) { + return Err(PumasError::Validation { + field: "acquisition.custody".into(), + message: "Multiple active acquisition demands share one workspace".into(), + }); + } } Ok(()) } @@ -324,3 +349,168 @@ fn require_durable(publication: AtomicPublishResult) -> Result<()> { Err(failure) => Err(failure.into_error()), } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::acquisition::{ + AcquisitionDemand, AcquisitionPhase, AcquisitionRecord, ArtifactFile, ArtifactManifest, + ArtifactRevisionEvidence, ArtifactSourceIdentity, FileVerificationRequirement, + RevisionStrength, VerifiedFile, WorkspaceIdentity, + }; + + fn record( + id: Uuid, + operation: &str, + workspace: &str, + phase: AcquisitionPhase, + ) -> AcquisitionRecord { + let manifest = ArtifactManifest::new( + ArtifactSourceIdentity::new( + "fixture", + "object", + ArtifactRevisionEvidence::new( + "fixture.revision", + "v1", + RevisionStrength::Immutable, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + "payload.bin", + "payload", + Some(1), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap()], + ) + .unwrap(); + let files = if matches!( + phase, + AcquisitionPhase::FilesReady + | AcquisitionPhase::Using { .. } + | AcquisitionPhase::Adopted { .. } + ) { + vec![VerifiedFile { + path: "payload.bin".into(), + bytes: 1, + sha256: "0".repeat(64), + }] + } else { + Vec::new() + }; + AcquisitionRecord { + id, + demand: AcquisitionDemand { + consumer: "fixture.consumer".into(), + operation: operation.into(), + }, + manifest, + workspace: WorkspaceIdentity { + root_identity: "fixture.root".into(), + relative_target: workspace.into(), + }, + phase, + files, + } + } + + fn document(records: Vec) -> AcquisitionDocument { + AcquisitionDocument { + schema_version: SCHEMA_VERSION, + acquisitions: records + .into_iter() + .map(|record| (record.id, record)) + .collect(), + legacy: BTreeMap::new(), + } + } + + fn persisted_document_is_refused_without_rewrite(document: AcquisitionDocument) { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("downloads.json"); + let bytes = serde_json::to_vec(&document).unwrap(); + std::fs::write(&path, &bytes).unwrap(); + + assert!(matches!( + AcquisitionStore::new(temp.path()).require_acquisition_schema(), + Err(PumasError::Validation { ref field, .. }) if field == "acquisition.custody" + )); + assert_eq!(std::fs::read(path).unwrap(), bytes); + } + + #[test] + fn schema_six_rejects_duplicate_demand_even_when_one_row_is_using() { + let demand = "same-operation"; + let document = document(vec![ + record( + Uuid::from_u128(1), + demand, + "model/path", + AcquisitionPhase::Transferring, + ), + record( + Uuid::from_u128(2), + demand, + "model/path", + AcquisitionPhase::Using { + lease: Uuid::from_u128(3), + }, + ), + ]); + + persisted_document_is_refused_without_rewrite(document); + } + + #[test] + fn schema_six_rejects_distinct_active_demands_for_one_workspace() { + let document = document(vec![ + record( + Uuid::from_u128(1), + "first-operation", + "model/path", + AcquisitionPhase::Transferring, + ), + record( + Uuid::from_u128(2), + "second-operation", + "model/path", + AcquisitionPhase::Using { + lease: Uuid::from_u128(3), + }, + ), + ]); + + persisted_document_is_refused_without_rewrite(document); + } + + #[test] + fn terminal_history_may_share_a_workspace_with_one_active_successor() { + let document = document(vec![ + record( + Uuid::from_u128(1), + "adopted-operation", + "model/path", + AcquisitionPhase::Adopted { + lease: Uuid::from_u128(3), + }, + ), + record( + Uuid::from_u128(2), + "withdrawn-operation", + "model/path", + AcquisitionPhase::Withdrawn, + ), + record( + Uuid::from_u128(4), + "successor-operation", + "model/path", + AcquisitionPhase::Transferring, + ), + ]); + + document.validate().unwrap(); + } +} From 8b6c5f70c55a3d124189d0cd88dc85c780f47c84 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Tue, 29 Sep 2026 22:54:40 -0700 Subject: [PATCH 15/20] fix(acquisition): guard in-place imports with exact custody proofs --- .../pumas-core/src/acquisition/service.rs | 127 +- .../pumas-core/src/acquisition/store.rs | 24 + .../pumas-core/src/acquisition/workspace.rs | 2 +- .../src/model_library/download_store.rs | 62 +- .../src/model_library/hf/download.rs | 120 +- .../pumas-core/src/model_library/importer.rs | 1278 ++++++++++++++--- .../src/model_library/importer/recovery.rs | 56 + .../pumas-core/src/model_library/library.rs | 185 ++- .../src/model_library/mutation_authority.rs | 339 ++++- 9 files changed, 1940 insertions(+), 253 deletions(-) diff --git a/rust/crates/pumas-core/src/acquisition/service.rs b/rust/crates/pumas-core/src/acquisition/service.rs index 20b18663..b32f918f 100644 --- a/rust/crates/pumas-core/src/acquisition/service.rs +++ b/rust/crates/pumas-core/src/acquisition/service.rs @@ -111,6 +111,7 @@ impl AcquisitionRecord { #[derive(Clone)] pub(crate) struct AcquisitionOperation { record: AcquisitionRecord, + context: TaskContext, } impl AcquisitionOperation { @@ -131,6 +132,64 @@ pub(crate) struct AcquisitionUseLease { operation: AcquisitionOperation, workspace: AcquisitionWorkspace, lease: Uuid, + context: TaskContext, +} + +/// Source-neutral, exact runtime proofs. A retained row or workspace locator +/// cannot construct either proof or reopen unresolved consumer use. +pub(crate) struct AcquisitionUseProof(AcquisitionProof); +pub(crate) struct AcquisitionTransferProof(AcquisitionProof); + +pub(crate) struct AcquisitionProof { + context: TaskContext, + store: Arc, + record: AcquisitionRecord, + workspace: AcquisitionWorkspace, +} + +impl AcquisitionUseProof { + pub(crate) fn into_proof(self) -> AcquisitionProof { + self.0 + } +} +impl AcquisitionTransferProof { + pub(crate) fn into_proof(self) -> AcquisitionProof { + self.0 + } +} + +impl AcquisitionProof { + pub(crate) fn record(&self) -> &AcquisitionRecord { + &self.record + } + + pub(crate) fn validate_current( + &self, + store: &Arc, + current: &std::collections::BTreeMap, + ) -> Result<()> { + if !self + .context + .is_current_role(super::task_custody::TaskRole::Worker) + || !Arc::ptr_eq(&self.store, store) + || current.get(&self.record.id) != Some(&self.record) + || self.workspace.identity() != &self.record.workspace + { + return Err(invalid( + "Acquisition effect proof is stale or belongs to another store/workspace", + )); + } + self.validate_binding() + } + + pub(crate) fn verify_receipts(&self) -> Result<()> { + self.workspace + .verify_receipts(&self.record.manifest, &self.record.files) + } + + pub(crate) fn validate_binding(&self) -> Result<()> { + self.workspace.validate() + } } #[derive(Clone)] @@ -185,6 +244,51 @@ impl AcquisitionService { self.supervisor.request_shutdown().wait().await } + pub(crate) fn use_proof( + &self, + context: &TaskContext, + lease: &AcquisitionUseLease, + ) -> Result { + if !context.shares_scope(&lease.context) + || !context.generation().matches(lease.context.generation()) + || !context.is_current_role(super::task_custody::TaskRole::Worker) + || !matches!(lease.operation.record.phase, AcquisitionPhase::Using { lease: token } if token == lease.lease) + { + return Err(invalid( + "Consumer effect requires its current verified use lease", + )); + } + Ok(AcquisitionUseProof(AcquisitionProof { + context: context.clone(), + store: self.store.clone(), + record: lease.operation.record.clone(), + workspace: lease.workspace.clone(), + })) + } + + pub(crate) fn transfer_proof( + &self, + context: &TaskContext, + operation: &AcquisitionOperation, + workspace: &AcquisitionWorkspace, + ) -> Result { + if !context.shares_scope(&operation.context) + || !context.generation().matches(operation.context.generation()) + || !context.is_current_role(super::task_custody::TaskRole::Worker) + || operation.record.phase != AcquisitionPhase::Transferring + { + return Err(invalid( + "Transfer effect requires its current transferring operation", + )); + } + Ok(AcquisitionTransferProof(AcquisitionProof { + context: context.clone(), + store: self.store.clone(), + record: operation.record.clone(), + workspace: workspace.clone(), + })) + } + pub(crate) async fn require_schema(&self, context: &TaskContext) -> Result<()> { let store = self.store.clone(); owned( @@ -233,6 +337,7 @@ impl AcquisitionService { reconciliation: Option, ) -> Result { let store = self.store.clone(); + let origin = context.clone(); owned(context, "admit durable acquisition", move || { store.update_acquisitions(|records| { if let Some(existing) = records.values().find(|record| record.demand == demand) { @@ -248,7 +353,7 @@ impl AcquisitionService { if !matches!(existing.phase, AcquisitionPhase::Transferring | AcquisitionPhase::FilesReady) && !reconciling { return Err(invalid("Acquisition demand has unresolved or terminal consumer custody")); } - return Ok(AcquisitionOperation { record: existing.clone() }); + return Ok(AcquisitionOperation { record: existing.clone(), context: origin.clone() }); } if records.values().any(|record| record.workspace == workspace && !matches!(record.phase, AcquisitionPhase::Adopted { .. } | AcquisitionPhase::Withdrawn)) { return Err(PumasError::DownloadRootBusy); @@ -256,7 +361,7 @@ impl AcquisitionService { let record = AcquisitionRecord { id: Uuid::new_v4(), demand, manifest, workspace, phase: AcquisitionPhase::Transferring, files: Vec::new() }; record.validate(record.id)?; records.insert(record.id, record.clone()); - Ok(AcquisitionOperation { record }) + Ok(AcquisitionOperation { record, context: origin.clone() }) }) }).await } @@ -526,6 +631,13 @@ impl AcquisitionService { operation: AcquisitionOperation, workspace: AcquisitionWorkspace, ) -> Result { + if !context.shares_scope(&operation.context) + || !context.generation().matches(operation.context.generation()) + { + return Err(invalid( + "Verified handoff belongs to another operation generation", + )); + } let seal = workspace.clone(); let manifest = operation.record.manifest.clone(); let files = owned(context, "seal verified acquisition file set", move || { @@ -583,10 +695,19 @@ impl AcquisitionService { } _ => return Err(invalid("Acquisition is not ready for consumer use")), }; + let mut record = operation.record; + record.files = files; + if !matches!(record.phase, AcquisitionPhase::Adopted { .. }) { + record.phase = AcquisitionPhase::Using { lease }; + } Ok(AcquisitionUseLease { - operation, + operation: AcquisitionOperation { + record, + context: context.clone(), + }, workspace, lease, + context: context.clone(), }) } diff --git a/rust/crates/pumas-core/src/acquisition/store.rs b/rust/crates/pumas-core/src/acquisition/store.rs index 6ac08da5..2965da54 100644 --- a/rust/crates/pumas-core/src/acquisition/store.rs +++ b/rust/crates/pumas-core/src/acquisition/store.rs @@ -3,6 +3,7 @@ //! Model custody is an opaque partition decoded by its model-owned facade. //! This module never interprets model requests, statuses, or recovery policy. use super::service::AcquisitionRecord; +pub(crate) use super::service::{AcquisitionProof, AcquisitionTransferProof, AcquisitionUseProof}; use crate::metadata::{ AtomicJsonTarget, AtomicPublication, AtomicPublishFailure, AtomicPublishFailureKind, AtomicPublishResult, AtomicPublishStage, StagingCleanup, @@ -275,6 +276,29 @@ impl AcquisitionTransaction<'_> { Ok(document) } + /// Read both custody partitions from one document revision while this + /// canonical transaction excludes every cooperative store writer. + pub(crate) fn import_custody_partition( + &self, + ) -> Result<(Option, BTreeMap)> { + let Some(value) = self.target.read_json::()? else { + return Ok((None, BTreeMap::new())); + }; + if matches!(schema(&value), Some(4 | 5)) && self.legacy_read_only { + return Ok((Some(value), BTreeMap::new())); + } + let document: AcquisitionDocument = serde_json::from_value(value)?; + document.validate()?; + let partition = if document.legacy.is_empty() { + None + } else { + let mut value: serde_json::Map = document.legacy.into_iter().collect(); + value.insert("schema_version".into(), 5.into()); + Some(Value::Object(value)) + }; + Ok((partition, document.acquisitions)) + } + pub(crate) fn model_partition(&self) -> Result> { let Some(value) = self.target.read_json::()? else { return Ok(None); diff --git a/rust/crates/pumas-core/src/acquisition/workspace.rs b/rust/crates/pumas-core/src/acquisition/workspace.rs index 1df4a99d..d589fc9b 100644 --- a/rust/crates/pumas-core/src/acquisition/workspace.rs +++ b/rust/crates/pumas-core/src/acquisition/workspace.rs @@ -123,7 +123,7 @@ impl AcquisitionWorkspace { &self.locator } - fn validate(&self) -> Result<()> { + pub(crate) fn validate(&self) -> Result<()> { (self.held.validate)()?; if identity(&self.held.directory.dir_metadata()?)? != self.held.binding { return Err(changed()); diff --git a/rust/crates/pumas-core/src/model_library/download_store.rs b/rust/crates/pumas-core/src/model_library/download_store.rs index 2c10d67b..d726d9c9 100644 --- a/rust/crates/pumas-core/src/model_library/download_store.rs +++ b/rust/crates/pumas-core/src/model_library/download_store.rs @@ -590,6 +590,25 @@ impl DownloadPersistence { ) -> Result<()> { let transaction = self.transaction(StoreOperation::Load)?; let data = self.load_data_strict(&transaction)?; + self.validate_queue_execution_data( + &data, + download_id, + attempt_id, + domain, + destination, + execution_files, + ) + } + + fn validate_queue_execution_data( + &self, + data: &DownloadStoreData, + download_id: &str, + attempt_id: &str, + domain: DownloadAdmissionDomain, + destination: &PersistedDestinationIdentity, + execution_files: &[String], + ) -> Result<()> { let invalid = |message: &str| crate::PumasError::Validation { field: "downloads.queue_admissions".into(), message: message.into(), @@ -737,7 +756,48 @@ impl DownloadPersistence { pub(crate) fn load_lifecycle_inventory_strict(&self) -> Result { let transaction = self.transaction(StoreOperation::Load)?; - let mut data = self.load_data_strict(&transaction)?; + let data = self.load_data_strict(&transaction)?; + self.lifecycle_inventory(data) + } + + /// One coherent custody read for the importer boundary. Execution identity + /// is present only for a private HF stage capability. + pub(crate) fn load_import_custody_strict( + &self, + execution: Option<( + &str, + &str, + DownloadAdmissionDomain, + &PersistedDestinationIdentity, + &[String], + )>, + ) -> Result<( + PersistedDownloadInventory, + BTreeMap, + )> { + let transaction = self.transaction(StoreOperation::Load)?; + let (partition, acquisitions) = transaction.target.import_custody_partition()?; + let data = match partition { + Some(value) => normalize_legacy_store(value, &self.path)?, + None => DownloadStoreData::empty(), + }; + if let Some((download_id, attempt, domain, destination, files)) = execution { + self.validate_queue_execution_data( + &data, + download_id, + attempt, + domain, + destination, + files, + )?; + } + Ok((self.lifecycle_inventory(data)?, acquisitions)) + } + + fn lifecycle_inventory( + &self, + mut data: DownloadStoreData, + ) -> Result { let confirmed = self.confirmed_admission_ids()?; let mut hidden_admissions: BTreeMap = data .admission_attempts diff --git a/rust/crates/pumas-core/src/model_library/hf/download.rs b/rust/crates/pumas-core/src/model_library/hf/download.rs index 81dfb51b..37972f31 100644 --- a/rust/crates/pumas-core/src/model_library/hf/download.rs +++ b/rust/crates/pumas-core/src/model_library/hf/download.rs @@ -20,6 +20,9 @@ use crate::model_library::download_store::{ DownloadPersistence, LifecycleCleanupDisposition, LifecycleQuarantine, LifecycleQuarantineDomain, PersistedDownload, PersistedDownloadInventory, }; +use crate::model_library::mutation_authority::{ + ModelFinalImportCapability, ModelPartialImportCapability, +}; use crate::model_library::sharding; use crate::model_library::types::{DownloadRequest, DownloadStatus, ModelDownloadProgress}; use crate::model_library::SelectedArtifactIdentity; @@ -750,6 +753,7 @@ async fn import_completed_download( context: &TaskContext, info: Option, revision: DownloadRevision, + capability: Option, ) -> Result<()> { let Some(importer) = importer.clone() else { return Ok(()); @@ -757,10 +761,19 @@ async fn import_completed_download( let info = info.ok_or_else(|| PumasError::Config { message: "Download import requires completion metadata".into(), })?; + let capability = capability.ok_or_else(|| PumasError::Config { + message: "Download import requires its exact verified-use capability".into(), + })?; + let import_context = context.clone(); context .run_fallible_async_named("finalize downloaded model import", move || async move { importer - .finalize_downloaded_directory_at_revision(&info, &revision) + .finalize_downloaded_directory_with_capability( + &info, + &revision, + &import_context, + capability, + ) .await .map(|_| ()) }) @@ -1119,6 +1132,21 @@ impl PreparedDownloadTask { context, info, self.revision.clone(), + self.download_importer + .as_ref() + .map(|_| { + Ok::<_, PumasError>(ModelFinalImportCapability::new( + self.acquisition.use_proof(context, &lease)?, + &self.download_id, + if self.destination.is_recovery() { + DownloadAdmissionDomain::Recovery + } else { + DownloadAdmissionDomain::Ambient + }, + context.held_root_execution_grant()?, + )) + }) + .transpose()?, ) .await .map_err(RestoredFinalizationError::Import)?; @@ -4433,14 +4461,31 @@ impl HuggingFaceClient { if let Some(importer) = download_importer.clone() { let import_info = info.clone(); let import_revision = revision.clone(); + let capability = ModelPartialImportCapability::new( + acquisition.transfer_proof( + &task_context, + &operation, + &workspace, + )?, + download_id, + if destination.is_recovery() { + DownloadAdmissionDomain::Recovery + } else { + DownloadAdmissionDomain::Ambient + }, + task_context.held_root_execution_grant()?, + ); + let import_context = task_context.clone(); task_context .run_fallible_async_named( "persist auxiliary download metadata", move || async move { importer - .upsert_download_metadata_stub_at_revision( + .upsert_download_metadata_stub_with_capability( &import_info, &import_revision, + &import_context, + capability, ) .await }, @@ -4653,6 +4698,21 @@ impl HuggingFaceClient { &task_context, completion_info, revision.clone(), + download_importer + .as_ref() + .map(|_| { + Ok::<_, PumasError>(ModelFinalImportCapability::new( + acquisition.use_proof(&task_context, &use_lease)?, + download_id, + if destination.is_recovery() { + DownloadAdmissionDomain::Recovery + } else { + DownloadAdmissionDomain::Ambient + }, + task_context.held_root_execution_grant()?, + )) + }) + .transpose()?, ) .await?; } @@ -8019,9 +8079,8 @@ mod tests { client .configure_download_destination_root(library.library_root()) .unwrap(); - client.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( - library.clone(), - ))); + let importer = importer_with_authority_for_test(library.clone(), &client).await; + client.set_download_importer(importer); let revision = DownloadRevision::from_commit("0123456789abcdef0123456789abcdef01234567").unwrap(); let revision_value = revision.as_str().to_owned(); @@ -8113,9 +8172,8 @@ mod tests { reopened .configure_download_destination_root(library.library_root()) .unwrap(); - reopened.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( - library.clone(), - ))); + let importer = importer_with_authority_for_test(library.clone(), &reopened).await; + reopened.set_download_importer(importer); assert!(reopened .restore_persisted_downloads() .await @@ -11160,6 +11218,37 @@ mod tests { client.download_tasks.set_blocking_observer(None); } + async fn importer_with_authority_for_test( + library: Arc, + client: &HuggingFaceClient, + ) -> Arc { + // Reopening installs the new composition's exact store rather than + // reusing a prior invocation's store/admission confirmations. + let library = if library.mutation_authority().is_ok() { + Arc::new( + crate::model_library::ModelLibrary::new(library.library_root()) + .await + .unwrap(), + ) + } else { + library + }; + library + .install_mutation_authority( + crate::api::RuntimeTasks::new(), + crate::model_library::download_recovery::DownloadDestinationRoot::open( + library.library_root(), + ) + .unwrap(), + client + .persistence + .clone() + .expect("managed import fixture requires its store"), + ) + .unwrap(); + Arc::new(crate::model_library::ModelImporter::new(library)) + } + async fn imported_download_fixture( root: &Path, ) -> ( @@ -11181,9 +11270,8 @@ mod tests { .configure_download_destination_root(library.library_root()) .unwrap(); client.set_persistence(Arc::new(DownloadPersistence::new(root))); - client.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( - library.clone(), - ))); + let importer = importer_with_authority_for_test(library.clone(), &client).await; + client.set_download_importer(importer); let mut request = recovery_test_request("acme/model", &["model.onnx".into()]); request.model_type = Some("vision".into()); request.pipeline_tag = Some("image-classification".into()); @@ -18819,9 +18907,8 @@ mod tests { .configure_download_destination_root(library.library_root()) .unwrap(); client.set_persistence(persistence.clone()); - client.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( - library.clone(), - ))); + let importer = importer_with_authority_for_test(library.clone(), &client).await; + client.set_download_importer(importer); client.set_test_download_base_url(base_url.clone()); *client.auth_token.write().await = None; cache_pinned_repo_tree( @@ -18881,9 +18968,8 @@ mod tests { .unwrap(); let reopened_persistence = Arc::new(DownloadPersistence::new(temp.path())); reopened.set_persistence(reopened_persistence.clone()); - reopened.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( - library.clone(), - ))); + let importer = importer_with_authority_for_test(library.clone(), &reopened).await; + reopened.set_download_importer(importer); reopened.set_test_download_base_url(base_url); *reopened.auth_token.write().await = None; reopened.restore_persisted_downloads().await.unwrap(); diff --git a/rust/crates/pumas-core/src/model_library/importer.rs b/rust/crates/pumas-core/src/model_library/importer.rs index b0c6a8e9..3c61f01d 100644 --- a/rust/crates/pumas-core/src/model_library/importer.rs +++ b/rust/crates/pumas-core/src/model_library/importer.rs @@ -15,6 +15,10 @@ use crate::model_library::external_assets::{ use crate::model_library::hashing::{compute_dual_hash, DualHash}; use crate::model_library::identifier::{identify_model_type, ModelTypeInfo}; use crate::model_library::library::ModelLibrary; +use crate::model_library::mutation_authority::owned_mutation_outcome; +use crate::model_library::mutation_authority::{ + ModelFinalImportCapability, ModelPartialImportCapability, +}; use crate::model_library::naming::{normalize_filename, normalize_name}; use crate::model_library::sharding; use crate::model_library::types::{ @@ -115,35 +119,41 @@ async fn path_is_dir(path: &Path) -> Result { } async fn resolve_model_type_with_rules_async( - index: crate::index::ModelIndex, + library: Arc, model_dir: PathBuf, pipeline_tag: Option, model_type_hint: Option, huggingface_evidence: Option, ) -> Result { - tokio::task::spawn_blocking(move || { - resolve_model_type_with_rules( - &index, - &model_dir, - pipeline_tag.as_deref(), - model_type_hint.as_deref(), - huggingface_evidence.as_ref(), - ) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join in-place model-type resolution task: {}", - err - )) - })? + let index = library.index().clone(); + library + .run_import_blocking("resolve imported model type", move || { + resolve_model_type_with_rules( + &index, + &model_dir, + pipeline_tag.as_deref(), + model_type_hint.as_deref(), + huggingface_evidence.as_ref(), + ) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join in-place model-type resolution task: {}", + err + )) + })? } async fn load_model_metadata_or_default( library: Arc, model_dir: PathBuf, ) -> Result { - tokio::task::spawn_blocking(move || Ok(library.load_metadata(&model_dir)?.unwrap_or_default())) + let execution = library.clone(); + execution + .run_import_blocking("load importer metadata", move || { + Ok(library.load_metadata(&model_dir)?.unwrap_or_default()) + }) .await .map_err(|err| { PumasError::Other(format!( @@ -203,15 +213,18 @@ impl ModelImporter { // Detect file type and model info let importer = self.clone(); let source_path_for_detection = source_path.clone(); - let type_info = - tokio::task::spawn_blocking(move || importer.detect_type(&source_path_for_detection)) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join import type detection task: {}", - err - )) - })??; + let type_info = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.detect_type(&source_path_for_detection) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join import type detection task: {}", + err + )) + })??; // Check security tier let security_tier = type_info.format.security_tier(); @@ -229,16 +242,17 @@ impl ModelImporter { let bundle_validation = if source_metadata.is_dir() { let validation_source_path = source_path.clone(); Some( - tokio::task::spawn_blocking(move || { - validate_diffusers_directory_for_import(&validation_source_path) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join import diffusers validation task: {}", - err - )) - })?, + self.library + .run_import_blocking("importer blocking effect", move || { + validate_diffusers_directory_for_import(&validation_source_path) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join import diffusers validation task: {}", + err + )) + })?, ) } else { None @@ -372,16 +386,18 @@ impl ModelImporter { } let validation_source_path = source_path.clone(); - let validation = tokio::task::spawn_blocking(move || { - validate_diffusers_directory_for_import(&validation_source_path) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join external diffusers validation task: {}", - err - )) - })?; + let validation = self + .library + .run_import_blocking("importer blocking effect", move || { + validate_diffusers_directory_for_import(&validation_source_path) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join external diffusers validation task: {}", + err + )) + })?; tokio::fs::create_dir_all(&target_dir).await?; let model_id = self.library.get_model_id(&target_dir).ok_or_else(|| { PumasError::Other(format!( @@ -421,16 +437,19 @@ impl ModelImporter { let temp_dir = self.create_temp_import_dir().await?; let source_path_for_copy = source_path.to_path_buf(); let temp_dir_for_copy = temp_dir.clone(); - if let Err(err) = tokio::task::spawn_blocking(move || { - copy_directory_preserving_layout(&source_path_for_copy, &temp_dir_for_copy) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join copied diffusers directory copy task: {}", - err - )) - })? { + if let Err(err) = self + .library + .run_import_blocking("importer blocking effect", move || { + copy_directory_preserving_layout(&source_path_for_copy, &temp_dir_for_copy) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join copied diffusers directory copy task: {}", + err + )) + })? + { let _ = tokio::fs::remove_dir_all(&temp_dir).await; return Err(err); } @@ -446,16 +465,18 @@ impl ModelImporter { } let target_dir_for_expected_files = target_dir.to_path_buf(); - let expected_files = tokio::task::spawn_blocking(move || { - collect_relative_file_paths(&target_dir_for_expected_files) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join copied diffusers expected-files task: {}", - err - )) - })??; + let expected_files = self + .library + .run_import_blocking("importer blocking effect", move || { + collect_relative_file_paths(&target_dir_for_expected_files) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join copied diffusers expected-files task: {}", + err + )) + })??; let in_place_spec = InPlaceImportSpec { model_dir: target_dir.to_path_buf(), @@ -589,6 +610,84 @@ impl ModelImporter { &self, info: &DownloadCompletionInfo, revision: &DownloadRevision, + ) -> Result { + let authority = self.library.mutation_authority()?; + if self.library.get_model_id(&info.dest_dir).is_none() { + return Err(PumasError::Validation { + field: "download_finalization".into(), + message: "Downloaded directory has no library model identity".into(), + }); + } + let tasks = authority.tasks(); + let importer = self.clone(); + let info = info.clone(); + let revision = revision.clone(); + tasks + .run_owned( + "finalize in-place model import", + move |context| async move { + owned_mutation_outcome( + async move { + let model_dir = info.dest_dir.clone(); + let guard_context = context.clone(); + let guard = context + .run_blocking("protect model import", move || { + authority.protect_import(&model_dir, guard_context) + }) + .await??; + let importer = Self { + library: Arc::new(importer.library.with_import_guard(guard)), + }; + importer + .finalize_downloaded_directory_guarded(&info, &revision) + .await + } + .await, + ) + }, + ) + .await? + } + + /// Only the managed HF owner can supply this exact verified-use proof. + pub(crate) async fn finalize_downloaded_directory_with_capability( + &self, + info: &DownloadCompletionInfo, + revision: &DownloadRevision, + context: &crate::acquisition::task_custody::TaskContext, + capability: ModelFinalImportCapability, + ) -> Result { + capability.validate_provenance( + &info.download_id, + &info.download_request.repo_id, + revision.as_str(), + &info.filenames, + )?; + // Re-wrap without broadening the proof's stage; the guard factory owns + // the coherent current-store decision under its existing root grant. + let authority = self.library.mutation_authority()?; + let destination = info.dest_dir.clone(); + let guard_context = context.clone(); + let guard = context + .run_fallible_blocking_named("protect verified model import", move || { + authority.protect_final_import(&destination, capability, guard_context) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Model import guard observation failed: {error}")) + })??; + let importer = Self { + library: Arc::new(self.library.with_import_guard(guard)), + }; + importer + .finalize_downloaded_directory_guarded(info, revision) + .await + } + + async fn finalize_downloaded_directory_guarded( + &self, + info: &DownloadCompletionInfo, + revision: &DownloadRevision, ) -> Result { let model_id = self.library @@ -616,7 +715,7 @@ impl ModelImporter { license_status: info.download_request.license_status.clone(), }; let result = self - .import_in_place_with_mode(&spec, InPlaceImportMode::FinalizeDownload, revision) + .import_in_place_effects(&spec, InPlaceImportMode::FinalizeDownload, revision) .await?; if !result.success || result.model_id.as_deref() != Some(model_id.as_str()) { return Err(PumasError::Validation { @@ -646,6 +745,77 @@ impl ModelImporter { &self, info: &AuxFilesCompleteInfo, revision: &DownloadRevision, + ) -> Result<()> { + let authority = self.library.mutation_authority()?; + let tasks = authority.tasks(); + let importer = self.clone(); + let info = info.clone(); + let revision = revision.clone(); + tasks + .run_owned( + "upsert in-place model metadata", + move |context| async move { + owned_mutation_outcome( + async move { + let model_dir = info.dest_dir.clone(); + let guard_context = context.clone(); + let guard = context + .run_blocking("protect partial model metadata", move || { + authority.protect_import(&model_dir, guard_context) + }) + .await??; + let importer = Self { + library: Arc::new(importer.library.with_import_guard(guard)), + }; + importer + .upsert_download_metadata_stub_effects(&info, &revision) + .await + } + .await, + ) + }, + ) + .await? + } + + pub(crate) async fn upsert_download_metadata_stub_with_capability( + &self, + info: &AuxFilesCompleteInfo, + revision: &DownloadRevision, + context: &crate::acquisition::task_custody::TaskContext, + capability: ModelPartialImportCapability, + ) -> Result<()> { + capability.validate_provenance( + &info.download_id, + &info.download_request.repo_id, + revision.as_str(), + &info.filenames, + )?; + let authority = self.library.mutation_authority()?; + let destination = info.dest_dir.clone(); + let guard_context = context.clone(); + let guard = context + .run_fallible_blocking_named("protect transferring model metadata", move || { + authority.protect_partial_import(&destination, capability, guard_context) + }) + .await + .map_err(|error| { + PumasError::Other(format!( + "Partial metadata guard observation failed: {error}" + )) + })??; + let importer = Self { + library: Arc::new(self.library.with_import_guard(guard)), + }; + importer + .upsert_download_metadata_stub_effects(info, revision) + .await + } + + async fn upsert_download_metadata_stub_effects( + &self, + info: &AuxFilesCompleteInfo, + revision: &DownloadRevision, ) -> Result<()> { let model_dir = &info.dest_dir; let model_type = info @@ -780,15 +950,18 @@ impl ModelImporter { let importer = self.clone(); let source_path_for_detection = source_path.clone(); - let type_info = - tokio::task::spawn_blocking(move || importer.detect_type(&source_path_for_detection)) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join progress import type detection task: {}", - err - )) - })??; + let type_info = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.detect_type(&source_path_for_detection) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join progress import type detection task: {}", + err + )) + })??; let security_tier = type_info.format.security_tier(); // Security check @@ -849,16 +1022,18 @@ impl ModelImporter { let importer = self.clone(); let source_path_for_copy = source_path.clone(); let temp_dir_for_copy = temp_dir.clone(); - let files = tokio::task::spawn_blocking(move || { - importer.copy_files(&source_path_for_copy, &temp_dir_for_copy) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join progress import file copy task: {}", - err - )) - })??; + let files = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.copy_files(&source_path_for_copy, &temp_dir_for_copy) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join progress import file copy task: {}", + err + )) + })??; // Compute hashes let _ = progress_tx @@ -871,20 +1046,25 @@ impl ModelImporter { let importer = self.clone(); let temp_dir_for_primary = temp_dir.clone(); - let primary_file = tokio::task::spawn_blocking(move || { - importer.choose_primary_file(&temp_dir_for_primary) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join progress import primary file selection task: {}", - err - )) - })??; + let primary_file = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.choose_primary_file(&temp_dir_for_primary) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join progress import primary file selection task: {}", + err + )) + })??; let hashes = if let Some(ref primary) = primary_file { let primary_for_hash = primary.clone(); Some( - tokio::task::spawn_blocking(move || compute_dual_hash(&primary_for_hash)) + self.library + .run_import_blocking("importer blocking effect", move || { + compute_dual_hash(&primary_for_hash) + }) .await .map_err(|err| { PumasError::Other(format!( @@ -1074,34 +1254,41 @@ impl ModelImporter { let importer = self.clone(); let source_for_copy = source.to_path_buf(); let temp_dir_for_copy = temp_dir.to_path_buf(); - let files = tokio::task::spawn_blocking(move || { - importer.copy_files(&source_for_copy, &temp_dir_for_copy) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join temp import file copy task: {}", - err - )) - })??; + let files = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.copy_files(&source_for_copy, &temp_dir_for_copy) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join temp import file copy task: {}", + err + )) + })??; // Compute hashes for primary file let importer = self.clone(); let temp_dir_for_primary = temp_dir.to_path_buf(); - let primary_file = tokio::task::spawn_blocking(move || { - importer.choose_primary_file(&temp_dir_for_primary) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join temp import primary file selection task: {}", - err - )) - })??; + let primary_file = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.choose_primary_file(&temp_dir_for_primary) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join temp import primary file selection task: {}", + err + )) + })??; let hashes = if let Some(ref primary) = primary_file { let primary_for_hash = primary.clone(); Some( - tokio::task::spawn_blocking(move || compute_dual_hash(&primary_for_hash)) + self.library + .run_import_blocking("importer blocking effect", move || { + compute_dual_hash(&primary_for_hash) + }) .await .map_err(|err| { PumasError::Other(format!( @@ -1368,6 +1555,42 @@ impl ModelImporter { spec: &InPlaceImportSpec, mode: InPlaceImportMode, revision: &DownloadRevision, + ) -> Result { + let authority = self.library.mutation_authority()?; + let tasks = authority.tasks(); + let importer = self.clone(); + let spec = spec.clone(); + let revision = revision.clone(); + tasks + .run_owned("import model in place", move |context| async move { + owned_mutation_outcome( + async move { + let model_dir = spec.model_dir.clone(); + let guard_context = context.clone(); + let guard = context + .run_blocking("protect model import", move || { + authority.protect_import(&model_dir, guard_context) + }) + .await??; + let importer = Self { + library: Arc::new(importer.library.with_import_guard(guard)), + }; + importer + .import_in_place_effects(&spec, mode, &revision) + .await + } + .await, + ) + }) + .await? + } + + /// Runs only on a private library clone carrying the admitted effect lease. + async fn import_in_place_effects( + &self, + spec: &InPlaceImportSpec, + mode: InPlaceImportMode, + revision: &DownloadRevision, ) -> Result { let model_dir = &spec.model_dir; let metadata_path = model_dir.join("metadata.json"); @@ -1403,16 +1626,18 @@ impl ModelImporter { } let bundle_validation_dir = model_dir.to_path_buf(); - let bundle_validation = tokio::task::spawn_blocking(move || { - validate_diffusers_directory_for_import(&bundle_validation_dir) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join in-place diffusers validation task: {}", - err - )) - })?; + let bundle_validation = self + .library + .run_import_blocking("importer blocking effect", move || { + validate_diffusers_directory_for_import(&bundle_validation_dir) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join in-place diffusers validation task: {}", + err + )) + })?; if bundle_validation.validation_state == crate::models::AssetValidationState::Valid { return self .import_library_owned_diffusers_directory(spec, &bundle_validation, mode, revision) @@ -1422,16 +1647,18 @@ impl ModelImporter { // Find primary model file let importer = self.clone(); let model_dir_for_primary = model_dir.to_path_buf(); - let primary_file = tokio::task::spawn_blocking(move || { - importer.choose_primary_file(&model_dir_for_primary) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join in-place primary file selection task: {}", - err - )) - })??; + let primary_file = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.choose_primary_file(&model_dir_for_primary) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join in-place primary file selection task: {}", + err + )) + })??; if primary_file.is_none() { return Ok(ModelImportResult { path: model_dir.display().to_string(), @@ -1446,20 +1673,23 @@ impl ModelImporter { // Detect file type from primary file. let primary_file_for_type = primary_file.clone(); - let type_info = - tokio::task::spawn_blocking(move || identify_model_type(&primary_file_for_type)) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join in-place type detection task: {}", - err - )) - })??; + let type_info = self + .library + .run_import_blocking("importer blocking effect", move || { + identify_model_type(&primary_file_for_type) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join in-place type detection task: {}", + err + )) + })??; // Resolve model type from hard source signals via SQLite rule tables. // Medium hints (pipeline_tag/spec.model_type) only adjust confidence. let resolved_model_type = resolve_model_type_with_rules_async( - self.library.index().clone(), + self.library.clone(), model_dir.to_path_buf(), spec.pipeline_tag.clone(), spec.model_type.clone(), @@ -1470,16 +1700,18 @@ impl ModelImporter { // Detect dLLM subtype from config.json let resolved_subtype = if resolved_model_type.model_type == ModelType::Llm { let model_dir_for_subtype = model_dir.to_path_buf(); - let is_dllm = tokio::task::spawn_blocking(move || { - detect_dllm_from_config_json(&model_dir_for_subtype) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join in-place dLLM subtype detection task: {}", - err - )) - })?; + let is_dllm = self + .library + .run_import_blocking("importer blocking effect", move || { + detect_dllm_from_config_json(&model_dir_for_subtype) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join in-place dLLM subtype detection task: {}", + err + )) + })?; if is_dllm { Some("dllm".to_string()) } else { @@ -1492,16 +1724,18 @@ impl ModelImporter { // Enumerate existing files (no copy needed) let importer = self.clone(); let model_dir_for_enumeration = model_dir.to_path_buf(); - let files = tokio::task::spawn_blocking(move || { - importer.enumerate_model_files(&model_dir_for_enumeration) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join in-place file enumeration task: {}", - err - )) - })??; + let files = self + .library + .run_import_blocking("importer blocking effect", move || { + importer.enumerate_model_files(&model_dir_for_enumeration) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join in-place file enumeration task: {}", + err + )) + })??; // Validate shard completeness — reject if any file is part of an incomplete set. // Uses extract_shard_info per file to catch even single-shard-of-set cases @@ -1553,7 +1787,10 @@ impl ModelImporter { } else if spec.compute_hashes { let primary_file_for_hash = primary_file.clone(); Some( - tokio::task::spawn_blocking(move || compute_dual_hash(&primary_file_for_hash)) + self.library + .run_import_blocking("importer blocking effect", move || { + compute_dual_hash(&primary_file_for_hash) + }) .await .map_err(|err| { PumasError::Other(format!( @@ -1967,10 +2204,709 @@ mod tests { async fn setup() -> (TempDir, Arc) { let temp_dir = TempDir::new().unwrap(); let library = Arc::new(ModelLibrary::new(temp_dir.path()).await.unwrap()); + library + .install_mutation_authority( + crate::api::RuntimeTasks::new(), + crate::model_library::download_recovery::DownloadDestinationRoot::open( + library.library_root(), + ) + .unwrap(), + Arc::new( + crate::model_library::download_store::DownloadPersistence::new(temp_dir.path()), + ), + ) + .unwrap(); (temp_dir, library) } - fn completed_download(model_dir: &Path, diffusers: bool) -> DownloadCompletionInfo { + pub(super) async fn custody_fixture() -> ( + TempDir, + Arc, + Arc, + crate::api::RuntimeTasks, + crate::model_library::download_recovery::DownloadDestinationRoot, + ) { + let temp = TempDir::new().unwrap(); + let library = Arc::new( + ModelLibrary::new(temp.path().join("library")) + .await + .unwrap(), + ); + let downloads = + Arc::new(crate::model_library::download_store::DownloadPersistence::new(temp.path())); + let tasks = crate::api::RuntimeTasks::new(); + let root = crate::model_library::download_recovery::DownloadDestinationRoot::open( + library.library_root(), + ) + .unwrap(); + library + .install_mutation_authority(tasks.clone(), root.clone(), downloads.clone()) + .unwrap(); + (temp, library, downloads, tasks, root) + } + + pub(super) fn orphan_spec(path: &Path) -> InPlaceImportSpec { + InPlaceImportSpec { + model_dir: path.into(), + official_name: "model".into(), + family: "publisher".into(), + model_type: Some("vision".into()), + repo_id: None, + download_request: None, + known_sha256: None, + compute_hashes: false, + expected_files: None, + pipeline_tag: None, + huggingface_evidence: None, + release_date: None, + download_url: None, + model_card_json: None, + license_status: None, + } + } + + pub(super) fn real_admission( + downloads: &crate::model_library::download_store::DownloadPersistence, + root: &crate::model_library::download_recovery::DownloadDestinationRoot, + info: &DownloadCompletionInfo, + id: &str, + ) -> String { + use crate::model_library::download_store::*; + let attempt = uuid::Uuid::new_v4().to_string(); + let request = DownloadAdmissionRequest { + snapshot: PersistedDownload { + download_id: id.into(), + repo_id: info.download_request.repo_id.clone(), + filename: info.filename.clone(), + filenames: info.filenames.clone(), + dest_dir: info.dest_dir.clone(), + total_bytes: Some(4), + status: crate::models::DownloadStatus::Queued, + download_request: info.download_request.clone(), + revision: None, + created_at: chrono::Utc::now().to_rfc3339(), + known_sha256: None, + huggingface_evidence: None, + }, + domain: DownloadAdmissionDomain::Ambient, + destination: root + .resolve(&info.dest_dir) + .unwrap() + .persisted_identity() + .unwrap(), + requested_payload_files: info.filenames.clone(), + execution_files: info.filenames.clone(), + }; + downloads + .admit_download(&attempt, &request) + .unwrap() + .into_result() + .unwrap(); + attempt + } + + #[tokio::test] + async fn custody_guard_refuses_unconfigured_direct_import_even_with_metadata() { + let temp = TempDir::new().unwrap(); + let library = Arc::new(ModelLibrary::new(temp.path()).await.unwrap()); + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("metadata.json"), b"{}").unwrap(); + let error = ModelImporter::new(library.clone()) + .import_in_place(&orphan_spec(&model)) + .await + .unwrap_err(); + assert!(matches!(error, PumasError::Config { .. })); + assert!(error.to_string().contains("authority unavailable")); + assert_eq!(std::fs::read(model.join("metadata.json")).unwrap(), b"{}"); + assert!(library + .index() + .get("vision/publisher/model") + .unwrap() + .is_none()); + } + + #[tokio::test] + async fn custody_guard_precedes_metadata_shortcuts_and_diffusers_for_all_retained_phases() { + use crate::acquisition::*; + for phase in [ + AcquisitionPhase::Transferring, + AcquisitionPhase::FilesReady, + AcquisitionPhase::Using { + lease: uuid::Uuid::new_v4(), + }, + ] { + for branch in ["plain", "metadata", "diffusers"] { + let (_temp, library, downloads, tasks, root) = custody_fixture().await; + let model = if branch == "diffusers" { + create_external_diffusers_bundle( + &library.library_root().join("diffusion/publisher"), + ) + } else { + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("detector.onnx"), b"data").unwrap(); + model + }; + if branch == "metadata" { + std::fs::write(model.join("metadata.json"), b"{}").unwrap(); + } + let file = if branch == "diffusers" { + "model_index.json" + } else { + "detector.onnx" + }; + let manifest = ArtifactManifest::new( + ArtifactSourceIdentity::new( + "fixture", + "object", + ArtifactRevisionEvidence::new( + "fixture.rev", + "v1", + RevisionStrength::Immutable, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + file, + file, + None, + None, + FileVerificationRequirement::CompleteRepresentation, + ) + .unwrap()], + ) + .unwrap(); + let grant = Arc::new(root.try_acquire_execution_grant().unwrap()); + let workspace = root + .resolve(&model) + .unwrap() + .acquisition_workspace(grant.clone()) + .unwrap(); + let files = if phase == AcquisitionPhase::Transferring { + Vec::new() + } else { + workspace.seal(&manifest).unwrap() + }; + let id = uuid::Uuid::new_v4(); + downloads + .acquisition_store() + .update_acquisitions(|records| { + records.insert( + id, + AcquisitionRecord { + id, + demand: AcquisitionDemand { + consumer: "fixture.consumer".into(), + operation: id.to_string(), + }, + manifest, + workspace: workspace.identity().clone(), + phase: phase.clone(), + files, + }, + ); + Ok(()) + }) + .unwrap(); + drop(workspace); + drop(grant); + let before = std::fs::read(model.join("metadata.json")).ok(); + assert!( + matches!( + ModelImporter::new(library.clone()) + .import_in_place(&orphan_spec(&model)) + .await, + Err(PumasError::DownloadRootBusy) + ), + "{phase:?}/{branch}" + ); + assert_eq!(std::fs::read(model.join("metadata.json")).ok(), before); + assert!(library + .index() + .get(&library.get_model_id(&model).unwrap()) + .unwrap() + .is_none()); + tasks.shutdown_owned().await.unwrap(); + } + } + } + + #[tokio::test] + async fn custody_guard_retains_metadata_worker_and_shutdown_receipt_after_caller_drop() { + for (panic_worker, index_shortcut) in [(false, false), (true, false), (false, true)] { + let (_temp, library, _downloads, tasks, root) = custody_fixture().await; + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("detector.onnx"), b"data").unwrap(); + if index_shortcut { + library + .save_metadata( + &model, + &ModelMetadata { + model_id: Some("vision/publisher/model".into()), + model_type: Some("vision".into()), + pipeline_tag: Some("image-classification".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + } + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let entered_tx = std::sync::Mutex::new(Some(entered_tx)); + let (release_tx, release) = std::sync::mpsc::channel(); + let release = std::sync::Mutex::new(release); + library.set_metadata_write_notifier(Some(Arc::new(move |_| { + if let Some(sender) = entered_tx.lock().unwrap().take() { + sender.send(()).unwrap(); + release.lock().unwrap().recv().unwrap(); + assert!(!panic_worker, "held import metadata sentinel panic"); + } + }))); + let importer = ModelImporter::new(library.clone()); + let info = completed_download(&model, false); + let caller = tokio::spawn(async move { + if index_shortcut { + importer.import_in_place(&orphan_spec(&info.dest_dir)).await + } else { + importer.finalize_downloaded_directory(&info).await + } + }); + tokio::time::timeout(std::time::Duration::from_secs(5), entered) + .await + .unwrap() + .unwrap(); + caller.abort(); + assert!(caller.await.unwrap_err().is_cancelled()); + let shutdown_tasks = tasks.clone(); + let shutdown = tokio::spawn(async move { shutdown_tasks.shutdown_owned().await }); + tokio::task::yield_now().await; + assert!(!shutdown.is_finished()); + assert!(matches!( + root.try_acquire_execution_grant(), + Err(PumasError::DownloadRootBusy) + )); + release_tx.send(()).unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(5), shutdown) + .await + .unwrap() + .unwrap(); + assert_eq!(result.is_err(), panic_worker); + assert!(root.try_acquire_execution_grant().is_ok()); + assert_eq!( + library + .index() + .get("vision/publisher/model") + .unwrap() + .is_some(), + !panic_worker + ); + assert_eq!(std::fs::read(model.join("detector.onnx")).unwrap(), b"data"); + library.set_metadata_write_notifier(None); + } + } + + #[tokio::test] + async fn custody_guard_refuses_root_and_destination_replacement_at_real_metadata_effect() { + for replace_root in [false, true] { + let (temp, library, _downloads, tasks, _root) = custody_fixture().await; + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("detector.onnx"), b"data").unwrap(); + let target = if replace_root { + library.library_root().to_path_buf() + } else { + model.clone() + }; + let displaced = temp.path().join("displaced"); + let replacement_model = if replace_root { + target.join("vision/publisher/model") + } else { + target.clone() + }; + library.set_metadata_write_notifier(Some(Arc::new(move |_| { + std::fs::rename(&target, &displaced).unwrap(); + std::fs::create_dir_all(&replacement_model).unwrap(); + std::fs::write(replacement_model.join("sentinel"), b"replacement").unwrap(); + }))); + let error = ModelImporter::new(library.clone()) + .finalize_downloaded_directory(&completed_download(&model, false)) + .await + .unwrap_err(); + assert!(matches!(error, PumasError::Io { .. }), "{error}"); + assert!(!model.join("metadata.json").exists()); + assert_eq!( + std::fs::read(model.join("sentinel")).unwrap(), + b"replacement" + ); + assert!(tasks.shutdown_owned().await.is_err()); + library.set_metadata_write_notifier(None); + } + } + + #[tokio::test] + async fn custody_guard_hf_stage_proofs_preserve_followers_and_refuse_stale_use() { + use crate::acquisition::task_custody::TaskRole; + use crate::acquisition::*; + use crate::model_library::download_store::DownloadAdmissionDomain; + let (_temp, library, downloads, tasks, root) = custody_fixture().await; + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("detector.onnx"), b"data").unwrap(); + let mut info = completed_download(&model, false); + info.download_id = "owner".into(); + let attempt = real_admission(&downloads, &root, &info, "owner"); + let follower = real_admission(&downloads, &root, &info, "follower"); + let acquisition = Arc::new(AcquisitionService::new(downloads.acquisition_store())); + let scope = acquisition + .supervisor() + .open_scope(|| async { Ok(()) }) + .unwrap(); + let (result_tx, result) = tokio::sync::oneshot::channel(); + let worker_acquisition = acquisition.clone(); + let worker_library = library.clone(); + let external_store = + crate::model_library::download_store::DownloadPersistence::new(_temp.path()); + let worker_downloads = downloads.clone(); + let prepared = scope + .prepare( + "owner".into(), + TaskRole::Worker, + move |context| async move { + let result = async { + let grant = Arc::new(root.try_acquire_execution_grant()?); + let context = context.with_effect_lease(Some(grant.clone())); + let workspace = + root.resolve(&model)?.acquisition_workspace(grant.clone())?; + let manifest = ArtifactManifest::new( + ArtifactSourceIdentity::new( + "huggingface", + &info.download_request.repo_id, + ArtifactRevisionEvidence::new( + "huggingface.commit", + "main", + RevisionStrength::Weak, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + "detector.onnx", + "detector.onnx", + Some(4), + None, + FileVerificationRequirement::CompleteRepresentation, + ) + .unwrap()], + ) + .unwrap(); + let operation = worker_acquisition + .begin( + &context, + AcquisitionDemand { + consumer: "hf.model".into(), + operation: attempt.clone(), + }, + manifest, + workspace.identity().clone(), + None, + ) + .await?; + let partial = ModelPartialImportCapability::new( + worker_acquisition.transfer_proof(&context, &operation, &workspace)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + let importer = ModelImporter::new(worker_library.clone()); + let aux = AuxFilesCompleteInfo { + download_id: "owner".into(), + dest_dir: model.clone(), + filenames: info.filenames.clone(), + download_request: info.download_request.clone(), + total_bytes: Some(4), + huggingface_evidence: None, + }; + importer + .upsert_download_metadata_stub_with_capability( + &aux, + &DownloadRevision::legacy_main(), + &context, + partial, + ) + .await?; + assert_eq!( + worker_library + .load_metadata(&model)? + .unwrap() + .match_source + .as_deref(), + Some("download_partial") + ); + assert!(worker_library + .index() + .get("vision/publisher/model")? + .is_some()); + // Stage isolation is enforced even on the privately scoped clone. + let partial = ModelPartialImportCapability::new( + worker_acquisition.transfer_proof(&context, &operation, &workspace)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + let guard = worker_library + .mutation_authority()? + .protect_partial_import(&model, partial, context.clone())?; + let scoped = worker_library.with_import_guard(guard); + assert!(scoped + .resolve_model_package_facts("vision/publisher/model") + .await + .unwrap_err() + .to_string() + .contains("Partial metadata authority")); + drop(scoped); + let lease = worker_acquisition + .files_ready(&context, operation, workspace) + .await?; + // A proof for the right lease cannot authorize another selection. + let wrong = ModelFinalImportCapability::new( + worker_acquisition.use_proof(&context, &lease)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + assert!(wrong + .validate_provenance("owner", "other/repo", "main", &info.filenames) + .is_err()); + // Synthetic wrong-token fault on the actual canonical + // store: an authentic runtime lease cannot execute if + // current durable consumer custody names another use. + let proof = worker_acquisition.use_proof(&context, &lease)?; + let expected = proof.into_proof().record().clone(); + let mismatched = ModelFinalImportCapability::new( + worker_acquisition.use_proof(&context, &lease)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + worker_downloads + .acquisition_store() + .update_acquisitions(|records| { + records.get_mut(&expected.id).unwrap().phase = + AcquisitionPhase::Using { + lease: uuid::Uuid::new_v4(), + }; + Ok(()) + })?; + let before_wrong_token = std::fs::read(model.join("metadata.json"))?; + assert!(importer + .finalize_downloaded_directory_with_capability( + &info, + &DownloadRevision::legacy_main(), + &context, + mismatched, + ) + .await + .unwrap_err() + .to_string() + .contains("stale")); + assert_eq!( + std::fs::read(model.join("metadata.json"))?, + before_wrong_token + ); + worker_downloads + .acquisition_store() + .update_acquisitions(|records| { + records.insert(expected.id, expected); + Ok(()) + })?; + let capability = ModelFinalImportCapability::new( + worker_acquisition.use_proof(&context, &lease)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + importer + .finalize_downloaded_directory_with_capability( + &info, + &DownloadRevision::legacy_main(), + &context, + capability, + ) + .await?; + assert_ne!( + worker_library + .load_metadata(&model)? + .unwrap() + .match_source + .as_deref(), + Some("download_partial") + ); + let hidden_attempt = + real_admission(&external_store, &root, &info, "hidden-follower"); + assert!(worker_downloads + .load_lifecycle_inventory_strict()? + .hidden_admissions + .contains_key("hidden-follower")); + let hidden = ModelFinalImportCapability::new( + worker_acquisition.use_proof(&context, &lease)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + let before_hidden = std::fs::read(model.join("metadata.json"))?; + assert!(matches!( + importer + .finalize_downloaded_directory_with_capability( + &info, + &DownloadRevision::legacy_main(), + &context, + hidden + ) + .await, + Err(PumasError::DownloadRootBusy) + )); + assert_eq!(std::fs::read(model.join("metadata.json"))?, before_hidden); + assert_eq!( + external_store + .load_lifecycle_inventory_strict()? + .queue_admissions["hidden-follower"] + .attempt_id, + hidden_attempt + ); + let stale = ModelFinalImportCapability::new( + worker_acquisition.use_proof(&context, &lease)?, + "owner", + DownloadAdmissionDomain::Ambient, + grant.clone(), + ); + // Actual durable settlement invalidates an earlier unconsumed proof. + worker_acquisition.acknowledge(&context, lease).await?; + let before = std::fs::read(model.join("metadata.json"))?; + let error = importer + .finalize_downloaded_directory_with_capability( + &info, + &DownloadRevision::legacy_main(), + &context, + stale, + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("stale"), "{error}"); + assert_eq!(std::fs::read(model.join("metadata.json"))?, before); + let inventory = worker_downloads.load_lifecycle_inventory_strict()?; + assert_eq!(inventory.queue_admissions["follower"].attempt_id, follower); + assert_eq!(inventory.queue_admissions["owner"].attempt_id, attempt); + Ok::<_, PumasError>(()) + } + .await; + result_tx.send(result).unwrap(); + }, + ) + .unwrap(); + scope.install_gated(prepared).unwrap().start(); + tokio::time::timeout(std::time::Duration::from_secs(5), result) + .await + .unwrap() + .unwrap() + .unwrap(); + // The negative stale-use probe is a synthetic invocation on real custody; + // it is deliberately reported as a failed owned effect by this owner. + assert!(acquisition.shutdown().await.is_err()); + tasks.shutdown_owned().await.unwrap(); + let records = downloads.acquisition_store().acquisitions().unwrap(); + assert!(matches!( + records.values().next().unwrap().phase, + AcquisitionPhase::Adopted { .. } + )); + } + + #[tokio::test] + async fn custody_guard_refuses_hidden_pending_and_settled_quarantine_without_effects() { + use crate::model_library::download_store::*; + for state in ["hidden", "pending", "quarantine"] { + let temp = TempDir::new().unwrap(); + let library = Arc::new( + ModelLibrary::new(temp.path().join("library")) + .await + .unwrap(), + ); + let root = crate::model_library::download_recovery::DownloadDestinationRoot::open( + library.library_root(), + ) + .unwrap(); + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("detector.onnx"), b"data").unwrap(); + let original = Arc::new(DownloadPersistence::new(temp.path())); + let attempt = real_admission( + &original, + &root, + &completed_download(&model, false), + "retained", + ); + if state != "hidden" { + let snapshot = original + .load_lifecycle_inventory_strict() + .unwrap() + .downloads[0] + .clone(); + original + .begin_lifecycle_quarantine( + &snapshot, + LifecycleQuarantineDomain::Ambient, + state == "quarantine", + Some(&attempt), + ) + .unwrap(); + if state == "quarantine" { + assert!(original.verify_lifecycle_quarantine("retained").unwrap()); + } + } + let downloads = if state == "hidden" { + Arc::new(DownloadPersistence::new(temp.path())) + } else { + original + }; + let inventory = downloads.load_lifecycle_inventory_strict().unwrap(); + if state == "hidden" { + assert!(inventory.hidden_admissions.contains_key("retained")); + } else { + assert_eq!( + inventory.quarantines["retained"].disposition, + if state == "pending" { + LifecycleCleanupDisposition::Pending + } else { + LifecycleCleanupDisposition::Verified + } + ); + } + let tasks = crate::api::RuntimeTasks::new(); + library + .install_mutation_authority(tasks.clone(), root, downloads) + .unwrap(); + assert!( + matches!( + ModelImporter::new(library.clone()) + .import_in_place(&orphan_spec(&model)) + .await, + Err(PumasError::DownloadRootBusy) + ), + "{state}" + ); + assert!(!model.join("metadata.json").exists()); + assert!(library + .index() + .get("vision/publisher/model") + .unwrap() + .is_none()); + tasks.shutdown_owned().await.unwrap(); + } + } + + pub(super) fn completed_download(model_dir: &Path, diffusers: bool) -> DownloadCompletionInfo { DownloadCompletionInfo { download_id: "truthful-finalization".into(), dest_dir: model_dir.to_path_buf(), diff --git a/rust/crates/pumas-core/src/model_library/importer/recovery.rs b/rust/crates/pumas-core/src/model_library/importer/recovery.rs index ba3228a2..8bd7816c 100644 --- a/rust/crates/pumas-core/src/model_library/importer/recovery.rs +++ b/rust/crates/pumas-core/src/model_library/importer/recovery.rs @@ -459,3 +459,59 @@ struct InferredSpec { family: String, official_name: String, } + +#[cfg(test)] +mod custody_tests { + use super::super::tests::{completed_download, custody_fixture, orphan_spec, real_admission}; + use super::*; + use crate::PumasError; + #[tokio::test] + async fn custody_guard_rechecks_a_real_stale_orphan_scan_after_admission() { + let (_temp, library, downloads, tasks, root) = custody_fixture().await; + let model = library.build_model_path("vision", "publisher", "model"); + std::fs::create_dir_all(&model).unwrap(); + std::fs::write(model.join("detector.onnx"), b"data").unwrap(); + let importer = ModelImporter::new(library.clone()); + let (scanned_tx, scanned) = tokio::sync::oneshot::channel(); + let (admitted_tx, admitted) = tokio::sync::oneshot::channel(); + let candidate_importer = importer.clone(); + let scan = tokio::spawn(async move { + let candidates = candidate_importer + .find_orphan_dirs(candidate_importer.library.library_root(), false); + scanned_tx.send(candidates.clone()).unwrap(); + admitted.await.unwrap(); + candidate_importer + .import_in_place(&orphan_spec(&candidates[0])) + .await + }); + assert_eq!(scanned.await.unwrap(), vec![model.clone()]); + // The competing queue admission commits before the stale scan reaches + // the actual common importer boundary, while no download can yet write. + real_admission( + &downloads, + &root, + &completed_download(&model, false), + "racing-download", + ); + admitted_tx.send(()).unwrap(); + assert!(matches!( + scan.await.unwrap(), + Err(PumasError::DownloadRootBusy) + )); + assert!(!model.join("metadata.json").exists()); + assert!(library + .index() + .get("vision/publisher/model") + .unwrap() + .is_none()); + assert_eq!( + downloads + .load_lifecycle_inventory_strict() + .unwrap() + .queue_admissions + .len(), + 1 + ); + tasks.shutdown_owned().await.unwrap(); + } +} diff --git a/rust/crates/pumas-core/src/model_library/library.rs b/rust/crates/pumas-core/src/model_library/library.rs index e8ac994b..1e0d2191 100644 --- a/rust/crates/pumas-core/src/model_library/library.rs +++ b/rust/crates/pumas-core/src/model_library/library.rs @@ -28,7 +28,7 @@ use crate::model_library::hashing::{verify_blake3, verify_sha256}; use crate::model_library::identifier::{identify_model_type, ModelTypeInfo}; use crate::model_library::importer::detect_dllm_from_config_json; use crate::model_library::mutation_authority::{ - authority_unavailable, owned_mutation_outcome, LibraryMutationAuthority, + authority_unavailable, owned_mutation_outcome, LibraryImportGuard, LibraryMutationAuthority, }; use crate::model_library::naming::normalize_name; use crate::model_library::package_facts::{ @@ -208,6 +208,8 @@ pub struct ModelLibrary { /// Installed once by the composition owner. Standalone libraries remain /// read-only for destructive operations until trusted authority is supplied. mutation_authority: Arc>, + /// Present only on the private clone used by one admitted importer effect. + import_guard: Option>, } impl ModelLibrary { @@ -248,6 +250,7 @@ impl ModelLibrary { package_facts_locks: Arc::new(Mutex::new(HashMap::new())), metadata_write_notifier: Arc::new(StdMutex::new(None)), mutation_authority: Arc::new(OnceLock::new()), + import_guard: None, }; // Rebuild index from existing metadata files on disk @@ -307,6 +310,44 @@ impl ModelLibrary { .ok_or_else(|| authority_unavailable("trusted composition was not installed")) } + pub(crate) fn with_import_guard(&self, guard: Arc) -> Self { + let mut library = self.clone(); + library.import_guard = Some(guard); + library + } + + pub(crate) async fn run_import_blocking( + &self, + operation: &'static str, + work: impl FnOnce() -> T + Send + 'static, + ) -> Result { + if let Some(guard) = &self.import_guard { + return guard.run_blocking(operation, work).await; + } + tokio::task::spawn_blocking(work) + .await + .map_err(|error| PumasError::Other(format!("Library blocking effect failed: {error}"))) + } + + async fn validate_import_effect_async(&self, model_dir: &Path) -> Result<()> { + if self.import_guard.is_none() { + return Ok(()); + } + let library = self.clone(); + let path = model_dir.to_path_buf(); + self.run_import_blocking("validate import binding", move || { + library.validate_import_effect(&path) + }) + .await? + } + + fn validate_import_effect(&self, model_dir: &Path) -> Result<()> { + if let Some(guard) = &self.import_guard { + guard.validate(model_dir)?; + } + Ok(()) + } + /// Return the canonical SQLite-backed model count. pub fn model_count(&self) -> Result { self.index.count() @@ -564,6 +605,9 @@ impl ModelLibrary { /// /// * `model_dir` - Path to the model directory pub fn load_metadata(&self, model_dir: &Path) -> Result> { + if let Some(guard) = &self.import_guard { + return guard.read_metadata(model_dir); + } let path = model_dir.join(METADATA_FILENAME); atomic_read_json(&path) } @@ -612,6 +656,7 @@ impl ModelLibrary { /// /// * `model_dir` - Path to the model directory pub async fn index_model_dir(&self, model_dir: &Path) -> Result<()> { + self.validate_import_effect_async(model_dir).await?; let prepared = self.prepare_index_projection_async(model_dir).await?; self.persist_index_projection(model_dir, prepared).await?; @@ -2284,18 +2329,19 @@ impl ModelLibrary { // Keep file-signature detection independent from resolver rules and use it as fallback. let model_dir_for_type = model_dir.clone(); - let type_info = tokio::task::spawn_blocking(move || { - find_primary_model_file(&model_dir_for_type) - .as_ref() - .and_then(|f| identify_model_type(f).ok()) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join redetect type inspection task: {}", - err - )) - })?; + let type_info = self + .run_import_blocking("classify imported model", move || { + find_primary_model_file(&model_dir_for_type) + .as_ref() + .and_then(|f| identify_model_type(f).ok()) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join redetect type inspection task: {}", + err + )) + })?; let resolved = resolve_local_model_type_with_persisted_hints_async( self.index().clone(), model_dir.clone(), @@ -2311,16 +2357,17 @@ impl ModelLibrary { .map(|f| f.as_str().to_string()); let new_subtype = if resolved.model_type == ModelType::Llm { let model_dir_for_subtype = model_dir.clone(); - let is_dllm = tokio::task::spawn_blocking(move || { - detect_dllm_from_config_json(&model_dir_for_subtype) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join redetect dLLM subtype task: {}", - err - )) - })?; + let is_dllm = self + .run_import_blocking("classify imported model", move || { + detect_dllm_from_config_json(&model_dir_for_subtype) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join redetect dLLM subtype task: {}", + err + )) + })?; if is_dllm { Some("dllm".to_string()) } else { @@ -2671,6 +2718,9 @@ impl ModelLibrary { &self, model_id: &str, ) -> Result { + if let Some(guard) = &self.import_guard { + guard.require_package_facts()?; + } let descriptor = self.resolve_model_execution_descriptor(model_id).await?; let model_dir = self.library_root.join(model_id); let metadata = load_effective_metadata_by_id_async(self.clone(), model_id.to_string()) @@ -2708,6 +2758,8 @@ impl ModelLibrary { { match serde_json::from_str::(&cached.facts_json) { Ok(facts) => { + self.validate_import_effect_async(context.model_dir()) + .await?; self.upsert_model_package_facts_summary_cache( &context, &source_fingerprint, @@ -2870,6 +2922,8 @@ impl ModelLibrary { .unwrap_or_default(), }; if can_persist_package_facts { + self.validate_import_effect_async(context.model_dir()) + .await?; self.upsert_model_package_facts_summary_cache(&context, &source_fingerprint, &facts)?; let now = chrono::Utc::now().to_rfc3339(); self.index @@ -3950,7 +4004,11 @@ async fn load_model_metadata_async( library: ModelLibrary, model_dir: PathBuf, ) -> Result> { - tokio::task::spawn_blocking(move || library.load_metadata(&model_dir)) + library + .clone() + .run_import_blocking("import library metadata effect", move || { + library.load_metadata(&model_dir) + }) .await .map_err(|err| PumasError::Other(format!("Failed to join metadata load task: {}", err)))? } @@ -3959,7 +4017,11 @@ async fn load_effective_metadata_by_id_async( library: ModelLibrary, model_id: String, ) -> Result> { - tokio::task::spawn_blocking(move || library.load_effective_metadata_by_id(&model_id)) + library + .clone() + .run_import_blocking("import library metadata effect", move || { + library.load_effective_metadata_by_id(&model_id) + }) .await .map_err(|err| { PumasError::Other(format!( @@ -3989,39 +4051,41 @@ async fn save_metadata_projection_async( model_dir: PathBuf, metadata: ModelMetadata, ) -> Result<()> { - tokio::task::spawn_blocking(move || { - let mut normalized = metadata; - if let Some(model_id) = library.get_model_id(&model_dir) { - normalized.model_id = Some(model_id); - } - let active_bindings = normalized - .model_id - .as_deref() - .map(|model_id| { - library - .index - .list_active_model_dependency_bindings(model_id, None) - }) - .transpose()? - .unwrap_or_default(); - apply_recommended_backend_hint(&mut normalized, &active_bindings); - let path = model_dir.join(METADATA_FILENAME); - if let Some(existing) = atomic_read_json::(&path)? { - let existing_json = serde_json::to_value(existing).unwrap_or(Value::Null); - let next_json = serde_json::to_value(&normalized).unwrap_or(Value::Null); - if existing_json == next_json { - return Ok(()); + library + .clone() + .run_import_blocking("import library metadata effect", move || { + let mut normalized = metadata; + if let Some(model_id) = library.get_model_id(&model_dir) { + normalized.model_id = Some(model_id); } - } - library.write_metadata_projection(&path, &normalized) - }) - .await - .map_err(|err| { - PumasError::Other(format!( - "Failed to join metadata projection save task: {}", - err - )) - })? + let active_bindings = normalized + .model_id + .as_deref() + .map(|model_id| { + library + .index + .list_active_model_dependency_bindings(model_id, None) + }) + .transpose()? + .unwrap_or_default(); + apply_recommended_backend_hint(&mut normalized, &active_bindings); + let path = model_dir.join(METADATA_FILENAME); + if let Some(existing) = library.load_metadata(&model_dir)? { + let existing_json = serde_json::to_value(existing).unwrap_or(Value::Null); + let next_json = serde_json::to_value(&normalized).unwrap_or(Value::Null); + if existing_json == next_json { + return Ok(()); + } + } + library.write_metadata_projection(&path, &normalized) + }) + .await + .map_err(|err| { + PumasError::Other(format!( + "Failed to join metadata projection save task: {}", + err + )) + })? } async fn save_overrides_projection_async( @@ -4235,6 +4299,7 @@ impl ModelLibrary { } } + self.validate_import_effect_async(model_dir).await?; let mut mutated = self.index.upsert(&prepared.record)?; if let Some(projection) = prepared.projection.as_ref() { mutated |= self.ensure_custom_runtime_binding(&prepared.model_id, projection)?; @@ -4312,6 +4377,12 @@ impl ModelLibrary { fn write_metadata_projection(&self, path: &Path, metadata: &ModelMetadata) -> Result<()> { self.notify_metadata_projection_write(path); + if let Some(guard) = &self.import_guard { + let model_dir = path + .parent() + .ok_or_else(|| authority_unavailable("import metadata has no parent"))?; + return guard.write_metadata(model_dir, metadata); + } atomic_write_json(path, metadata, true) } diff --git a/rust/crates/pumas-core/src/model_library/mutation_authority.rs b/rust/crates/pumas-core/src/model_library/mutation_authority.rs index dc857ad4..da6bb709 100644 --- a/rust/crates/pumas-core/src/model_library/mutation_authority.rs +++ b/rust/crates/pumas-core/src/model_library/mutation_authority.rs @@ -1,14 +1,198 @@ //! Shared authority for destructive model-library mutations. +use crate::acquisition::store::{AcquisitionProof, AcquisitionTransferProof, AcquisitionUseProof}; use crate::api::RuntimeTasks; use crate::index::{IntentDeletionClaimResult, ModelIndex}; -use crate::model_library::download_recovery::{DownloadDestinationRoot, RootExecutionGrant}; +use crate::model_library::download_recovery::{ + DownloadDestinationRoot, DownloadRecoveryDestination, RootExecutionGrant, +}; +use crate::model_library::download_store::DownloadAdmissionDomain; use crate::model_library::download_store::{DownloadPersistence, PersistedDestinationIdentity}; use crate::{PumasError, Result}; use std::path::{Path, PathBuf}; use std::sync::Arc; use uuid::Uuid; +/// Model policy composes exact neutral proof with the current HF admission +/// and its already-held native grant. Stage types expose no generic bypass. +pub(crate) struct ModelFinalImportCapability(ModelImportProof); +pub(crate) struct ModelPartialImportCapability(ModelImportProof); + +struct ModelImportProof { + acquisition: AcquisitionProof, + grant: Arc, + download_id: String, + domain: DownloadAdmissionDomain, +} + +impl ModelFinalImportCapability { + pub(crate) fn new( + proof: AcquisitionUseProof, + download_id: &str, + domain: DownloadAdmissionDomain, + grant: Arc, + ) -> Self { + Self(ModelImportProof { + acquisition: proof.into_proof(), + grant, + download_id: download_id.into(), + domain, + }) + } + + pub(crate) fn validate_provenance( + &self, + download_id: &str, + repo_id: &str, + revision: &str, + files: &[String], + ) -> Result<()> { + self.0 + .validate_provenance(download_id, repo_id, revision, files) + } +} + +impl ModelPartialImportCapability { + pub(crate) fn new( + proof: AcquisitionTransferProof, + download_id: &str, + domain: DownloadAdmissionDomain, + grant: Arc, + ) -> Self { + Self(ModelImportProof { + acquisition: proof.into_proof(), + grant, + download_id: download_id.into(), + domain, + }) + } + + pub(crate) fn validate_provenance( + &self, + download_id: &str, + repo_id: &str, + revision: &str, + files: &[String], + ) -> Result<()> { + self.0 + .validate_provenance(download_id, repo_id, revision, files) + } +} + +impl ModelImportProof { + fn validate( + &self, + downloads: &DownloadPersistence, + destination: &PersistedDestinationIdentity, + partial: bool, + ) -> Result<()> { + let record = self.acquisition.record(); + if record.demand.consumer != "hf.model" + || record.workspace.root_identity != destination.library_root + || record.workspace.relative_target != destination.relative_target + || if partial { + record.phase != crate::acquisition::AcquisitionPhase::Transferring + } else { + !matches!( + record.phase, + crate::acquisition::AcquisitionPhase::Using { .. } + ) + } + { + return Err(import_invalid( + "Model import proof does not identify its stage/workspace", + )); + } + let files = record + .manifest + .files() + .iter() + .map(|file| file.logical_path().to_string()) + .collect::>(); + let (inventory, current) = downloads.load_import_custody_strict(Some(( + &self.download_id, + &record.demand.operation, + self.domain, + destination, + &files, + )))?; + self.acquisition + .validate_current(&downloads.acquisition_store(), ¤t)?; + let admission = inventory + .queue_admissions + .get(&self.download_id) + .ok_or_else(|| import_invalid("Model import admission is unavailable"))?; + // Confirmed FIFO successors wait for this predecessor's release. A + // hidden owner or quarantine supplies no equivalent execution proof. + if !inventory.quarantines.is_empty() + || inventory + .hidden_admissions + .values() + .any(|hidden| &hidden.request.destination == destination) + || inventory.queue_admissions.iter().any(|(id, other)| { + id != &self.download_id + && &other.destination == destination + && other.position.ordinal <= admission.position.ordinal + }) + || current.values().any(|other| { + other.id != record.id + && other.workspace == record.workspace + && !matches!( + other.phase, + crate::acquisition::AcquisitionPhase::Adopted { .. } + | crate::acquisition::AcquisitionPhase::Withdrawn + ) + }) + { + return Err(PumasError::DownloadRootBusy); + } + if !partial { + self.acquisition.verify_receipts()?; + } + Ok(()) + } + + fn validate_binding(&self) -> Result<()> { + self.acquisition.validate_binding() + } + + fn validate_provenance( + &self, + download_id: &str, + repo_id: &str, + revision: &str, + files: &[String], + ) -> Result<()> { + let manifest = &self.acquisition.record().manifest; + let mut selected = manifest + .files() + .iter() + .map(|file| file.logical_path().to_string()) + .collect::>(); + let mut requested = files.to_vec(); + selected.sort(); + requested.sort(); + if download_id != self.download_id + || selected != requested + || manifest.source().provider() != "huggingface" + || manifest.source().source_id() != repo_id + || manifest.source().revision().value() != revision + { + return Err(import_invalid( + "Model import provenance does not match its exact manifest", + )); + } + Ok(()) + } +} + +fn import_invalid(message: &str) -> PumasError { + PumasError::Validation { + field: "model_library.mutation".into(), + message: message.into(), + } +} + #[derive(Clone)] pub(crate) struct LibraryMutationAuthority { tasks: RuntimeTasks, @@ -44,6 +228,70 @@ impl LibraryMutationAuthority { &self.root } + /// Import admission is checked under native root exclusion, including + /// idempotent imports whose indexing can still rewrite metadata. + pub(crate) fn protect_import( + &self, + model_dir: &Path, + context: crate::api::RuntimeTaskContext, + ) -> Result> { + let grant = Arc::new(self.root.try_acquire_execution_grant()?); + let destination = self.root.resolve(model_dir)?; + let identity = destination.persisted_identity()?; + self.require_no_download_custody(&[identity])?; + // Bind the existing directory now, rather than accepting a replacement + // first encountered by a later metadata worker. + destination.read_model_metadata()?; + Ok(Arc::new(LibraryImportGuard { + root: self.root.clone(), + grant, + destination, + proof: None, + partial: false, + context: ImportEffectContext::Runtime(context), + })) + } + + pub(crate) fn protect_final_import( + &self, + model_dir: &Path, + capability: ModelFinalImportCapability, + context: crate::acquisition::task_custody::TaskContext, + ) -> Result> { + self.protect_hf_import(model_dir, capability.0, false, context) + } + + pub(crate) fn protect_partial_import( + &self, + model_dir: &Path, + capability: ModelPartialImportCapability, + context: crate::acquisition::task_custody::TaskContext, + ) -> Result> { + self.protect_hf_import(model_dir, capability.0, true, context) + } + + fn protect_hf_import( + &self, + model_dir: &Path, + proof: ModelImportProof, + partial: bool, + context: crate::acquisition::task_custody::TaskContext, + ) -> Result> { + let grant = proof.grant.clone(); + grant.validate_root(&self.root)?; + let destination = self.root.resolve(model_dir)?; + proof.validate(&self.downloads, &destination.persisted_identity()?, partial)?; + destination.read_model_metadata()?; + Ok(Arc::new(LibraryImportGuard { + root: self.root.clone(), + grant, + destination, + proof: Some(proof), + partial, + context: ImportEffectContext::Acquisition(context), + })) + } + pub(crate) fn acquire( &self, index: &ModelIndex, @@ -152,7 +400,7 @@ impl LibraryMutationAuthority { } fn require_no_download_custody(&self, targets: &[PersistedDestinationIdentity]) -> Result<()> { - let inventory = self.downloads.load_lifecycle_inventory_strict()?; + let (inventory, acquisitions) = self.downloads.load_import_custody_strict(None)?; let queued = inventory .queue_admissions .values() @@ -161,7 +409,6 @@ impl LibraryMutationAuthority { .hidden_admissions .values() .any(|admission| targets.contains(&admission.request.destination)); - let acquisitions = self.downloads.acquisition_store().acquisitions()?; let acquiring = acquisitions.values().any(|record| { !matches!( record.phase, @@ -182,6 +429,92 @@ impl LibraryMutationAuthority { } } +/// One scoped import retains the existing exclusion and held destination. +/// This is an effect lease, not an owner or a persistent custody record. +pub(crate) struct LibraryImportGuard { + root: DownloadDestinationRoot, + grant: Arc, + destination: DownloadRecoveryDestination, + proof: Option, + partial: bool, + context: ImportEffectContext, +} + +enum ImportEffectContext { + Runtime(crate::api::RuntimeTaskContext), + Acquisition(crate::acquisition::task_custody::TaskContext), +} + +impl LibraryImportGuard { + pub(crate) async fn run_blocking( + self: &Arc, + operation: &'static str, + work: impl FnOnce() -> T + Send + 'static, + ) -> Result { + let guard = self.clone(); + let work = move || { + let _guard = guard; + work() + }; + match &self.context { + ImportEffectContext::Runtime(context) => context.run_blocking(operation, work).await, + // HF registers the complete non-abortable importer future as one + // TaskContext async effect. That owner awaits every worker join, + // including after cancellation replaces the worker generation. + // Re-registering children against the retired generation would + // manufacture a failure while that already-admitted effect drains. + ImportEffectContext::Acquisition(_context) => { + tokio::task::spawn_blocking(work).await.map_err(|error| { + PumasError::Other(format!("Import effect observation failed: {error}")) + }) + } + } + } + + pub(crate) fn validate(&self, path: &Path) -> Result<()> { + self.grant.validate_root(&self.root)?; + if let Some(proof) = &self.proof { + proof.validate_binding()?; + } + let current = self.root.resolve(path)?; + if current.persisted_identity()? != self.destination.persisted_identity()? { + return Err(PumasError::Validation { + field: "model_library.mutation".into(), + message: "Import effect does not identify its admitted destination".into(), + }); + } + self.destination.read_model_metadata()?; + Ok(()) + } + + pub(crate) fn require_package_facts(&self) -> Result<()> { + if self.partial { + return Err(PumasError::Validation { + field: "model_library.mutation".into(), + message: "Partial metadata authority cannot resolve package facts".into(), + }); + } + Ok(()) + } + + pub(crate) fn read_metadata( + &self, + path: &Path, + ) -> Result> { + self.validate(path)?; + self.destination.read_model_metadata() + } + + pub(crate) fn write_metadata( + &self, + path: &Path, + metadata: &crate::models::ModelMetadata, + ) -> Result<()> { + self.validate(path)?; + self.destination.write_model_metadata(metadata) + } +} + pub(crate) struct LibraryMutationGuard { index: Option, claims: Vec<(String, Uuid)>, From e46fc9a47440b021539e1c2ab600c3ca4949854d Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 30 Sep 2026 16:17:32 -0700 Subject: [PATCH 16/20] feat(acquisition): complete shared consumer receipt handoff --- docs/contracts/artifact-acquisition.md | 12 +- .../artifact-acquisition/execution-ledger.md | 76 +- docs/plans/artifact-acquisition/issues.md | 7 +- docs/plans/artifact-acquisition/plan.md | 6 +- .../reports/acceptance-matrix.md | 2 +- .../reports/coding-standards-mcp-usability.md | 139 ++ .../reports/dependency-gates.md | 2 +- .../reports/passeur-mcp-usability.md | 84 + .../reports/write-sets.md | 51 +- rust/crates/pumas-app-manager/Cargo.toml | 1 + .../src/version_manager/installer.rs | 1554 ++++++++++++++--- .../src/version_manager/mod.rs | 453 ++++- .../crates/pumas-core/src/acquisition/http.rs | 2 +- rust/crates/pumas-core/src/acquisition/mod.rs | 9 +- .../pumas-core/src/acquisition/service.rs | 618 ++++++- .../pumas-core/src/acquisition/store.rs | 783 ++++++++- .../src/acquisition/task_custody.rs | 40 + .../pumas-core/src/acquisition/workspace.rs | 158 +- .../src/model_library/download_recovery.rs | 13 + .../src/model_library/download_store.rs | 745 +++++++- .../src/model_library/hf/download.rs | 974 +++++++++-- .../pumas-core/src/model_library/hf/types.rs | 6 +- .../pumas-core/src/model_library/importer.rs | 64 + .../pumas-core/src/model_library/library.rs | 176 +- .../src/model_library/mutation_authority.rs | 251 ++- rust/crates/pumas-core/src/network/github.rs | 62 + rust/crates/pumas-core/src/tests.rs | 56 +- .../pumas-core/tests/artifact_acquisition.rs | 170 +- rust/crates/pumas-rpc/src/main.rs | 15 +- rust/crates/pumas-rpc/src/server.rs | 21 +- 30 files changed, 6099 insertions(+), 451 deletions(-) create mode 100644 docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md diff --git a/docs/contracts/artifact-acquisition.md b/docs/contracts/artifact-acquisition.md index e335648f..6673cb1b 100644 --- a/docs/contracts/artifact-acquisition.md +++ b/docs/contracts/artifact-acquisition.md @@ -101,14 +101,20 @@ An installer holds the handoff until package/extraction workers and their cleanu Persist the selected manifest/specification identity, current demand/attempt, source-scoped nonsecret revision evidence, workspace identity, validated resume progress and lifecycle dispositions. Reuse the owned atomic store/publication machinery after separating model-specific records. Runtime artifacts must not require a fake repo/model/library UUID. One shared acquisition implementation can retain a supported legacy decoding boundary, but two stores must not both authorize the same transfer. -The current model-download document is schema 5, with schema 4 as its supported legacy input. Q1 adds neutral acquisition facts to that same physical store under schema 6; it does not create a second transfer store or reinterpret old model records as neutral manifests. Schema 6 retains the complete legacy model-download, recovery, queue, hidden-admission, release-proof and quarantine partitions, including unresolved dispositions, alongside neutral acquisition state. New empty roots use schema 6. An ordinary open of schema 4 or 5 reports that explicit migration is required and does not publish or authorize neutral work. A separate, explicit offline migration operation may convert only supported schema 4 or 5 state to schema 6, preserving legacy facts and publishing the new document atomically under the existing store coordination. Its operational precondition is that every process capable of reading or writing the old store has been stopped; the file lock serializes store operations but cannot prove that an older process will not resume with cached state. Unsupported, corrupt, or unknown state fails closed without a guessed conversion. Pending cleanup remains non-authorizing and is never replayed by migration. No automatic downgrade or old-binary rollback is supported after schema 6 publication; older strict readers must reject schema 6 without rewriting it. +The current model-download document is schema 5, with schema 4 as its supported legacy input. Q1 initially added neutral acquisition facts to that same physical store under schema 6; the exact importer-receipt extension advances the acquisition document to schema 7. This does not create a second transfer store or reinterpret old model records as neutral manifests. Schema 7 retains the complete legacy model-download, recovery, queue, hidden-admission, release-proof and quarantine partitions, including unresolved dispositions, alongside neutral acquisition state and a separate versioned model-consumer receipt partition. That partition is not part of the strict schema-5 model projection. New empty roots use schema 7. Ordinary open of schema 4, 5, or pre-receipt schema 6 reports that explicit migration is required and does not publish or authorize neutral work. A separate, explicit offline migration may convert supported schema 4/5 state or pre-receipt schema 6 to schema 7, preserving all model and neutral acquisition facts, leases, queues, revocations, hidden admissions, quarantines, and cleanup dispositions while creating no historical consumer receipts. Every old reader/writer, including one holding cached state, must be stopped before conversion; the new file lock does not exclude an old process that resumes later. Unsupported, corrupt, or unknown state fails closed without guessed conversion. All supported readers reject unknown schema/receipt versions without rebuilding or publishing. Pending cleanup remains non-authorizing and is never replayed by migration. No automatic downgrade or old-binary rollback is supported after schema 7 publication; older strict readers must reject it without rewriting it. Actual deployed schema-6 population and writer/rollback state are unknown; only disposable fixtures qualify this transition here. Persisting progress is not the same as making the file bytes durable. After a crash, validate source identity and actual partial files under reopened authority; do not trust a saved byte counter as proof. Complete-file verification must re-establish readiness when prior verification cannot safely be reused. Unsupported recovery remains non-authorizing. -Acquisition FilesReady and model/runtime publication are distinct commits. Consumers record their own idempotent finalization linked to the exact request/generation. If a crash occurs after consumer commit but before acquisition settlement, retain inputs conservatively and reconcile through the consumer's authoritative result; do not automatically repeat installation/import or delete final output. There is no assumed cross-store atomic transaction or exactly-once remote-I/O guarantee. +Acquisition FilesReady and model/runtime publication are distinct commits. The managed HF importer issues receipt version 1 only after complete finalization and durable metadata, index and applicable pinned package-facts outputs. The receipt binds the acquisition ID, persisted `Using` lease, HF demand and operation, current queue admission, manifest and ordered verified-file receipts, destination/workspace identity, resulting model ID, and a versioned canonical output projection. Canonical JSON sorts object keys recursively, preserves array order and explicit nulls, and rejects unsupported values. Its projection contract explicitly lists metadata, model-index and package-facts fields; package-facts content includes its independent contract version. Cold recovery compares current read-only outputs to the issuer-published proof, never computes new proof from current outputs alone. Partial imports and ordinary model-import callers have no receipt authority. A crash before receipt publication, unsupported/malformed receipt, or changed/missing output remains recovery-required and never replays import effects. Unknown publication visibility is failure, not success. + +The current Q1 candidate also routes llama.cpp archive acquisition through the shared consumer. Its native receipt binds the exact tag/metadata and hashes of the extracted output tree and launcher. The installer claims its cancellation/publication arbitration before returning the prepared receipt payload, so an accepted cancellation cannot later be replayed as an installation; after the claim, cancellation is refused and restart may finish the exact staged publication. If cancellation wins before receipt issuance, the installer durably revokes the exact attempt, drains and removes its owned workspace under the native lock, then withdraws only the unchanged, receipt-free `Using` lease. Cleanup or withdrawal failure retains recovery custody and prevents same-tag retry from selecting the unresolved attempt. Cold recovery verifies or completes publication from a committed receipt, settles the same acquisition, and explicitly reclaims its owned workspace. A retained `Using` acquisition without a receipt or exact withdrawal remains unresolved. Source-only composed review found no substantiated P0–P3 issue; objective-level consumer/platform evidence is still required. + +Receipt publication conditionally validates the exact durable `Using` lease, demand/manifest/files, non-revoked queue admission, workspace/destination, and held root grant in the same canonical store transaction. A cold worker does not renew or replace that lease before checking its receipt. Only after read-only validation may it obtain a private receipt-qualified settlement capability. Acquisition transition to `Adopted` and exact queue release are one atomic `AcquisitionStore` document publication; the immutable receipt remains paired with the adopted acquisition as completion history. Thus recovery has no intermediate acknowledgement-with-unreleased-queue state. An identical already-published receipt is idempotent; conflicting, orphaned, duplicated, malformed, or unknown-version receipts fail closed. No implicit receipt pruning exists; any future terminal-record compaction must remove the exact acquisition and receipt together under a separately selected retention policy. Migration never manufactures receipts, so pre-receipt `Using` remains unresolved even if output files match. No cross-store exactly-once remote-I/O guarantee is assumed. Consumer settlement is idempotent for its exact generation. Old acknowledgements cannot release a successor. Durable unfinished consumption retains custody across restart even though in-memory RAII handles no longer exist. If confirmation is unavailable, surface recovery-required and bounded retained state, not deletion by age. +Every effectful in-place model import, including orphan adoption, acquires the configured model-root execution authority and rechecks current durable model/acquisition custody immediately before metadata-present shortcuts, index publication, Diffusers delegation, or other importer effects. A scan is advisory and does not reserve the target. Ordinary imports receive no custody exemption. Managed Hugging Face has two private, operation-scoped stages: its partial metadata/index stub may proceed only under the current exact queue admission, `Transferring` generation, manifest/demand, workspace, destination, and held root grant; full finalization may proceed only under the current verified-file use lease and exact matching model admission. The partial-stage capability authorizes only the existing stub upsert and its index projection, including any metadata projection performed by indexing; it cannot authorize full import, package-fact resolution, another target, or a caller-supplied path or operation string. Neither stage bypasses unrelated queue/hidden custody or unresolved Pending/quarantined state. The held root grant and existing operation owner remain live until all importer/index effects settle, including after a caller stops waiting. Missing configured authority or unresolved custody fails closed. The current candidate's HF receipt now permits a reopened `Using` record to settle only after exact receipt and read-only output validation; missing or mismatched proof remains recovery-required and never replays importer effects. + ### Durable handoff transition ownership | Transition | Durable writer and exact identity | Filesystem authority and cancellation | Restart and reclamation | @@ -142,4 +148,4 @@ The decisive test denies network during the final installation leg and rejects h Internal coordinated DTOs, public Rust/IPC APIs, persisted formats, optional source implementations and consumer install/model records have different evolution obligations. Update actual generated consumers with their producer. New source support within this contract does not change model-adapter registration or runtime installation identity. Unknown schema/protocol versions are explicitly rejected, not decoded into weaker defaults. -Before independently deploying a breaking contract, disposition every supported public client and retained data state. Schema 6 is a breaking retained-store transition: migration is opt-in and offline, schema 4 and 5 remain untouched until that operation is invoked, and old writers must be stopped before publication. The actual deployed root population, retirement of old writers, and operational rollback policy are not established by disposable fixtures; those facts remain required for live-root qualification. Gate status is owned by the acquisition plan, not by a version number in this document. This document remains proposed until the implemented producer/consumer evidence exists. +Before independently deploying a breaking contract, disposition every supported public client and retained data state. Schema 7 is a breaking retained-store transition: migration is opt-in and offline, schema 4, 5, and pre-receipt 6 remain untouched until that operation is invoked, and old writers must be stopped before publication. The actual deployed root population, retirement of old writers, and operational rollback policy are not established by disposable fixtures; those facts remain required for live-root qualification. Gate status is owned by the acquisition plan, not by a version number in this document. This document remains proposed until the implemented producer/consumer evidence exists. diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 789dfb5f..a92d5698 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -10,7 +10,7 @@ Created this acquisition plan, a proposed shared contract, gate record, source a **Evidence:** source/docs inspection and mechanical delivery checks only. All AC production claims and AQ gates remain pending/not ready. No Rust/Python production suite, source-service integration, network-denied package installation, GUI, migration, GPU workload or native release was executed. No independent reviewer was run; independent review is a later explicit acceptance requirement. -**Next slice:** Q1 after current checkout/consumer/retained-state preparation and exact source-work admission. Runtime R1 remains gated by AQ-HTTP. +**Next slice:** Q1 — guard every effectful in-place importer and orphan-adoption path against unresolved model/acquisition custody. This slice keeps acquisition completion fail-closed; receipt/reopen and the llama.cpp consumer remain later Q1 work. Runtime R1 remains gated by AQ-HTTP. ## 2026-09-29 — Q1 started from accepted current main @@ -94,18 +94,72 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - A read-only architecture investigation of `c3052583` confirmed the supervisor extraction remains necessary preparation and does not create a shared durable owner. Its source trace identified the v4 migration-on-load, hidden custody and deletion-authority coupling, GitHub exact-tag/asset bridge, separate HF/native shutdown composition, and public constructors that need explicit lifecycle dispositions. The full write-set recommendation is retained in its handoff and will be narrowed to the exact production changes and tests before the next edits. - User-owned untracked `docs/breif/future.md` remains untouched. No live retained store, installed environment, model, or release was opened or changed. -### Exact next worker admission — HF consumer of the durable owner +### Completed worker admission — HF consumer of the durable owner - Parent milestone: `work/acquisition-q1-http` / draft PR #7, targeting `main`; exact worker base: `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`. - Worker role: one GPT-6.1 Sol High implementation writer for the canonical acquisition lifecycle/store/workspace and the existing HF consumer cutover. It is a Q1 sub-slice; it does not open AQ-HTTP. -- Exact primary paths, test requirements and exclusions are recorded in [the Q1 write-set admission](reports/write-sets.md#exact-next-worker-admission--durable-acquisition-owner-and-hugging-face-cutover). The shared semantic contract and gate/plan records remain integrator-owned. The admitted durable format is schema 6 in the existing canonical store; fresh roots use it, normal open does not migrate schema 4/5, and a separate explicit offline migration operation is required. Stop every old reader/writer before migration. No live retained root is authorized as a fixture, and deployed population/rollback qualification remains open. +- Exact primary paths, test requirements and exclusions are recorded in [the completed Q1 write-set admission](reports/write-sets.md#completed-worker-admission--durable-acquisition-owner-and-hugging-face-cutover). The shared semantic contract and gate/plan records remain integrator-owned. The admitted durable format is schema 6 in the existing canonical store; fresh roots use it, normal open does not migrate schema 4/5, and a separate explicit offline migration operation is required. Stop every old reader/writer before migration. No live retained root is authorized as a fixture, and deployed population/rollback qualification remains open. - Required proof is the actual normal HF workflow through the acquisition owner, schema-6 migration/reopen against disposable supported schema-4 and schema-5 fixtures, all hidden custody/Pending refusal, lease retention through importer cleanup, concurrent writer isolation, cancellation/restart and shutdown evidence. Normal open must not mutate retained schema-4/5 stores; migration is a separate explicit offline operation, and older writers/readers must be stopped before it is invoked. Fresh schema-6 roots and reopened schema-6 state must be covered. A helper-only service, mock consumer, or passing unit test is not sufficient for this handoff. Disposable fixtures do not qualify the unknown deployed retained-state population or rollback to old binaries. -- After this handoff is reviewed and integrated, the next Q1 sub-slice is the existing exact-tag llama.cpp installer using that same service and verified-input lease, with shared RPC composition and consumer-before-owner shutdown. AQ-HTTP remains blocked until both consumers and the plan acceptance matrix are satisfied. +- This handoff was reviewed and integrated. The next Q1 sub-slice is the common effect-boundary importer/orphan-adoption custody guard; its exact write set and evidence are admitted in [the current primary write-set record](reports/write-sets.md#current-primary-admission--importer-and-orphan-adoption-custody-guard). This guard does not recover a reopened `Using` phase. After it is complete and reviewed, a separate Q1 sub-slice must establish exact HF finalization proof/reopen before the existing exact-tag llama.cpp installer consumes the same service and verified-input lease. AQ-HTTP remains blocked until both consumers and the plan acceptance matrix are satisfied. - The completed custody worker worktree `/tmp/pumas-q1-hf-custody` was clean at `584b5f45a17a80cd584b16ce77f7be5d9688a9b8`, which remains reachable from primary `work/acquisition-q1-http` (`7129d51631d7257742573d4e4873009f20158bec`) through the preserved merge. Its task-owned worktree and redundant local branch were removed after the reachability check (`removed-reachable`). Other task-owned and user-owned worktrees remain untouched. -- The current worker worktree is `/tmp/pumas-q1-hf-acquisition`, branch `agent/q1-hf-acquisition`, based exactly on `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`. The primary advanced only by plan/write-set documentation after that base was created; production source is identical. Integrate with the history-preserving mechanism selected after reviewing its actual commit graph, then record its terminal disposition separately. - -### Coding-Standards route for the next shared-custody slice - -- Fresh route snapshot `snapshot:v1:2ea2f5e4-70d3-4a32-97c6-13a93989fb3f` selected 34 standards with zero unresolved fact categories after explicitly marking the unrelated framework fact known-absent. The first route omitted that fact and returned one required unresolved category; correcting the route preserved the same change scope. -- Focused reads were requested in three batches for 17 standards: Core, Architecture, Code Design, Concurrency, Rust Async/API, Persistence, Contracts/Evolution/Protocols, Resilience, Security/Untrusted Execution, Dependencies, Verification, Commit, and Concurrent Plan Integration. The combined tool presentation exceeded the response limit and was truncated, so exact follow-up policy reads will be kept smaller and limited to the owner boundaries actually changed. This route is guidance for the upcoming extraction; it is not a compliance result or implementation evidence. -- Repository inspection remains necessary to identify that HF's current task ID/projection scans and shutdown callback are owner-wide, and that its destination root creates model-library marker/deletion authority. The extraction slice will retain the current store and HF model policy while adding scoped custody under one source-neutral supervisor. +- Worker worktree `/tmp/pumas-q1-hf-acquisition` is clean at `65274dffd90e9c5272a89ec5a3e9c1ae564bfa0c`; its commit is reachable from primary `work/acquisition-q1-http` through merge `bc9e9da4147a3e64f2a5e67093604eddeddc5391`. It remains task-owned pending the recorded terminal worktree disposition; unrelated worktrees are untouched. + +## 2026-09-29 — Q1 exact HF receipt and cold-reopen design + +- The primary started this slice on `work/acquisition-q1-http` at `9719ecb50d84df796db887832b87b42e8992988a` (tree `0c3e472c8b5f05073ede4ddbe2252a2a34e54baf`). The custody-guard prerequisite is integrated and reviewed; no receipt/reopen source changes have been made yet. +- Coding-Standards snapshot `snapshot:v1:a802e13a-248b-40ce-8233-45efaae444d6` routed the persistence/replay/schema-evolution/concurrency/Rust/security/verification/implementation/commit scope with 28 standards and zero unresolved fact categories. Focused reads covered Persistence, Architecture Replay, Schemas, Contract Evolution, Concurrency, Code Design, Rust Async, Untrusted Execution, Verification Oracles, Resilience, Rust API, Commit, Platform Verification, Implementation and Documentation. The route guided ownership/evidence work; it does not establish code or acceptance evidence. +- Source audit found that all production `downloads.json` writes converge on `AcquisitionStore`, while model writes decode the strict schema-5 projection and replace the flattened legacy partition. The receipt therefore needs a distinct versioned envelope partition. Normal HF completion removes its marker before importer effects, then publishes guarded metadata, index projection and applicable package facts, issues the exact `Using` receipt, and atomically adopts the acquisition while releasing the exact queue admission. The receipt remains paired with the adopted acquisition as immutable completion history. Existing schema-6 metadata publication requires `Durable` when it writes, but equality may skip a new publication. The package-facts source fingerprint is a cache freshness key (metadata/descriptor/dependency JSON and file length/mtime), not payload identity; `ModelMetadata` also contains a `HashMap`. Neither fact permits treating that fingerprint or metadata/index presence as a completion receipt. +- A read-only feasibility review at the exact source commit found schema-4/5 read-only and explicit-offline migration, current schema-6 reads/writes, and no exact importer completion record. The proposed implementation advances the envelope to schema 7, stores a strictly validated, opaque versioned model receipt outside the schema-5 projection, and explicitly converts pre-receipt schema 6 plus supported schemas 4/5 without fabricating receipts. Receipt v1 binds the exact acquisition/use IDs, demand, queue admission, manifest, ordered file receipts, destination and model identity. Its separately versioned canonical output proof covers declared metadata/index projections and required pinned package-facts content/version: recursively sorted object keys, preserved array order/nulls, and rejected unsupported values. Receipt publication is conditional on the persisted exact Using lease and current confirmed admission under the held root grant. Cold recovery validates the historic lease, queue, file receipts, receipt and outputs under fresh root custody without renewing/replacing the Using lease and without importer, metadata repair, index mutation or package-facts hydration. Exact settlement atomically transitions acquisition to Adopted and releases the queue admission in one canonical document publication while retaining the receipt with the adopted acquisition; no acknowledgement-before-release intermediate state is supported. Conflicting/orphaned/malformed/unknown-version receipts fail closed, and only disposable fixtures may qualify schema migration because deployed schema-6 population and old-writer exclusion remain unknown. +- Unknown actual deployed schema-6 population, overlapping old processes, rollback requirements, separately retained model/index state, and index-only writers remain unverified. No live retained root is authorized or modified. The schema-7 transition and reopen tests will use disposable fixtures; no deployed migration-compatibility claim is made. +- The feasibility reviewer performed no edits, Git mutation or tests. Its Coding-Standards routing used snapshot `snapshot:v1:968b380d-c352-496b-9a5f-f522222e09ab`, with an initial and final route both selecting 21 standards and zero unresolved questions, plus focused reads for Persistence, Replay, Contract Evolution and Schemas. Its separate actual MCP usability feedback is recorded in [the MCP report](reports/coding-standards-mcp-usability.md). +- Independent design review `q1_custody_final_review` found no source defect, but identified implementation-blocking conditions: conditional receipt publication over the complete Using/admission identity; no unrecoverable receipt→acknowledgement→queue-release window; private cold-settlement authority that does not renew the persisted lease; precise independent receipt/output/package-facts versions and canonical projections; offline migration that operationally stops cached old writers and preserves every partition without manufacturing receipts; and fail-closed duplicate/orphan/malformed/unknown receipt rules. The exact contract/write-set now records all conditions. Reviewer ran no source tests and performed no edits; its actual MCP experience remains in the separate MCP usability report. +- The integrator additionally read Platform Verification, Implementation and Documentation on the root receipt-slice snapshot after the initial compact batches; this did not change the source boundary. Exact implementation, restart, output-mismatch, cancellation and fault-injection evidence remains pending. AQ-I12 is recorded; AQ-HTTP remains not ready. + +### Coding-Standards routing for the HF receipt design and current custody-guard slice + +- Fresh route snapshot `snapshot:v1:79c1c8b5-7064-48df-93a9-17b1baf9f8cf` selected 32 standards with zero unresolved facts. The first route omitted the required empty framework fact and returned `needs-facts`; correcting it retained the same scope. The complete route covers Rust library/API/async/security/cross-platform, persistence, consumer receipts/replay, concurrency, contract evolution/schema, security/delegated execution, independent oracles, documentation, verification and commit. +- A broad `read_many` asked for 21 compact policies but exceeded the visible output window. Focused follow-up reads covered Persistence, Contract Evolution, Architecture Replay, Concurrency, Untrusted Execution, Independent Test Oracles and Commit Workflow. The source-level receipt/adoption proposal was independently reviewed separately; MCP calls supply obligations, not implementation or acceptance evidence. +- Repository inspection remains necessary to establish schema-6 partition behavior, exact acquisition `Using` lease, importer side effects, metadata durability, current mutation grants, and all orphan-adoption entry points. The selected next implementation keeps the existing physical store and model importer as owners; it does not introduce another store, importer authority, or task registry. + +- The initial guard route used fresh snapshot `snapshot:v1:9881b6be-4a0e-4d72-b0cd-4e4307cdc710`, selected 26 standards, and had zero unresolved fact categories. Source tracing then found the existing partial-HF metadata/index write during `Transferring`, so the design boundary changed and was rerouted against fresh snapshot `snapshot:v1:0ed9c25a-060e-4970-a5ca-a7286eda1601`: 28 selected standards, zero unresolved. The expanded explicit facts include Rust library/API/async/security, persistence, contracts, architecture/replay, concurrency, security, implementation, planning, uncertainty reduction, verification, and commit; frameworks are explicitly known absent. Focused policy reads covered Independent Test Oracles, Code Design and Ownership, Commit Workflow, Verification, Rust, Architecture, and Security, in addition to the earlier Concurrency, Rust Async, Persistence, Rust API, and Untrusted Execution reads. The narrowed implementation remains a guard, not receipt/output-proof or reopen recovery. The obligations applied are one lifecycle owner, root custody through actual child effects and cancellation, current proof at the effect boundary, fail-closed unavailable authority, and independent negative and positive stage fixtures. Routing records applicability, not source compliance evidence. + +## 2026-09-29 — Q1 durable acquisition and HF consumer integration + +- Worker candidate `65274dffd90e9c5272a89ec5a3e9c1ae564bfa0c`, based on `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`, was reviewed and integrated by a history-preserving merge into `work/acquisition-q1-http`. Root candidate is `d0b71b51075815cf307cea95c9364be0ed25472d` after a focused retained-custody repair. The schema-6 document and publication remain in the existing `downloads.json` authority; the normal Hugging Face path now acquires through the shared HTTP/store/workspace service and holds verified-file custody through awaited model import. Native llama.cpp remains on its independent transfer path. +- Exact worker source set: `rust/crates/pumas-core/src/acquisition/{mod.rs,service.rs,store.rs,workspace.rs}`, `src/api/{builder.rs,hf.rs,state.rs}`, `src/lib.rs`, `src/model_library/{download_recovery.rs,download_store.rs,mutation_authority.rs}`, `src/model_library/hf/{download.rs,lifecycle.rs,mod.rs}`, and `tests/artifact_acquisition.rs`. Root's focused repair changes only `src/acquisition/store.rs`. No app-manager/native consumer, metadata schema, generated/RPC/frontend interface, manifest/lockfile, or live retained root changed. +- Independent review found a P2 in schema-6 cross-record validation: duplicate demand or conflicting active workspace rows could let `begin` select around retained `Using` custody. The root repair rejects duplicate `(consumer, operation)` identities and duplicate workspaces across active `Transferring`, `FilesReady`, and `Using` records. Terminal Adopted/Withdrawn history can still share a workspace with its active successor. Independent read-only follow-up at exact `d0b71b5` confirmed the finding closed and found no new issue. +- Exact candidate tests at `d0b71b51075815cf307cea95c9364be0ed25472d`: `cargo test --manifest-path rust/Cargo.toml --locked --offline -p pumas-library --no-default-features --lib acquisition:: -- --test-threads=4` passed 63; `... --lib model_library::download_store::tests -- --test-threads=4` passed 62 (2 ignored); `... --lib model_library::mutation_authority::tests -- --test-threads=4` passed 2; `... --test artifact_acquisition` passed 1. All-target pumas-library Clippy with `-D warnings`, no-default-features check, rustfmt check and `git diff --check` passed. Loopback HTTP and disposable temporary roots exercise actual local protocol/filesystem/store behavior; they are not live upstream Hugging Face or native-service evidence. +- The importer has no exact durable generation/use-lease completion receipt. A reopened `Using` acquisition currently returns `acquisition.consumer_recovery_required`; it cannot reconcile a committed import without replay. A crash after the HF marker is removed but before metadata exists can also reach ordinary orphan adoption, whose scan does not consult retained acquisition custody. Source tracing also found that the existing partial HF stub persists metadata and runs index projection while the acquisition is `Transferring`; preserving this behavior requires its own narrowly scoped private stage capability, not a broad phase exemption. Startup and public adoption entry points need one fail-closed admission guard. Metadata/index presence is not exact-generation proof. Migration/reopen evidence is disposable-fixture evidence only; actual deployed schema-4/5 population and old-writer retirement remain unknown. +- Hosted CI run `36661110500` completed successfully on the prior exact PR head `5ef3e0530a3b2c050d35cc302a22b9f783ccc777`: Workflow/release contracts, Rust quality, headless-without-inference, frontend/desktop contracts, and all three Torch native QA jobs passed. Several release/package/archive/RPC-E2E jobs were skipped conditionally. This run predates `65274dff` and `d0b71b5`, so it is not evidence for the current source candidate. PR #7 remains open and draft; the new primary source commits still need push and fresh hosted checks. +- AQ-HTTP remains not ready. The importer custody guard is now implemented, independently reviewed, and integrated on local source tree `0c3e472c8b5f05073ede4ddbe2252a2a34e54baf`; it does not provide receipt/reopen reconciliation. No native shared-acquisition consumer, dedicated unqueued-worker withdrawal regression, public app-manager acquisition capability, live-source/manual UI, or final plan-matrix evidence is accepted. AQ-PACKAGES, AQ-S3, AQ-COMPLETE and runtime R1 remain gated. +- User-owned `docs/breif/future.md`, `pumas-coordinated-plans.zip`, unrelated worktrees and retained environments remain untouched. No live download store, model, installed runtime, or release was opened or changed. + +## 2026-09-29 — importer and orphan-adoption custody guard + +- Worker commit `8b6c5f70c55a3d124189d0cd88dc85c780f47c84` was built directly on `d0b71b51075815cf307cea95c9364be0ed25472d`, then integrated history-preservingly by merge commit `9719ecb50d84df796db887832b87b42e8992988a` on `work/acquisition-q1-http`. The integrated tree is exactly `0c3e472c8b5f05073ede4ddbe2252a2a34e54baf`, the reviewed/tested worker tree. The merge contains nine admitted Rust files and no schema, migration, public/generated interface, dependency, app-manager, or frontend change. +- The guard obtains existing root exclusion and rechecks one coherent canonical custody snapshot before import shortcuts/effects. Ordinary and orphan imports have no custody exemption. Private partial-HF and full-HF capabilities bind their respective `Transferring` stage or current verified-use lease, matching queue admission, selected files/workspace/destination, and held root grant. Existing `RuntimeTasks` and acquisition task custody retain and drain effects through caller cancellation and worker failure. Reopened unresolved `Using` remains non-executable. +- Writer evidence on exact tree `0c3e472c8b5f05073ede4ddbe2252a2a34e54baf`: importer/recovery default 32 passed; importer/recovery no-default 32 passed; managed HF 170 passed; acquisition 63 passed; download-store 62 passed with 2 existing ignored helpers; mutation-authority 2 passed; focused HF import cancellation 1 passed; no-default check, default/no-default all-target Clippy with `-D warnings`, formatting, diff checks and commit hooks passed. The tests use real temporary filesystems, canonical stores, SQLite and actual owner contexts. Local HTTP fixtures exercise real localhost transport; stale-token/store corruption, seeded phases, panics and scheduling are synthetic fault probes on those real components. This is not live HF/native acceptance. +- Independent review checked exact head `8b6c5f70c55a3d124189d0cd88dc85c780f47c84`, exact parent `d0b71b51075815cf307cea95c9364be0ed25472d`, and nine-file diff identity `5991327bf4df196cd379f70d65ed30298a51f2e3`. It found no substantiated P0–P3 issue and ran `git diff --check`; it ran no tests/build/Clippy/formatting. Reviewer findings and worker-reported test results are distinct evidence. +- Coding-Standards routing for the implementation used snapshot `snapshot:v1:9881b6be-4a0e-4d72-b0cd-4e4307cdc710`, 24 selected standards and zero unresolved facts, with focused Concurrency, Rust Async, Persistence, Rust API, Untrusted Execution, Verification, Security, and independent-oracle obligations. The material partial-stage/aggregate-effect design changes were rerouted by the worker; root will retain a final same-tree route. Full per-agent MCP usability feedback is separate in [the usability report](reports/coding-standards-mcp-usability.md). +- The exact source commit/merge has not been pushed yet, so no hosted check is claimed for this candidate. PR #7 remains the Q1 draft to `main`; update/push and inspect fresh CI before treating any check as current. AQ-HTTP remains not ready. +- Next Q1 slice: exact model-owned HF finalization receipt/output proof and no-replay `Using` reopen. Then native shared-acquisition consumer cutover, public app-manager handoff, remaining withdrawal regression, and objective-level acceptance. Q2 and runtime R1 remain gated on AQ-HTTP. + +## 2026-09-30 — Q1 exact receipt and shared native consumer candidate + +- The local candidate advances the canonical acquisition document to schema 7 with exact HF consumer receipts and cold `Using` reconciliation. HF completion publishes a durable receipt only after importer effects settle; cold recovery verifies the receipt/output projection and settles without reimport. The llama.cpp installer now uses the shared HTTP acquisition consumer, extracts from the verified file handle, binds its receipt to the exact native output tree and metadata, and composes consumer shutdown before global acquisition shutdown in RPC. +- Native recovery now removes its owned input workspace after exact settlement. Startup also handles the already-`Adopted` cleanup window idempotently: under the native versions lock it validates the bound receipt, installed tree and metadata before retrying removal. The live native installer claims its publication/cancellation barrier before receipt issuance, so an accepted cancellation cannot leave a receipt that startup later treats as install intent. A local loopback restart test leaves an adopted workspace behind and verifies startup removes it. +- **Current local test evidence:** acquisition store tests passed 8/8; HF receipt tests passed 24/24; `artifact_acquisition` integration tests passed 2/2; the loopback-only GitHub fixture-seam test passed 1/1; the workspace identity-only lookup test passed 1/1; native installer/manager tests passed 24/24 with 235 filtered. The native test run includes install, restart cleanup of a stale adopted workspace, explicit removal, and same-tag reinstall with distinct acquisition identities. `cargo clippy --manifest-path rust/Cargo.toml -p pumas-library -p pumas-app-manager -p pumas-rpc --all-targets -- -D warnings`, `cargo check --manifest-path rust/Cargo.toml -p pumas-library --no-default-features`, `cargo fmt --manifest-path rust/Cargo.toml --all -- --check`, and `git diff --check` all pass on the current local source tree. +- Independent composed review of the previous candidate found the adopted-workspace retry gap (P2), plus a cancellation/receipt race subsequently closed by moving the arbitration claim before receipt issuance. The adopted cleanup retry checks exact record/receipt/lease, active attempt, persisted workspace root+relative identity, installed output and metadata while holding the native lock. Final read-only follow-up found no substantiated P0–P3 findings; reviewers ran no checks. The new restart regression proves the local seeded stale-workspace case, not arbitrary crash timing. Coding-Standards final routing used snapshot `snapshot:v1:425e276a-fc0b-4282-9889-a76742027c42`, 40 selected standards and zero unresolved fact categories; routing/policy reads inform obligations but do not establish code acceptance. +- The Passeur issue was reproduced without submitting work: agent/task limits are undocumented in tool declarations, a valid agent query returns `PATH_NOT_FOUND` from `profile.open`, a valid task query returns `SERVICE_PROFILE_CONFLICT`, and status confirms repository service unavailable. The steps and targeted tool improvements are recorded in [the Passeur usability report](reports/passeur-mcp-usability.md); this session did not notify maintainers. +- These are local disposable-state and controlled loopback checks. They do not establish live HF/native publisher behavior, desktop workflow, current-head hosted CI, deployed schema-6 population/old-writer isolation, public compatibility dispositions, or Windows/macOS/Linux durability. No live retained root was mutated. AQ-HTTP remains not ready; Q2 and runtime R1 remain gated. +- `docs/breif/future.md` remains user-owned and untouched. No commit, push, merge, deploy or Passeur task submission was made. + +## 2026-09-30 — native pre-receipt withdrawal and composed verification + +- Closed the native cancellation gap before a completion receipt exists. `AcquiredArtifactUse::withdraw_after_cleanup` drains registered effects, then requires the exact unchanged receipt-free `Using` record/lease before withdrawal. The native installer durably revokes the attempt, removes and syncs only its bound workspace while holding the versions lock, then withdraws that lease. Cleanup failure retains custody and prevents retry. The local RPC/manager regression cancels an admitted install, reopens a fresh API/manager, verifies no stale workspace/output/receipt is exposed, and retries the same tag with a new acquisition identity. +- The first composed serial all-target candidate run exposed a stack overflow in `test_serving_torch_missing_checkpoint_is_non_critical_and_not_loaded`; the same exact test passed on accepted `main`. Temporary child-process markers isolated recursion depth in the async descriptor-refresh path. Boxing that existing refresh future preserves its order, errors and cancellation, and starts no detached task. Temporary marker/test edits and stack-size workarounds were removed. +- **Final composed candidate test:** `cargo test --manifest-path rust/Cargo.toml --locked -p pumas-library -p pumas-app-manager -p pumas-rpc --all-targets -- --test-threads=1` exited 0 after the fix. Core unit tests: 1,506 passed, 6 ignored; API tests: 36 passed; `artifact_acquisition`: 2 passed; app-manager: 260 tests completed successfully; RPC unit tests: 265 passed; RPC integration: 17 passed, 10 ignored; intent integration: 2 passed, 2 ignored. Other library/RPC fixture targets also passed. The formerly failing Torch serving RPC case passed at the normal worker stack size. `cargo fmt --manifest-path rust/Cargo.toml --all -- --check` and `git diff --check` passed on this tree. Post-fix all-target Clippy with `-D warnings` and the pumas-library no-default-features check had also passed after the source edits. +- Final independent source-only composed review checked exact current Rust diff SHA-256 `5e10716362784caa8a3146bb519b391a233960ed1a83fca4932bb1d577afe618` and found no substantiated P0–P3 findings. The reviewer did not run tests; test outcomes above are integrator-run. Review limitations remain: no injected crash/error at every revocation/cleanup/withdrawal boundary, no live source/desktop/deployed-state or cross-platform qualification. +- The resumed Coding-Standards route used fresh snapshot `snapshot:v1:10b3e01e-6a86-4185-83e0-65ca7b0db870`, selected 44 standards with zero unresolved questions, and read focused `core`, persistence and concurrency policies in small batches. It records obligations, not implementation compliance or acceptance. See the usability record. +- Passeur remains unavailable: current `passeur_status({})` returns `SERVICE_PROFILE_CONFLICT`, and valid `passeur_agents({ limit: 4 })` fails at `profile.open`. No task was submitted; implementation used the authorized GPT-6.1 Sol fallback. Reproduction/follow-up details remain in the separate Passeur usability record. +- Exact source write set is enumerated in the current Q1 section of [write sets](reports/write-sets.md). The owning records updated in this resumed slice are `docs/contracts/artifact-acquisition.md`, the acquisition plan/issue/acceptance/dependency/write-set/ledger records, and both MCP usability reports. `docs/breif/future.md` remains untouched. AQ-HTTP stays not ready; live HF/GitHub/native acquisition, desktop workflow, deployed schema-6 inventory and old-writer exclusion, public compatibility, current-head hosted CI and cross-platform durability remain unqualified. No commit, push, PR update, merge or deploy has yet occurred at this ledger point. diff --git a/docs/plans/artifact-acquisition/issues.md b/docs/plans/artifact-acquisition/issues.md index f797eff9..315836cc 100644 --- a/docs/plans/artifact-acquisition/issues.md +++ b/docs/plans/artifact-acquisition/issues.md @@ -6,13 +6,16 @@ Q1 is active; production repairs and acceptance claims remain pending until evid | --- | --- | --- | --- | | AQ-I01 | High: model-specific destination authority cannot become the generic artifact root | Acquisition/core: fix in Q1 | AC01/AC04/AC09; re-plan if safe neutral capability extraction cannot preserve model custody | | AQ-I02 | High: retained model/HF records and recovery transitions need explicit migration | Core/recovery: fix in Q1 | AC04–AC06; actual source/deployment inventory before mutation | -| AQ-I03 | High: cancellation/restart must retain worker and byte-use ownership | Acquisition: fix in Q1 | AC05/AC06; no dropped-future or timeout-as-cleanup proof | -| AQ-I04 | High: acquisition/import/install completion and duplicate transfer owners | Acquisition plus consumers: consolidate in Q1 | AC03/AC05/AC08/AC18 | +| AQ-I03 | High: cancellation/restart must retain worker and byte-use ownership | The local Q1 candidate closes the native pre-receipt cancellation gap: it durably revokes the exact attempt, drains/cleans its workspace under the native lock, and withdraws only the exact unchanged receipt-free `Using` lease. Cold reopen and same-tag retry are covered. Broader cancellation/crash-window acceptance remains pending. | AC05/AC06; retain exact-generation worker drainage and cleanup-failure/cold-reopen evidence; never equate caller cancellation with worker stop | +| AQ-I04 | High: acquisition/import/install completion and duplicate transfer owners | Current local Q1 candidate routes HF and llama.cpp through the shared transfer owner and gives both consumers exact completion receipts; composed source review found no remaining issue, while objective acceptance evidence remains pending | AC03/AC05/AC08/AC18; retain cold-reopen, cancellation, and no-duplicate-transfer proof; never infer import/install completion from path or metadata presence | | AQ-I05 | High: cross-plan duplicate authority or prerequisite cycle | Integrator: fixed in plan design; implementation evidence pending | Gate graph and Q1/Q2 old-consumer evidence; review after any milestone dependency change | | AQ-I06 | High: generic retrieval can be mistaken for package compatibility or executable trust | Runtime/package/security: Q2, with Q1 trust contract | AC07/AC11/AC12; keep origin and byte digest separate | | AQ-I07 | Medium: new generic module could accidentally require inference or overclaim feature isolation | Core/composition: Q1/Q3/Q4 | AC09/AC15/AC17; no unsupported minimal-build claim | | AQ-I08 | Medium: source roadmap can expand prerequisite beyond useful consumer result | Integrator: Q1 contract first; Q3 required S3 scope; future features deferred below | AC18 and gate-status record | | AQ-I09 | Medium: hashless source comparison is not isolation from an uncooperative local writer that can modify a file in place and restore metadata before model import | Acquisition/import handoff: Q1 claim is explicitly limited; resolve through a same-owner handle/lifetime handoff or document and verify the supported local-writer threat model before any broader claim | AC05–AC07; revisit at shared durable handoff and final composed review | +| AQ-I10 | High, fixed in local Q1 candidate `d0b71b5`: acquisition-document validation accepted duplicate demands and overlapping active workspace custody, allowing `begin` to select around a retained `Using` record | Acquisition store: fixed by cross-record `AcquisitionDocument::validate`; independent exact-candidate review found no new issue | Three persisted-document fixtures cover duplicate demand, active-workspace collision, and terminal-history/successor coexistence; retain these invariants through schema-7 receipt work | +| AQ-I11 | High, fixed in local Q1 source tree `0c3e472c`: effectful in-place import and orphan adoption could act on a destination after an acquisition or hidden model admission retained custody | Model-library/importer: fixed by guard at the effect boundary with exact private partial/full HF stages; independent review found no P0–P3 issue | Worker reports focused real-store/filesystem/owner tests and cancellation/panic/replacement probes; reopened `Using` remains unresolved and AQ-HTTP is still not ready | +| AQ-I12 | High: metadata/index presence and package-facts cache freshness alone do not prove which `Using` lease completed import | Current local Q1 candidate implements the schema-7 HF receipt and exact cold reconciliation; composed source review found no remaining issue, while objective acceptance evidence remains pending. Never replay import after ambiguous publication. | AC05/AC08; retain proof that the receipt binds acquisition/use identity and deterministic output content, survives every writer/migration, and cold-reopens without importer invocation | | AQ-D01 | Deferred: native Xet reconstruction | Acquisition/source owner | Revisit when a required HF source cannot use the supported existing file path or Xet transfer benefits are an admitted goal; evaluate maintained Rust implementation, no homemade protocol | | AQ-D02 | Deferred: peers and concurrent multi-source failover | Acquisition/distribution owner | Revisit with actual node/authorization contract and content-equivalence proof; no cross-origin ETag comparison | | AQ-D03 | Deferred: chunk CAS, reflink optimization and coalescing | Acquisition/storage owner | Revisit with measured duplication/network/storage need and explicit retention consumers; ordinary files and safe release are required now | diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index 1c1ed3dd..efdd9c4c 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,8 +2,8 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** Q1 manifest/HTTP protocol and the ordinary Hugging Face consumer are implemented. The exact-tag GitHub resolver maps publisher asset identity/digest evidence into a manifest without changing the public/cache DTO. HF physical-root grants coalesce independent opens of one root while retaining distinct grants. Native installation shutdown custody is committed, and the shared task/effect supervisor has been extracted with consumer-scoped admission, observation and drainage at `c3052583860baa76dc114edf6420805e9b06f327`. These are preparation/current-consumer slices: HF still owns transfer retry/effects, persistence and handoff; llama.cpp still uses its independent downloader. A durable source-neutral acquisition owner/store, exact-generation consumer reconciliation, real HF/native cutovers, desktop path and source/platform qualification remain pending. -**Exactly one next slice:** Continue **Q1 — one durable, source-neutral acquisition lifecycle and verified-file handoff consumed by both the normal Hugging Face workflow and the existing llama.cpp installer.** Keep consumer publication and model/runtime policy at their existing owners. Change the retained store only after supported schema/state and writer coordination are explicitly accounted for; do not use a live retained root as an implementation fixture. +**Current phase:** The local Q1 candidate advances the shared `downloads.json` authority to schema 7 with exact HF completion receipts and cold `Using` reconciliation. The Hugging Face workflow imports through the shared verified-file owner, and the llama.cpp archive installer uses that same owner, extracts from its verified file handle, and records/reconciles a native installation receipt. Cancellation that wins before the native receipt now revokes the exact attempt, cleans its workspace, and withdraws the exact receipt-free lease; cold reopen and same-tag retry are covered locally. RPC composition injects the shared consumer and drains consumers before acquisition shutdown. Local disposable-state and loopback tests cover these paths; they do not qualify live Hugging Face behavior, desktop behavior, deployed migration safety, or cross-platform durability. Independent composed review of the cancellation repair found no substantiated source correctness issue. AQ-HTTP remains not ready; objective acceptance evidence, public consumer compatibility, current-candidate hosted CI, real-source/manual behavior, deployed schema-6 population and old-writer isolation, and supported-platform qualification remain pending. +**Exactly one next slice:** Collect **Q1 objective acceptance evidence** for this candidate. Record current-head hosted checks and affected consumer evidence. Exercise the real HF import and llama.cpp source/consumer workflows, desktop controls, public API compatibility dispositions, and supported-platform durability. Keep deployed retained-root mutation blocked until schema-6 population, overlapping old writers and rollback requirements are inventoried. Do not open Q2 or runtime R1 until AQ-HTTP's objective gate is accepted; local fixtures alone do not satisfy that gate. **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. **Operation:** `start` this exact plan on `work/acquisition-q1-http`. @@ -98,7 +98,7 @@ Runtime R2 adds arbitrary registered adapters later using that same accepted fil ### AD8 — own evolution before mutation -The existing download schema, recovery admissions, root markers, hidden history and cleanup custody have retained consumers. Q1 must reconcile the active Rust recovery owner and enumerate supported source states before migration. Preserve legacy IDs through explicit mapping where needed for existing UI/consumer operations. Unsupported/corrupt or custody-unresolved states remain visible and non-authorizing. The source-neutral fields extend the existing canonical store as schema 6; they do not reuse schema 5 with an expanded shape or add a second writer. Fresh roots use schema 6. Existing schema 4/5 roots require a separate explicit offline migration operation, while normal open/admission fails closed without modifying them. Every old reader/writer must be stopped before that operation; disposable fixture migration is implementation evidence, not proof that the deployed root population or rollback policy is qualified. +The existing download schema, recovery admissions, root markers, hidden history and cleanup custody have retained consumers. The Q1 candidate extends the existing canonical store through schema 7; it does not reuse schema 5 with an expanded shape or add a second writer. Schema 7 includes the source-neutral acquisition fields and the versioned consumer-receipt partition. Receipt-free legacy schema 4/5/6 conversion is explicit and offline; normal open/admission fails closed without modifying those formats. Every old reader/writer must be stopped before conversion. Disposable fixture migration is implementation evidence, not proof that the deployed root population or rollback policy is qualified. Preserve legacy IDs through explicit mapping where needed for existing UI/consumer operations; unsupported, corrupt or custody-unresolved states remain visible and non-authorizing. Reopen after each interrupted publication boundary to source, destination or an explicit uncertainty state. Do not mark completion, delete partials, auto-retry imports/installations, or replay Pending cleanup from guessed facts. New readers reject unknown future formats. Older writers must be retired or isolated; a new format number cannot constrain an old binary. Rollback requires evidence about both stores and subsequent authored changes, not just a backup file. diff --git a/docs/plans/artifact-acquisition/reports/acceptance-matrix.md b/docs/plans/artifact-acquisition/reports/acceptance-matrix.md index 603b2ecd..b7e85961 100644 --- a/docs/plans/artifact-acquisition/reports/acceptance-matrix.md +++ b/docs/plans/artifact-acquisition/reports/acceptance-matrix.md @@ -1,6 +1,6 @@ # Acquisition acceptance matrix -**Production status: all AC01–AC18 pending.** No code, production tests, source service, GUI, migration or installed artifact was executed in this planning delivery. A required unavailable environment changes its claim to blocked, not satisfied. All claim owners are roles for the assigned integrator to resolve. +**Acceptance status: all AC01–AC18 pending.** The local Q1 candidate has focused Rust tests and controlled loopback/filesystem evidence, but the required real-source, desktop, retained-deployment, public-interface, and supported-platform evidence has not been collected. A required unavailable environment changes its claim to blocked, not satisfied. All claim owners are roles for the assigned integrator to resolve. | ID | Observable claim / deciding procedure | Evidence kind | Environment | Mode | Milestone / owner | Status | | --- | --- | --- | --- | --- | --- | --- | diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index f8095931..a29af49d 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -176,3 +176,142 @@ This report records agent experience using the Coding-Standards MCP during Acqui The primary integrator, independent architecture reviewer, native-custody contributor, native-custody independent reviewer, shared-owner design reviewer, and Q1 next-slice architecture investigator used the MCP and are represented above. The bounded source-inventory reviewer did not use it, so there is no MCP usability report from that agent. The architecture reviewer’s narrow follow-up inspected the repaired current source without making a new MCP call and reused the prior routed obligations; its findings confirm the two identified integrity paths are closed at source level, not that Q1 is accepted. The MCP `review` operation is an authoring workflow for changes to the standards corpus; it does not review application source. Application compliance was checked by final routing, standards reads, source/test inspection, and the independent code review. + +## Import-receipt feasibility investigator — GPT-6.1 Sol High + +- **Useful calls:** Reused the existing routing facts and selected snapshot for a focused route over persistence, architecture/replay, contract evolution/schema, concurrency, Rust/API/async and verification concerns. Snapshot `snapshot:v1:869927cd-0134-4833-89c9-c545393f1dff` selected 21 standards with zero unresolved facts. Focused reads of Persistence and Architecture informed the separation between the model-owned completion proof and acquisition's exact lease/workspace custody. A final same-snapshot route confirmed the design scope. +- **Confusing or redundant steps:** No confusing or redundant operation was reported beyond long full policy text; source-specific questions were not answered by another MCP call. +- **Missing context:** The MCP did not expose the exact schema-6 partition writer, importer effects, held root grant, or orphan adoption call graph. Those required source inspection. +- **Where the investigator left MCP:** Candidate identity, schema decoder behavior, finalizer publication ordering, and concrete required tests were established in repository source, not by routing. +- **Smallest sufficient workflow:** Reuse the fact vocabulary/snapshot, route the exact slice, read two focused policies, reroute the final design, then verify the actual source boundaries and evidence separately. +- **Recommendations:** Offer section-scoped policy reads and make the distinction between a complete standards route and source/evidence qualification explicit. + +## Primary integrator — initial combined receipt/guard routing, later split by source review + +- **Useful calls:** I refreshed `routing_facts` at snapshot `snapshot:v1:af0c1e53-2199-4992-9b93-a00d526934fe`, then routed persistence, schema/evolution, replay, concurrency, delegated-execution security, test-oracle, verification, documentation and commit facts. The final same-snapshot route selected 32 standards with zero unresolved facts. Focused reads of Persistence, Contract Evolution, Architecture Replay, Concurrency, Untrusted Execution, Independent Test Oracles and Commit Workflow made conditional publication, exact replay authority, effect-boundary guards and candidate-matched tests explicit. +- **Confusing or redundant steps:** `ALL_TOOLS` keyword discovery returned long descriptions for both app-code routing/read operations and standards-authoring operations. The first route omitted the explicitly required empty framework category and returned `needs-facts`; correcting that on the same snapshot was straightforward. A 21-policy `read_many` produced more text than the visible output window, so I read the seven relevant policies in two smaller batches. Repeating the same complete route to inspect the selected count was unnecessary but provided a precise record. +- **Missing context:** The MCP did not know the current `d0b71b5` source boundary, schema-6 partition decoder, exact `Using` lease, importer side effects, metadata durability, mutation authority, or orphan-adoption entry points. Repository source and the separate exact-candidate design review are establishing those facts. Routing does not decide receipt schema/ownership or prove behavior. +- **Where I left MCP:** Git/PR/CI state, store and importer ownership, publication failure semantics, all adoption paths, and local test evidence came from repository/GitHub inspection. No standards-corpus authoring, attestation, or approval operation was performed. +- **Smallest sufficient workflow:** Capture the fact vocabulary once, route the exact library/persistence/lifecycle/security/verification/commit scope with every required category explicit, read three or four applicable policies at a time, inspect source and tests, and reroute the final candidate after implementation. +- **Recommendations:** Filter initial tool discovery to application-routing/read operations before exposing authoring tools; keep selection-only routing compact; surface required empty applicability facts before policy evaluation; cap `read_many` by a practical byte budget; show the selected count without another call; and provide section-scoped policy reads. Continue to distinguish a complete applicability route from implementation evidence and acceptance. + +This is MCP usability feedback only. The Q1 product boundary and its acceptance state are recorded in the acquisition plan and execution ledger, not inferred from this report. + +## Primary integrator — current importer/adoption custody-guard slice + +- **Useful calls:** A fresh `routing_facts` snapshot (`snapshot:v1:9881b6be-4a0e-4d72-b0cd-4e4307cdc710`) and a complete route selected 26 standards with zero unresolved facts. Focused `read_many` covered Concurrency, Rust Async, Persistence, Rust API, Untrusted Execution, and Verification. This narrower route made effect-boundary ownership, cancellation retention, configured-authority refusal, and negative lifecycle evidence applicable without carrying receipt/output-proof policies into the current implementation slice. +- **Confusing or redundant steps:** The same empty `routing.frameworks` value had to be supplied explicitly despite no framework being used. The earlier combined route and broad 21-policy read remain separate historical discovery for receipt feasibility; neither is needed to implement this guard. The current focused reads were sufficient. +- **Missing context:** Routing did not identify the common importer shortcut, Diffusers delegation, HF use lease, already-held root grant, exact queue admission, or the unconfigured `ModelLibrary::new` path. Source inspection and a separate read-only architecture review supplied those facts. +- **Where I left MCP:** The precise private capability contents, caller cancellation behavior, and stale-scan race are grounded in Rust source and tests. The MCP supplied obligations, not the design choice or evidence. +- **Smallest sufficient workflow:** Refresh the facts once when the slice boundary changes, route the exact guard scope without inline policy text, read the six focused obligations in one bounded call, inspect source and lifecycle tests, then route the final implemented candidate. +- **Recommendations:** Make applicability-empty facts visible before route evaluation; retain the selection-only route and compact policy reads; allow a slice-oriented policy view that separates immediate obligations from later-slice closure; keep route completion distinct from source/test compliance. + +This MCP usability report is separate from product acceptance. The guard was subsequently implemented on candidate `8b6c5f70c55a3d124189d0cd88dc85c780f47c84`; it remains under independent review and is not an accepted Q1 gate. + +## Primary integrator — revised partial-HF-stage custody guard route + +- **Useful calls:** After source inspection changed the guard boundary, I refreshed `routing_facts` and captured a new snapshot (`snapshot:v1:0ed9c25a-060e-4970-a5ca-a7286eda1601`). A complete route for the revised implementation scope selected 28 standards with zero unresolved fact categories. Focused reads included Independent Test Oracles, Code Design and Ownership, Commit Workflow, Verification, Rust, Architecture, and Security; earlier reads on the same guard work covered Concurrency, Rust Async, Persistence, Rust API, and Untrusted Execution. This surfaced the one-owner, exact-effect-boundary, cancellation-custody, typed-unavailable, and decision-fixture obligations relevant to the partial metadata stage. +- **Confusing or redundant steps:** The no-framework fact still needed an explicit `known-absent` value. A seven-policy `read_many` returned roughly 40,000 tokens of full policy text, exceeding the visible output window and truncating the result; compact reads do not provide a practical section-level summary for a narrow code slice. Routing the complete set was still useful, but rereading long whole policies was not. +- **Missing context:** The MCP did not identify the HF partial-stub call before weights, the fact it also indexes, or that index projection can rewrite metadata and custom runtime projections. Repository tracing established those concrete effects and showed that the design needed a separate exact `Transferring` stage capability. +- **Where I left MCP:** Source call chains, queue admission, stage timing, existing owner contexts, root grants, index side effects, tests, and Git/PR state were established from repository and service evidence. The MCP supplied applicable obligations; it did not choose or validate the code design. +- **Smallest sufficient workflow:** Refresh the fact vocabulary at the changed slice boundary; complete one exact route with all required categories; read only the handful of policy sections for ownership, persistence, async cancellation, delegated authority, and verification; inspect the source; then route the final implemented candidate. +- **Recommendations:** Add section-scoped obligation reads with an output-byte budget; show required empty applicability values before route evaluation; make the selected set and unresolved count visible in the initial response; and distinguish immediate slice obligations from downstream closure. Keep source and test evidence explicitly outside the route result. + +This MCP usability feedback is separate from product acceptance. The revised partial-stage guard was implemented on candidate `8b6c5f70c55a3d124189d0cd88dc85c780f47c84`; implementation evidence does not certify its acceptance. + +## Q1 importer custody-guard implementer — GPT-6.1 Sol High + +- **Useful calls:** `describe_input` and `routing_facts` exposed the exact registered fact vocabulary; snapshot-bound routes made the scope reproducible. Focused reads covered Concurrency, Rust Async, Persistence, Rust API, Untrusted Execution and Verification. The worker rerouted after the partial-stage capability and aggregate-effect lifecycle boundaries changed, then routed the final candidate. The final route selected 24 standards with zero unresolved categories. +- **Confusing or redundant steps:** Discovery and schema descriptions were large, and some MCP results duplicated content between textual and structured forms. This was cumbersome; consuming the structured result avoided part of the duplication. No standards-corpus mutation or application-source attestation operation was useful or performed. +- **Missing context:** The MCP did not know the repository call graph, exact `TaskContext` lifecycle, admitted write set or candidate diff. Those were established locally and translated into registered routing facts. +- **Where the worker left MCP:** It used source inspection, tests, Git and local evidence for implementation and verification, returning to MCP when the design boundary materially changed and for final routing. Routing supplied obligations but did not certify the code. +- **Smallest sufficient workflow:** Discover input shape and routing facts once; route known facts on a pinned snapshot; read focused applicable policies; implement and verify locally; reroute when ownership changes and against the final candidate. +- **Recommendations:** Make discovery/results compact, avoid duplicate text and structured bodies, allow concise routing summaries with selected IDs and unresolved questions, and provide a clearly non-authoritative place to associate candidate/tree/write-set context with an evidence record. + +## Q1 importer custody-guard independent reviewer — GPT-6.1 Sol High + +- **Useful calls:** The reviewer reused `routing_facts` snapshot `snapshot:v1:9881b6be-4a0e-4d72-b0cd-4e4307cdc710`, ran a selection-only route with implementation/verification, library, persistence, Rust/API/async/security/cross-platform, architecture/concurrency/contracts/security/resilience/diagnostics and code-design/evolution/delegated-authority/oracle details, and received 24 selected standards with zero unresolved facts. Two focused `read_many` calls covered Rust Security, Untrusted Execution, Independent Test Oracles, Concurrency, Rust Async and Persistence. These obligations helped assess resource/stage authority, coherent reads, held capabilities, effect drainage, failure ownership and no-effect tests. +- **Confusing or redundant steps:** No MCP errors or schema-discovery calls were needed. Route explanations and repeated continuation suggestions were verbose. +- **Missing context:** The MCP did not expose the repository HF effect envelope or capability-relative metadata publisher. Source inspection was needed to trace those actual boundaries. +- **Where the reviewer left MCP:** Candidate identity, diff, source lifecycle and test claims were checked with Git and repository tools. The reviewer ran `git diff --check`, but no build or tests. +- **Smallest sufficient workflow:** Reuse the pinned snapshot, route the concrete review scope, read focused policies, then inspect the exact diff and tests independently. +- **Recommendations:** Return a compact selected-policy/count view by default and provide task-focused policy excerpts while keeping source and evidence qualification outside the routing result. + +The review found no P0–P3 issue in its slice. That source-review conclusion and the writer-reported test evidence remain separate. + +## Q1 receipt/reopen design reviewer + +- **Useful calls:** The reviewer used one initial route and a final route on snapshot `snapshot:v1:968b380d-c352-496b-9a5f-f522222e09ab`; both selected 21 standards with zero unresolved questions. Two focused `read_many` calls read Persistence and Replay, then Contract Evolution and Schemas. Those obligations clarified that a receipt follows durable output publication, that current projections cannot stand in for exact historical completion, and that the receipt format must be explicit and versioned. +- **Confusing or redundant steps:** No vocabulary rediscovery was needed. Whole-policy reads were long, and the final route repeated the same facts because the slice workflow requires a final route. +- **Missing context:** The MCP could not identify actual schema writers, metadata no-op publication, startup index rebuild, cache fingerprint inputs, custody guard, or cancellation order. Source inspection was necessary and cannot establish deployed retained-state population. +- **Where the reviewer left MCP:** It inspected the repository for each persistence writer, output effect, migration/reopen path, startup rebuild, package-facts cache, and cancellation boundary; no tests or edits were performed. +- **Smallest sufficient workflow:** Route retained facts, read four focused policies, inspect source lifecycle, then route the final design. +- **Recommendations:** Add section-scoped policy reads, compact obligation references, and an explicit result label saying routing is complete while implementation and evidence remain unverified. + +This design review is not product acceptance. Its source audit found no existing exact importer completion receipt; schema-7 implementation, cold-reopen tests, and independent final-candidate review remain required. + +## Primary integrator — Q1 receipt/reopen slice routing + +- **Useful calls:** A fresh `routing_facts` snapshot and route selected 28 standards with zero unresolved categories for persistence/replay, schema evolution, concurrency, Rust APIs/async/security, implementation, verification, planning and commit obligations. Focused reads on the same snapshot covered Persistence, Replay, Schemas, Contract Evolution, Concurrency, Code Design, Rust Async, Untrusted Execution, Verification Oracles, Resilience, Rust API, Commit, Platform Verification, Implementation and Documentation. Explicit policy text made the transaction boundary, old-reader handling, exact publication, platform evidence scope and commit obligations concrete. +- **Confusing or redundant steps:** The first five-policy read response was too large for the visible output window, duplicating full policy text across structured and text results; later small batches were more useful. The complete route is necessary at slice start, but verbose continuation explanations added little. +- **Missing context:** Routing could not tell that schema-6 `legacy` flattening would either drop or contaminate a receipt, that current schema-6 import metadata is capability-published but can no-op without a fresh barrier, or that package-facts fingerprints omit payload bytes and serialize `HashMap` metadata. Repository source and a read-only feasibility review established those facts. +- **Where I left MCP:** Exact reader/writer inventories, output formats, startup rebuild effects, marker order, and current CI/PR state came from source/GitHub tools. The package-facts fingerprint investigation and feasibility review are design evidence, not MCP evidence. +- **Smallest sufficient workflow:** Snapshot-bound route once; read the selected ownership, schema, replay, async, security and oracle policies in small focused batches; inspect exact source; reroute the implemented candidate before handoff. +- **Recommendations:** Add section-scoped reads with a practical byte budget; avoid returning policy bodies twice; surface the selected count and unresolved questions in compact form; keep routing status distinct from implementation evidence and acceptance. + +This usability record is separate from product acceptance. At the time of this entry, the local candidate did not yet implement a receipt or reopen path; a later uncommitted worktree update is now being verified. + +## Primary integrator — resumed Q1 acquisition and native-consumer route + +- **Useful calls:** Reused the accepted snapshot `snapshot:v1:425e276a-fc0b-4282-9889-a76742027c42`, refreshed its fact definitions with `routing_facts`, and completed the route for launcher/library consumers, durable receipts, async lifecycle, security, platform/UI evidence, implementation, planning, release and commit work. The route selected 43 standards with zero unresolved facts. Paginated policy reads returned the full selected set in six bounded batches. +- **Confusing or redundant steps:** The first expanded route remained `needs-facts` until the workflow-profile condition was answered. The valid answer was `known-absent` for concurrent plan integration in this serial Q1 slice. The route accepted that fact and completed on the same snapshot. Policy retrieval is bounded by record count, but each selected item still contains its full policy text; reading 43 policies required six calls and substantial output handling. +- **Missing context:** The standards service did not expose repository state, current acquisition schema, the previously uncommitted receipt implementation, production RPC composition, or the Passeur service failure. These came from Git/source and separate Passeur diagnostics. Routing did not establish any implementation or acceptance claim. +- **Where I left MCP:** The route and policy reads supplied obligations only. Source review, test evidence, GitHub PR/CI state and exact plan-gate status remain separate repository or service observations. +- **Smallest sufficient workflow:** Reuse the accepted snapshot when its authority still applies; refresh facts, state every conditional applicability question explicitly, route, then page selected policy text in small batches. Reroute the completed source/evidence boundary after implementation. +- **Recommendations:** Let callers request focused policy sections from a completed route, preserve the compact selected-ID/count summary while paging, and show the exact missing fact prompt in a directly actionable form. Continue to label routing as obligations rather than implementation certification. + +This usability feedback is separate from product acceptance. The schema-7 receipt implementation and native consumer changes remain subject to source review and their required real-consumer evidence. + +## Independent Q1 composed reviewer — GPT-6.1 Sol High + +- **Useful calls:** The pinned standards snapshot supported fact discovery, routing, and focused reads. The reviewer routed 29 standards with zero unresolved facts. A focused three-policy `read_many` returned complete, usable policy text. +- **Confusing or redundant steps:** Requesting 32 full policy texts exceeded the visible output window. The smaller focused read was easier to use. +- **Missing context:** Routing supplied applicable obligations but did not provide repository-specific source facts or establish implementation compliance or product acceptance. +- **Where the reviewer left MCP:** The reviewer inspected the exact current diff and source independently. It did not run tests or builds. +- **Smallest sufficient workflow:** Reuse the pinned snapshot, route the focused review scope, read a small number of relevant policies, then verify source and evidence separately. +- **Recommendations:** Keep complete policy reads bounded by output size; label a completed route as obligations only, separate from source compliance and acceptance. + +This reviewer’s source findings are recorded separately from the MCP observations and do not certify the candidate. + +## Shared native-consumer contributor — GPT-6.1 Sol Medium fallback + +- **Useful calls:** After correcting the route tool's input shape, the contributor completed a focused standards route and used its selected obligations to inform implementation. +- **Confusing or redundant steps:** `describe_input({tool: "route"})` failed because the operation discriminator was missing and `tool` was an extra field; `describe_input({operation: "route"})` worked. Invented fact keys `language` and `scope` were rejected with a generic `/facts` message that did not identify the invalid keys; the contributor had to discover `routing_facts`. Route output repeated full content in both text and structured results, risking truncation. +- **Missing context:** The route did not identify the repository-specific acquisition service boundary; the contributor found that in source. Its route selected 28 standards and left two facts unresolved; the primary integrator separately completed the broader accepted route with zero unresolved facts. +- **Where the contributor left MCP:** It reported route obligations, not whether source changes complied or passed product acceptance. +- **Smallest sufficient workflow:** Describe the operation using its actual discriminator, retrieve the supported fact vocabulary before routing, resolve all returned facts explicitly, and keep full policy reads bounded. +- **Recommendations:** Make input descriptions work with the public tool name or provide a direct schema index; report each invalid fact key and valid alternatives; avoid duplicating large route bodies in text and structured output; retain the route's unresolved-fact count and distinguish obligations from evidence. + +This feedback documents actual MCP usability only; it is not an implementation approval. + +## Primary integrator — final Q1 candidate route + +- **Useful calls:** Reused the accepted snapshot `snapshot:v1:425e276a-fc0b-4282-9889-a76742027c42`, refreshed its registered fact vocabulary, and routed the final implementation/verification scope. It selected 40 standards with zero unresolved questions. The route remained compact and returned actionable policy IDs. +- **Confusing or redundant steps:** A six-policy compact `read_many` still returned full policy bodies, exceeding the visible output window. Earlier bounded batches and the already-read policies supplied the remaining applicable obligations. +- **Missing context:** The route did not know the final native attempt identity, local HTTP fixture, current CI head, or platform execution results; those were checked in source, test output, and repository state. +- **Where I left MCP:** The route supplied obligations only. Review, exact source checks and test evidence were performed independently. +- **Smallest sufficient workflow:** Refresh facts, complete a snapshot-bound route, read only the relevant small policy subset, and check implementation/evidence separately. +- **Recommendations:** Add section-scoped reads and budget the response by serialized bytes, not number of policy records. Return selected IDs/count without repeating full bodies across structured and text output. + +This is final-route usability evidence, not a Coding-Standards compliance or product-acceptance result. + +## Primary integrator — resumed exact-candidate route + +- **Useful calls:** A fresh route on snapshot `snapshot:v1:10b3e01e-6a86-4185-83e0-65ca7b0db870` selected 44 standards with zero unresolved questions. Two bounded `read_many` calls read the focused `core`, persistence, and concurrency policies. +- **Confusing or redundant steps:** The earlier snapshot handle was unavailable after session restart, so the route facts had to be refreshed and the same scope routed on a fresh handle. The small policy reads returned complete policy text and remained usable. +- **Missing context:** The standards service still supplied obligations rather than the actual Git diff, candidate test output, Passeur service health, or hosted CI status; these were checked separately. +- **Where I left MCP:** The current route and focused policy reads inform the final candidate obligations. They do not establish source compliance or gate acceptance. +- **Smallest sufficient workflow:** Refresh routing facts after a session restart, route the actual implementation/verification scope on the new snapshot, then read only the relevant policies in small batches. +- **Recommendations:** Provide a safe snapshot-refresh path after session loss and retain the clear distinction between standards obligations and independent source/test evidence. + +This entry records the resumed candidate route only; implementation review and acceptance evidence remain separate. diff --git a/docs/plans/artifact-acquisition/reports/dependency-gates.md b/docs/plans/artifact-acquisition/reports/dependency-gates.md index 45f71b9a..0c4a1241 100644 --- a/docs/plans/artifact-acquisition/reports/dependency-gates.md +++ b/docs/plans/artifact-acquisition/reports/dependency-gates.md @@ -1,7 +1,7 @@ # Acquisition prerequisites and runtime handoffs **Owner:** acquisition integration, with one serial cross-plan integrator. This is the single status record for acquisition-provided gates. The runtime plan references these rows and does not independently declare them ready. -**All gates:** not ready. No production evidence exists for the proposed implementation. +**All gates:** not ready. Local candidate tests exist, but no objective-level production acceptance evidence has been recorded for this candidate. | Gate | Provider milestone / claims | Required consumer observation | Unblocks | Current status | | --- | --- | --- | --- | --- | diff --git a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md new file mode 100644 index 00000000..9cf09000 --- /dev/null +++ b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md @@ -0,0 +1,84 @@ +# Passeur MCP usability feedback + +This report records observed contributor-tool usability, separately from +artifact-acquisition implementation and acceptance evidence. It does not claim +that Passeur has been notified or that any proposed improvement has shipped. + +## 2026-09-30 — repository service unavailable + +### Reproduction + +From the Pumas-Library MCP session: + +1. `passeur_agents({ limit: 100 })` and + `passeur_tasks({ schema_version: 1, limit: 100 })` return argument + validation errors. The backend reports maximum limits of 4 and 16, + respectively, while the published tool declarations do not state those + bounds. +2. Retry with `passeur_agents({ limit: 4 })`. It returns + `PATH_NOT_FOUND` from `profile.open` without identifying the missing + profile resource. +3. Retry with `passeur_tasks({ schema_version: 1, limit: 16 })`. It returns + `SERVICE_PROFILE_CONFLICT` with “Use the same approved profile path for + clients of this repository service”. +4. Call `passeur_status({})`. The frontend is running, but the repository + service is unavailable with `SERVICE_PROFILE_CONFLICT`. + +No task was submitted or started during this reproduction. Agent availability +could not be inspected because the valid agent-list request failed. + +### Suggested improvements + +- Publish the actual maximums in the tool schemas so callers can validate + requests before sending them. +- Make `PATH_NOT_FOUND` identify which profile lookup failed and give a safe, + concrete recovery action. +- Make profile-conflict diagnostics identify the conflicting approved profile + selection, or provide a non-sensitive correlation identifier and a supported + way for the operator to find the required value. The current message says to + use the same path but does not reveal which path the service expects. +- Return one consistent service-unavailable diagnosis from agent discovery, + task listing, and status. In this run those calls produced two different + errors for what status reported as an unavailable repository service. + +### Evidence limits + +These observations are from one repository/session configuration. They show +the errors returned by the tools in that session; they do not establish the +cause of the approved-profile mismatch or whether another configured profile +would work. + +## 2026-09-30 — retry after reported service recovery + +After the user reported that Passeur should work, the current session retried +`passeur_agents({ limit: 4 })`, +`passeur_tasks({ schema_version: 1, limit: 16 })`, +`passeur_status({})`, `passeur_prepare({})`, and the coordination identity +read. Agent discovery again returned `PATH_NOT_FOUND` from `profile.open`; +task listing, prepare, and identity returned `SERVICE_PROFILE_CONFLICT`. +Status now reports the repository service as unavailable with that conflict. +No task was submitted. Because the available tools could not provide an agent +ID or bind a coordinated request, this implementation continued through the +authorized GPT-6.1 Sol Medium fallback instead of guessing an agent or +submitting to an unverified worker. + +## 2026-09-30 — status after session restart + +After the user restarted the session, `passeur_status({})` reported a running +frontend with a new process/build identity, but the repository service still +returned `SERVICE_PROFILE_CONFLICT` and “Use the same approved profile path for +clients of this repository service”. No agent discovery or task submission was +attempted in this check. The user authorized GPT-6.1 Sol implementation until +the repository service becomes available, so the cancellation recovery repair +continues under that fallback. + +## 2026-09-30 — fresh repository-service check + +In the resumed session, `passeur_status({})` still reports the repository service +unavailable with `SERVICE_PROFILE_CONFLICT`. A valid `passeur_agents({ limit: 4 })` +request still fails at `profile.open`, now including the generic next action to +check configured path/access and preserve the existing namespace; it still does +not identify the unavailable profile. An initial request above the documented +maximum (`limit: 50`) was rejected by validation, confirming the agents bound is +4. No task was submitted, and implementation remains on the user-authorized +GPT-6.1 Sol fallback until the repository service becomes available. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index ae9f3a7d..c3f7be69 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -4,7 +4,7 @@ These are the admitted exact paths/closed path families. The actual source files ## Q1: one HTTP acquisition owner with real consumers -**Canonical module paths:** `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs,http.rs}` currently hold validated source-neutral selections and the HTTP representation/body-streaming protocol. The GitHub release adapter is `acquisition/github_release.rs`, with its fresh asset-metadata resolver in the existing `network/github.rs` owner; the public/cache release DTO remains unchanged. The adapter maps publisher asset identity/digest evidence into a verified manifest while keeping the retrieval URL ephemeral. The remaining Q1 design still needs one durable transfer owner and capability workspace; place those with the canonical module rather than creating another downloader. Source-reader adaptation may remain in that module or a focused `acquisition/sources/` child if the actual design supports it. +**Canonical module paths:** `rust/crates/pumas-core/src/acquisition/{mod.rs,manifest.rs,http.rs,service.rs,store.rs,workspace.rs}` now hold validated source-neutral selections, the HTTP representation/body-streaming protocol, the durable lifecycle/store, and capability workspace. The GitHub release adapter is `acquisition/github_release.rs`, with its fresh asset-metadata resolver in the existing `network/github.rs` owner; the public/cache release DTO remains unchanged. The adapter maps publisher asset identity/digest evidence into a verified manifest while keeping the retrieval URL ephemeral. The local Q1 candidate has one durable transfer owner; consumer-specific receipt interpretation stays with its existing model or native owner. Source-reader adaptation may remain in this module or a focused `acquisition/sources/` child when an accepted source requires it. **Existing owners allowed to change:** - `rust/crates/pumas-core/src/lib.rs`, `network/{mod.rs,download.rs,github.rs}`, `models/github.rs`, `model_library/hf/{mod.rs,download.rs,lifecycle.rs,types.rs,metadata.rs}`; @@ -15,17 +15,17 @@ These are the admitted exact paths/closed path families. The actual source files - `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,ollama.rs,progress.rs,state.rs}` for the acquisition bridge and transfer progress, not installed-unit identity migration; - current core atomic JSON/capability-filesystem modules only where the same selected invariant requires a targeted change, never as an unrelated filesystem rewrite. -The current root-owned extraction-preparation sub-slice is limited to `rust/crates/pumas-core/src/model_library/{download_recovery.rs,hf/lifecycle.rs}` and the co-located lifecycle regression. It adds a physical-root equality key and lets the existing supervised lifecycle owner hold distinct root grants concurrently while coalescing independently reopened handles for the same root. It does not move the lifecycle owner, add a transfer/persistence authority, or advance AQ-HTTP. +The completed root-owned extraction-preparation sub-slice changed `rust/crates/pumas-core/src/model_library/{download_recovery.rs,hf/lifecycle.rs}` and its co-located lifecycle regression. It adds a physical-root equality key and lets the existing supervised lifecycle owner hold distinct root grants concurrently while coalescing independently reopened handles for the same root. It did not move the lifecycle owner or add another transfer/persistence authority. -The isolated native-custody worker proposal is complete and integrated from `work/q1-native-custody` commit `560cec71` into the Q1 branch. Its exact five-file write set is `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs,ollama.rs,state.rs}` and `rust/crates/pumas-rpc/src/server.rs`. The `ollama.rs` extension closes the public wrapper's owned state-mutation drain. It repairs native install/removal custody, metadata coordination, pending file-I/O settlement, and fallible stage cleanup; it does not change the source/download selection algorithm or replace the independent Ollama downloader. Shared acquisition consumer cutover remains outstanding. +The native-custody prerequisite was integrated from `work/q1-native-custody` commit `560cec71`. Its exact five-file write set was `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs,ollama.rs,state.rs}` and `rust/crates/pumas-rpc/src/server.rs`. The `ollama.rs` extension closes the public wrapper's owned state-mutation drain. It repairs native install/removal custody, metadata coordination, pending file-I/O settlement, and fallible stage cleanup; it did not replace the independent Ollama downloader. The current candidate separately routes llama.cpp through the shared acquisition consumer; current-head review and acceptance remain pending. **Tests:** proposed `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/tests/artifact_acquisition_install.rs`, plus existing co-located GitHub metadata, HF lifecycle/recovery, and installer regression tests. Fixtures must reach the owner under test with independent expected byte/effect outcomes. **Serial adjacent writes:** `rust/crates/pumas-rpc/src/contract.rs`, `contract/export.rs`, affected HF/version/status handlers, `electron/src/{preload.ts,rpc-method-registry.ts,ipc-validation.ts}`, actual corresponding frontend download/install/source views and generated DTOs. Enumerate outputs from the actual exporter rather than guess or edit generated files manually. Reuse the existing model/native UI; this is not a dashboard redesign. -### Exact next worker admission — durable acquisition owner and Hugging Face cutover +### Completed worker admission — durable acquisition owner and Hugging Face cutover -Parent milestone: `work/acquisition-q1-http`, draft PR #7, target `main`. The worker starts from the exact primary documentation head `efc4d20bde6ae426c96f6a5fcb55ce0022788d09`, in a task-owned branch/worktree. The primary integrator owns PR history, the shared semantic contract, gate state, later native composition, review and final candidate evidence. +Parent milestone: `work/acquisition-q1-http`, draft PR #7, target `main`. Worker branch `agent/q1-hf-acquisition` started at `efc4d20bde6ae426c96f6a5fcb55ce0022788d09` and delivered `65274dffd90e9c5272a89ec5a3e9c1ae564bfa0c`. It was integrated by merge commit `bc9e9da4147a3e64f2a5e67093604eddeddc5391`; root then committed the cross-record custody repair `d0b71b51075815cf307cea95c9364be0ed25472d`. The primary integrator owns PR history, the shared semantic contract, gate state, later native composition, review and final candidate evidence. Primary worker write set: @@ -37,12 +37,51 @@ Primary worker write set: The worker implements one durable neutral acquisition owner/store and makes the ordinary HF path consume its verified-file handoff through awaited import/finalization. Schema 6 extends the same canonical file and preserves all current model snapshots, hidden admission attempts, revocations, queues, release proofs and quarantines. Fresh roots use schema 6; ordinary open/admission does not migrate schema 4/5. A separate explicit offline migration operation is required, exposed independently from normal API construction; its caller must ensure all old readers and writers are stopped. Without migration, acquisition admission reports a typed migration-required result and performs no transfer effects. The model mutation authority must read the canonical store's custody view. One store file/transaction authority remains; no second transfer writer, model marker initializer in neutral workspace code, inferred v4/v5 compatibility, cross-store exactly-once claim, or new public wire representation is allowed. The worker does not modify `docs/contracts/artifact-acquisition.md`, plan/gate/ledger files, RPC/generated/frontend outputs, `pumas-app-manager`, Cargo manifests/lockfiles, or package/S3/runtime/adapter state. Report any discovered contract or scope change before editing outside this set. -Required handoff evidence: exact branch/base/head and path list; representative schema 4/5 fixture inventory; prove ordinary open/admission leaves legacy fixture bytes unchanged and returns migration-required; invoke the separate explicit migration operation and prove v6 reopen without live roots; single-writer and concurrent-instance tests; preservation/refusal of every hidden custody and Pending-cleanup state; controlled HTTP acquisition through the actual HF workflow; cancellation, dropped observer, shutdown and lease-through-import cleanup; consumer commit/reopen settlement behavior; focused store/recovery/HF/acquisition tests; `pumas-library` Clippy, no-default-features and formatting; and limitations separate from mocks/local fixtures. This HF-only vertical cutover is not AQ-HTTP acceptance; native llama.cpp must later consume the same service on the same PR before that gate can be considered. +Worker handoff evidence completed for schema-4/5 fixture inventory, read-only normal admission, explicit offline fixture migration/reopen, legacy custody/Pending preservation, controlled HTTP through ordinary HF acquisition/import, cancellation and lease drainage, focused store/recovery/HF/acquisition tests, Clippy, no-default-features and formatting. The exact consumer-commit/reopen proof remains open: `Using` without a durable importer receipt fails closed, but no receipt yet exists. Root added and tested the schema-6 duplicate-demand/active-workspace validator after independent review. This HF-only vertical cutover is not AQ-HTTP acceptance; importer receipt/reopen, orphan guard, native shared acquisition, and the plan matrix remain outstanding. **Docs:** the canonical shared contract, acquisition plan records, bounded handoffs in the existing HF/Rust remediation and upstream-runtime plans, and current architecture/development documentation where behavior has landed. Preserve the `docs/breif` path spelling; do not rename unrelated source-intent files. **Forbidden:** runtime installation-ID/profile migration, new model-adapter registry, package dependency reinterpretation, arbitrary source/plugin execution, consumer data deletion, claims of completed Pending replay, and concurrent second writers to the same transfer state. +### Completed primary slice — importer and orphan-adoption custody guard + +Parent milestone: `work/acquisition-q1-http`, draft PR #7 to `main`. Worker commit `8b6c5f70c55a3d124189d0cd88dc85c780f47c84` was based exactly on primary candidate `d0b71b51075815cf307cea95c9364be0ed25472d`, reviewed read-only, and integrated by merge commit `9719ecb50d84df796db887832b87b42e8992988a`. The resulting tree `0c3e472c8b5f05073ede4ddbe2252a2a34e54baf` is identical to the worker tree. The worker branch/worktree remain task-owned; no source was copied or rewritten. Independent review found no substantiated P0–P3 issue and ran only `git diff --check`; all test results are worker-reported and recorded in the execution ledger. The root integrator owns PR/Git history, docs, composed verification and the next slice. + +Primary write set: + +- Source-neutral exact operation/use identity evidence and current-record validation: `rust/crates/pumas-core/src/acquisition/{service.rs,store.rs}`; permit the one crate-private validation visibility change in `acquisition/workspace.rs` so a proof can revalidate held workspace identity without model-specific paths. HF admission and stage capabilities remain owned by `model_library` and do not add model-specific policy to acquisition. +- One model-side snapshot of current legacy queue/hidden/quarantine and acquisition custody through the existing canonical publisher: `rust/crates/pumas-core/src/model_library/download_store.rs`. +- The shared guard and exact managed-HF exception using the existing root grant: `rust/crates/pumas-core/src/model_library/mutation_authority.rs`. +- Owned ordinary import and narrowly admitted managed-HF finalization before idempotency shortcuts or Diffusers delegation: `rust/crates/pumas-core/src/model_library/importer.rs`. +- Retain the existing root grant through model metadata/index/classification effects: `rust/crates/pumas-core/src/model_library/library.rs`. +- Pass exact current HF operation evidence and the already-held execution grant at the existing partial metadata/index stage and normal/restored finalization: `rust/crates/pumas-core/src/model_library/hf/download.rs`. +- Canonical guard behavior and these owning records: `docs/contracts/artifact-acquisition.md`, `docs/plans/artifact-acquisition/{plan.md,execution-ledger.md,issues.md,reports/write-sets.md}`. Standards usability remains in its separate report. +- Co-located ownership/lifecycle regressions; `rust/crates/pumas-core/tests/artifact_acquisition.rs` only if the builder-level production path needs a cross-module fixture. + +The worker edited only the listed production paths and directly co-located unit tests. It did not edit gate/plan records, generated/RPC/frontend files, public API contracts, schemas/migrations, manifests/lockfiles, unrelated lifecycle owners, or shared files assigned to another writer. The handoff includes exact source commit/tree and diff, real temporary-filesystem/store/owner evidence distinguished from synthetic faults, cancellation/root-grant/failure observations, final standards route, review focus, and a clean task worktree. The branch was integrated through a history-preserving merge. + +No startup/state/reconciliation caller change is admitted unless source evidence shows it can still bypass the common importer effect boundary. `importer/recovery.rs` may receive co-located scan/test changes, but the decisive recheck stays in the common in-place import path. Existing `RuntimeTasks` owns ordinary imports; the acquisition `TaskContext` owns managed HF finalization. Do not add a registry, runtime, sidecar store, public DTO, SQLite/metadata schema, lockfile, generated interface, app-manager, adapter, or frontend path. Direct importer calls built from `ModelLibrary::new` without installed trusted mutation authority fail closed with the typed authority-unavailable result; no unguarded fallback is permitted. + +The guard must acquire/reuse the configured root grant, revalidate the exact model destination, and inspect durable custody immediately before any metadata-present shortcut, indexing, Diffusers delegation, or other importer effect. A public/orphan import has no custody exemption. Preserve the existing partial HF metadata/index stub: it may use only a private capability matching the current queue admission and `Transferring` generation, manifest/demand, workspace, destination, and held root grant, and that capability is limited to the stub upsert plus its index projection (including metadata projection performed by indexing). It cannot authorize full import, package-fact resolution, or another target. Full HF finalization separately requires its current verified-file use lease and matching admission. Neither HF capability may bypass unrelated hidden/queue custody or Pending/quarantined state. Stale stage, lease, wrong demand/workspace/manifest/destination, or unrelated custody refuses. Legitimate FIFO followers must not prevent their admitted predecessor from finalizing. The held owner context and root grant remain alive until all nested importer effects settle, including when the caller stops waiting or a worker panics. + +Required evidence on this completed slice: actual stale orphan-scan/acquisition-admission race; matching and stale/wrong partial and final HF capabilities; matching/unrelated retained phases; hidden, Pending and quarantine refusal; metadata-present and Diffusers branches; FIFO follower preservation; cancellation/drop during held metadata/index effects; shutdown/panic drainage; root/destination replacement; unconfigured direct importer refusal; and existing HF regressions. Worker results and evidence-kind distinctions are recorded in `execution-ledger.md`. This guard closes the importer custody bypass only; it does not make reopened `Using` recoverable or open AQ-HTTP. + +### Completed Q1 slice — exact HF finalization receipt and no-replay reopen + +The custody guard is reviewed and integrated. The root integrator routed this exact replay/persistence slice through Coding-Standards before source edits and inspected the document/output lifecycle. All production `downloads.json` publication converges on `AcquisitionStore` transactions: neutral updates preserve model state; `DownloadPersistence` decodes the strict schema-5 projection and republishes it through `publish_model_partition`; `document_with_partition` replaces the entire flattened legacy map. Therefore the receipt must be a separate envelope partition, never a key in the legacy map or a `ModelMetadata` field. The source currently supports missing state, schema 6, and read-only schema 4/5; pending cleanup remains non-authorizing. Normal HF completion order is durable `FilesReady`, marker removal, guarded importer metadata/index work, optional pinned package-facts resolution, exact `Using` acknowledgment, then durable queue release. Marker removal or metadata/index presence alone cannot establish completion. + +The schema-6 consumer path publishes importer metadata through the held capability-backed atomic target and requires `Durable`, but metadata equality can skip a new publication. Package-facts resolution writes summary/detail rows, while its source fingerprint is only cache freshness (metadata/descriptor/dependency JSON and file length/mtime; not payload bytes) and typed metadata contains a `HashMap`. Do not treat that fingerprint as exact output identity. Receipt version 1 binds the exact acquisition ID, persisted `Using` lease, HF demand/operation, current admitted queue identity, manifest and ordered verified-file receipts, resulting model ID and destination identity. Its output proof is versioned independently from the envelope and package-facts schemas and uses canonical JSON: recursively sort object keys; preserve array order, explicit nulls and every field in the declared metadata/index/package-facts projection; serialize integers and strings with the repository JSON serializer; reject non-finite numbers and unknown proof versions. The projection includes model identity and HF provenance, imported metadata fields, the model-index row needed to resolve that identity, and—when the pinned revision path requires package facts—the exact detail facts content and package-facts contract version. The projection contract names its included and excluded fields in source, and cold validation compares current outputs against the issuer-published digest rather than deriving a new expected digest from current outputs alone. Before issuing it, the importer must explicitly publish the observed final metadata durably, including when it equals the prior bytes, and confirm required output projections. Receipt issuance is private to complete managed-HF finalization after all importer and package-facts effects succeed; partial imports and ordinary import callers cannot issue one. + +Receipt publication is conditional on the same canonical transaction observing the exact current `Using` record/lease, demand, manifest, verified files, non-revoked current queue admission, destination and workspace, with the held root grant still valid. It rejects an existing conflicting receipt and treats only an identical receipt as idempotent. A fresh root grant cannot renew or replace a persisted `Using` lease before cold validation. A private receipt-qualified settlement capability is minted only after read-only validation of the historical lease, exact queue identity, selected-file receipts, receipt and output projections. It performs the `Using` → `Adopted` acquisition transition and exact queue release in one `AcquisitionStore` document publication, retaining the immutable receipt alongside the adopted record as completion history. There is no separately durable receipt/ack/release intermediate state to reopen. Unknown publication outcome reports failure, preserves custody and is resolved by cold read-only observation; importer replay is forbidden. + +The receipt requires a schema-7 acquisition document with a distinct, strictly validated model-consumer receipt partition. An explicit offline conversion supports schema 4/5 and receipt-free pre-receipt schema 6; it preserves all acquisitions, model custody, leases, queue/revocation history, hidden admissions, quarantines and cleanup dispositions and never fabricates a receipt. The operator must stop every old reader/writer, including processes with cached state, before conversion; holding the new file lock alone is not evidence of exclusion. All supported entry points reject unsupported schemas without rebuilding, defaulting, or publishing. Receipt/acquisition key mismatch, orphan or duplicate receipts, malformed/unknown receipt versions, or a receipt paired with an invalid acquisition phase fail closed. A receipt remains with its `Adopted` acquisition as immutable completion history; no implicit pruning is added. Any future terminal-record compaction must remove acquisition and receipt evidence in one exact publication after its owning retention policy accepts the disposition. A pre-receipt `Using` record remains recovery-required even when output files happen to match. Actual deployed schema-6 population, overlapping processes, rollback policy, separately retained model/index state and index-only writers remain unknown; no live retained root is authorized, so this branch's migration and reopen claims use disposable fixtures only. + +Admitted source write set: `acquisition/{service.rs,store.rs}` for the schema-7 envelope, exact use/receipt transaction and receipt-gated cold lease; `model_library/{download_store.rs,importer.rs,library.rs,mutation_authority.rs,hf/download.rs}` for model-owned receipt proof, durable output checks, retained-store projection and normal/restored public-workflow reconciliation; directly necessary `download_recovery.rs` or index accessors only if held-destination or exact output observation cannot use current APIs; and their direct tests. Update only these owning contracts/plan records and the separate MCP usability report. No new task owner, sidecar, metadata field, generated interface, model/API branch, or live retained-state mutation. If the current source cannot expose one exact atomic acknowledgement-plus-queue-release publication, that is a design stop requiring a corrected owner contract before implementation; do not emulate it with sequential writes. Final source routing must be repeated against the actual implemented design before this slice is complete. + +### Current Q1 candidate slice — native receipt settlement and cancellation withdrawal + +Exact current source paths are `rust/crates/pumas-core/src/acquisition/{http.rs,mod.rs,service.rs,store.rs,task_custody.rs,workspace.rs}`, `rust/crates/pumas-core/src/model_library/{download_recovery.rs,download_store.rs,importer.rs,library.rs,mutation_authority.rs}`, `rust/crates/pumas-core/src/model_library/hf/{download.rs,types.rs}`, `rust/crates/pumas-core/src/network/github.rs`, `rust/crates/pumas-core/src/tests.rs`, `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/Cargo.toml`, `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs}`, and `rust/crates/pumas-rpc/src/{main.rs,server.rs}`. These cover the shared transfer/store/custody owner, exact receipt and output projections, HF reopen reconciliation, native shared-service consumer and attempt withdrawal, RPC lifetime ordering, and direct regressions. This slice gives llama.cpp the existing shared acquisition service, binds a durable receipt to exact extracted output and metadata, reconciles committed publication after reopen, and withdraws only an unchanged unreceipted native `Using` lease after its attempt is revoked and workspace cleanup succeeds. The `library.rs` descriptor-refresh future is boxed because the exact Torch serving RPC regression overflowed the default worker stack in the composed candidate; the existing integration test now passes with the normal stack size. Temporary diagnostic logs/test edits were removed. This source and its local fixtures do not qualify live sources, desktop behavior, deployed schema-6 population/old-writer isolation, or cross-platform durability. Keep AQ-HTTP not ready until objective acceptance is complete. + ## Q2: exact package-file handoff Allowed: Q1 acquisition types/service only for demonstrated missing file-set/lease semantics; `torch-server/resolve_runtime.py`, retained preview/lock consumers in `rust/crates/pumas-app-manager/src/version_manager/{torch_preview.rs,installer/torch.rs}`, corresponding existing package/integrity/progress tests, and `artifact_acquisition_install.rs`. Keep package resolution/install semantics with those consumers. The exact managed-Python/provider files are first traced for a migrate-versus-retain traffic disposition; no speculative private integration is authorized. diff --git a/rust/crates/pumas-app-manager/Cargo.toml b/rust/crates/pumas-app-manager/Cargo.toml index 60b1e1a0..3f6401bd 100644 --- a/rust/crates/pumas-app-manager/Cargo.toml +++ b/rust/crates/pumas-app-manager/Cargo.toml @@ -59,6 +59,7 @@ bytes = { workspace = true } nix = { workspace = true } [dev-dependencies] +pumas-library = { workspace = true, features = ["test-support"] } tokio = { workspace = true, features = ["test-util", "macros"] } [target.'cfg(windows)'.dev-dependencies] diff --git a/rust/crates/pumas-app-manager/src/version_manager/installer.rs b/rust/crates/pumas-app-manager/src/version_manager/installer.rs index 1d10dfbb..5a2e1294 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/installer.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/installer.rs @@ -29,17 +29,23 @@ use crate::version_manager::progress::{InstallationProgressTracker, ProgressUpda use chrono::Utc; use futures::future::{BoxFuture, Shared}; use futures::FutureExt; +use pumas_library::acquisition::{ + AcquisitionConsumer, AcquisitionDemand, AcquisitionHttpRequest, AcquisitionHttpSource, + AcquisitionRetryPolicy, AcquisitionWorkspace, +}; use pumas_library::config::{AppId, InstallationConfig, PathsConfig}; use pumas_library::metadata::{InstalledVersionMetadata, MetadataManager}; use pumas_library::models::InstallationStage; -use pumas_library::network::{GitHubAsset, GitHubRelease}; +use pumas_library::network::{GitHubAsset, GitHubClient, GitHubRelease, RetryConfig}; use pumas_library::{PumasError, Result}; +use sha2::{Digest, Sha256}; use std::fs::File; use std::io::BufReader; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicU8, Ordering}; use std::sync::Arc; use std::sync::Mutex as StdMutex; +use std::time::Instant; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, Mutex, RwLock}; @@ -54,6 +60,295 @@ async fn path_exists(path: &Path) -> Result { }) } +fn sync_native_directory(path: &Path) -> Result<()> { + #[cfg(unix)] + let directory = File::open(path); + #[cfg(windows)] + let directory = { + use std::os::windows::fs::OpenOptionsExt; + const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x02000000; + std::fs::OpenOptions::new() + .read(true) + .write(true) + .custom_flags(FILE_FLAG_BACKUP_SEMANTICS) + .open(path) + }; + #[cfg(not(any(unix, windows)))] + let directory: std::io::Result = Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Native directory durability is unsupported", + )); + directory + .and_then(|directory| directory.sync_all()) + .map_err(|error| PumasError::io_with_path(error, path)) +} + +fn sync_native_file(path: &Path) -> Result<()> { + let metadata = + std::fs::symlink_metadata(path).map_err(|error| PumasError::io_with_path(error, path))?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(PumasError::InstallationFailed { + message: format!( + "Native durability requires a regular file: {}", + path.display() + ), + }); + } + #[cfg(unix)] + let file = File::open(path); + #[cfg(windows)] + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path); + #[cfg(not(any(unix, windows)))] + let file: std::io::Result = Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "Native file durability is unsupported", + )); + file.and_then(|file| file.sync_all()) + .map_err(|error| PumasError::io_with_path(error, path)) +} + +fn sync_native_tree(root: &Path) -> Result<()> { + VersionInstaller::validate_native_output(root)?; + let mut directories = Vec::new(); + for entry in walkdir::WalkDir::new(root).follow_links(false) { + let entry = entry.map_err(|error| PumasError::InstallationFailed { + message: format!("Native durability traversal failed: {error}"), + })?; + if entry.file_type().is_file() { + sync_native_file(entry.path())?; + } else if entry.file_type().is_dir() { + directories.push(entry.path().to_path_buf()); + } + } + for directory in directories.into_iter().rev() { + sync_native_directory(&directory)?; + } + Ok(()) +} + +fn sync_native_metadata(versions: &Path, app_id: AppId) -> Result<()> { + let launcher = versions + .parent() + .ok_or_else(|| PumasError::InstallationFailed { + message: "Native versions have no launcher root".into(), + })?; + let metadata_dir = launcher.join("launcher-data/metadata"); + let path = metadata_dir.join(format!( + "versions-{}.json", + app_id.to_string().to_lowercase() + )); + sync_native_file(&path)?; + sync_native_directory(&metadata_dir)?; + sync_native_directory(&launcher.join("launcher-data"))?; + sync_native_directory(launcher) +} + +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct NativeAttemptIdentity { + schema_version: u32, + tag: String, + attempt: String, + removed: bool, +} + +fn native_attempt_path(versions: &Path, tag: &str) -> PathBuf { + let digest = format!("{:x}", Sha256::digest(tag.as_bytes())); + versions.join(format!(".llama-attempt-{}.json", &digest[..24])) +} + +fn native_workspace_path(versions: &Path, tag: &str, attempt: &str) -> PathBuf { + let digest = format!("{:x}", Sha256::digest(tag.as_bytes())); + versions.join(format!(".llama-install-{}-{attempt}", &digest[..24])) +} + +fn cleanup_native_workspace_if_present(directory: &Path, versions: &Path) -> Result<()> { + if !path_exists_sync(directory)? { + return Ok(()); + } + let metadata = std::fs::symlink_metadata(directory) + .map_err(|error| PumasError::io_with_path(error, directory))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PumasError::InstallationFailed { + message: "Native workspace is not a safe owned directory; reclamation required".into(), + }); + } + std::fs::remove_dir_all(directory).map_err(|error| PumasError::Io { + message: format!("Native workspace cleanup incomplete; reclamation required: {error}"), + path: Some(directory.to_path_buf()), + source: Some(error), + })?; + sync_native_directory(versions) +} + +fn read_native_attempt(versions: &Path, tag: &str) -> Result> { + let path = native_attempt_path(versions, tag); + if path_exists_sync(&path)? { + let metadata = std::fs::symlink_metadata(&path) + .map_err(|error| PumasError::io_with_path(error, &path))?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Native attempt identity is not a regular file".into(), + }); + } + } + let identity: Option = pumas_library::metadata::atomic_read_json(&path)?; + if let Some(identity) = &identity { + if identity.schema_version != 1 + || identity.tag != tag + || identity.attempt.len() != 32 + || !identity + .attempt + .bytes() + .all(|byte| byte.is_ascii_hexdigit()) + { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Native attempt identity is malformed".into(), + }); + } + } + Ok(identity) +} + +fn write_native_attempt(versions: &Path, identity: &NativeAttemptIdentity) -> Result<()> { + pumas_library::metadata::atomic_write_json( + &native_attempt_path(versions, &identity.tag), + identity, + false, + )?; + sync_native_directory(versions) +} + +/// Called only while removal holds the permanent native versions lease. +/// Persist revocation before deleting output so restart can never republish it. +pub(crate) fn mark_native_attempt_removed(versions: &Path, tag: &str) -> Result<()> { + VersionInstaller::validate_native_tag(tag)?; + if let Some(mut identity) = read_native_attempt(versions, tag)? { + identity.removed = true; + write_native_attempt(versions, &identity)?; + let directory = native_workspace_path(versions, tag, &identity.attempt); + cleanup_native_workspace_if_present(&directory, versions)?; + } + Ok(()) +} + +fn path_exists_sync(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(PumasError::io_with_path(error, path)), + } +} + +fn hash_metadata(metadata: &InstalledVersionMetadata) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(metadata)?) + )) +} + +fn metadata_matches(left: &InstalledVersionMetadata, right: &InstalledVersionMetadata) -> bool { + matches!((hash_metadata(left), hash_metadata(right)), (Ok(left), Ok(right)) if left == right) +} + +fn hash_regular_file(path: &Path) -> Result { + let metadata = + std::fs::symlink_metadata(path).map_err(|error| PumasError::io_with_path(error, path))?; + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(PumasError::InstallationFailed { + message: format!("Native receipt requires a regular file: {}", path.display()), + }); + } + let mut file = File::open(path).map_err(|error| PumasError::io_with_path(error, path))?; + let mut digest = Sha256::new(); + std::io::copy(&mut file, &mut digest).map_err(|error| PumasError::io_with_path(error, path))?; + Ok(format!("{:x}", digest.finalize())) +} + +fn hash_native_tree(root: &Path) -> Result { + let metadata = + std::fs::symlink_metadata(root).map_err(|error| PumasError::io_with_path(error, root))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PumasError::InstallationFailed { + message: "Native output root is not a directory".into(), + }); + } + VersionInstaller::validate_native_output(root)?; + let mut digest = Sha256::new(); + for entry in walkdir::WalkDir::new(root) + .follow_links(false) + .sort_by_file_name() + { + let entry = entry.map_err(|error| PumasError::InstallationFailed { + message: format!("Could not hash native output: {error}"), + })?; + let relative = entry + .path() + .strip_prefix(root) + .map_err(|error| PumasError::Other(error.to_string()))?; + let name = relative + .to_str() + .ok_or_else(|| PumasError::InstallationFailed { + message: "Native output name is not UTF-8".into(), + })? + .replace('\\', "/"); + digest.update((name.len() as u64).to_le_bytes()); + digest.update(name.as_bytes()); + let metadata = std::fs::symlink_metadata(entry.path()) + .map_err(|error| PumasError::io_with_path(error, entry.path()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + digest.update(metadata.permissions().mode().to_le_bytes()); + } + if metadata.file_type().is_symlink() { + digest.update(b"link"); + let target = std::fs::read_link(entry.path()) + .map_err(|error| PumasError::io_with_path(error, entry.path()))?; + let target = target + .to_str() + .ok_or_else(|| PumasError::InstallationFailed { + message: "Native link target is not UTF-8".into(), + })?; + digest.update((target.len() as u64).to_le_bytes()); + digest.update(target.as_bytes()); + } else if metadata.is_file() { + digest.update(b"file"); + digest.update(hash_regular_file(entry.path())?.as_bytes()); + } else if metadata.is_dir() { + digest.update(b"directory"); + } else { + return Err(PumasError::InstallationFailed { + message: "Native output contains a special file".into(), + }); + } + } + Ok(format!("{:x}", digest.finalize())) +} + +fn verify_llama_cpp_output(root: &Path, proof: &LlamaCppInstallReceiptV1) -> Result<()> { + let expected_launcher = if cfg!(windows) { + "bin/llama-server.exe" + } else { + "bin/llama-server" + }; + if proof.launcher_relative_path != expected_launcher + || hash_native_tree(root)? != proof.output_tree_sha256 + || hash_regular_file(&root.join(expected_launcher))? != proof.launcher_sha256 + { + return Err(PumasError::InstallationFailed { + message: "Native output differs from its durable llama.cpp receipt; recovery required" + .into(), + }); + } + Ok(()) +} + /// Coordinates Torch cancellation with the irreversible publication boundary. pub(crate) struct TorchInstallControl(AtomicU8); @@ -64,6 +359,7 @@ struct NativeInstallWorkspace { // Implicit destruction must preserve uncertain work. Reclamation is an // explicit fallible operation after all file/extraction effects settle. directory: PathBuf, + attempt: String, _lock: NativeVersionsLock, } @@ -107,15 +403,68 @@ impl NativeVersionsLock { } impl NativeInstallWorkspace { - fn create(versions: &Path) -> Result { + #[cfg(test)] + fn create(versions: &Path, tag: &str) -> Result { + Self::create_for_attempt(versions, tag, None) + } + + fn create_for_attempt(versions: &Path, tag: &str, expected: Option<&str>) -> Result { + VersionInstaller::validate_native_tag(tag)?; let lock = NativeVersionsLock::try_acquire(versions)?; - let directory = tempfile::Builder::new() - .prefix(".llama-install-") - .tempdir_in(versions) - .map_err(|error| PumasError::io_with_path(error, versions))? - .keep(); + let current = read_native_attempt(versions, tag)?; + let identity = match (current, expected) { + (Some(identity), Some(expected)) + if !identity.removed && identity.attempt == expected => + { + identity + } + (_, Some(_)) => { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Native recovery attempt was removed or changed".into(), + }) + } + (Some(identity), None) if !identity.removed => identity, + (previous, None) => { + if path_exists_sync(&versions.join(tag))? { + return Err(PumasError::VersionAlreadyInstalled { tag: tag.into() }); + } + if previous.is_some() { + mark_native_attempt_removed(versions, tag)?; + } + let mut entropy = [0u8; 16]; + getrandom::fill(&mut entropy).map_err(|error| { + PumasError::Other(format!("Native attempt identity failed: {error}")) + })?; + let identity = NativeAttemptIdentity { + schema_version: 1, + tag: tag.into(), + attempt: entropy.iter().map(|byte| format!("{byte:02x}")).collect(), + removed: false, + }; + write_native_attempt(versions, &identity)?; + identity + } + }; + let directory = native_workspace_path(versions, tag, &identity.attempt); + std::fs::create_dir_all(&directory) + .map_err(|error| PumasError::io_with_path(error, &directory))?; + let metadata = std::fs::symlink_metadata(&directory) + .map_err(|error| PumasError::io_with_path(error, &directory))?; + if metadata.file_type().is_symlink() || !metadata.is_dir() { + return Err(PumasError::InstallationFailed { + message: "Native acquisition stage is not a held directory".into(), + }); + } + sync_native_directory(versions)?; + sync_native_directory( + versions + .parent() + .ok_or_else(|| PumasError::Other("Native versions parent absent".into()))?, + )?; Ok(Self { directory, + attempt: identity.attempt, _lock: lock, }) } @@ -129,10 +478,132 @@ impl NativeInstallWorkspace { message: format!("Native staging cleanup incomplete; retained workspace requires reconciliation: {error}"), path: Some(self.directory.clone()), source: Some(error), - }) + })?; + sync_native_directory( + self.directory + .parent() + .ok_or_else(|| PumasError::Other("Native workspace parent absent".into()))?, + ) + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct LlamaCppInstallReceiptV1 { + schema_version: u32, + tag: String, + metadata: InstalledVersionMetadata, + metadata_sha256: String, + output_tree_sha256: String, + launcher_relative_path: String, + launcher_sha256: String, +} + +struct PreparedLlamaCppInstall { + use_set: pumas_library::acquisition::AcquiredArtifactUse, + stage: PathBuf, + destination: PathBuf, + custody: Arc, +} + +struct LlamaCppPublication { + versions: PathBuf, + app_id: AppId, + manager: Arc, + custody: Arc, + stage: PathBuf, + destination: PathBuf, + proof: LlamaCppInstallReceiptV1, +} + +struct LlamaCppHttpAttemptHost { + cancel_flag: Arc, + progress_tracker: Arc>, + progress_tx: mpsc::Sender, + total_size: Option, + started: Instant, +} + +#[async_trait::async_trait] +impl pumas_library::acquisition::HttpAttemptHost for LlamaCppHttpAttemptHost { + async fn pause_requested(&self) { + super::wait_for_install_cancel(self.cancel_flag.clone()).await; + } + + fn pause_requested_now(&self) -> bool { + false + } + + fn cancel_requested(&self) -> bool { + self.cancel_flag.load(Ordering::SeqCst) + } + + async fn record_progress(&mut self, downloaded_for_file: u64) -> Result<()> { + let elapsed = self.started.elapsed().as_secs_f64(); + let speed = (elapsed > 0.0).then_some(downloaded_for_file as f64 / elapsed); + self.progress_tracker + .write() + .await + .update_download_progress(downloaded_for_file, self.total_size, speed); + let _ = self + .progress_tx + .send(ProgressUpdate::Download { + downloaded_bytes: downloaded_for_file, + total_bytes: self.total_size, + speed_bytes_per_sec: speed, + }) + .await; + Ok(()) } } +#[async_trait::async_trait] +impl pumas_library::acquisition::AcquisitionHost for LlamaCppHttpAttemptHost { + async fn retry( + &mut self, + _attempt: u32, + _delay: Option, + _error: Option<&str>, + ) -> Result<()> { + Ok(()) + } +} + +async fn open_native_acquisition_workspace( + versions: PathBuf, + tag: String, + expected: Option, +) -> Result<(Arc, AcquisitionWorkspace)> { + tokio::task::spawn_blocking(move || { + let custody = Arc::new(NativeInstallWorkspace::create_for_attempt( + &versions, + &tag, + expected.as_deref(), + )?); + let relative = custody + .path() + .strip_prefix(&versions) + .map_err(|_| PumasError::InstallationFailed { + message: "Native acquisition stage escaped the versions root".into(), + })? + .to_path_buf(); + let execution_lease: Arc = Arc::new(custody._lock.clone()); + let validator_lease = execution_lease.clone(); + let workspace = AcquisitionWorkspace::from_reserved_directory( + &versions, + &relative, + execution_lease, + move || { + let _held = &validator_lease; + Ok(()) + }, + )?; + Ok((custody, workspace)) + }) + .await + .map_err(|error| PumasError::Other(format!("Failed to join native staging: {error}")))? +} + fn settled_result(outcome: Result, settlement: Result<()>) -> Result { match (outcome, settlement) { (result, Ok(())) => result, @@ -502,10 +973,14 @@ pub struct VersionInstaller { /// Cancellation flag. cancel_flag: Arc, shutdown_flag: Arc, + github_client: Option>, + acquisition_consumer: Option>, torch_control: Arc, torch_cleanup: Arc, torch_attempt_lock: Mutex<()>, #[cfg(test)] + native_receipt_pause: Option>, + #[cfg(test)] torch_stage_override: Option, #[cfg(test)] torch_publication_pause: Option>, @@ -536,10 +1011,14 @@ impl VersionInstaller { progress_tracker, cancel_flag, shutdown_flag: Arc::new(AtomicBool::new(false)), + github_client: None, + acquisition_consumer: None, torch_control: Arc::new(TorchInstallControl::new()), torch_cleanup: Arc::new(TorchCleanupTasks::default()), torch_attempt_lock: Mutex::new(()), #[cfg(test)] + native_receipt_pause: None, + #[cfg(test)] torch_stage_override: None, #[cfg(test)] torch_publication_pause: None, @@ -548,6 +1027,233 @@ impl VersionInstaller { } } + #[cfg(test)] + pub(crate) fn with_native_receipt_pause(mut self, pause: Arc) -> Self { + self.native_receipt_pause = Some(pause); + self + } + + pub(crate) fn with_github_client(mut self, client: Arc) -> Self { + self.github_client = Some(client); + self + } + + pub(crate) fn with_acquisition_consumer( + mut self, + consumer: Option>, + ) -> Self { + self.acquisition_consumer = consumer; + self + } + + /// Reconcile historic native uses from the shared store before admitting + /// new work. Publisher lookup cannot change a retained manifest or receipt. + pub(crate) async fn reconcile_retained_llama_cpp( + &self, + records: Vec, + ) -> Result<()> { + let consumer = self + .acquisition_consumer + .as_ref() + .ok_or_else(|| PumasError::Config { + message: "Native recovery requires the shared acquisition consumer".into(), + })?; + for record in records { + if record.demand.consumer != consumer.owner() + || !matches!( + &record.phase, + pumas_library::acquisition::AcquisitionPhase::Using { .. } + | pumas_library::acquisition::AcquisitionPhase::Adopted { .. } + ) + { + continue; + } + let (version_platform, attempt) = + record + .demand + .operation + .rsplit_once(':') + .ok_or_else(|| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Retained native demand has no exact attempt identity".into(), + })?; + let (tag, platform) = + version_platform + .rsplit_once(':') + .ok_or_else(|| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Retained native demand has no exact version/platform".into(), + })?; + Self::validate_native_tag(tag)?; + if platform != format!("{}-{}", std::env::consts::OS, std::env::consts::ARCH) { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Retained native acquisition belongs to another platform".into(), + }); + } + let tag = tag.to_owned(); + let versions = self.versions_dir(); + let current = + read_native_attempt(&versions, &tag)?.ok_or_else(|| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Retained native acquisition has no attempt identity".into(), + })?; + if current.removed || current.attempt != attempt { + // Removal revokes the attempt before output reclamation. A new + // current attempt proves explicit removal/reinstall occurred. + continue; + } + if matches!( + &record.phase, + pumas_library::acquisition::AcquisitionPhase::Adopted { .. } + ) { + let consumer = consumer.clone(); + let retained = record.clone(); + let receipt = + tokio::task::spawn_blocking(move || consumer.completion_receipt(&retained)) + .await + .map_err(|error| { + PumasError::Other(format!("Native receipt lookup failed: {error}")) + })?? + .ok_or_else(|| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Adopted native installation has no exact completion receipt" + .into(), + })?; + let proof: LlamaCppInstallReceiptV1 = serde_json::from_value(receipt.payload) + .map_err(|error| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: format!("Adopted native receipt is malformed: {error}"), + })?; + let versions_for_cleanup = versions.clone(); + let tag_for_cleanup = tag.to_owned(); + let attempt_for_cleanup = attempt.to_owned(); + let expected_workspace = record.workspace.clone(); + let manager = self.metadata_manager.clone(); + let app_id = self.app_id; + let lock = NativeVersionsLock::acquire(versions.clone()).await?; + tokio::task::spawn_blocking(move || { + let _lock = lock; + let current = read_native_attempt(&versions_for_cleanup, &tag_for_cleanup)?; + let Some(current) = current else { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Adopted native receipt has no attempt identity".into(), + }); + }; + if current.removed || current.attempt != attempt_for_cleanup { + return Ok(()); + } + let workspace = native_workspace_path( + &versions_for_cleanup, + &tag_for_cleanup, + &attempt_for_cleanup, + ); + let relative_workspace = workspace + .strip_prefix(&versions_for_cleanup) + .map_err(|_| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Adopted native workspace escaped its versions root".into(), + })?; + let actual_workspace = + pumas_library::acquisition::AcquisitionWorkspace::identity_for_reserved_directory( + &versions_for_cleanup, + relative_workspace, + )?; + if actual_workspace != expected_workspace { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Adopted native workspace identity changed".into(), + }); + } + let destination = versions_for_cleanup.join(&tag_for_cleanup); + let metadata = std::fs::symlink_metadata(&destination) + .map_err(|error| PumasError::io_with_path(error, &destination))?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PumasError::InstallationFailed { + message: "Adopted llama.cpp output is not a safe directory".into(), + }); + } + Self::validate_llama_cpp_receipt(&tag_for_cleanup, &proof, &destination)?; + verify_llama_cpp_output(&destination, &proof)?; + match manager.get_installed_version(&tag_for_cleanup, Some(app_id))? { + Some(installed) if metadata_matches(&installed, &proof.metadata) => {} + _ => { + return Err(PumasError::InstallationFailed { + message: "Adopted llama.cpp metadata does not match its receipt" + .into(), + }) + } + } + cleanup_native_workspace_if_present(&workspace, &versions_for_cleanup) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Native adopted cleanup task failed: {error}")) + })??; + continue; + } + let (custody, workspace) = open_native_acquisition_workspace( + versions.clone(), + tag.clone(), + Some(attempt.to_owned()), + ) + .await?; + let custody_for_reconcile = custody.clone(); + let manager = self.metadata_manager.clone(); + let app_id = self.app_id; + consumer + .reconcile( + record.demand, + record.manifest, + workspace, + move |receipt, use_set| async move { + let proof: LlamaCppInstallReceiptV1 = + serde_json::from_value(receipt.payload).map_err(|error| { + PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: format!( + "Retained native receipt is malformed: {error}" + ), + } + })?; + use_set + .run_blocking("reconcile retained native installation", move || { + Self::reconcile_llama_cpp_installation( + &versions, + &tag, + app_id, + &manager, + custody_for_reconcile, + proof, + ) + }) + .await + }, + ) + .await? + .ok_or_else(|| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Retained native use disappeared before reconciliation".into(), + })?; + tokio::task::spawn_blocking(move || { + Arc::try_unwrap(custody) + .map_err(|workspace| PumasError::InstallationFailed { + message: format!( + "Reconciled native workspace remains in use: {}", + workspace.path().display() + ), + })? + .cleanup() + }) + .await + .map_err(|error| { + PumasError::Other(format!("Native recovery cleanup task failed: {error}")) + })??; + } + Ok(()) + } + /// Drain Torch quarantine cleanup after the last direct install call. /// Owners using `VersionInstaller` without `VersionManager` must await this /// before shutting down their Tokio runtime. @@ -795,13 +1501,40 @@ impl VersionInstaller { info!("Starting llama.cpp binary installation for {}", tag); let asset = self.select_llama_cpp_asset(&release.assets)?; - let download_url = &asset.download_url; - let total_size = asset.size; let asset_name = asset.name.clone(); + let github = self + .github_client + .as_ref() + .ok_or_else(|| PumasError::Config { + message: "llama.cpp requires the configured shared GitHub asset resolver".into(), + })?; + // GitHub discovery appends a runtime flavor to the upstream release + // tag. Resolve the publisher release, while retaining the full local tag. + let publisher_tag = release + .tag_name + .rsplit_once('+') + .map_or(release.tag_name.as_str(), |(publisher, _)| publisher); + let selection = github + .resolve_release_asset(self.app_id.github_repo(), publisher_tag, &asset_name) + .await + .map_err(|error| PumasError::InstallationFailed { + message: format!("Could not verify llama.cpp release asset: {error}"), + })?; + let manifest = selection.manifest().clone(); + let selected_file = + manifest + .files() + .first() + .ok_or_else(|| PumasError::InstallationFailed { + message: "Verified llama.cpp release selection has no file".into(), + })?; + let total_size = selected_file.expected_size(); + let download_url = selection.download_url().to_owned(); info!( - "Selected llama.cpp asset: {} ({} bytes)", - asset_name, total_size + "Selected publisher-verified llama.cpp asset: {} ({} bytes)", + asset_name, + total_size.unwrap_or_default() ); let log_dir = self.logs_dir(); @@ -816,52 +1549,293 @@ impl VersionInstaller { let mut tracker = self.progress_tracker.write().await; tracker.start_installation( tag, - Some(total_size), + total_size, None, Some(log_path.to_string_lossy().as_ref()), ); } - // No digest is supplied by GitHubAsset. Filename and size are not - // integrity evidence: do not reuse or mutate retained download caches. let versions = self.versions_dir(); fs::create_dir_all(&versions) .await .map_err(|e| PumasError::io_with_path(e, &versions))?; - let custody = Arc::new( - tokio::task::spawn_blocking(move || NativeInstallWorkspace::create(&versions)) - .await - .map_err(|error| { - PumasError::Other(format!("Failed to join native staging: {error}")) - })??, - ); - let archive_path = custody.path().join("archive"); - let result = self - .do_llama_cpp_install( - tag, - release, - download_url, - &asset_name, - &archive_path, - custody.clone(), - &progress_tx, + let (custody, workspace) = + open_native_acquisition_workspace(versions.clone(), tag.to_owned(), None).await?; + let consumer = self + .acquisition_consumer + .clone() + .ok_or_else(|| PumasError::Config { + message: "llama.cpp requires the shared artifact acquisition consumer".into(), + })?; + let demand = AcquisitionDemand { + consumer: consumer.owner().to_owned(), + operation: format!( + "{tag}:{}-{}:{}", + std::env::consts::OS, + std::env::consts::ARCH, + custody.attempt + ), + }; + let custody_for_reconcile = custody.clone(); + let manager_for_reconcile = self.metadata_manager.clone(); + let versions_for_reconcile = versions.clone(); + let tag_for_reconcile = tag.to_owned(); + let cancel_for_reconcile = self.cancel_flag.clone(); + let control_for_reconcile = self.torch_control.clone(); + let app_id = self.app_id; + let reconcile = consumer + .reconcile( + demand.clone(), + manifest.clone(), + workspace.clone(), + move |receipt, use_set| async move { + let proof: LlamaCppInstallReceiptV1 = serde_json::from_value(receipt.payload) + .map_err(|error| { + PumasError::Validation { + field: "acquisition.consumer_receipt".into(), + message: format!("llama.cpp receipt is malformed: {error}"), + } + })?; + if cancel_for_reconcile.load(Ordering::SeqCst) + || !control_for_reconcile.try_begin_publication() + { + return Err(VersionInstaller::cancellation_error()); + } + use_set + .run_blocking("reconcile verified llama.cpp installation", move || { + Self::reconcile_llama_cpp_installation( + &versions_for_reconcile, + &tag_for_reconcile, + app_id, + &manager_for_reconcile, + custody_for_reconcile, + proof, + ) + }) + .await?; + Ok(()) + }, ) .await; - // Stage reclamation can traverse a large archive. The supervised - // installation awaits that blocking cleanup before releasing admission. - let cleanup = tokio::task::spawn_blocking(move || { - Arc::try_unwrap(custody) - .map_err(|workspace| PumasError::InstallationFailed { - message: format!("Native workspace remains in use; retained stage requires reconciliation: {}", workspace.path().display()), - })? - .cleanup() - }) + let result = match reconcile { + Ok(Some(())) => Ok(()), + Ok(None) => { + let client = reqwest::Client::builder() + .connect_timeout(InstallationConfig::URL_FETCH_TIMEOUT) + .user_agent("pumas-library") + .build() + .map_err(|error| PumasError::Network { + message: "Failed to create shared artifact HTTP client".into(), + cause: Some(error.to_string()), + })?; + let host = LlamaCppHttpAttemptHost { + cancel_flag: self.cancel_flag.clone(), + progress_tracker: self.progress_tracker.clone(), + progress_tx: progress_tx.clone(), + total_size, + started: Instant::now(), + }; + let retry = AcquisitionRetryPolicy { + attempts: Some(InstallationConfig::DOWNLOAD_RETRY_ATTEMPTS), + elapsed: std::time::Duration::ZERO, + backoff: RetryConfig::new() + .with_max_attempts(InstallationConfig::DOWNLOAD_RETRY_ATTEMPTS) + .with_base_delay(std::time::Duration::from_secs(2)), + }; + let stage = custody.path().join("output"); + let destination = versions.join(tag); + let asset_for_prepare = asset_name.clone(); + let tag_for_prepare = tag.to_owned(); + let release_date = Some(release.published_at.clone()); + let release_notes = release.body.clone(); + let metadata = InstalledVersionMetadata { + path: tag.to_owned(), + installed_date: Utc::now().to_rfc3339(), + release_tag: tag.to_owned(), + python_version: None, + git_commit: None, + release_date, + release_notes, + // Retrieval URLs are ephemeral source access, not durable + // installation provenance. + download_url: None, + size: total_size, + requirements_hash: None, + dependencies_installed: Some(true), + }; + let metadata_for_prepare = metadata.clone(); + let versions_for_publish = versions.clone(); + let manager_for_publish = self.metadata_manager.clone(); + let progress_for_publish = self.progress_tracker.clone(); + let progress_tx_for_publish = progress_tx.clone(); + #[cfg(test)] + let native_receipt_pause = self.native_receipt_pause.clone(); + let cancel_for_prepare = self.cancel_flag.clone(); + let control_for_prepare = self.torch_control.clone(); + let custody_for_prepare = custody.clone(); + let versions_for_cancel = versions.clone(); + let tag_for_cancel = tag.to_owned(); + let request = AcquisitionHttpRequest { + demand, + manifest, + workspace, + sources: vec![AcquisitionHttpSource { + url: download_url, + authorization: None, + }], + retry, + }; + consumer + .acquire_http( + request, + client, + Box::new(host), + move |use_set| async move { + let archive = use_set.open_file(0).await?; + let stage_for_extract = stage.clone(); + let asset_name = asset_for_prepare.clone(); + let extracted = use_set + .run_blocking("extract publisher-verified llama.cpp archive", move || { + if path_exists_sync(&stage_for_extract)? { + return Err(PumasError::InstallationFailed { + message: "llama.cpp output stage already contains unresolved work".into(), + }); + } + std::fs::create_dir(&stage_for_extract) + .map_err(|error| PumasError::io_with_path(error, &stage_for_extract))?; + VersionInstaller::extract_llama_cpp_binary_from( + archive, + &stage_for_extract, + &asset_name, + )?; + sync_native_tree(&stage_for_extract)?; + sync_native_directory(stage_for_extract.parent().ok_or_else(|| PumasError::Other("Native output parent absent".into()))?)?; + let output_tree_sha256 = hash_native_tree(&stage_for_extract)?; + let launcher = stage_for_extract.join(if cfg!(windows) { + "bin/llama-server.exe" + } else { + "bin/llama-server" + }); + let launcher_sha256 = hash_regular_file(&launcher)?; + Ok((output_tree_sha256, launcher_sha256)) + }) + .await?; + let metadata_sha256 = hash_metadata(&metadata_for_prepare)?; + let proof = LlamaCppInstallReceiptV1 { + schema_version: 1, + tag: tag_for_prepare, + metadata: metadata_for_prepare, + metadata_sha256, + output_tree_sha256: extracted.0, + launcher_relative_path: if cfg!(windows) { + "bin/llama-server.exe".into() + } else { + "bin/llama-server".into() + }, + launcher_sha256: extracted.1, + }; + #[cfg(test)] + if let Some(pause) = native_receipt_pause { + pause.reached.notify_one(); + let permit = pause.resume.acquire().await.map_err(|_| { + PumasError::Other("Native receipt test pause closed".into()) + })?; + permit.forget(); + } + // Once this receipt can be persisted, restart will + // finish publication from its staged output. Win + // the cancellation race first so an accepted + // cancel cannot turn into a later installation. + if cancel_for_prepare.load(Ordering::SeqCst) + || !control_for_prepare.try_begin_publication() + { + use_set.withdraw_after_cleanup(move || { + let mut identity = read_native_attempt(&versions_for_cancel, &tag_for_cancel)? + .ok_or_else(|| PumasError::Other("Cancelled native attempt identity absent".into()))?; + if identity.attempt != custody_for_prepare.attempt || identity.removed { + return Err(PumasError::Other("Cancelled native attempt identity changed".into())); + } + identity.removed = true; + write_native_attempt(&versions_for_cancel, &identity)?; + cleanup_native_workspace_if_present(custody_for_prepare.path(), &versions_for_cancel) + }).await?; + return Err(VersionInstaller::cancellation_error()); + } + let prepared = PreparedLlamaCppInstall { + use_set, + stage, + destination: destination.clone(), + custody: custody_for_prepare, + }; + Ok((prepared, serde_json::to_value(proof)?)) + }, + move |prepared, issued| async move { + let proof: LlamaCppInstallReceiptV1 = + serde_json::from_value(issued.payload).map_err(|error| { + PumasError::Validation { + field: "acquisition.consumer_receipt".into(), + message: format!("llama.cpp publication receipt is malformed: {error}"), + } + })?; + prepared + .use_set + .run_blocking("publish verified llama.cpp installation", move || { + VersionInstaller::publish_llama_cpp_installation( + LlamaCppPublication { + versions: versions_for_publish, + app_id, + manager: manager_for_publish, + custody: prepared.custody, + stage: prepared.stage, + destination: prepared.destination, + proof, + }, + ) + }) + .await?; + progress_for_publish.write().await.update_stage( + InstallationStage::Setup, + 100.0, + Some("Installation complete"), + ); + let _ = progress_tx_for_publish + .send(ProgressUpdate::Setup { + message: "Installation complete".into(), + }) + .await; + Ok(()) + }, + ) + .await + } + Err(error) => Err(error), + }; + + let result = match result { + Err(PumasError::DownloadPaused) if self.cancel_flag.load(Ordering::SeqCst) => { + Err(PumasError::DownloadCancelled) + } + result => result, + }; + + let result = if result.is_ok() { + let cleanup = tokio::task::spawn_blocking(move || { + Arc::try_unwrap(custody) + .map_err(|workspace| PumasError::InstallationFailed { + message: format!( + "Native workspace remains in use: {}", + workspace.path().display() + ), + })? + .cleanup() + }) .await - .unwrap_or_else(|error| Err( - PumasError::Other(format!("Failed to join native cleanup: {error}")) - )); - let result = settled_result(result, cleanup); + .map_err(|error| PumasError::Other(format!("Native cleanup join failed: {error}")))?; + settled_result(result, cleanup) + } else { + result + }; { let mut tracker = self.progress_tracker.write().await; @@ -977,69 +1951,6 @@ impl VersionInstaller { Ok(()) } - /// Execute llama.cpp installation steps. - #[allow(clippy::too_many_arguments)] - async fn do_llama_cpp_install( - &self, - tag: &str, - release: &GitHubRelease, - download_url: &str, - asset_name: &str, - archive_path: &Path, - custody: Arc, - progress_tx: &mpsc::Sender, - ) -> Result<()> { - Self::validate_native_tag(tag)?; - let version_dir = self.versions_dir().join(tag); - // Replacement requires a durable directory/metadata transaction owned - // elsewhere. Never delete or replace retained output here. - if path_exists(&version_dir).await? { - return Err(PumasError::VersionAlreadyInstalled { tag: tag.into() }); - } - self.check_cancelled()?; - self.download_archive(download_url, archive_path, progress_tx) - .await?; - self.check_cancelled()?; - let stage = custody.path().join("output"); - fs::create_dir(&stage) - .await - .map_err(|e| PumasError::io_with_path(e, &stage))?; - { - let mut tracker = self.progress_tracker.write().await; - tracker.update_stage( - InstallationStage::Extract, - 0.0, - Some("Extracting binary archive..."), - ); - } - self.send_progress( - progress_tx, - ProgressUpdate::StageChanged { - stage: InstallationStage::Extract, - message: "Extracting binary archive...".into(), - }, - ) - .await; - let archive_path = archive_path.to_path_buf(); - let extracted = stage.clone(); - let asset_name = asset_name.to_owned(); - // The blocking worker holds custody even if a direct caller drops its - // future. The manager never aborts its registered installation task. - let held = custody.clone(); - tokio::task::spawn_blocking(move || { - let _held = held; - Self::extract_llama_cpp_binary(&archive_path, &extracted, &asset_name) - }) - .await - .map_err(|e| { - PumasError::Other(format!("Failed to join llama.cpp extraction task: {e}")) - })??; - self.check_cancelled()?; - self.finalize_llama_cpp_installation(tag, release, &stage, custody, progress_tx) - .await?; - Ok(()) - } - fn publish_native_stage( stage: &Path, destination: &Path, @@ -1054,8 +1965,19 @@ impl VersionInstaller { Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => return Err(PumasError::io_with_path(error, destination)), } + sync_native_tree(stage)?; std::fs::rename(stage, destination) .map_err(|e| PumasError::io_with_path(e, destination))?; + sync_native_directory( + stage + .parent() + .ok_or_else(|| PumasError::Other("Native stage parent absent".into()))?, + )?; + sync_native_directory( + destination + .parent() + .ok_or_else(|| PumasError::Other("Native destination parent absent".into()))?, + )?; if let Err((error, can_withdraw)) = finalize() { if !can_withdraw { return Err(error); @@ -1068,11 +1990,161 @@ impl VersionInstaller { ), } })?; + sync_native_directory( + stage + .parent() + .ok_or_else(|| PumasError::Other("Native stage parent absent".into()))?, + )?; + sync_native_directory( + destination + .parent() + .ok_or_else(|| PumasError::Other("Native destination parent absent".into()))?, + )?; return Err(error); } Ok(()) } + fn publish_llama_cpp_installation(publication: LlamaCppPublication) -> Result<()> { + let LlamaCppPublication { + versions, + app_id, + manager, + custody, + stage, + destination, + proof, + } = publication; + Self::validate_llama_cpp_receipt(&proof.tag, &proof, &stage)?; + if manager + .get_installed_version(&proof.tag, Some(app_id))? + .is_some() + { + return Err(PumasError::VersionAlreadyInstalled { + tag: proof.tag.clone(), + }); + } + Self::publish_native_stage(&stage, &destination, || { + match manager.update_installed_version( + &proof.tag, + proof.metadata.clone(), + Some(app_id), + ) { + Ok(()) => Ok(()), + Err(error) => match manager.get_installed_version(&proof.tag, Some(app_id)) { + Ok(None) => Err((error, true)), + _ => Err(( + PumasError::InstallationFailed { + message: format!("Native metadata publication is uncertain; retained output requires reconciliation: {error}"), + }, + false, + )), + }, + } + })?; + sync_native_metadata(&versions, app_id)?; + drop(custody); + Ok(()) + } + + fn reconcile_llama_cpp_installation( + versions: &Path, + tag: &str, + app_id: AppId, + manager: &MetadataManager, + custody: Arc, + proof: LlamaCppInstallReceiptV1, + ) -> Result<()> { + Self::validate_llama_cpp_receipt(tag, &proof, custody.path().join("output").as_path())?; + let destination = versions.join(tag); + let staged_output = custody.path().join("output"); + match std::fs::symlink_metadata(&destination) { + Ok(metadata) => { + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(PumasError::InstallationFailed { + message: "Retained llama.cpp destination is not a directory".into(), + }); + } + verify_llama_cpp_output(&destination, &proof)?; + if path_exists_sync(&staged_output)? { + return Err(PumasError::InstallationFailed { + message: "Both staged and published llama.cpp output exist".into(), + }); + } + match manager.get_installed_version(tag, Some(app_id))? { + Some(current) if metadata_matches(¤t, &proof.metadata) => {} + Some(_) => { + return Err(PumasError::InstallationFailed { + message: "Installed llama.cpp metadata conflicts with its receipt" + .into(), + }) + } + None => manager.update_installed_version( + tag, + proof.metadata.clone(), + Some(app_id), + )?, + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + verify_llama_cpp_output(&staged_output, &proof)?; + if manager.get_installed_version(tag, Some(app_id))?.is_some() { + return Err(PumasError::InstallationFailed { + message: "Installed metadata exists without its llama.cpp output".into(), + }); + } + Self::publish_native_stage(&staged_output, &destination, || { + match manager.update_installed_version( + tag, + proof.metadata.clone(), + Some(app_id), + ) { + Ok(()) => Ok(()), + Err(error) => match manager.get_installed_version(tag, Some(app_id)) { + Ok(None) => Err((error, true)), + _ => Err(( + PumasError::InstallationFailed { + message: format!( + "Native metadata recovery is uncertain: {error}" + ), + }, + false, + )), + }, + } + })?; + } + Err(error) => return Err(PumasError::io_with_path(error, &destination)), + } + sync_native_tree(&destination)?; + sync_native_directory(versions)?; + sync_native_metadata(versions, app_id)?; + drop(custody); + Ok(()) + } + + fn validate_llama_cpp_receipt( + tag: &str, + proof: &LlamaCppInstallReceiptV1, + output: &Path, + ) -> Result<()> { + if proof.schema_version != 1 + || proof.tag != tag + || proof.metadata.path != tag + || proof.metadata.release_tag != tag + || hash_metadata(&proof.metadata)? != proof.metadata_sha256 + { + return Err(PumasError::Validation { + field: "acquisition.consumer_receipt".into(), + message: "llama.cpp receipt does not match its exact version or metadata".into(), + }); + } + if path_exists_sync(output)? { + verify_llama_cpp_output(output, proof)?; + } + Ok(()) + } + fn validate_native_tag(tag: &str) -> Result<()> { if tag.is_empty() || !tag.bytes().all(|byte| { @@ -1277,19 +2349,30 @@ impl VersionInstaller { } /// Extract a llama.cpp binary archive and ensure llama-server is executable. + #[cfg(test)] fn extract_llama_cpp_binary( archive_path: &Path, version_dir: &Path, asset_name: &str, + ) -> Result<()> { + let archive = File::open(archive_path) + .map_err(|error| PumasError::io_with_path(error, archive_path))?; + Self::extract_llama_cpp_binary_from(archive, version_dir, asset_name) + } + + fn extract_llama_cpp_binary_from( + archive: File, + version_dir: &Path, + asset_name: &str, ) -> Result<()> { info!("Extracting llama.cpp binary from {}", asset_name); if asset_name.ends_with(".tar.zst") { - Self::extract_tar_zst(archive_path, version_dir)?; + Self::extract_tar_zst_from(archive, version_dir)?; } else if asset_name.ends_with(".tgz") || asset_name.ends_with(".tar.gz") { - Self::extract_tarball(archive_path, version_dir)?; + Self::extract_tarball_from(archive, version_dir)?; } else if asset_name.ends_with(".zip") { - Self::extract_zip(archive_path, version_dir)?; + Self::extract_zip_from(archive, version_dir)?; } else { return Err(PumasError::InstallationFailed { message: format!("Unsupported llama.cpp archive format: {}", asset_name), @@ -1435,10 +2518,13 @@ impl VersionInstaller { path: Some(archive_path.to_path_buf()), source: Some(e), })?; + Self::extract_tar_zst_from(file, dest_dir) + } + fn extract_tar_zst_from(file: File, dest_dir: &Path) -> Result<()> { let decoder = zstd::Decoder::new(BufReader::new(file)).map_err(|e| PumasError::Io { message: format!("Failed to create zstd decoder: {}", e), - path: Some(archive_path.to_path_buf()), + path: Some(dest_dir.to_path_buf()), source: Some(std::io::Error::other(e)), })?; @@ -1686,106 +2772,6 @@ impl VersionInstaller { Ok(()) } - /// Finalize llama.cpp installation metadata. - async fn finalize_llama_cpp_installation( - &self, - tag: &str, - release: &GitHubRelease, - stage: &Path, - custody: Arc, - progress_tx: &mpsc::Sender, - ) -> Result<()> { - info!("Finalizing llama.cpp installation for {}", tag); - - { - let mut tracker = self.progress_tracker.write().await; - tracker.update_stage( - InstallationStage::Setup, - 0.0, - Some("Finalizing installation..."), - ); - } - self.send_progress( - progress_tx, - ProgressUpdate::StageChanged { - stage: InstallationStage::Setup, - message: "Finalizing installation...".to_string(), - }, - ) - .await; - - let selected = self.select_llama_cpp_asset(&release.assets)?; - let (download_url, size) = (Some(selected.download_url.clone()), Some(selected.size)); - - let metadata = InstalledVersionMetadata { - path: tag.to_string(), - installed_date: Utc::now().to_rfc3339(), - release_tag: tag.to_string(), - python_version: None, - git_commit: None, - release_date: Some(release.published_at.clone()), - release_notes: release.body.clone(), - download_url, - size, - requirements_hash: None, - dependencies_installed: Some(true), - }; - - self.check_cancelled()?; - if !self.torch_control.try_begin_publication() { - return Err(PumasError::InstallationFailed { - message: "Installation cancelled before publication".into(), - }); - } - let stage = stage.to_path_buf(); - let destination = self.versions_dir().join(tag); - let manager = self.metadata_manager.clone(); - let app_id = self.app_id; - let tag = tag.to_owned(); - tokio::task::spawn_blocking(move || { - let _custody = custody; - if manager.get_installed_version(&tag, Some(app_id))?.is_some() { - return Err(PumasError::VersionAlreadyInstalled { tag }); - } - Self::publish_native_stage(&stage, &destination, || { - match manager.update_installed_version(&tag, metadata, Some(app_id)) { - Ok(()) => Ok(()), - Err(error) => { - // atomic_write_json may fail after replacement. Withdraw - // output only when the metadata owner proves no entry. - match manager.get_installed_version(&tag, Some(app_id)) { - Ok(None) => Err((error, true)), - _ => Err((PumasError::InstallationFailed { - message: format!("Native metadata publication failed with retained complete output; reconciliation required: {error}"), - }, false)), - } - } - } - }) - }) - .await - .map_err(|e| PumasError::Other(format!("Failed to join native publication: {e}")))??; - - { - let mut tracker = self.progress_tracker.write().await; - tracker.update_stage( - InstallationStage::Setup, - 100.0, - Some("Installation complete"), - ); - } - self.send_progress( - progress_tx, - ProgressUpdate::Setup { - message: "Installation complete".to_string(), - }, - ) - .await; - - info!("llama.cpp installation finalized"); - Ok(()) - } - async fn download_archive( &self, url: &str, @@ -1959,7 +2945,10 @@ impl VersionInstaller { path: Some(archive_path.to_path_buf()), source: Some(e), })?; + Self::extract_zip_from(file, extract_dir) + } + fn extract_zip_from(file: File, extract_dir: &Path) -> Result<()> { let mut archive = zip::ZipArchive::new(file).map_err(|e| PumasError::InstallationFailed { message: format!("Invalid zip archive: {}", e), @@ -2026,7 +3015,10 @@ impl VersionInstaller { path: Some(archive_path.to_path_buf()), source: Some(e), })?; + Self::extract_tarball_from(file, extract_dir) + } + fn extract_tarball_from(file: File, extract_dir: &Path) -> Result<()> { let decoder = flate2::read::GzDecoder::new(BufReader::new(file)); let mut archive = tar::Archive::new(decoder); @@ -2183,6 +3175,127 @@ fn shell_single_quote(value: &str) -> String { mod tests { use super::*; + #[test] + fn native_attempt_reopen_preserves_identity_and_remove_reinstall_renews_it() { + let root = tempfile::tempdir().unwrap(); + let first = NativeInstallWorkspace::create(root.path(), "b1234+cpu").unwrap(); + let first_identity = first.attempt.clone(); + let first_path = first.path().to_path_buf(); + drop(first); + let reopened = NativeInstallWorkspace::create(root.path(), "b1234+cpu").unwrap(); + assert_eq!(reopened.attempt, first_identity); + assert_eq!(reopened.path(), first_path); + drop(reopened); + let lock = NativeVersionsLock::try_acquire(root.path()).unwrap(); + mark_native_attempt_removed(root.path(), "b1234+cpu").unwrap(); + assert!(!first_path.exists()); + drop(lock); + assert!(NativeInstallWorkspace::create_for_attempt( + root.path(), + "b1234+cpu", + Some(&first_identity) + ) + .is_err()); + let reinstalled = NativeInstallWorkspace::create(root.path(), "b1234+cpu").unwrap(); + assert_ne!(reinstalled.attempt, first_identity); + assert_ne!(reinstalled.path(), first_path); + reinstalled.cleanup().unwrap(); + } + + #[test] + fn native_receipt_reconciles_staged_publication_and_rejects_changed_output() { + let root = tempfile::tempdir().unwrap(); + let versions = root.path().join("llama-cpp-versions"); + let custody = Arc::new(NativeInstallWorkspace::create(&versions, "b1234+cpu").unwrap()); + let stage = custody.path().join("output"); + std::fs::create_dir_all(stage.join("bin")).unwrap(); + let launcher = stage.join("bin").join(if cfg!(windows) { + "llama-server.exe" + } else { + "llama-server" + }); + std::fs::write(&launcher, b"verified native output").unwrap(); + let metadata = InstalledVersionMetadata { + path: "b1234+cpu".into(), + installed_date: "2026-09-30T00:00:00Z".into(), + release_tag: "b1234+cpu".into(), + python_version: None, + git_commit: None, + release_date: None, + release_notes: None, + download_url: None, + size: Some(22), + requirements_hash: None, + dependencies_installed: Some(true), + }; + let proof = LlamaCppInstallReceiptV1 { + schema_version: 1, + tag: "b1234+cpu".into(), + metadata_sha256: hash_metadata(&metadata).unwrap(), + metadata, + output_tree_sha256: hash_native_tree(&stage).unwrap(), + launcher_relative_path: if cfg!(windows) { + "bin/llama-server.exe".into() + } else { + "bin/llama-server".into() + }, + launcher_sha256: hash_regular_file(&launcher).unwrap(), + }; + // Reopen the stable workspace after a prepare-before-publish interruption. + drop(custody); + let custody = Arc::new(NativeInstallWorkspace::create(&versions, "b1234+cpu").unwrap()); + let manager = MetadataManager::new(root.path()); + manager.ensure_directories().unwrap(); + VersionInstaller::reconcile_llama_cpp_installation( + &versions, + "b1234+cpu", + AppId::LlamaCpp, + &manager, + custody.clone(), + proof.clone(), + ) + .unwrap(); + assert!(!stage.exists()); + let destination = versions.join("b1234+cpu"); + assert!(metadata_matches( + &manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .unwrap(), + &proof.metadata + )); + // Already-published recovery checks exact bytes and metadata before success. + VersionInstaller::reconcile_llama_cpp_installation( + &versions, + "b1234+cpu", + AppId::LlamaCpp, + &manager, + custody.clone(), + proof.clone(), + ) + .unwrap(); + std::fs::write( + destination.join(&proof.launcher_relative_path), + b"changed output", + ) + .unwrap(); + let published_launcher = destination.join(&proof.launcher_relative_path); + let error = VersionInstaller::reconcile_llama_cpp_installation( + &versions, + "b1234+cpu", + AppId::LlamaCpp, + &manager, + custody, + proof, + ) + .unwrap_err(); + assert!(error.to_string().contains("recovery required")); + assert_eq!( + std::fs::read(published_launcher).unwrap(), + b"changed output" + ); + } + #[test] fn torch_cancel_and_publication_are_mutually_exclusive() { let control = TorchInstallControl::new(); @@ -2471,7 +3584,7 @@ mod tests { #[tokio::test] async fn native_worker_retains_custody_after_waiter_is_cancelled() { let root = tempfile::tempdir().unwrap(); - let custody = Arc::new(NativeInstallWorkspace::create(root.path()).unwrap()); + let custody = Arc::new(NativeInstallWorkspace::create(root.path(), "fixture").unwrap()); let path = custody.path().to_path_buf(); std::fs::write(path.join("archive"), "owned").unwrap(); let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); @@ -2490,21 +3603,21 @@ mod tests { drop(custody); worker.abort(); // spawn_blocking is already running and cannot be aborted. assert!(path.exists()); - assert!(NativeInstallWorkspace::create(root.path()).is_err()); + assert!(NativeInstallWorkspace::create(root.path(), "fixture").is_err()); release_tx.send(()).unwrap(); worker.await.unwrap(); // Cancellation loses the receipt, so implicit destruction preserves // bytes for reconciliation even after the blocking worker has settled. assert!(path.exists()); - let next = NativeInstallWorkspace::create(root.path()).unwrap(); + let next = NativeInstallWorkspace::create(root.path(), "fixture").unwrap(); next.cleanup().unwrap(); - std::fs::remove_dir_all(path).unwrap(); + assert!(!path.exists()); } #[test] fn native_cleanup_failure_reports_and_retains_its_path() { let root = tempfile::tempdir().unwrap(); - let custody = NativeInstallWorkspace::create(root.path()).unwrap(); + let custody = NativeInstallWorkspace::create(root.path(), "fixture").unwrap(); let path = custody.path().to_owned(); std::fs::remove_dir(&path).unwrap(); std::fs::write(&path, "retained uncertainty").unwrap(); @@ -2512,16 +3625,13 @@ mod tests { assert!(error.contains("cleanup incomplete")); assert!(error.contains(&path.display().to_string())); assert_eq!(std::fs::read(&path).unwrap(), b"retained uncertainty"); - NativeInstallWorkspace::create(root.path()) - .unwrap() - .cleanup() - .unwrap(); + assert!(NativeInstallWorkspace::create(root.path(), "fixture").is_err()); } #[test] fn native_cancelled_archive_settles_queued_writes_before_reclamation() { let root = tempfile::tempdir().unwrap(); - let custody = NativeInstallWorkspace::create(root.path()).unwrap(); + let custody = NativeInstallWorkspace::create(root.path(), "fixture").unwrap(); let path = custody.path().join("archive"); let file = std::fs::File::create(&path).unwrap(); let runtime = tokio::runtime::Builder::new_current_thread() diff --git a/rust/crates/pumas-app-manager/src/version_manager/mod.rs b/rust/crates/pumas-app-manager/src/version_manager/mod.rs index 85d07d1f..debd35c7 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/mod.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/mod.rs @@ -72,6 +72,7 @@ pub use torch_preview::{ TorchArtifact, TorchPreview, TorchPreviewOutcome, TorchPreviewRejectionReason, }; +use pumas_library::acquisition::{AcquisitionConsumer, AcquisitionService}; use pumas_library::config::{AppId, PathsConfig}; use pumas_library::metadata::MetadataManager; use pumas_library::models::InstallationProgress; @@ -182,6 +183,8 @@ pub struct VersionManager { metadata_manager: Arc, /// GitHub client for fetching releases. github_client: Arc, + /// Native release consumers share PumasApi's single acquisition service. + acquisition_consumer: Option>, /// Version state tracker. state: Arc>, /// Installation progress tracker. @@ -200,6 +203,8 @@ pub struct VersionManager { #[cfg(test)] torch_publication_pause: Option>, #[cfg(test)] + native_receipt_pause: Option>, + #[cfg(test)] torch_stage_override: Option, #[cfg(test)] removal_pause: Option>, @@ -326,6 +331,7 @@ impl VersionManager { app_id, metadata_manager, github_client, + acquisition_consumer: None, state, progress_tracker, cancel_flag: Arc::new(AtomicBool::new(false)), @@ -341,6 +347,8 @@ impl VersionManager { #[cfg(test)] torch_publication_pause: None, #[cfg(test)] + native_receipt_pause: None, + #[cfg(test)] torch_stage_override: None, #[cfg(test)] removal_pause: None, @@ -395,6 +403,51 @@ impl VersionManager { Ok(manager) } + /// Construct the llama.cpp manager with the application's existing shared + /// acquisition owner. A second service/store is never created here. + pub async fn new_with_acquisition( + launcher_root: impl Into, + app_id: AppId, + acquisition: Arc, + ) -> Result { + if app_id != AppId::LlamaCpp { + return Err(PumasError::Config { + message: "Shared artifact acquisition is currently required for llama.cpp".into(), + }); + } + let mut manager = Self::new(launcher_root, app_id).await?; + manager.acquisition_consumer = + Some(Arc::new(acquisition.open_consumer("runtime.llama.cpp")?)); + let store = acquisition.store().clone(); + let records = tokio::task::spawn_blocking(move || store.acquisitions()) + .await + .map_err(|error| { + PumasError::Other(format!("Native recovery lookup failed: {error}")) + })??; + let installer = VersionInstaller::new( + manager.launcher_root.clone(), + app_id, + manager.metadata_manager.clone(), + manager.progress_tracker.clone(), + manager.cancel_flag.clone(), + ) + .with_acquisition_consumer(manager.acquisition_consumer.clone()); + if let Err(error) = installer + .reconcile_retained_llama_cpp(records.into_values().collect()) + .await + { + let settlement = manager.shutdown_installations().await; + return Err(match settlement { + Ok(()) => error, + Err(settlement) => PumasError::InstallationFailed { + message: format!("{error}; native recovery shutdown: {settlement}"), + }, + }); + } + manager.state.write().await.refresh().await?; + Ok(manager) + } + // ======================================== // Path helpers // ======================================== @@ -806,6 +859,11 @@ impl VersionManager { errors.push(error.to_string()); } } + if let Some(consumer) = &manager.acquisition_consumer { + if let Err(error) = consumer.shutdown().await { + errors.push(error.to_string()); + } + } if errors.is_empty() { Ok(()) } else { @@ -849,6 +907,12 @@ impl VersionManager { tag: &str, preview_id: Option<&str>, ) -> Result> { + if self.app_id == AppId::LlamaCpp && self.acquisition_consumer.is_none() { + return Err(PumasError::Config { + message: "llama.cpp installation requires the shared artifact acquisition service" + .into(), + }); + } if self.app_id == AppId::Torch && torch_alternatives::stable_release_version(tag).is_none() { return Err(PumasError::VersionNotFound { @@ -968,7 +1032,15 @@ impl VersionManager { ) .with_torch_control(self.torch_control.clone()) .with_torch_cleanup(self.torch_cleanup.clone()) - .with_shutdown_flag(self.torch_shutting_down.clone()); + .with_shutdown_flag(self.torch_shutting_down.clone()) + .with_github_client(self.github_client.clone()) + .with_acquisition_consumer(self.acquisition_consumer.clone()); + #[cfg(test)] + let installer = if let Some(pause) = &self.native_receipt_pause { + installer.with_native_receipt_pause(pause.clone()) + } else { + installer + }; #[cfg(test)] let installer = if let Some(pause) = &self.torch_publication_pause { installer.with_torch_publication_pause(pause.clone()) @@ -1258,7 +1330,11 @@ impl VersionManager { let metadata = self.metadata_manager.clone(); let removed_tag = tag.to_owned(); let app_id = self.app_id; + let versions_for_removal = self.versions_dir(); let remove = move || { + if app_id == AppId::LlamaCpp { + installer::mark_native_attempt_removed(&versions_for_removal, &removed_tag)?; + } info!("Removing version directory: {}", version_path.display()); match std::fs::remove_dir_all(&version_path) { Ok(()) => {} @@ -1424,7 +1500,9 @@ mod tests { tokio::task::JoinHandle<()>, tokio::sync::oneshot::Receiver<()>, tokio::sync::oneshot::Sender, + String, ) { + use sha2::Digest; use tokio::io::{AsyncReadExt, AsyncWriteExt}; let payload = b"#!/bin/sh\nprintf 'native-fixture'\n"; let compressed = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); @@ -1439,7 +1517,9 @@ mod tests { let bytes = archive.into_inner().unwrap().finish().unwrap(); let size = bytes.len() as u64; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let url = format!("http://{}/archive", listener.local_addr().unwrap()); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let url = format!("{base_url}/archive"); + let digest = format!("{:x}", sha2::Sha256::digest(&bytes)); let releases = pumas_library::network::ReleasesCache::new( root.join("launcher-data/cache"), Duration::from_secs(3600), @@ -1458,7 +1538,7 @@ mod tests { assets: vec![pumas_library::network::GitHubAsset { name: "llama-b1234-bin-ubuntu-x64.tar.gz".into(), size, - download_url: url, + download_url: url.clone(), content_type: Some("application/gzip".into()), }], html_url: "https://github.com/ggml-org/llama.cpp/releases/tag/b1234".into(), @@ -1471,6 +1551,33 @@ mod tests { let (entered, observed) = tokio::sync::oneshot::channel(); let (release, wait) = tokio::sync::oneshot::channel(); let server = tokio::spawn(async move { + // Real HTTP release metadata supplies the exact publisher asset ID + // and SHA; cached discovery has neither and cannot authorize bytes. + let (mut metadata_stream, _) = listener.accept().await.unwrap(); + let mut request = [0; 4096]; + let read = metadata_stream.read(&mut request).await.unwrap(); + assert!(String::from_utf8_lossy(&request[..read]) + .starts_with("GET /repos/ggml-org/llama.cpp/releases/tags/b1234 HTTP/1.1")); + let body = serde_json::to_vec(&serde_json::json!({ + "tag_name": "b1234", "assets": [{ + "id": 1234, "name": "llama-b1234-bin-ubuntu-x64.tar.gz", + "size": size, "browser_download_url": url, + "digest": format!("sha256:{digest}") + }] + })) + .unwrap(); + metadata_stream + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + metadata_stream.write_all(&body).await.unwrap(); + drop(metadata_stream); let (mut stream, _) = listener.accept().await.unwrap(); let mut request = [0; 4096]; assert!(stream.read(&mut request).await.unwrap() > 0); @@ -1489,7 +1596,7 @@ mod tests { stream.write_all(&bytes).await.unwrap(); } }); - (server, observed, release) + (server, observed, release, base_url) } #[cfg(all(target_os = "linux", target_arch = "x86_64"))] @@ -1757,14 +1864,196 @@ mod tests { }); } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn native_cancel_before_completion_receipt_prevents_publication() { + let root = TempDir::new().unwrap(); + let (server, observed, release, base_url) = native_archive_fixture(root.path()).await; + let api = pumas_library::PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() + .await + .unwrap(); + let mut manager = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + manager.github_client = Arc::new( + GitHubClient::with_loopback_api( + manager.cache_dir(), + Duration::from_secs(3600), + base_url, + ) + .unwrap(), + ); + let pause = Arc::new(installer::TorchPublicationPause::new()); + manager.native_receipt_pause = Some(pause.clone()); + let mut updates = manager.install_version("b1234+cpu").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .unwrap() + .unwrap(); + release.send(true).unwrap(); + tokio::time::timeout(Duration::from_secs(5), pause.reached.notified()) + .await + .expect("native extraction and proof hashes must reach the receipt boundary"); + assert!(manager.cancel_installation().await.unwrap()); + pause.resume.add_permits(1); + let message = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match updates + .recv() + .await + .expect("installation must send its terminal result") + { + ProgressUpdate::Error { message } => break message, + ProgressUpdate::Completed { success } => { + panic!("cancelled installation reported completion: {success}") + } + _ => {} + } + } + }) + .await + .expect("cancelled attempt must settle"); + assert!(message.to_lowercase().contains("cancel"), "{message}"); + server.await.unwrap(); + // Shutdown joins the registered installer task and observes its failure. + assert!(manager.shutdown_installations().await.is_err()); + assert!(!manager.is_installing().await); + let document: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.path().join("launcher-data/downloads.json")).unwrap(), + ) + .unwrap(); + assert_eq!(document["acquisitions"].as_object().unwrap().len(), 1); + assert!(document["consumer_receipts"] + .as_object() + .unwrap() + .is_empty()); + assert!(!manager.version_path("b1234+cpu").exists()); + assert!(manager + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_none()); + assert_eq!( + document["acquisitions"] + .as_object() + .unwrap() + .values() + .next() + .unwrap()["phase"]["state"], + "withdrawn" + ); + assert!(!std::fs::read_dir(manager.versions_dir()) + .unwrap() + .any(|entry| { + entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".llama-install-") + })); + api.shutdown_acquisition().await.unwrap(); + drop(manager); + drop(api); + // A fresh service and manager must admit a new attempt for the same tag. + let (retry_server, retry_observed, retry_release, retry_base_url) = + native_archive_fixture(root.path()).await; + let reopened_api = pumas_library::PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() + .await + .unwrap(); + let mut reopened = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + reopened_api.acquisition().clone(), + ) + .await + .unwrap(); + reopened.github_client = Arc::new( + GitHubClient::with_loopback_api( + reopened.cache_dir(), + Duration::from_secs(3600), + retry_base_url, + ) + .unwrap(), + ); + let mut retry_updates = reopened.install_version("b1234+cpu").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), retry_observed) + .await + .unwrap() + .unwrap(); + retry_release.send(true).unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match retry_updates.recv().await.unwrap() { + ProgressUpdate::Completed { success: true } => break, + ProgressUpdate::Error { message } => panic!("same-tag retry failed: {message}"), + _ => {} + } + } + }) + .await + .unwrap(); + retry_server.await.unwrap(); + reopened.shutdown_installations().await.unwrap(); + assert!(reopened + .version_path("b1234+cpu") + .join("bin/llama-server") + .is_file()); + assert!(reopened + .metadata_manager + .get_installed_version("b1234+cpu", Some(AppId::LlamaCpp)) + .unwrap() + .is_some()); + let retry_document: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.path().join("launcher-data/downloads.json")).unwrap(), + ) + .unwrap(); + assert_eq!(retry_document["acquisitions"].as_object().unwrap().len(), 2); + assert_eq!( + retry_document["consumer_receipts"] + .as_object() + .unwrap() + .len(), + 1 + ); + reopened_api.shutdown_acquisition().await.unwrap(); + } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] #[tokio::test] async fn native_archive_publishes_complete_output_and_reopens_metadata() { let root = TempDir::new().unwrap(); - let (server, observed, release) = native_archive_fixture(root.path()).await; - let manager = VersionManager::new(root.path(), AppId::LlamaCpp) + let (server, observed, release, base_url) = native_archive_fixture(root.path()).await; + let api = pumas_library::PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() .await .unwrap(); + let mut manager = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + manager.github_client = Arc::new( + GitHubClient::with_loopback_api( + manager.cache_dir(), + Duration::from_secs(3600), + base_url, + ) + .unwrap(), + ); let mut updates = manager.install_version("b1234+cpu").await.unwrap(); tokio::time::timeout(Duration::from_secs(2), observed) .await @@ -1799,9 +2088,37 @@ mod tests { .unwrap(); assert!(output.status.success()); assert_eq!(output.stdout, b"native-fixture"); - let reopened = VersionManager::new(root.path(), AppId::LlamaCpp) - .await + use sha2::Digest; + let document: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.path().join("launcher-data/downloads.json")).unwrap(), + ) + .unwrap(); + let operation = document["acquisitions"] + .as_object() + .unwrap() + .values() + .next() + .unwrap()["demand"]["operation"] + .as_str() .unwrap(); + let attempt = operation.rsplit_once(':').unwrap().1; + let tag_digest = format!("{:x}", sha2::Sha256::digest(b"b1234+cpu")); + let stale_workspace = manager + .versions_dir() + .join(format!(".llama-install-{}-{attempt}", &tag_digest[..24])); + std::fs::create_dir(&stale_workspace).unwrap(); + std::fs::write( + stale_workspace.join("stale"), + b"retry cleanup after adoption", + ) + .unwrap(); + let reopened = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); assert_eq!( reopened.get_installed_versions().await.unwrap(), vec!["b1234+cpu"] @@ -1816,16 +2133,124 @@ mod tests { .starts_with(".llama-install-") })); reopened.shutdown_installations().await.unwrap(); + let document: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.path().join("launcher-data/downloads.json")).unwrap(), + ) + .unwrap(); + assert_eq!(document["acquisitions"].as_object().unwrap().len(), 1); + assert_eq!(document["consumer_receipts"].as_object().unwrap().len(), 1); + let receipt = document["consumer_receipts"] + .as_object() + .unwrap() + .values() + .next() + .unwrap(); + assert_eq!(receipt["owner"], "runtime.llama.cpp"); + // Exercise explicit removal through the real manager and reinstall the + // same tag with a new publisher-verified acquisition identity. + let reinstall = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + let mut keep = reinstall + .get_version_info("b1234+cpu") + .await + .unwrap() + .unwrap(); + keep.path = "b9999+cpu".into(); + keep.release_tag = "b9999+cpu".into(); + std::fs::create_dir(reinstall.version_path("b9999+cpu")).unwrap(); + reinstall + .state + .write() + .await + .add_installed_version("b9999+cpu", keep) + .unwrap(); + reinstall.set_active_version("b9999+cpu").await.unwrap(); + assert!(reinstall.remove_version("b1234+cpu").await.unwrap()); + reinstall.shutdown_installations().await.unwrap(); + let mut reinstall = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + assert!(!reinstall.version_path("b1234+cpu").exists()); + let (server, observed, release, base_url) = native_archive_fixture(root.path()).await; + reinstall.github_client = Arc::new( + GitHubClient::with_loopback_api( + reinstall.cache_dir(), + Duration::from_secs(3600), + base_url, + ) + .unwrap(), + ); + let mut updates = reinstall.install_version("b1234+cpu").await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), observed) + .await + .unwrap() + .unwrap(); + release.send(true).unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + while let Some(update) = updates.recv().await { + match update { + ProgressUpdate::Completed { success: true } => return, + ProgressUpdate::Error { message } => panic!("Reinstall failed: {message}"), + _ => {} + } + } + panic!("Reinstall ended without completion"); + }) + .await + .unwrap(); + server.await.unwrap(); + reinstall.shutdown_installations().await.unwrap(); + let document: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.path().join("launcher-data/downloads.json")).unwrap(), + ) + .unwrap(); + let records = document["acquisitions"].as_object().unwrap(); + assert_eq!(records.len(), 2); + let demands: std::collections::BTreeSet<_> = records + .values() + .map(|record| record["demand"]["operation"].as_str().unwrap()) + .collect(); + assert_eq!(demands.len(), 2); + assert_eq!(document["consumer_receipts"].as_object().unwrap().len(), 2); + api.shutdown_intent().await.unwrap(); + api.shutdown_acquisition().await.unwrap(); } #[cfg(all(target_os = "linux", target_arch = "x86_64"))] #[tokio::test] async fn native_shutdown_cancels_stalled_transfer_and_retains_failed_outcome() { let root = TempDir::new().unwrap(); - let (server, observed, release) = native_archive_fixture(root.path()).await; - let manager = VersionManager::new(root.path(), AppId::LlamaCpp) + let (server, observed, release, base_url) = native_archive_fixture(root.path()).await; + let api = pumas_library::PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() .await .unwrap(); + let mut manager = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + manager.github_client = Arc::new( + GitHubClient::with_loopback_api( + manager.cache_dir(), + Duration::from_secs(3600), + base_url, + ) + .unwrap(), + ); let _updates = manager.install_version("b1234+cpu").await.unwrap(); tokio::time::timeout(Duration::from_secs(2), observed) .await @@ -1836,7 +2261,7 @@ mod tests { .unwrap() .unwrap_err() .to_string(); - assert!(error.contains("cancelled")); + assert!(error.to_ascii_lowercase().contains("cancelled"), "{error}"); assert_eq!( manager .shutdown_installations() @@ -1855,13 +2280,15 @@ mod tests { .is_none()); assert!(std::fs::read_dir(manager.versions_dir()) .unwrap() - .all(|entry| { - !entry + .any(|entry| { + entry .unwrap() .file_name() .to_string_lossy() .starts_with(".llama-install-") })); + api.shutdown_intent().await.unwrap(); + api.shutdown_acquisition().await.unwrap(); assert!(!manager.is_installing().await); assert!(manager.get_installation_progress().await.unwrap().success == Some(false)); } diff --git a/rust/crates/pumas-core/src/acquisition/http.rs b/rust/crates/pumas-core/src/acquisition/http.rs index c3fb049f..5810b3d7 100644 --- a/rust/crates/pumas-core/src/acquisition/http.rs +++ b/rust/crates/pumas-core/src/acquisition/http.rs @@ -26,7 +26,7 @@ pub(crate) trait HttpArtifactSink: Send { /// Existing supervised operation supplies cancellation and progress projection /// without transferring its lifecycle ownership to the protocol adapter. #[async_trait::async_trait] -pub(crate) trait HttpAttemptHost: Send { +pub trait HttpAttemptHost: Send { async fn pause_requested(&self); fn pause_requested_now(&self) -> bool; fn cancel_requested(&self) -> bool; diff --git a/rust/crates/pumas-core/src/acquisition/mod.rs b/rust/crates/pumas-core/src/acquisition/mod.rs index 3ddf0a81..20138414 100644 --- a/rust/crates/pumas-core/src/acquisition/mod.rs +++ b/rust/crates/pumas-core/src/acquisition/mod.rs @@ -13,7 +13,7 @@ pub(crate) mod store; pub(crate) mod task_custody; mod workspace; -pub(crate) use http::HttpAttemptHost; +pub use http::HttpAttemptHost; pub(crate) use github_release::{select_github_release_asset, GitHubReleaseAssetMetadata}; pub use github_release::{ @@ -25,7 +25,10 @@ pub use manifest::{ CURRENT_MANIFEST_VERSION, }; -pub use service::{AcquisitionDemand, AcquisitionPhase, AcquisitionRecord, AcquisitionService}; -pub(crate) use service::{AcquisitionHost, AcquisitionRetryPolicy}; +pub use service::{ + AcquiredArtifactUse, AcquisitionConsumer, AcquisitionConsumerReceipt, AcquisitionDemand, + AcquisitionHost, AcquisitionHttpRequest, AcquisitionHttpSource, AcquisitionPhase, + AcquisitionRecord, AcquisitionRetryPolicy, AcquisitionService, +}; pub use store::AcquisitionStore; pub use workspace::{AcquisitionWorkspace, VerifiedFile, WorkspaceIdentity}; diff --git a/rust/crates/pumas-core/src/acquisition/service.rs b/rust/crates/pumas-core/src/acquisition/service.rs index b32f918f..e6f548b3 100644 --- a/rust/crates/pumas-core/src/acquisition/service.rs +++ b/rust/crates/pumas-core/src/acquisition/service.rs @@ -8,6 +8,8 @@ use super::workspace::{write_chunk, AcquisitionWorkspace, VerifiedFile, Workspac use super::ArtifactManifest; use crate::{PumasError, Result}; use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::future::Future; use std::sync::Arc; use std::time::{Duration, Instant}; use uuid::Uuid; @@ -48,6 +50,84 @@ pub struct AcquisitionRecord { pub files: Vec, } +/// Versioned consumer-owned completion data, paired with the exact acquisition +/// record and use lease that authorized publication. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AcquisitionConsumerReceipt { + pub receipt_kind: String, + pub receipt_version: u32, + pub owner: String, + pub acquisition_id: String, + pub use_lease: String, + pub demand: AcquisitionDemand, + pub manifest: ArtifactManifest, + pub workspace: WorkspaceIdentity, + pub verified_files: Vec, + pub payload: Value, +} + +impl AcquisitionConsumerReceipt { + const KIND: &'static str = "pumas.consumer-completion"; + + fn for_record(record: &AcquisitionRecord, lease: Uuid, payload: Value) -> Result { + let receipt = Self { + receipt_kind: Self::KIND.into(), + receipt_version: 1, + owner: record.demand.consumer.clone(), + acquisition_id: record.id.to_string(), + use_lease: lease.to_string(), + demand: record.demand.clone(), + manifest: record.manifest.clone(), + workspace: record.workspace.clone(), + verified_files: record.files.clone(), + payload, + }; + receipt.validate_for_record(record, lease)?; + Ok(receipt) + } + + pub(crate) fn validate_for_record( + &self, + record: &AcquisitionRecord, + lease: Uuid, + ) -> Result<()> { + if self.receipt_kind != Self::KIND + || self.receipt_version != 1 + || self.owner != record.demand.consumer + || self.acquisition_id != record.id.to_string() + || self.use_lease != lease.to_string() + || self.demand != record.demand + || self.manifest != record.manifest + || self.workspace != record.workspace + || self.verified_files != record.files + { + return Err(invalid( + "Consumer completion receipt does not bind the exact acquisition use", + )); + } + Ok(()) + } +} + +/// Ephemeral location and optional authorization for one manifest file. These +/// values are used for the request only and are never persisted. +#[derive(Clone, Default)] +pub struct AcquisitionHttpSource { + pub url: String, + pub authorization: Option, +} + +/// Request data for the shared HTTP acquisition lifecycle. +#[derive(Clone)] +pub struct AcquisitionHttpRequest { + pub demand: AcquisitionDemand, + pub manifest: ArtifactManifest, + pub workspace: AcquisitionWorkspace, + pub sources: Vec, + pub retry: AcquisitionRetryPolicy, +} + fn invalid(message: &str) -> PumasError { PumasError::Validation { field: "acquisition.custody".into(), @@ -135,6 +215,26 @@ pub(crate) struct AcquisitionUseLease { context: TaskContext, } +impl AcquisitionUseLease { + pub(crate) fn record(&self) -> &AcquisitionRecord { + &self.operation.record + } +} + +/// Cold observation of a retained `Using` lease. It proves only that the +/// historic selection and file receipts still match under the reopened +/// workspace; it cannot resume importer effects or renew the lease. +pub(crate) struct AcquisitionReopenProof { + record: AcquisitionRecord, + _workspace: AcquisitionWorkspace, +} + +impl AcquisitionReopenProof { + pub(crate) fn record(&self) -> &AcquisitionRecord { + &self.record + } +} + /// Source-neutral, exact runtime proofs. A retained row or workspace locator /// cannot construct either proof or reopen unresolved consumer use. pub(crate) struct AcquisitionUseProof(AcquisitionProof); @@ -193,14 +293,14 @@ impl AcquisitionProof { } #[derive(Clone)] -pub(crate) struct AcquisitionRetryPolicy { - pub(crate) attempts: Option, - pub(crate) elapsed: Duration, - pub(crate) backoff: crate::network::RetryConfig, +pub struct AcquisitionRetryPolicy { + pub attempts: Option, + pub elapsed: Duration, + pub backoff: crate::network::RetryConfig, } #[async_trait::async_trait] -pub(crate) trait AcquisitionHost: HttpAttemptHost { +pub trait AcquisitionHost: HttpAttemptHost { async fn retry( &mut self, attempt: u32, @@ -209,6 +309,93 @@ pub(crate) trait AcquisitionHost: HttpAttemptHost { ) -> Result<()>; } +/// Verified manifest file use held inside one current acquisition worker. +/// The workspace capability and exact `Using` lease remain held until the +/// consumer callback returns and its completion receipt is atomically settled. +pub struct AcquiredArtifactUse { + store: Arc, + context: TaskContext, + lease: AcquisitionUseLease, +} + +impl AcquiredArtifactUse { + /// Join registered effects, then durably revoke and reclaim consumer output. + /// Only successful cleanup permits exact, receipt-free Using withdrawal. + /// Cleanup must revoke this attempt before reclaiming it. Cleanup errors retain + /// Using; withdrawal publication failures propagate durability uncertainty. + pub async fn withdraw_after_cleanup( + self, + cleanup: impl FnOnce() -> Result<()> + Send + 'static, + ) -> Result<()> { + match self.context.drain_blocking().await { + Ok(0) => {} + result => { + return Err(PumasError::Other(format!( + "Consumer effects unsettled before cancellation cleanup: {result:?}" + ))) + } + } + let store = self.store.clone(); + let expected = self.lease.record().clone(); + owned( + &self.context, + "check exact unreceipted consumer use", + move || store.require_unreceipted_use(&expected), + ) + .await?; + owned( + &self.context, + "revoke and clean cancelled consumer output", + cleanup, + ) + .await?; + let store = self.store.clone(); + let expected = self.lease.record().clone(); + owned( + &self.context, + "withdraw exact cancelled consumer use", + move || store.withdraw_unreceipted_use(&expected), + ) + .await + } + + pub fn record(&self) -> &AcquisitionRecord { + self.lease.record() + } + + /// Open a file only after rechecking the receipt through the held directory + /// capability. The returned descriptor is read-only and no-follow. + pub async fn open_file(&self, file_index: usize) -> Result { + let record = self.lease.record(); + let selected = record + .manifest + .files() + .get(file_index) + .ok_or_else(|| invalid("Selected file is unavailable"))? + .clone(); + let verified = record + .files + .get(file_index) + .ok_or_else(|| invalid("Verified file receipt is unavailable"))? + .clone(); + let workspace = self.lease.workspace.clone(); + owned(&self.context, "open verified consumer input", move || { + workspace.open_verified_readonly(&selected, &verified) + }) + .await + } + + /// Run blocking consumer effects under the same custody worker. Dropping a + /// waiter cannot detach the blocking effect from acquisition shutdown. + pub async fn run_blocking( + &self, + name: &'static str, + work: impl FnOnce() -> Result + Send + 'static, + ) -> Result { + owned(&self.context, name, work).await + } +} + /// One durable lifecycle/store owner and the existing shared task supervisor. /// Source access is ephemeral and is never written to the durable manifest. pub struct AcquisitionService { @@ -234,10 +421,82 @@ impl AcquisitionService { pub fn store(&self) -> &Arc { &self.store } + + /// Open a narrow consumer scope on this service's one existing supervisor. + /// The returned consumer never creates a second store or task owner. + pub fn open_consumer( + self: &Arc, + owner: impl Into, + ) -> Result { + let owner = owner.into(); + if owner.trim().is_empty() { + return Err(invalid("Consumer identity is empty")); + } + let scope = self.supervisor.open_scope(|| async { Ok(()) })?; + Ok(AcquisitionConsumer { + service: self.clone(), + scope, + owner, + }) + } + + /// Read a consumer-owned receipt without granting filesystem or settlement + /// authority. Reopen callers must still validate their own durable outputs. + pub fn consumer_receipt(&self, id: Uuid) -> Result> { + self.store.consumer_receipt(id) + } pub(crate) fn supervisor(&self) -> Arc { self.supervisor.clone() } + async fn settle_consumer_use( + &self, + context: &TaskContext, + expected: &AcquisitionRecord, + lease: Uuid, + payload: Value, + ) -> Result<()> { + let receipt = AcquisitionConsumerReceipt::for_record(expected, lease, payload)?; + let store = self.store.clone(); + let expected = expected.clone(); + owned( + context, + "settle exact consumer completion receipt", + move || store.settle_consumer_use(&expected, lease, receipt), + ) + .await + } + + async fn issue_consumer_receipt( + &self, + context: &TaskContext, + expected: &AcquisitionRecord, + lease: Uuid, + receipt: AcquisitionConsumerReceipt, + ) -> Result<()> { + let store = self.store.clone(); + let expected = expected.clone(); + owned(context, "issue consumer completion receipt", move || { + store.issue_consumer_receipt(&expected, lease, &receipt) + }) + .await + } + + async fn settle_consumer_receipt( + &self, + context: &TaskContext, + expected: &AcquisitionRecord, + lease: Uuid, + receipt: AcquisitionConsumerReceipt, + ) -> Result<()> { + let store = self.store.clone(); + let expected = expected.clone(); + owned(context, "settle issued consumer receipt", move || { + store.settle_consumer_receipt(&expected, lease, &receipt) + }) + .await + } + /// Global closure after every consumer has stopped admitting work. Narrow /// consumer shutdown must close that consumer's scope first. pub async fn shutdown(self: &Arc) -> Result<()> { @@ -625,6 +884,68 @@ impl AcquisitionService { .await } + /// Reopen the exact historical `Using` record without changing its lease + /// or publishing state. The returned proof retains the verified workspace + /// through caller-owned, receipt-qualified settlement. + pub(crate) async fn reopen_using( + &self, + context: &TaskContext, + demand: &AcquisitionDemand, + manifest: &ArtifactManifest, + workspace: &AcquisitionWorkspace, + ) -> Result> { + if !context.is_current_role(super::task_custody::TaskRole::Worker) { + return Err(invalid("Cold reconciliation requires its active worker")); + } + let store = self.store.clone(); + let demand = demand.clone(); + let manifest = manifest.clone(); + let expected_workspace = workspace.identity().clone(); + let record = owned(context, "observe retained acquisition use", move || { + let Some(record) = store + .acquisitions()? + .into_values() + .find(|record| record.demand == demand) + else { + return Ok(None); + }; + if record.workspace != expected_workspace { + return Err(invalid( + "Retained acquisition does not match the reopened workspace", + )); + } + match &record.phase { + AcquisitionPhase::Using { .. } => { + if record.manifest != manifest { + return Err(invalid( + "Retained acquisition use does not match the reopened selection", + )); + } + Ok(Some(record)) + } + AcquisitionPhase::Transferring | AcquisitionPhase::FilesReady => Ok(None), + AcquisitionPhase::Adopted { .. } | AcquisitionPhase::Withdrawn => Err(invalid( + "Retained terminal acquisition cannot be replayed by a reopened consumer", + )), + } + }) + .await?; + let Some(record) = record else { + return Ok(None); + }; + let verify = workspace.clone(); + let manifest = record.manifest.clone(); + let files = record.files.clone(); + owned(context, "verify retained acquisition receipts", move || { + verify.verify_receipts(&manifest, &files) + }) + .await?; + Ok(Some(AcquisitionReopenProof { + record, + _workspace: workspace.clone(), + })) + } + pub(crate) async fn files_ready( &self, context: &TaskContext, @@ -753,6 +1074,293 @@ impl AcquisitionService { } } +/// Consumer-facing view of the existing shared acquisition lifecycle. +pub struct AcquisitionConsumer { + service: Arc, + scope: Arc, + owner: String, +} + +impl AcquisitionConsumer { + pub fn owner(&self) -> &str { + &self.owner + } + + /// Read and validate this consumer's completion receipt for an exact + /// retained record. This grants no filesystem or settlement authority; + /// adopted consumers use it to verify their own published output before + /// retrying local workspace cleanup. + pub fn completion_receipt( + &self, + record: &AcquisitionRecord, + ) -> Result> { + if record.demand.consumer != self.owner { + return Err(invalid("Completion receipt belongs to another consumer")); + } + let lease = match &record.phase { + AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } => *lease, + _ => { + return Err(invalid( + "Completion receipt requires a retained consumer use", + )) + } + }; + let Some(receipt) = self.service.consumer_receipt(record.id)? else { + return Ok(None); + }; + receipt.validate_for_record(record, lease)?; + Ok(Some(receipt)) + } + + /// Close this consumer scope and join every transfer and registered effect. + pub async fn shutdown(&self) -> Result<()> { + self.scope.shutdown().await + } + + /// Revalidate a retained consumer receipt under the supplied held + /// workspace and exact source selection. The callback owns interpretation + /// of its payload and must verify its durable output before returning. + /// This method never repeats transfer or consumer effects. + pub async fn reconcile( + &self, + demand: AcquisitionDemand, + manifest: ArtifactManifest, + workspace: AcquisitionWorkspace, + validate_output: F, + ) -> Result> + where + T: Send + 'static, + F: FnOnce(AcquisitionConsumerReceipt, AcquiredArtifactUse) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + if demand.consumer != self.owner { + return Err(invalid("Consumer demand identity does not match its scope")); + } + let service = self.service.clone(); + self.scope + .run_worker_invocation(move |context| async move { + service.require_schema(&context).await?; + let records = service.store.clone(); + let demand_lookup = demand.clone(); + let current = owned(&context, "observe consumer completion record", move || { + Ok(records + .acquisitions()? + .into_values() + .find(|record| record.demand == demand_lookup)) + }) + .await?; + let Some(record) = current else { + return Ok(None); + }; + if record.manifest != manifest || &record.workspace != workspace.identity() { + return Err(invalid( + "Retained consumer operation changed selection or workspace", + )); + } + let lease = match &record.phase { + AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } => { + *lease + } + AcquisitionPhase::Transferring | AcquisitionPhase::FilesReady => { + return Ok(None) + } + AcquisitionPhase::Withdrawn => { + return Err(invalid("Withdrawn consumer operation cannot be reconciled")) + } + }; + if matches!(&record.phase, AcquisitionPhase::Using { .. }) { + let proof = service + .reopen_using(&context, &demand, &manifest, &workspace) + .await? + .ok_or_else(|| invalid("Retained consumer use disappeared"))?; + if proof.record() != &record { + return Err(invalid("Retained consumer use changed during reopen")); + } + } else { + let verify = workspace.clone(); + let manifest = manifest.clone(); + let files = record.files.clone(); + owned( + &context, + "verify adopted consumer input receipts", + move || verify.verify_receipts(&manifest, &files), + ) + .await?; + } + let receipt_store = service.store.clone(); + let receipt = owned( + &context, + "read exact consumer completion receipt", + move || { + receipt_store.consumer_receipt(record.id)?.ok_or_else(|| { + PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: + "Retained consumer use has no authoritative completion receipt" + .into(), + } + }) + }, + ) + .await?; + receipt.validate_for_record(&record, lease)?; + let use_handle = AcquiredArtifactUse { + store: service.store.clone(), + context: context.clone(), + lease: AcquisitionUseLease { + operation: AcquisitionOperation { + record: record.clone(), + context: context.clone(), + }, + workspace: workspace.clone(), + lease, + context: context.clone(), + }, + }; + let result = validate_output(receipt.clone(), use_handle).await?; + match context.drain_blocking().await { + Ok(0) => {} + Ok(failures) => { + return Err(PumasError::Other(format!( + "Consumer recovery effects failed before settlement: {failures}" + ))) + } + Err(error) => { + return Err(PumasError::Other(format!( + "Consumer recovery effect drain failed before settlement: {error}" + ))) + } + } + service + .settle_consumer_use(&context, &record, lease, receipt.payload) + .await?; + Ok(Some(result)) + }) + .await + } + + /// Acquire the exact manifest using shared HTTP custody, then run the + /// consumer's extraction/publication while the verified use lease remains + /// held. The callback returns its owner-specific durable receipt payload. + pub async fn acquire_http( + &self, + request: AcquisitionHttpRequest, + client: reqwest::Client, + mut host: Box, + prepare: F, + publish: Publish, + ) -> Result + where + Staged: Send + 'static, + Output: Send + 'static, + F: FnOnce(AcquiredArtifactUse) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + Publish: FnOnce(Staged, AcquisitionConsumerReceipt) -> PublishFut + Send + 'static, + PublishFut: Future> + Send + 'static, + { + if request.demand.consumer != self.owner { + return Err(invalid("Consumer demand identity does not match its scope")); + } + if request.sources.len() != request.manifest.files().len() + || request + .sources + .iter() + .any(|source| source.url.trim().is_empty()) + { + return Err(invalid( + "HTTP sources must match the exact selected manifest files", + )); + } + let service = self.service.clone(); + self.scope + .run_worker_invocation(move |context| async move { + service.require_schema(&context).await?; + let operation = service + .begin( + &context, + request.demand, + request.manifest.clone(), + request.workspace.identity().clone(), + None, + ) + .await?; + if operation.is_adopted() { + return Err(PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "An adopted consumer operation cannot be replayed".into(), + }); + } + for (file_index, source) in request.sources.iter().enumerate() { + service + .acquire_file( + &context, + &operation, + &request.workspace, + file_index, + &client, + &source.url, + source.authorization.as_deref(), + &request.retry, + host.as_mut(), + ) + .await?; + } + let lease = service + .files_ready(&context, operation, request.workspace) + .await?; + let expected = lease.record().clone(); + let use_lease = match &expected.phase { + AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } => { + *lease + } + _ => return Err(invalid("Verified consumer handoff has no exact use lease")), + }; + let (staged, payload) = prepare(AcquiredArtifactUse { + store: service.store.clone(), + context: context.clone(), + lease, + }) + .await?; + match context.drain_blocking().await { + Ok(0) => {} + Ok(failures) => { + return Err(PumasError::Other(format!( + "Consumer effects failed before receipt settlement: {failures}" + ))) + } + Err(error) => { + return Err(PumasError::Other(format!( + "Consumer effect drain failed before receipt settlement: {error}" + ))) + } + } + let receipt = AcquisitionConsumerReceipt::for_record(&expected, use_lease, payload)?; + service + .issue_consumer_receipt(&context, &expected, use_lease, receipt.clone()) + .await?; + let result = publish(staged, receipt.clone()).await?; + match context.drain_blocking().await { + Ok(0) => {} + Ok(failures) => { + return Err(PumasError::Other(format!( + "Consumer publication effects failed before receipt settlement: {failures}" + ))) + } + Err(error) => { + return Err(PumasError::Other(format!( + "Consumer publication effect drain failed before receipt settlement: {error}" + ))) + } + } + service + .settle_consumer_receipt(&context, &expected, use_lease, receipt) + .await?; + Ok(result) + }) + .await + } +} + async fn owned( context: &TaskContext, name: &'static str, diff --git a/rust/crates/pumas-core/src/acquisition/store.rs b/rust/crates/pumas-core/src/acquisition/store.rs index 2965da54..cbb2de3a 100644 --- a/rust/crates/pumas-core/src/acquisition/store.rs +++ b/rust/crates/pumas-core/src/acquisition/store.rs @@ -2,6 +2,7 @@ //! //! Model custody is an opaque partition decoded by its model-owned facade. //! This module never interprets model requests, statuses, or recovery policy. +use super::service::AcquisitionConsumerReceipt; use super::service::AcquisitionRecord; pub(crate) use super::service::{AcquisitionProof, AcquisitionTransferProof, AcquisitionUseProof}; use crate::metadata::{ @@ -17,7 +18,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Mutex, MutexGuard}; use uuid::Uuid; -const SCHEMA_VERSION: u32 = 6; +const SCHEMA_VERSION: u32 = 7; const LOCK_FILE: &str = ".downloads.lock"; #[derive(Clone, Serialize, Deserialize)] @@ -25,6 +26,10 @@ pub(crate) struct AcquisitionDocument { schema_version: u32, #[serde(with = "super::service::uuid_map")] pub(crate) acquisitions: BTreeMap, + /// Opaque to the neutral acquisition owner; interpreted by the model + /// consumer that issued each exact completion proof. + #[serde(with = "uuid_value_map")] + pub(crate) consumer_receipts: BTreeMap, /// Existing model partitions retain their exact serialized custody shape. /// Only the trusted model facade may decode or replace these values. #[serde(flatten)] @@ -36,6 +41,7 @@ impl AcquisitionDocument { Self { schema_version: SCHEMA_VERSION, acquisitions: BTreeMap::new(), + consumer_receipts: BTreeMap::new(), legacy: BTreeMap::new(), } } @@ -43,38 +49,158 @@ impl AcquisitionDocument { if self.schema_version != SCHEMA_VERSION { return Err(invalid_schema()); } - let mut demands = BTreeSet::new(); - let mut active_workspaces = BTreeSet::new(); - for (id, record) in &self.acquisitions { - record.validate(*id)?; - if !demands.insert(( - record.demand.consumer.clone(), - record.demand.operation.clone(), - )) { - return Err(PumasError::Validation { - field: "acquisition.custody".into(), - message: "Acquisition demand is duplicated in the durable document".into(), - }); + validate_acquisition_records(&self.acquisitions)?; + for id in self.consumer_receipts.keys() { + let Some(record) = self.acquisitions.get(id) else { + return Err(invalid_receipt("Receipt has no acquisition record")); + }; + if !matches!( + &record.phase, + super::service::AcquisitionPhase::Using { .. } + | super::service::AcquisitionPhase::Adopted { .. } + ) { + return Err(invalid_receipt( + "Receipt is only valid for a current or adopted consumer use", + )); } - if matches!( - record.phase, - super::service::AcquisitionPhase::Transferring - | super::service::AcquisitionPhase::FilesReady - | super::service::AcquisitionPhase::Using { .. } - ) && !active_workspaces.insert(( - record.workspace.root_identity.clone(), - record.workspace.relative_target.clone(), - )) { - return Err(PumasError::Validation { - field: "acquisition.custody".into(), - message: "Multiple active acquisition demands share one workspace".into(), - }); + let value = &self.consumer_receipts[id]; + if value.get("receipt_kind").and_then(Value::as_str) + == Some("pumas.consumer-completion") + { + let receipt: AcquisitionConsumerReceipt = serde_json::from_value(value.clone()) + .map_err(|_| invalid_receipt("Consumer completion receipt is malformed"))?; + let lease = match &record.phase { + super::service::AcquisitionPhase::Using { lease } + | super::service::AcquisitionPhase::Adopted { lease } => *lease, + _ => return Err(invalid_receipt("Receipt has no consumer use lease")), + }; + receipt.validate_for_record(record, lease).map_err(|_| { + invalid_receipt("Consumer receipt does not match its acquisition") + })?; + } else if record.demand.consumer == "hf.model" { + validate_hf_receipt_binding(value, record)?; + } else { + return Err(invalid_receipt( + "Unwrapped receipts are reserved for the HF model consumer", + )); } } Ok(()) } } +/// Validate the stable identity envelope of the legacy HF-owned receipt while +/// leaving its queue and output proof interpretation to the HF facade. +fn validate_hf_receipt_binding(value: &Value, record: &AcquisitionRecord) -> Result<()> { + const FIELDS: [&str; 12] = [ + "receipt_version", + "output_proof_version", + "acquisition_id", + "use_lease", + "demand", + "manifest", + "workspace", + "verified_files", + "download_id", + "queue_admission", + "model_id", + "outputs", + ]; + + let object = value + .as_object() + .ok_or_else(|| invalid_receipt("HF completion receipt must be an object"))?; + let acquisition_id = record.id.to_string(); + let demand = serde_json::to_value(&record.demand)?; + let manifest = serde_json::to_value(&record.manifest)?; + let workspace = serde_json::to_value(&record.workspace)?; + let verified_files = serde_json::to_value(&record.files)?; + if object.len() != FIELDS.len() || FIELDS.iter().any(|field| !object.contains_key(*field)) { + return Err(invalid_receipt( + "HF completion receipt has an unsupported shape", + )); + } + if value.get("receipt_version").and_then(Value::as_u64) != Some(1) + || value.get("output_proof_version").and_then(Value::as_u64) != Some(1) + || value.get("acquisition_id").and_then(Value::as_str) != Some(acquisition_id.as_str()) + || value.get("demand") != Some(&demand) + || value.get("manifest") != Some(&manifest) + || value.get("workspace") != Some(&workspace) + || value.get("verified_files") != Some(&verified_files) + || value + .get("download_id") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + || !value.get("queue_admission").is_some_and(Value::is_object) + || value + .get("model_id") + .and_then(Value::as_str) + .is_none_or(str::is_empty) + || !value.get("outputs").is_some_and(Value::is_object) + { + return Err(invalid_receipt( + "HF completion receipt does not bind the supported acquisition identity", + )); + } + + let receipt_lease_text = value + .get("use_lease") + .and_then(Value::as_str) + .ok_or_else(|| invalid_receipt("HF completion receipt has an invalid use lease"))?; + let receipt_lease = Uuid::parse_str(receipt_lease_text) + .ok() + .filter(|lease| !lease.is_nil()) + .ok_or_else(|| invalid_receipt("HF completion receipt has an invalid use lease"))?; + if receipt_lease.to_string() != receipt_lease_text { + return Err(invalid_receipt( + "HF completion receipt use lease is not canonical", + )); + } + let record_lease = match &record.phase { + super::service::AcquisitionPhase::Using { lease } + | super::service::AcquisitionPhase::Adopted { lease } => *lease, + _ => return Err(invalid_receipt("HF receipt has no consumer use lease")), + }; + if receipt_lease != record_lease || record.demand.consumer != "hf.model" { + return Err(invalid_receipt( + "HF completion receipt lease or owner does not match its acquisition", + )); + } + Ok(()) +} + +fn validate_acquisition_records(acquisitions: &BTreeMap) -> Result<()> { + let mut demands = BTreeSet::new(); + let mut active_workspaces = BTreeSet::new(); + for (id, record) in acquisitions { + record.validate(*id)?; + if !demands.insert(( + record.demand.consumer.clone(), + record.demand.operation.clone(), + )) { + return Err(PumasError::Validation { + field: "acquisition.custody".into(), + message: "Acquisition demand is duplicated in the durable document".into(), + }); + } + if matches!( + record.phase, + super::service::AcquisitionPhase::Transferring + | super::service::AcquisitionPhase::FilesReady + | super::service::AcquisitionPhase::Using { .. } + ) && !active_workspaces.insert(( + record.workspace.root_identity.clone(), + record.workspace.relative_target.clone(), + )) { + return Err(PumasError::Validation { + field: "acquisition.custody".into(), + message: "Multiple active acquisition demands share one workspace".into(), + }); + } + } + Ok(()) +} + /// Source-neutral physical store authority. Construction has no I/O effects. pub struct AcquisitionStore { path: PathBuf, @@ -87,6 +213,8 @@ pub(crate) struct AcquisitionTransaction<'a> { _os_lock: Option, legacy_read_only: bool, consumer_settlement: Option<(String, String)>, + receipt_issuance: Option<(AcquisitionRecord, Value)>, + receipt_settlement: Option<(AcquisitionRecord, Value)>, } fn schema(value: &Value) -> Option { @@ -96,12 +224,53 @@ fn schema(value: &Value) -> Option { pub(crate) fn migration_required() -> PumasError { PumasError::Validation { field: "acquisition.migration_required".into(), - message: "Acquisition requires schema 6; stop all old readers/writers and explicitly migrate the supported v4/v5 store offline".into(), + message: "Acquisition requires schema 7; stop all old readers/writers and explicitly migrate the supported v4/v5 or schema-6 store offline".into(), } } fn invalid_schema() -> PumasError { - PumasError::Validation { field: "downloads.schema_version".into(), message: "Only complete supported schema 4/5 projection and schema 6 acquisition documents are accepted".into() } + PumasError::Validation { field: "downloads.schema_version".into(), message: "Only complete supported schema 4/5 model projections and schema 7 acquisition documents are accepted".into() } +} + +fn invalid_receipt(message: &str) -> PumasError { + PumasError::Validation { + field: "acquisition.consumer_receipts".into(), + message: message.into(), + } +} + +mod uuid_value_map { + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_json::Value; + use std::collections::BTreeMap; + use uuid::Uuid; + + pub(crate) fn serialize( + values: &BTreeMap, + serializer: S, + ) -> std::result::Result { + values + .iter() + .map(|(id, value)| (id.to_string(), value)) + .collect::>() + .serialize(serializer) + } + + pub(crate) fn deserialize<'de, D: Deserializer<'de>>( + deserializer: D, + ) -> std::result::Result, D::Error> { + let encoded = BTreeMap::::deserialize(deserializer)?; + let mut decoded = BTreeMap::new(); + for (key, value) in encoded { + let id = Uuid::parse_str(&key).map_err(serde::de::Error::custom)?; + if key != id.to_string() || decoded.insert(id, value).is_some() { + return Err(serde::de::Error::custom( + "acquisition receipt keys must be canonical and unique", + )); + } + } + Ok(decoded) + } } impl AcquisitionStore { @@ -116,7 +285,7 @@ impl AcquisitionStore { pub fn require_acquisition_schema(&self) -> Result<()> { let target = AtomicJsonTarget::open(&self.path)?; if let Some(value) = target.read_json::()? { - if matches!(schema(&value), Some(4 | 5)) { + if matches!(schema(&value), Some(4..=6)) { return Err(migration_required()); } let document: AcquisitionDocument = serde_json::from_value(value)?; @@ -140,9 +309,14 @@ impl AcquisitionStore { .lock() .map_err(|_| PumasError::Other("Acquisition store lock is poisoned".into()))?; let target = AtomicJsonTarget::open(&self.path)?; - let legacy = target - .read_json::()? - .is_some_and(|value| matches!(schema(&value), Some(4 | 5))); + let observed = target.read_json::()?; + if observed.as_ref().and_then(schema) == Some(6) { + return Err(migration_required()); + } + let legacy = observed + .as_ref() + .and_then(schema) + .is_some_and(|version| matches!(version, 4..=5)); if legacy { if !read_only { return Err(migration_required()); @@ -153,6 +327,8 @@ impl AcquisitionStore { _os_lock: None, legacy_read_only: true, consumer_settlement: None, + receipt_issuance: None, + receipt_settlement: None, }); } let os_lock = target.open_lock_file(LOCK_FILE)?; @@ -163,7 +339,7 @@ impl AcquisitionStore { if !read_only && target .read_json::()? - .is_some_and(|value| matches!(schema(&value), Some(4 | 5))) + .is_some_and(|value| matches!(schema(&value), Some(4..=6))) { return Err(migration_required()); } @@ -173,6 +349,8 @@ impl AcquisitionStore { _os_lock: Some(os_lock), legacy_read_only: false, consumer_settlement: None, + receipt_issuance: None, + receipt_settlement: None, }) } @@ -196,16 +374,49 @@ impl AcquisitionStore { _os_lock: Some(lock), legacy_read_only: false, consumer_settlement: None, + receipt_issuance: None, + receipt_settlement: None, }; let value = transaction .target .read_json::()? .ok_or_else(invalid_schema)?; - if !matches!(schema(&value), Some(4 | 5)) { + if !matches!(schema(&value), Some(4..=6)) { return Err(invalid_schema()); } - let legacy = convert(value)?; - let document = document_with_partition(AcquisitionDocument::empty(), legacy)?; + let document = match schema(&value) { + Some(4 | 5) => { + let legacy = convert(value)?; + document_with_partition(AcquisitionDocument::empty(), legacy)? + } + Some(6) => { + if value.get("consumer_receipts").is_some() { + return Err(invalid_schema()); + } + let mut upgraded_value = value; + let object = upgraded_value.as_object_mut().ok_or_else(invalid_schema)?; + object.insert("schema_version".into(), Value::from(SCHEMA_VERSION)); + object.insert( + "consumer_receipts".into(), + Value::Object(Default::default()), + ); + let mut document: AcquisitionDocument = serde_json::from_value(upgraded_value)?; + validate_acquisition_records(&document.acquisitions)?; + document.consumer_receipts.clear(); + if !document.legacy.is_empty() { + let mut model_projection: serde_json::Map = + document.legacy.clone().into_iter().collect(); + model_projection.insert("schema_version".into(), Value::from(5)); + let normalized = convert(Value::Object(model_projection))?; + let normalized = + document_with_partition(AcquisitionDocument::empty(), normalized)?; + document.legacy = normalized.legacy; + } + document.validate()?; + document + } + _ => return Err(invalid_schema()), + }; require_durable(transaction.publish_document(&document)) } @@ -240,13 +451,168 @@ impl AcquisitionStore { Ok(result) } - pub(crate) fn acquisitions(&self) -> Result> { + pub(crate) fn require_unreceipted_use(&self, expected: &AcquisitionRecord) -> Result<()> { + let transaction = self.transaction(true)?; + let document = transaction.document()?; + if !matches!( + expected.phase, + super::service::AcquisitionPhase::Using { .. } + ) || document.acquisitions.get(&expected.id) != Some(expected) + || document.consumer_receipts.contains_key(&expected.id) + { + return Err(invalid_receipt( + "Withdrawal requires an exact unreceipted Using lease", + )); + } + Ok(()) + } + + /// Commit cancellation only for the unchanged, unreceipted consumer lease. + pub(crate) fn withdraw_unreceipted_use(&self, expected: &AcquisitionRecord) -> Result<()> { + let transaction = self.transaction(false)?; + let mut document = transaction.document()?; + if !matches!( + expected.phase, + super::service::AcquisitionPhase::Using { .. } + ) || document.acquisitions.get(&expected.id) != Some(expected) + || document.consumer_receipts.contains_key(&expected.id) + { + return Err(invalid_receipt( + "Withdrawal requires an exact unreceipted Using lease", + )); + } + let record = document.acquisitions.get_mut(&expected.id).unwrap(); + record.phase = super::service::AcquisitionPhase::Withdrawn; + record.files.clear(); + document.validate()?; + require_durable(transaction.publish_document(&document)) + } + + pub fn acquisitions(&self) -> Result> { let transaction = self.transaction(true)?; if transaction.legacy_read_only { return Ok(BTreeMap::new()); } Ok(transaction.document()?.acquisitions) } + + /// Read a versioned generic consumer receipt. Legacy HF receipts remain + /// interpreted only by the HF persistence facade. + pub(crate) fn consumer_receipt(&self, id: Uuid) -> Result> { + let transaction = self.transaction(true)?; + if transaction.legacy_read_only { + return Ok(None); + } + let Some(value) = transaction.document()?.consumer_receipts.remove(&id) else { + return Ok(None); + }; + if value.get("receipt_kind").and_then(Value::as_str) != Some("pumas.consumer-completion") { + return Ok(None); + } + serde_json::from_value(value) + .map(Some) + .map_err(|_| invalid_receipt("Consumer completion receipt is malformed")) + } + + /// Atomically publish a consumer-owned completion receipt with the exact + /// lease's transition from `Using` to `Adopted`. + pub(crate) fn settle_consumer_use( + &self, + expected: &AcquisitionRecord, + lease: Uuid, + receipt: AcquisitionConsumerReceipt, + ) -> Result<()> { + receipt + .validate_for_record(expected, lease) + .map_err(|_| invalid_receipt("Consumer receipt does not match its acquisition"))?; + if matches!(&expected.phase, super::service::AcquisitionPhase::Using { lease: current } if *current == lease) + { + self.issue_consumer_receipt(expected, lease, &receipt)?; + } + self.settle_consumer_receipt(expected, lease, &receipt) + } + + pub(crate) fn issue_consumer_receipt( + &self, + expected: &AcquisitionRecord, + lease: Uuid, + receipt: &AcquisitionConsumerReceipt, + ) -> Result<()> { + receipt + .validate_for_record(expected, lease) + .map_err(|_| invalid_receipt("Consumer receipt does not match its acquisition"))?; + let transaction = self.transaction(false)?; + let mut document = transaction.document()?; + let Some(current) = document.acquisitions.get(&expected.id) else { + return Err(invalid_receipt("Consumer acquisition disappeared")); + }; + let receipt_value = serde_json::to_value(receipt)?; + if current == expected + && matches!(¤t.phase, super::service::AcquisitionPhase::Using { lease: current_lease } if *current_lease == lease) + { + if document + .consumer_receipts + .get(&expected.id) + .is_some_and(|stored| stored != &receipt_value) + { + return Err(invalid_receipt( + "A conflicting consumer receipt is already published", + )); + } + document + .consumer_receipts + .insert(expected.id, receipt_value); + document.validate()?; + return require_durable(transaction.publish_document(&document)); + } + if current == expected + && matches!(¤t.phase, super::service::AcquisitionPhase::Using { lease: current_lease } if *current_lease == lease) + && document.consumer_receipts.get(&expected.id) == Some(&receipt_value) + { + return Ok(()); + } + Err(invalid_receipt( + "Consumer receipt issuance does not match the exact active acquisition lease", + )) + } + + pub(crate) fn settle_consumer_receipt( + &self, + expected: &AcquisitionRecord, + lease: Uuid, + receipt: &AcquisitionConsumerReceipt, + ) -> Result<()> { + receipt + .validate_for_record(expected, lease) + .map_err(|_| invalid_receipt("Consumer receipt does not match its acquisition"))?; + let transaction = self.transaction(false)?; + let mut document = transaction.document()?; + let Some(current) = document.acquisitions.get(&expected.id) else { + return Err(invalid_receipt("Consumer acquisition disappeared")); + }; + let receipt_value = serde_json::to_value(receipt)?; + if current == expected + && matches!(¤t.phase, super::service::AcquisitionPhase::Using { lease: current_lease } if *current_lease == lease) + && document.consumer_receipts.get(&expected.id) == Some(&receipt_value) + { + document + .acquisitions + .get_mut(&expected.id) + .expect("checked acquisition exists") + .phase = super::service::AcquisitionPhase::Adopted { lease }; + document.validate()?; + return require_durable(transaction.publish_document(&document)); + } + if current == expected + && matches!(¤t.phase, super::service::AcquisitionPhase::Adopted { lease: current_lease } if *current_lease == lease) + && document.consumer_receipts.get(&expected.id) == Some(&receipt_value) + { + return Ok(()); + } + Err(invalid_receipt( + "Consumer settlement does not match the exact active acquisition lease", + )) + } } fn document_with_partition( @@ -268,7 +634,7 @@ impl AcquisitionTransaction<'_> { let Some(value) = self.target.read_json::()? else { return Ok(AcquisitionDocument::empty()); }; - if schema(&value) != Some(6) { + if schema(&value) != Some(7) { return Err(invalid_schema()); } let document: AcquisitionDocument = serde_json::from_value(value)?; @@ -299,6 +665,30 @@ impl AcquisitionTransaction<'_> { Ok((partition, document.acquisitions)) } + pub(crate) fn consumer_completion_state( + &self, + id: Uuid, + ) -> Result<(Option, Option)> { + if self.legacy_read_only { + return Ok((None, None)); + } + let document = self.document()?; + Ok(( + document.acquisitions.get(&id).cloned(), + document.consumer_receipts.get(&id).cloned(), + )) + } + + pub(crate) fn consumer_completion_partition( + &self, + ) -> Result<(BTreeMap, BTreeMap)> { + if self.legacy_read_only { + return Ok((BTreeMap::new(), BTreeMap::new())); + } + let document = self.document()?; + Ok((document.acquisitions, document.consumer_receipts)) + } + pub(crate) fn model_partition(&self) -> Result> { let Some(value) = self.target.read_json::()? else { return Ok(None); @@ -336,6 +726,54 @@ impl AcquisitionTransaction<'_> { } } } + if let Some((expected, receipt)) = &self.receipt_issuance { + let Some(record) = document.acquisitions.get(&expected.id) else { + return Err(invalid_receipt("Receipt acquisition disappeared")); + }; + if record != expected + || !matches!( + record.phase, + super::service::AcquisitionPhase::Using { .. } + | super::service::AcquisitionPhase::Adopted { .. } + ) + { + return Err(invalid_receipt( + "Receipt issuance requires the exact persisted consumer lease", + )); + } + if document + .consumer_receipts + .get(&expected.id) + .is_some_and(|current| current != receipt) + { + return Err(invalid_receipt( + "A conflicting consumer receipt is already published", + )); + } + document + .consumer_receipts + .insert(expected.id, receipt.clone()); + } + if let Some((expected, receipt)) = &self.receipt_settlement { + let Some(record) = document.acquisitions.get_mut(&expected.id) else { + return Err(invalid_receipt("Receipt acquisition disappeared")); + }; + if record != expected { + return Err(invalid_receipt( + "Receipt settlement acquisition identity is stale", + )); + } + let lease = match record.phase { + super::service::AcquisitionPhase::Using { lease } => lease, + _ => return Err(invalid_receipt("Receipt use is already settled")), + }; + if document.consumer_receipts.get(&expected.id) != Some(receipt) { + return Err(invalid_receipt( + "Receipt changed before exact consumer settlement", + )); + } + record.phase = super::service::AcquisitionPhase::Adopted { lease }; + } document.validate()?; Ok(document) })(); @@ -354,6 +792,22 @@ impl AcquisitionTransaction<'_> { self.consumer_settlement = Some((consumer.into(), operation.into())); } + pub(crate) fn stage_consumer_receipt_issuance( + &mut self, + expected: &AcquisitionRecord, + receipt: Value, + ) { + self.receipt_issuance = Some((expected.clone(), receipt)); + } + + pub(crate) fn stage_consumer_receipt_settlement( + &mut self, + expected: &AcquisitionRecord, + receipt: Value, + ) { + self.receipt_settlement = Some((expected.clone(), receipt)); + } + fn publish_document(&self, document: &AcquisitionDocument) -> AtomicPublishResult { self.target.publish_json(document) } @@ -448,10 +902,44 @@ mod tests { .into_iter() .map(|record| (record.id, record)) .collect(), + consumer_receipts: BTreeMap::new(), legacy: BTreeMap::new(), } } + fn generic_receipt(record: &AcquisitionRecord, lease: Uuid) -> Value { + serde_json::to_value(AcquisitionConsumerReceipt { + receipt_kind: "pumas.consumer-completion".into(), + receipt_version: 1, + owner: record.demand.consumer.clone(), + acquisition_id: record.id.to_string(), + use_lease: lease.to_string(), + demand: record.demand.clone(), + manifest: record.manifest.clone(), + workspace: record.workspace.clone(), + verified_files: record.files.clone(), + payload: serde_json::json!({"fixture": "complete"}), + }) + .unwrap() + } + + fn hf_receipt(record: &AcquisitionRecord, lease: Uuid) -> Value { + serde_json::json!({ + "receipt_version": 1, + "output_proof_version": 1, + "acquisition_id": record.id.to_string(), + "use_lease": lease.to_string(), + "demand": &record.demand, + "manifest": &record.manifest, + "workspace": &record.workspace, + "verified_files": &record.files, + "download_id": "download-1", + "queue_admission": {}, + "model_id": "model-1", + "outputs": {}, + }) + } + fn persisted_document_is_refused_without_rewrite(document: AcquisitionDocument) { let temp = tempfile::TempDir::new().unwrap(); let path = temp.path().join("downloads.json"); @@ -466,7 +954,46 @@ mod tests { } #[test] - fn schema_six_rejects_duplicate_demand_even_when_one_row_is_using() { + fn withdrawal_requires_exact_using_lease_and_absent_receipt() { + let temp = tempfile::TempDir::new().unwrap(); + let path = temp.path().join("downloads.json"); + let lease = Uuid::new_v4(); + let expected = record( + Uuid::new_v4(), + "cancelled", + "model/path", + AcquisitionPhase::Using { lease }, + ); + let store = AcquisitionStore::new(temp.path()); + let original = document(vec![expected.clone()]); + std::fs::write(&path, serde_json::to_vec(&original).unwrap()).unwrap(); + let mut wrong_lease = expected.clone(); + wrong_lease.phase = AcquisitionPhase::Using { + lease: Uuid::new_v4(), + }; + let before = std::fs::read(&path).unwrap(); + assert!(store.require_unreceipted_use(&wrong_lease).is_err()); + assert!(store.withdraw_unreceipted_use(&wrong_lease).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), before); + let mut receipted = original.clone(); + receipted + .consumer_receipts + .insert(expected.id, generic_receipt(&expected, lease)); + std::fs::write(&path, serde_json::to_vec(&receipted).unwrap()).unwrap(); + let before = std::fs::read(&path).unwrap(); + assert!(store.require_unreceipted_use(&expected).is_err()); + assert!(store.withdraw_unreceipted_use(&expected).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), before); + std::fs::write(&path, serde_json::to_vec(&original).unwrap()).unwrap(); + store.withdraw_unreceipted_use(&expected).unwrap(); + let result = store.acquisitions().unwrap(); + assert_eq!(result[&expected.id].phase, AcquisitionPhase::Withdrawn); + assert!(result[&expected.id].files.is_empty()); + assert!(store.consumer_receipt(expected.id).unwrap().is_none()); + } + + #[test] + fn schema_seven_rejects_duplicate_demand_even_when_one_row_is_using() { let demand = "same-operation"; let document = document(vec![ record( @@ -489,7 +1016,7 @@ mod tests { } #[test] - fn schema_six_rejects_distinct_active_demands_for_one_workspace() { + fn schema_seven_rejects_distinct_active_demands_for_one_workspace() { let document = document(vec![ record( Uuid::from_u128(1), @@ -537,4 +1064,180 @@ mod tests { document.validate().unwrap(); } + + #[test] + fn receipt_stays_with_using_or_adopted_record_in_one_owner_partition() { + let id = Uuid::from_u128(1); + let lease = Uuid::from_u128(2); + let using = record( + id, + "receipt-operation", + "model/path", + AcquisitionPhase::Using { lease }, + ); + let mut doc = document(vec![using.clone()]); + doc.consumer_receipts + .insert(id, generic_receipt(&using, lease)); + doc.validate().unwrap(); + + doc.consumer_receipts.insert( + id, + serde_json::json!({ + "receipt_kind": "pumas.consumer-completion", + "receipt_version": 999 + }), + ); + assert!(matches!( + doc.validate(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.consumer_receipts" + )); + doc.consumer_receipts + .insert(id, generic_receipt(&using, lease)); + + doc.acquisitions.get_mut(&id).unwrap().phase = AcquisitionPhase::Adopted { lease }; + doc.validate().unwrap(); + + doc.acquisitions.clear(); + assert!(matches!( + doc.validate(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.consumer_receipts" + )); + } + + #[test] + fn hf_receipts_require_supported_version_and_exact_acquisition_binding() { + let id = Uuid::from_u128(11); + let lease = Uuid::from_u128(12); + let mut hf_record = record( + id, + "hf-operation", + "model/path", + AcquisitionPhase::Using { lease }, + ); + hf_record.demand.consumer = "hf.model".into(); + + let mut doc = document(vec![hf_record.clone()]); + doc.consumer_receipts + .insert(id, hf_receipt(&hf_record, lease)); + doc.validate().unwrap(); + + let mut unsupported_version = hf_receipt(&hf_record, lease); + unsupported_version["receipt_version"] = 999.into(); + doc.consumer_receipts.insert(id, unsupported_version); + assert!(matches!( + doc.validate(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.consumer_receipts" + )); + + let mut mismatched_lease = hf_receipt(&hf_record, lease); + mismatched_lease["use_lease"] = Uuid::from_u128(13).to_string().into(); + doc.consumer_receipts.insert(id, mismatched_lease); + assert!(matches!( + doc.validate(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.consumer_receipts" + )); + + let mut truncated = hf_receipt(&hf_record, lease); + truncated.as_object_mut().unwrap().remove("outputs"); + doc.consumer_receipts.insert(id, truncated); + assert!(matches!( + doc.validate(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.consumer_receipts" + )); + } + + #[test] + fn receipt_partition_rejects_noncanonical_uuid_aliases() { + #[derive(Deserialize)] + struct ReceiptKeys { + #[serde(with = "uuid_value_map")] + values: BTreeMap, + } + + let id = Uuid::from_u128(1); + let input = serde_json::json!({ + "values": { + id.to_string(): {"receipt_version": 1}, + format!("urn:uuid:{id}"): {"receipt_version": 999} + } + }); + assert!(serde_json::from_value::(input).is_err()); + let canonical: ReceiptKeys = serde_json::from_value(serde_json::json!({ + "values": {id.to_string(): {"receipt_version": 1}} + })) + .unwrap(); + assert_eq!(canonical.values.len(), 1); + } + + #[test] + fn schema_six_migration_preserves_acquisition_and_model_partition_without_receipts() { + let temp = tempfile::TempDir::new().unwrap(); + let id = Uuid::from_u128(1); + let record = record( + id, + "retained-using-operation", + "model/path", + AcquisitionPhase::Using { + lease: Uuid::from_u128(2), + }, + ); + let mut old = document(vec![record.clone()]); + old.schema_version = 6; + old.legacy.insert( + "downloads".into(), + serde_json::json!([{"download_id":"retained-model-row"}]), + ); + let mut value = serde_json::to_value(old).unwrap(); + value.as_object_mut().unwrap().remove("consumer_receipts"); + std::fs::write( + temp.path().join("downloads.json"), + serde_json::to_vec(&value).unwrap(), + ) + .unwrap(); + + let store = AcquisitionStore::new(temp.path()); + let before = std::fs::read(temp.path().join("downloads.json")).unwrap(); + assert!(matches!( + store.require_acquisition_schema(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.migration_required" + )); + assert_eq!( + std::fs::read(temp.path().join("downloads.json")).unwrap(), + before + ); + + store.migrate_legacy_offline(Ok).unwrap(); + store.require_acquisition_schema().unwrap(); + let migrated = store.transaction(true).unwrap().document().unwrap(); + assert_eq!(migrated.schema_version, 7); + assert_eq!(migrated.acquisitions.get(&id), Some(&record)); + assert!(migrated.consumer_receipts.is_empty()); + assert_eq!( + migrated.legacy.get("downloads").unwrap()[0]["download_id"], + "retained-model-row" + ); + } + + #[test] + fn schema_six_rejects_normal_read_without_rewrite() { + let temp = tempfile::TempDir::new().unwrap(); + let mut old = document(Vec::new()); + old.schema_version = 6; + let bytes = serde_json::to_vec(&old).unwrap(); + let path = temp.path().join("downloads.json"); + std::fs::write(&path, &bytes).unwrap(); + + assert!(matches!( + AcquisitionStore::new(temp.path()).acquisitions(), + Err(PumasError::Validation { ref field, .. }) + if field == "acquisition.migration_required" + )); + assert_eq!(std::fs::read(path).unwrap(), bytes); + } } diff --git a/rust/crates/pumas-core/src/acquisition/task_custody.rs b/rust/crates/pumas-core/src/acquisition/task_custody.rs index c7d4809b..6cdcd807 100644 --- a/rust/crates/pumas-core/src/acquisition/task_custody.rs +++ b/rust/crates/pumas-core/src/acquisition/task_custody.rs @@ -1021,6 +1021,46 @@ impl TaskScope { result } + /// Run one consumer operation as an owner-held worker. The waiter may + /// disappear without detaching its registered effects, and operation + /// proofs remain current until the worker finishes publication. + pub(crate) async fn run_worker_invocation( + self: &Arc, + operation: F, + ) -> crate::Result + where + T: Send + 'static, + F: FnOnce(TaskContext) -> Fut + Send + 'static, + Fut: Future> + Send + 'static, + { + let id = format!("consumer-worker-{}", uuid::Uuid::new_v4()); + let (sender, receiver) = oneshot::channel(); + let prepared = self.prepare(id.clone(), TaskRole::Worker, move |context| async move { + let result = operation(context).await; + let _ = sender.send(result); + })?; + let generation = prepared.generation.clone(); + let installed = self + .install_gated(prepared) + .map_err(|_| crate::PumasError::DownloadLifecycleClosed)?; + let waiter = InvocationWaiter { + owner: self.clone(), + id, + generation, + }; + installed.start(); + let result = receiver.await.map_err(|_| { + if self.is_closed() { + crate::PumasError::DownloadLifecycleClosed + } else { + crate::PumasError::DownloadShutdownFailed { failures: 1 } + } + })?; + drop(waiter); + self.ensure_open()?; + result + } + pub(crate) async fn shutdown(self: &Arc) -> crate::Result<()> { self.request_shutdown().wait().await } diff --git a/rust/crates/pumas-core/src/acquisition/workspace.rs b/rust/crates/pumas-core/src/acquisition/workspace.rs index d589fc9b..0b8a594d 100644 --- a/rust/crates/pumas-core/src/acquisition/workspace.rs +++ b/rust/crates/pumas-core/src/acquisition/workspace.rs @@ -10,7 +10,7 @@ use cap_std::fs::{Dir, Metadata, OpenOptions}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::collections::BTreeMap; -use std::io::{Read, Write}; +use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Component, Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -98,6 +98,54 @@ pub struct AcquisitionWorkspace { } impl AcquisitionWorkspace { + /// Reconstruct the equality-only identity for a reserved child without + /// opening or creating that child. Consumers use this after durable + /// adoption, when owned input cleanup may already have removed it. + pub fn identity_for_reserved_directory( + root: &Path, + relative_target: &Path, + ) -> Result { + let relative = normalized_relative_path(relative_target)?; + let root_directory = crate::platform::capability_fs::open_directory(root)?; + let root_binding = identity(&root_directory.dir_metadata()?)?; + Ok(WorkspaceIdentity { + root_identity: format!("{:x}:{:x}", root_binding.0, root_binding.1), + relative_target: relative, + }) + } + + /// Open a previously reserved child of a consumer-owned root. + /// + /// `execution_lease` retains the consumer's reservation for the root, and + /// `validate_root` rechecks its authority on every workspace operation. + /// The locator is equality-only; the held directory capability, not the + /// serialized locator, authorizes reads and writes. + pub fn from_reserved_directory( + root: &Path, + relative_target: &Path, + execution_lease: Arc, + validate_root: impl Fn() -> Result<()> + Send + Sync + 'static, + ) -> Result { + let relative = normalized_relative_path(relative_target)?; + validate_root()?; + let root_directory = crate::platform::capability_fs::open_directory(root)?; + let root_binding = identity(&root_directory.dir_metadata()?)?; + let directory = open_relative_directory(&root_directory, relative_target)?; + let held_root = root_directory.try_clone()?; + let validate_root = move || { + validate_root()?; + if identity(&held_root.dir_metadata()?)? != root_binding { + return Err(changed()); + } + Ok(()) + }; + let locator = WorkspaceIdentity { + root_identity: format!("{:x}:{:x}", root_binding.0, root_binding.1), + relative_target: relative, + }; + Self::from_capability(directory, locator, execution_lease, validate_root) + } + pub(crate) fn from_capability( directory: Dir, locator: WorkspaceIdentity, @@ -314,6 +362,57 @@ impl AcquisitionWorkspace { }) } + /// Open the exact verified file through the held directory capability. + /// Hashing and identity checks use the returned file handle itself, so a + /// consumer can move it to a blocking worker without reopening a path. + pub(crate) fn open_verified_readonly( + &self, + file: &ArtifactFile, + expected: &VerifiedFile, + ) -> Result { + if expected.path != file.logical_path() { + return Err(changed()); + } + self.validate()?; + let (parent, name) = self.parent(file.logical_path(), false)?; + let mut open_options = options(); + open_options.read(true); + let mut handle = parent.open_with(&name, &open_options)?.into_std(); + let before = Metadata::from_file(&handle)?; + if !before.is_file() || before.len() != expected.bytes { + return Err(changed()); + } + let binding = identity(&before)?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 64 * 1024]; + loop { + let read = handle.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let digest = hex::encode(hasher.finalize()); + let after = Metadata::from_file(&handle)?; + let current = Metadata::from_file(&parent.open_with(&name, &open_options)?.into_std())?; + if digest != expected.sha256 + || file + .expected_sha256() + .is_some_and(|selected| selected.value() != digest) + || identity(&after)? != binding + || identity(¤t)? != binding + || before.len() != after.len() + || before.len() != current.len() + || before.modified()? != after.modified()? + || before.modified()? != current.modified()? + { + return Err(changed()); + } + handle.seek(SeekFrom::Start(0))?; + self.validate()?; + Ok(handle) + } + pub(crate) fn publish_part( &self, file: &ArtifactFile, @@ -368,6 +467,43 @@ impl AcquisitionWorkspace { } } +fn normalized_relative_path(path: &Path) -> Result { + let mut components = Vec::new(); + for component in path.components() { + let Component::Normal(part) = component else { + return Err(changed()); + }; + let part = part.to_str().ok_or_else(changed)?; + if part.is_empty() { + return Err(changed()); + } + components.push(part); + } + if components.is_empty() { + return Err(changed()); + } + Ok(components.join("/")) +} + +fn open_relative_directory(root: &Dir, relative: &Path) -> Result { + let mut current = root.try_clone()?; + for component in relative.components() { + let Component::Normal(component) = component else { + return Err(changed()); + }; + let metadata = current.symlink_metadata(component)?; + if !metadata.is_dir() || metadata.is_symlink() { + return Err(changed()); + } + let next = current.open_dir(component)?; + if identity(&metadata)? != identity(&next.dir_metadata()?)? { + return Err(changed()); + } + current = next; + } + Ok(current) +} + pub(crate) fn write_chunk(file: &mut std::fs::File, bytes: &[u8]) -> Result<()> { file.write_all(bytes)?; Ok(()) @@ -378,6 +514,26 @@ mod tests { use super::*; use crate::acquisition::FileVerificationRequirement; + #[test] + fn identity_only_lookup_matches_a_reserved_child_without_creating_it() { + let temp = tempfile::TempDir::new().unwrap(); + let relative = Path::new(".native-attempt"); + let identity = + AcquisitionWorkspace::identity_for_reserved_directory(temp.path(), relative).unwrap(); + assert_eq!(identity.relative_target, ".native-attempt"); + assert!(!temp.path().join(relative).exists()); + + std::fs::create_dir(temp.path().join(relative)).unwrap(); + let workspace = AcquisitionWorkspace::from_reserved_directory( + temp.path(), + relative, + Arc::new(()), + || Ok(()), + ) + .unwrap(); + assert_eq!(workspace.identity(), &identity); + } + #[test] fn concurrent_parent_creation_is_coalesced_and_replacement_is_refused() { let temp = tempfile::TempDir::new().unwrap(); diff --git a/rust/crates/pumas-core/src/model_library/download_recovery.rs b/rust/crates/pumas-core/src/model_library/download_recovery.rs index 2a76d3b6..f0de5ea5 100644 --- a/rust/crates/pumas-core/src/model_library/download_recovery.rs +++ b/rust/crates/pumas-core/src/model_library/download_recovery.rs @@ -757,10 +757,23 @@ impl DownloadRecoveryDestination { Ok(metadata) } + pub(crate) fn read_model_metadata_value(&self) -> Result> { + let Some(directory) = self.directory_if_present(false)? else { + return Ok(None); + }; + let metadata = Self::read_provenance_file(&directory, "metadata.json")?; + self.directory(false)?; + Ok(metadata) + } + pub(crate) fn write_model_metadata( &self, metadata: &crate::models::ModelMetadata, ) -> Result<()> { + self.write_model_metadata_value(&serde_json::to_value(metadata)?) + } + + pub(crate) fn write_model_metadata_value(&self, metadata: &Value) -> Result<()> { let directory = self.directory(false)?; let expected = directory_identity(&directory)?; let destination = self.clone(); diff --git a/rust/crates/pumas-core/src/model_library/download_store.rs b/rust/crates/pumas-core/src/model_library/download_store.rs index d726d9c9..38e9f87b 100644 --- a/rust/crates/pumas-core/src/model_library/download_store.rs +++ b/rust/crates/pumas-core/src/model_library/download_store.rs @@ -6,6 +6,10 @@ //! durable terminal proofs may outlive the resumable snapshot they protect. use crate::acquisition::store::{AcquisitionStore, AcquisitionTransaction}; +use crate::acquisition::{ + AcquisitionConsumerReceipt, AcquisitionDemand, AcquisitionPhase, AcquisitionRecord, + ArtifactManifest, VerifiedFile, WorkspaceIdentity, +}; use crate::error::Result; use crate::metadata::{ AtomicPublication, AtomicPublishFailure, AtomicPublishFailureKind, AtomicPublishResult, @@ -16,6 +20,7 @@ use crate::model_library::types::DownloadRequest; use crate::models::DownloadStatus; use crate::models::HuggingFaceEvidence; use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; @@ -132,7 +137,7 @@ pub(crate) struct DownloadAdmissionPosition { pub(crate) predecessor: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub(crate) struct PersistedQueueAdmission { pub(crate) attempt_id: String, @@ -143,6 +148,208 @@ pub(crate) struct PersistedQueueAdmission { pub(crate) position: DownloadAdmissionPosition, } +/// Model-owned proof that one exact Hugging Face import completed. The +/// acquisition document stores its serialized form opaquely; this model +/// facade alone interprets its version and output projection. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HfCompletionReceipt { + pub(crate) receipt_version: u16, + pub(crate) output_proof_version: u16, + pub(crate) acquisition_id: String, + pub(crate) use_lease: String, + pub(crate) demand: AcquisitionDemand, + pub(crate) manifest: ArtifactManifest, + pub(crate) workspace: WorkspaceIdentity, + pub(crate) verified_files: Vec, + pub(crate) download_id: String, + pub(crate) queue_admission: PersistedQueueAdmission, + pub(crate) model_id: String, + pub(crate) outputs: HfCompletionOutputProof, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HfCompletionOutputProof { + pub(crate) metadata_sha256: String, + pub(crate) index_sha256: String, + pub(crate) package_facts: Option, +} + +pub(crate) struct HfCompletionReceiptRequest<'a> { + pub(crate) expected: &'a AcquisitionRecord, + pub(crate) use_lease: Uuid, + pub(crate) download_id: &'a str, + pub(crate) domain: DownloadAdmissionDomain, + pub(crate) destination: &'a PersistedDestinationIdentity, + pub(crate) model_id: &'a str, + pub(crate) outputs: HfCompletionOutputProof, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct HfPackageFactsProof { + pub(crate) contract_version: i64, + pub(crate) content_sha256: String, +} + +impl HfCompletionReceipt { + pub(crate) fn validate_for_record(&self, record: &AcquisitionRecord) -> Result<()> { + let receipt_lease = Uuid::parse_str(&self.use_lease) + .map_err(|_| invalid_completion_receipt("Receipt use-lease identity is malformed"))?; + let lease_matches = matches!( + record.phase, + AcquisitionPhase::Using { lease } | AcquisitionPhase::Adopted { lease } + if lease == receipt_lease + ); + let selected_files = record + .manifest + .files() + .iter() + .map(|file| file.logical_path().to_string()) + .collect::>(); + if self.receipt_version != 1 + || self.output_proof_version != 1 + || self.acquisition_id != record.id.to_string() + || receipt_lease.is_nil() + || self.demand != record.demand + || self.demand.consumer != "hf.model" + || self.demand.operation != self.queue_admission.attempt_id + || self.manifest != record.manifest + || self.workspace != record.workspace + || self.verified_files != record.files + || self.download_id.is_empty() + || self.queue_admission.execution_files != selected_files + || self.queue_admission.destination.library_root != self.workspace.root_identity + || self.queue_admission.destination.relative_target != self.workspace.relative_target + || self.model_id.is_empty() + || !lease_matches + { + return Err(invalid_completion_receipt( + "Receipt identity does not match the exact HF acquisition and queue admission", + )); + } + validate_sha256(&self.outputs.metadata_sha256)?; + validate_sha256(&self.outputs.index_sha256)?; + if let Some(package_facts) = &self.outputs.package_facts { + if package_facts.contract_version <= 0 { + return Err(invalid_completion_receipt( + "Receipt package-facts contract version is invalid", + )); + } + validate_sha256(&package_facts.content_sha256)?; + } + Ok(()) + } + + fn validate_versioned(&self) -> Result<()> { + if self.receipt_version != 1 || self.output_proof_version != 1 { + return Err(invalid_completion_receipt( + "Receipt or output-proof version is unsupported", + )); + } + validate_sha256(&self.outputs.metadata_sha256)?; + validate_sha256(&self.outputs.index_sha256)?; + if let Some(package_facts) = &self.outputs.package_facts { + if package_facts.contract_version <= 0 { + return Err(invalid_completion_receipt( + "Receipt package-facts contract version is invalid", + )); + } + validate_sha256(&package_facts.content_sha256)?; + } + Ok(()) + } +} + +fn invalid_completion_receipt(message: &str) -> crate::PumasError { + crate::PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: message.into(), + } +} + +fn validate_hf_completion_receipts( + acquisitions: &BTreeMap, + receipts: &BTreeMap, +) -> Result<()> { + let mut identities = HashSet::new(); + for (key, value) in receipts { + let value = if value + .get("receipt_kind") + .and_then(serde_json::Value::as_str) + == Some("pumas.consumer-completion") + { + let receipt: AcquisitionConsumerReceipt = serde_json::from_value(value.clone()) + .map_err(|_| invalid_completion_receipt("Stored consumer receipt is malformed"))?; + if receipt.owner != "hf.model" { + continue; + } + receipt.payload + } else { + value.clone() + }; + let receipt: HfCompletionReceipt = serde_json::from_value(value) + .map_err(|_| invalid_completion_receipt("Stored completion receipt is malformed"))?; + if receipt.acquisition_id != key.to_string() + || !identities.insert(receipt.acquisition_id.clone()) + { + return Err(invalid_completion_receipt( + "Stored completion receipt identity is duplicated or mismatched", + )); + } + let record = acquisitions + .get(key) + .ok_or_else(|| invalid_completion_receipt("Stored completion receipt is orphaned"))?; + receipt.validate_for_record(record)?; + } + Ok(()) +} + +fn validate_sha256(value: &str) -> Result<()> { + if value.len() != 64 + || !value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(invalid_completion_receipt( + "Receipt contains a malformed SHA-256 digest", + )); + } + Ok(()) +} + +pub(crate) fn canonical_json_sha256(value: &serde_json::Value) -> Result { + fn normalize(value: &serde_json::Value) -> Result { + match value { + serde_json::Value::Object(values) => { + let sorted = values.iter().collect::>(); + let mut canonical = serde_json::Map::new(); + for (key, value) in sorted { + canonical.insert(key.clone(), normalize(value)?); + } + Ok(serde_json::Value::Object(canonical)) + } + serde_json::Value::Array(values) => values + .iter() + .map(normalize) + .collect::>>() + .map(serde_json::Value::Array), + serde_json::Value::Number(number) + if number.as_f64().is_some_and(|value| !value.is_finite()) => + { + Err(invalid_completion_receipt( + "Receipt output projection contains a non-finite number", + )) + } + _ => Ok(value.clone()), + } + } + + let canonical = serde_json::to_vec(&normalize(value)?)?; + Ok(format!("{:x}", Sha256::digest(canonical))) +} + #[derive(Debug, Clone)] pub(crate) struct HiddenDownloadAdmission { pub(crate) request: DownloadAdmissionRequest, @@ -514,6 +721,14 @@ impl DownloadPersistence { self } + #[cfg(test)] + pub(crate) fn with_receipt_settlement_failure_publisher_for_test( + self, + publisher: Arc, + ) -> Self { + self.with_test_publisher(publisher) + } + #[cfg(test)] fn with_test_observer(mut self, observer: Arc) -> Self { self.observer = observer; @@ -727,6 +942,182 @@ impl DownloadPersistence { Ok(true) } + /// Durably issue a complete-HF-import receipt together with a locked + /// revalidation of its exact active queue admission and acquisition lease. + pub(crate) fn publish_hf_completion_receipt( + &self, + request: HfCompletionReceiptRequest<'_>, + ) -> Result { + let HfCompletionReceiptRequest { + expected, + use_lease, + download_id, + domain, + destination, + model_id, + outputs, + } = request; + if !matches!(expected.phase, AcquisitionPhase::Using { lease } if lease == use_lease) + || expected.demand.consumer != "hf.model" + || expected.demand.operation.is_empty() + || model_id.is_empty() + { + return Err(invalid_completion_receipt( + "Receipt issuance lacks the exact active HF use lease", + )); + } + let mut transaction = self.transaction(StoreOperation::UpdateStatus)?; + let data = self.load_data_strict(&transaction)?; + let admission = data.queue_admissions.get(download_id).ok_or_else(|| { + invalid_completion_receipt("Receipt queue admission is no longer active") + })?; + if admission.domain != domain || &admission.destination != destination { + return Err(invalid_completion_receipt( + "Receipt queue admission has another domain or destination", + )); + } + self.validate_queue_execution_data( + &data, + download_id, + &expected.demand.operation, + admission.domain, + &admission.destination, + &admission.execution_files, + )?; + let receipt = HfCompletionReceipt { + receipt_version: 1, + output_proof_version: 1, + acquisition_id: expected.id.to_string(), + use_lease: use_lease.to_string(), + demand: expected.demand.clone(), + manifest: expected.manifest.clone(), + workspace: expected.workspace.clone(), + verified_files: expected.files.clone(), + download_id: download_id.into(), + queue_admission: admission.clone(), + model_id: model_id.into(), + outputs, + }; + receipt.validate_for_record(expected)?; + transaction + .target + .stage_consumer_receipt_issuance(expected, serde_json::to_value(&receipt)?); + let mut unchanged = data; + self.write_data(&transaction, &mut unchanged)?; + Ok(receipt) + } + + /// Read and strictly decode the model-owned receipt. Unsupported versions + /// and malformed contents never fall back to output-path inference. + pub(crate) fn read_hf_completion_receipt( + &self, + acquisition_id: Uuid, + ) -> Result> { + let transaction = self.transaction(StoreOperation::Load)?; + self.load_data_strict(&transaction)?; + let (_, receipts) = transaction.target.consumer_completion_partition()?; + let Some(value) = receipts.get(&acquisition_id).cloned() else { + return Ok(None); + }; + let value = if value + .get("receipt_kind") + .and_then(serde_json::Value::as_str) + == Some("pumas.consumer-completion") + { + let envelope: AcquisitionConsumerReceipt = serde_json::from_value(value)?; + if envelope.owner != "hf.model" { + return Ok(None); + } + envelope.payload + } else { + value + }; + let receipt: HfCompletionReceipt = serde_json::from_value(value)?; + receipt.validate_versioned()?; + if receipt.acquisition_id != acquisition_id.to_string() { + return Err(invalid_completion_receipt( + "Receipt key does not match its acquisition identity", + )); + } + Ok(Some(receipt)) + } + + /// Atomically acknowledge the exact imported acquisition and release its + /// FIFO admission. The immutable receipt remains paired with the Adopted + /// acquisition as durable completion history. + pub(super) fn settle_hf_completion( + &self, + expected: &AcquisitionRecord, + receipt: &HfCompletionReceipt, + ) -> Result { + receipt.validate_for_record(expected)?; + let mut transaction = self.transaction(StoreOperation::Remove)?; + let mut data = self.load_data_strict(&transaction)?; + let receipt_value = serde_json::to_value(receipt)?; + let (current_record, current_receipt) = + transaction.target.consumer_completion_state(expected.id)?; + if current_record.as_ref() != Some(expected) + || current_receipt.as_ref() != Some(&receipt_value) + { + return Err(invalid_completion_receipt( + "Receipt or acquisition changed before exact settlement", + )); + } + + if let Some(admission) = data.queue_admissions.get(&receipt.download_id) { + if admission != &receipt.queue_admission { + return Err(invalid_completion_receipt( + "Receipt queue admission changed before settlement", + )); + } + self.validate_queue_execution_data( + &data, + &receipt.download_id, + &receipt.demand.operation, + admission.domain, + &admission.destination, + &admission.execution_files, + )?; + if let Some(admission) = data.queue_admissions.remove(&receipt.download_id) { + data.released_queue_admissions + .insert(receipt.download_id.clone(), admission); + } + data.downloads + .retain(|download| download.download_id != receipt.download_id); + data.lifecycle_quarantines + .retain(|id, quarantine| id != &receipt.download_id || quarantine.sticky_failure); + match &expected.phase { + AcquisitionPhase::Using { .. } => transaction + .target + .stage_consumer_receipt_settlement(expected, receipt_value), + AcquisitionPhase::Adopted { .. } => {} + _ => { + return Err(invalid_completion_receipt( + "Receipt settlement requires an active or already-adopted use", + )); + } + } + self.write_data(&transaction, &mut data)?; + return Ok(true); + } + + let already_released = data + .released_queue_admissions + .get(&receipt.download_id) + .is_some_and(|admission| admission == &receipt.queue_admission); + let already_adopted = matches!( + current_record.map(|record| record.phase), + Some(AcquisitionPhase::Adopted { lease }) + if lease.to_string() == receipt.use_lease + ); + if already_released && already_adopted { + return Ok(true); + } + Err(invalid_completion_receipt( + "Receipt settlement has no exact active or already released queue admission", + )) + } + /// Load all persisted downloads. pub fn load_all(&self) -> Vec { match self.load_all_strict() { @@ -1550,9 +1941,9 @@ impl DownloadPersistence { Ok(true) } - /// Explicit one-shot offline migration of the complete supported v4/v5 - /// model custody representation into schema 6. The caller must stop all - /// old readers and writers first; the advisory lock cannot prove that. + /// Explicit one-shot offline migration of supported v4/v5 model custody + /// or pre-receipt schema 6 into schema 7. The caller must stop every old + /// reader and writer first; the advisory lock cannot prove that. /// Normal construction/open never invokes this operation. pub fn migrate_legacy_offline(data_dir: impl AsRef) -> Result<()> { let store = Self::new(data_dir.as_ref()); @@ -1576,6 +1967,8 @@ impl DownloadPersistence { } fn load_data_strict(&self, transaction: &StoreTransaction<'_>) -> Result { + let (acquisitions, receipts) = transaction.target.consumer_completion_partition()?; + validate_hf_completion_receipts(&acquisitions, &receipts)?; let Some(value) = transaction.target.model_partition()? else { return Ok(DownloadStoreData::empty()); }; @@ -2299,9 +2692,76 @@ fn validate_store_data(data: &DownloadStoreData) -> Result<()> { Ok(()) } +#[cfg(test)] +pub(crate) struct FailAfterReceiptSettlementPublisher { + failed: std::sync::atomic::AtomicBool, +} + +#[cfg(test)] +impl FailAfterReceiptSettlementPublisher { + pub(crate) fn new() -> Self { + Self { + failed: std::sync::atomic::AtomicBool::new(false), + } + } + + pub(crate) fn was_triggered(&self) -> bool { + self.failed.load(std::sync::atomic::Ordering::SeqCst) + } +} + +#[cfg(test)] +impl DownloadStorePublisher for FailAfterReceiptSettlementPublisher { + fn publish( + &self, + target: &AcquisitionTransaction<'_>, + data: &DownloadStoreData, + ) -> AtomicPublishResult { + let (_, receipts) = target.consumer_completion_partition().map_err(|error| { + Box::new(AtomicPublishFailure { + stage: AtomicPublishStage::Serialization, + kind: AtomicPublishFailureKind::InvalidData, + error, + cleanup: StagingCleanup::NotRequired, + }) + })?; + let receipt_qualified_release = receipts.values().any(|value| { + serde_json::from_value::(value.clone()).is_ok_and(|receipt| { + data.released_queue_admissions.get(&receipt.download_id) + == Some(&receipt.queue_admission) + }) + }); + if receipt_qualified_release + && self + .failed + .compare_exchange( + false, + true, + std::sync::atomic::Ordering::SeqCst, + std::sync::atomic::Ordering::SeqCst, + ) + .is_ok() + { + return Err(Box::new(AtomicPublishFailure { + stage: AtomicPublishStage::Staging, + kind: AtomicPublishFailureKind::Filesystem, + error: crate::PumasError::Other( + "injected failure after HF receipt publication".to_string(), + ), + cleanup: StagingCleanup::NotRequired, + })); + } + target.publish_model_partition(data) + } +} + #[cfg(test)] mod tests { use super::*; + use crate::acquisition::{ + ArtifactFile, ArtifactRevisionEvidence, ArtifactSourceIdentity, + FileVerificationRequirement, RevisionStrength, + }; use std::collections::VecDeque; use std::io::{BufRead, BufReader, Write}; use std::process::{Command, Stdio}; @@ -2609,6 +3069,269 @@ mod tests { assert_eq!(store.load_all().len(), 0); } + #[test] + fn receipt_output_hash_canonicalizes_objects_but_preserves_arrays_and_null() { + let left = serde_json::json!({"z": [1, null], "a": {"y": true, "x": 2}}); + let reordered = serde_json::json!({"a": {"x": 2, "y": true}, "z": [1, null]}); + let array_reordered = serde_json::json!({"a": {"x": 2, "y": true}, "z": [null, 1]}); + let missing_null = serde_json::json!({"a": {"x": 2, "y": true}, "z": [1]}); + + assert_eq!( + canonical_json_sha256(&left).unwrap(), + canonical_json_sha256(&reordered).unwrap() + ); + assert_ne!( + canonical_json_sha256(&left).unwrap(), + canonical_json_sha256(&array_reordered).unwrap() + ); + assert_ne!( + canonical_json_sha256(&left).unwrap(), + canonical_json_sha256(&missing_null).unwrap() + ); + } + + #[test] + fn persisted_completion_reader_rejects_unknown_receipt_version_without_rewrite() { + let tmp = TempDir::new().unwrap(); + let store = DownloadPersistence::new(tmp.path()); + let snapshot = persisted("unknown-receipt-version"); + let attempt = store.admit_test_download(&snapshot).unwrap(); + let (record, lease) = using_hf_acquisition(&store, &snapshot, &attempt); + let destination = PersistedDestinationIdentity { + library_root: "store-test-root".into(), + relative_target: snapshot.dest_dir.to_string_lossy().into_owned(), + }; + store + .publish_hf_completion_receipt(HfCompletionReceiptRequest { + expected: &record, + use_lease: lease, + download_id: &snapshot.download_id, + domain: DownloadAdmissionDomain::Ambient, + destination: &destination, + model_id: "fixture-model-id", + outputs: HfCompletionOutputProof { + metadata_sha256: "b".repeat(64), + index_sha256: "c".repeat(64), + package_facts: None, + }, + }) + .unwrap(); + + let path = tmp.path().join("downloads.json"); + let mut document: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + document["consumer_receipts"][record.id.to_string()]["receipt_version"] = 2.into(); + let corrupt = serde_json::to_vec(&document).unwrap(); + std::fs::write(&path, &corrupt).unwrap(); + let reopened = DownloadPersistence::new(tmp.path()); + let transaction = reopened.transaction(StoreOperation::Load).unwrap(); + assert!(matches!( + reopened.load_data_strict(&transaction), + Err(crate::PumasError::Validation { ref field, .. }) + if field == "acquisition.consumer_receipts" + )); + drop(transaction); + assert_eq!(std::fs::read(&path).unwrap(), corrupt); + } + + fn using_hf_acquisition( + store: &DownloadPersistence, + snapshot: &PersistedDownload, + attempt: &str, + ) -> (AcquisitionRecord, Uuid) { + let lease = Uuid::new_v4(); + let id = Uuid::new_v4(); + let manifest = ArtifactManifest::new( + ArtifactSourceIdentity::new( + "huggingface", + snapshot.repo_id.clone(), + ArtifactRevisionEvidence::new( + "huggingface.commit", + "candidate-commit", + RevisionStrength::Immutable, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + "model.gguf", + "model.gguf", + Some(1000), + None, + FileVerificationRequirement::SizeAndImmutableRevision, + ) + .unwrap()], + ) + .unwrap(); + let record = AcquisitionRecord { + id, + demand: AcquisitionDemand { + consumer: "hf.model".into(), + operation: attempt.into(), + }, + manifest, + workspace: WorkspaceIdentity { + root_identity: "store-test-root".into(), + relative_target: snapshot.dest_dir.to_string_lossy().into_owned(), + }, + phase: AcquisitionPhase::Using { lease }, + files: vec![VerifiedFile { + path: "model.gguf".into(), + bytes: 1000, + sha256: "a".repeat(64), + }], + }; + store + .store + .update_acquisitions(|records| { + records.insert(id, record.clone()); + Ok(()) + }) + .unwrap(); + (record, lease) + } + + #[test] + fn receipt_partition_rejects_the_same_acquisition_receipt_under_two_keys() { + let tmp = TempDir::new().unwrap(); + let store = DownloadPersistence::new(tmp.path()); + let snapshot = persisted("duplicate-receipt"); + let attempt = store.admit_test_download(&snapshot).unwrap(); + let (record, lease) = using_hf_acquisition(&store, &snapshot, &attempt); + let destination = PersistedDestinationIdentity { + library_root: "store-test-root".into(), + relative_target: snapshot.dest_dir.to_string_lossy().into_owned(), + }; + let receipt = store + .publish_hf_completion_receipt(HfCompletionReceiptRequest { + expected: &record, + use_lease: lease, + download_id: &snapshot.download_id, + domain: DownloadAdmissionDomain::Ambient, + destination: &destination, + model_id: "fixture-model-id", + outputs: HfCompletionOutputProof { + metadata_sha256: "b".repeat(64), + index_sha256: "c".repeat(64), + package_facts: None, + }, + }) + .unwrap(); + let duplicate_id = Uuid::new_v4(); + let mut duplicate_record = record.clone(); + duplicate_record.id = duplicate_id; + duplicate_record.demand.operation = "other-hf-operation".into(); + duplicate_record.phase = AcquisitionPhase::Adopted { + lease: Uuid::new_v4(), + }; + store + .store + .update_acquisitions(|records| { + records.insert(duplicate_id, duplicate_record.clone()); + Ok(()) + }) + .unwrap(); + + let acquisitions = store.store.acquisitions().unwrap(); + let receipt_value = serde_json::to_value(receipt).unwrap(); + let duplicate_partition = BTreeMap::from([ + (record.id, receipt_value.clone()), + (duplicate_id, receipt_value), + ]); + assert!(matches!( + validate_hf_completion_receipts(&acquisitions, &duplicate_partition), + Err(crate::PumasError::Validation { ref field, .. }) + if field == "downloads.hf_completion_receipts" + )); + } + + #[test] + fn managed_hf_receipt_settlement_is_one_restart_safe_publication() { + let tmp = TempDir::new().unwrap(); + let store = DownloadPersistence::new(tmp.path()); + let snapshot = persisted("managed-hf-receipt"); + let attempt = store.admit_test_download(&snapshot).unwrap(); + let (record, lease) = using_hf_acquisition(&store, &snapshot, &attempt); + let destination = PersistedDestinationIdentity { + library_root: "store-test-root".into(), + relative_target: snapshot.dest_dir.to_string_lossy().into_owned(), + }; + let outputs = HfCompletionOutputProof { + metadata_sha256: "b".repeat(64), + index_sha256: "c".repeat(64), + package_facts: None, + }; + let receipt = store + .publish_hf_completion_receipt(HfCompletionReceiptRequest { + expected: &record, + use_lease: lease, + download_id: &snapshot.download_id, + domain: DownloadAdmissionDomain::Ambient, + destination: &destination, + model_id: "fixture-model-id", + outputs, + }) + .unwrap(); + assert_eq!( + store + .read_hf_completion_receipt(record.id) + .unwrap() + .as_ref(), + Some(&receipt) + ); + + // Simulate restart after receipt publication but before settlement. + let interrupted = DownloadPersistence::new(tmp.path()).with_test_publisher(Arc::new( + ScriptedPublisher::new([ + ScriptedPublication::Durable, + ScriptedPublication::NotPublished, + ]), + )); + interrupted.reconcile_lifecycle_inventory_strict().unwrap(); + assert!(interrupted.settle_hf_completion(&record, &receipt).is_err()); + let (still_using, still_receipt) = interrupted + .store + .transaction(true) + .unwrap() + .consumer_completion_state(record.id) + .unwrap(); + assert_eq!(still_using.as_ref(), Some(&record)); + assert_eq!(still_receipt, Some(serde_json::to_value(&receipt).unwrap())); + assert!(interrupted + .load_lifecycle_inventory_strict() + .unwrap() + .queue_admissions + .contains_key(&snapshot.download_id)); + + // A later writer reopens the same Using lease and commits adoption, + // receipt retention, and queue release together. + let reopened = DownloadPersistence::new(tmp.path()); + reopened.reconcile_lifecycle_inventory_strict().unwrap(); + assert!(reopened.settle_hf_completion(&record, &receipt).unwrap()); + let adopted = reopened.store.acquisitions().unwrap(); + assert!(matches!( + adopted[&record.id].phase, + AcquisitionPhase::Adopted { lease: observed } if observed == lease + )); + assert_eq!( + reopened.read_hf_completion_receipt(record.id).unwrap(), + Some(receipt.clone()) + ); + let inventory = reopened.load_lifecycle_inventory_strict().unwrap(); + assert!(!inventory + .queue_admissions + .contains_key(&snapshot.download_id)); + assert!(inventory.queue_admissions.is_empty()); + assert_eq!( + reopened + .load_data_strict(&reopened.transaction(StoreOperation::Load).unwrap()) + .unwrap() + .released_queue_admissions[&snapshot.download_id] + .attempt_id, + attempt + ); + } + #[test] fn revoked_download_rejects_new_admission_and_status_mutation() { let tmp = TempDir::new().unwrap(); @@ -3402,7 +4125,7 @@ mod tests { assert_eq!(reopened.downloads[0].revision.as_deref(), Some(commit)); let document = std::fs::read_to_string(&store.path).unwrap(); assert!(document.contains(&format!("\"revision\": \"{commit}\""))); - assert!(document.contains("\"schema_version\": 6")); + assert!(document.contains("\"schema_version\": 7")); } #[test] @@ -3491,6 +4214,7 @@ mod tests { serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); let mut legacy = expected.clone(); legacy.as_object_mut().unwrap().remove("acquisitions"); + legacy.as_object_mut().unwrap().remove("consumer_receipts"); legacy["schema_version"] = 5.into(); std::fs::write(&store.path, serde_json::to_vec_pretty(&legacy).unwrap()).unwrap(); if version == 4 { @@ -3507,7 +4231,7 @@ mod tests { assert!(fresh.reconcile_lifecycle_inventory_strict().is_err()); assert_eq!(std::fs::read(&store.path).unwrap(), original); DownloadPersistence::migrate_legacy_offline(tmp.path()).unwrap(); - expected["schema_version"] = 6.into(); + expected["schema_version"] = 7.into(); let migrated_bytes = std::fs::read(&store.path).unwrap(); let actual: serde_json::Value = serde_json::from_slice(&migrated_bytes).unwrap(); assert_eq!( @@ -3543,6 +4267,7 @@ mod tests { .unwrap(); legacy["schema_version"] = serde_json::Value::from(4); legacy.as_object_mut().unwrap().remove("acquisitions"); + legacy.as_object_mut().unwrap().remove("consumer_receipts"); legacy["downloads"][0] .as_object_mut() .unwrap() @@ -3561,7 +4286,7 @@ mod tests { let durable: serde_json::Value = serde_json::from_slice(&std::fs::read(&reopened.path).unwrap()).unwrap(); - assert_eq!(durable["schema_version"], 6); + assert_eq!(durable["schema_version"], 7); assert!(durable["downloads"][0]["revision"].is_null()); } @@ -3673,6 +4398,10 @@ mod tests { serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); document["schema_version"] = serde_json::Value::from(4); document.as_object_mut().unwrap().remove("acquisitions"); + document + .as_object_mut() + .unwrap() + .remove("consumer_receipts"); for snapshot in document["downloads"].as_array_mut().unwrap() { snapshot.as_object_mut().unwrap().remove("revision"); } @@ -3735,6 +4464,7 @@ mod tests { serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); legacy["schema_version"] = serde_json::Value::from(4); legacy.as_object_mut().unwrap().remove("acquisitions"); + legacy.as_object_mut().unwrap().remove("consumer_receipts"); let original = serde_json::to_vec_pretty(&legacy).unwrap(); std::fs::write(&store.path, &original).unwrap(); @@ -3758,6 +4488,7 @@ mod tests { serde_json::from_slice(&std::fs::read(&store.path).unwrap()).unwrap(); legacy["schema_version"] = serde_json::Value::from(4); legacy.as_object_mut().unwrap().remove("acquisitions"); + legacy.as_object_mut().unwrap().remove("consumer_receipts"); legacy["downloads"][0] .as_object_mut() .unwrap() diff --git a/rust/crates/pumas-core/src/model_library/hf/download.rs b/rust/crates/pumas-core/src/model_library/hf/download.rs index 37972f31..1ac5507f 100644 --- a/rust/crates/pumas-core/src/model_library/hf/download.rs +++ b/rust/crates/pumas-core/src/model_library/hf/download.rs @@ -21,7 +21,7 @@ use crate::model_library::download_store::{ LifecycleQuarantineDomain, PersistedDownload, PersistedDownloadInventory, }; use crate::model_library::mutation_authority::{ - ModelFinalImportCapability, ModelPartialImportCapability, + DownloadCancellation, ModelFinalImportCapability, ModelPartialImportCapability, }; use crate::model_library::sharding; use crate::model_library::types::{DownloadRequest, DownloadStatus, ModelDownloadProgress}; @@ -579,7 +579,7 @@ struct PreparedDownloadTask { files: Vec, destination: DownloadDestination, configured_root: Option, - cancel_flag: Arc, + cancel_flag: Arc, pause_flag: Arc, completion_callback: Option, aux_complete_callback: Option, @@ -1007,7 +1007,7 @@ impl PreparedDownloadTask { } async fn finalize_restored( - &self, + &mut self, context: &TaskContext, initial_status: DownloadStatus, ) -> std::result::Result<(), RestoredFinalizationError> { @@ -1064,15 +1064,136 @@ impl PreparedDownloadTask { consumer: "hf.model".into(), operation: attempt.clone(), }; - let reconciliation = self - .acquisition - .reconciliation_lease(context, &demand) - .await?; - let manifest = super::acquisition_source::manifest_for_download( + let mut manifest = super::acquisition_source::manifest_for_download( &self.repo_id, &self.revision, &self.files, )?; + // A retained `Using` lease means import effects may already have + // committed before the previous worker stopped. Reopen that exact + // lease read-only and require its issued output receipt; never begin a + // replacement lease or replay import effects on this path. + if let Some(reopened) = self + .acquisition + .reopen_using(context, &demand, &manifest, &workspace) + .await? + { + let expected = reopened.record().clone(); + let persistence = self.persistence.clone().ok_or_else(|| PumasError::Config { + message: "Restored managed HF completion persistence is unavailable".into(), + })?; + let acquisition_id = expected.id; + let receipt = context + .run_fallible_blocking_named("read restored HF completion receipt", move || { + persistence.read_hf_completion_receipt(acquisition_id) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Restored receipt observation failed: {error}")) + })?? + .ok_or_else(|| PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: + "Retained HF use has no completion receipt; custody remains unresolved" + .into(), + })?; + if receipt.download_id != self.download_id + || receipt.demand.operation != attempt + || receipt.manifest != manifest + || &receipt.workspace != workspace.identity() + { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Restored receipt does not match the retained download admission" + .into(), + } + .into()); + } + let info = self + .downloads + .read() + .await + .get(&self.download_id) + .and_then(download_completion_info) + .ok_or_else(|| PumasError::Config { + message: "Restored receipt validation requires completion metadata".into(), + })?; + let importer = self + .download_importer + .clone() + .ok_or_else(|| PumasError::Config { + message: "Restored managed HF completion requires its model importer".into(), + })?; + let record = expected.clone(); + let revision = self.revision.clone(); + let validation_context = context.clone(); + let importer = importer.clone(); + let info_for_settlement = info.clone(); + let receipt_for_settlement = receipt.clone(); + let settled = context + .run_fallible_async_named( + "validate and settle restored HF completion receipt", + move || async move { + importer + .settle_hf_completion_receipt( + &info_for_settlement, + &revision, + &record, + &receipt_for_settlement, + &validation_context, + ) + .await + }, + ) + .await + .map_err(|error| { + PumasError::Other(format!("Restored receipt settlement failed: {error}")) + })??; + if !matches!(context.drain_blocking().await, Ok(0)) { + return Err(PumasError::Other( + "Restored receipt validation effects did not drain".into(), + ) + .into()); + } + if !settled || !matches!(context.drain_blocking().await, Ok(0)) { + return Err(PumasError::Other( + "Restored receipt settlement was not confirmed".into(), + ) + .into()); + } + { + let mut states = self.downloads.write().await; + let state = current_worker_state( + &mut states, + &self.download_id, + context, + &[DownloadStatus::Downloading], + )?; + state.status = DownloadStatus::Completed; + state.progress = 1.0; + state.files_completed = state.files.len(); + state.task_registered = false; + state.speed = 0.0; + } + drop(destination_guard); + publish_download_snapshot_from_parts(&self.download_publications).await; + return Ok(()); + } + if let Some(files) = self.revalidate_pinned_execution_tree(context).await? { + self.install_validated_pinned_files(context, &files).await?; + self.files = files; + manifest = super::acquisition_source::manifest_for_download( + &self.repo_id, + &self.revision, + &self.files, + )?; + } + let expected_manifest = manifest.clone(); + let expected_workspace = workspace.identity().clone(); + let reconciliation = self + .acquisition + .reconciliation_lease(context, &demand) + .await?; let operation = self .acquisition .begin( @@ -1144,6 +1265,7 @@ impl PreparedDownloadTask { DownloadAdmissionDomain::Ambient }, context.held_root_execution_grant()?, + self.cancel_flag.clone(), )) }) .transpose()?, @@ -1164,19 +1286,82 @@ impl PreparedDownloadTask { if !matches!(context.drain_blocking().await, Ok(0)) { return Err(PumasError::Other("Restored consumer effects did not drain".into()).into()); } - self.acquisition.acknowledge(context, lease).await?; - let persistence = self.persistence.clone().ok_or_else(|| PumasError::Config { - message: "Restore finalization persistence is unavailable".into(), - })?; - let id = self.download_id.clone(); - let settled = context - .run_fallible_blocking_named("settle restored download admission", move || { - persistence.settle_queue_admission(&id, &attempt) - }) - .await - .map_err(|error| { - PumasError::Other(format!("Restore settlement owner failed: {error}")) - })??; + let settled = if let Some(importer) = self.download_importer.clone() { + let expected = lease.record().clone(); + let persistence = self.persistence.clone().ok_or_else(|| PumasError::Config { + message: "Managed HF restore persistence is unavailable".into(), + })?; + let acquisition_id = expected.id; + let receipt = context + .run_fallible_blocking_named("read restored managed HF receipt", move || { + persistence.read_hf_completion_receipt(acquisition_id) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Restored receipt observation failed: {error}")) + })?? + .ok_or_else(|| PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Restored managed HF import completed without a durable receipt" + .into(), + })?; + if receipt.download_id != self.download_id + || receipt.demand.operation != attempt + || receipt.manifest != expected_manifest + || receipt.workspace != expected_workspace + { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Restored receipt does not match the current download admission" + .into(), + } + .into()); + } + let info = self + .downloads + .read() + .await + .get(&self.download_id) + .and_then(download_completion_info) + .ok_or_else(|| PumasError::Config { + message: "Restored managed HF settlement metadata is unavailable".into(), + })?; + let revision = self.revision.clone(); + let validation_context = context.clone(); + context + .run_fallible_async_named( + "validate and settle restored managed HF receipt", + move || async move { + importer + .settle_hf_completion_receipt( + &info, + &revision, + &expected, + &receipt, + &validation_context, + ) + .await + }, + ) + .await + .map_err(|error| { + PumasError::Other(format!("Restore receipt settlement failed: {error}")) + })?? + } else { + self.acquisition.acknowledge(context, lease).await?; + let persistence = self.persistence.clone().ok_or_else(|| PumasError::Config { + message: "Restore finalization persistence is unavailable".into(), + })?; + let id = self.download_id.clone(); + context + .run_fallible_blocking_named("settle restored download admission", move || { + persistence.settle_queue_admission(&id, &attempt) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Restore settlement owner failed: {error}")) + })?? + }; if !settled || !matches!(context.drain_blocking().await, Ok(0)) { return Err(PumasError::Other( "Restore finalization settlement was not confirmed".into(), @@ -1291,7 +1476,7 @@ impl PreparedDownloadTask { } // A pause before execution does not need upstream evidence. Observation // refusals enter the ordinary owned error settlement, preserving custody. - let evidence_result = if acquired == Some(true) { + let evidence_result = if acquired == Some(true) && self.restore_finalization.is_none() { match self.revalidate_pinned_execution_tree(&task_context).await { Ok(Some(files)) => match self .install_validated_pinned_files(&task_context, &files) @@ -1865,7 +2050,7 @@ struct HuggingFaceHttpAttemptHost<'a> { destination_guard: &'a mut Option>, download_id: &'a str, destination: &'a DownloadDestination, - cancel_flag: &'a AtomicBool, + cancel_flag: &'a DownloadCancellation, pause_flag: &'a AtomicBool, task_context: &'a TaskContext, bytes_offset: u64, @@ -1887,7 +2072,7 @@ impl crate::acquisition::HttpAttemptHost for HuggingFaceHttpAttemptHost<'_> { fn cancel_requested(&self) -> bool { #[cfg(test)] self.task_context.observe_cancellation_check(); - self.cancel_flag.load(Ordering::Relaxed) + self.cancel_flag.is_cancelled() } async fn record_progress(&mut self, downloaded_for_file: u64) -> Result<()> { @@ -2051,7 +2236,7 @@ fn interrupted_state_matches( || state.recovery_destination().is_some() || state.admission.is_none() || state.pause_flag.load(Ordering::Acquire) - || state.cancel_flag.load(Ordering::Acquire) + || state.cancel_flag.is_cancelled() || destination.capability().library_model_id() != expected.model_id || state.revision.as_persisted() != Some(commit) { @@ -2316,7 +2501,7 @@ async fn publish_worker_snapshot_and_revalidate( let mut states = downloads.write().await; current_worker_state(&mut states, download_id, task_context, expected_statuses).map( |state| { - !state.cancel_flag.load(Ordering::Relaxed) + !state.cancel_flag.is_cancelled() && state.matches_destination(&destination.identity()) }, ) @@ -3124,13 +3309,92 @@ impl HuggingFaceClient { self.publish_download_snapshot().await; let mut completed = Vec::new(); for id in restored_ids { - if let Some(info) = self.finalize_restored_download(context, &id).await? { - completed.push(info); + match self.finalize_restored_download(context, &id).await { + Ok(Some(info)) => completed.push(info), + Ok(None) => {} + Err(error) => { + if !self + .has_receiptless_managed_using_custody(context, &id) + .await? + { + return Err(error); + } + warn!("Download {id} remains in Error with durable recovery custody: {error}"); + } } } Ok(completed) } + async fn has_receiptless_managed_using_custody( + &self, + context: &TaskContext, + download_id: &str, + ) -> Result { + let Some(persistence) = self.persistence.clone() else { + return Ok(false); + }; + let Some((attempt_id, destination)) = self + .downloads + .read() + .await + .get(download_id) + .filter(|state| { + state.status == DownloadStatus::Error + && state.error.is_some() + && state.admission.is_some() + }) + .and_then(|state| { + Some(( + state.admission.as_ref()?.attempt_id.clone(), + state.destination.clone()?, + )) + }) + else { + return Ok(false); + }; + let acquisitions = self.acquisition.store().clone(); + let id = download_id.to_string(); + context + .run_fallible_blocking_named( + "inspect receiptless managed HF recovery custody", + move || -> Result { + let destination = destination.persisted_identity()?; + let inventory = persistence.load_lifecycle_inventory_strict()?; + if !inventory + .queue_admissions + .get(&id) + .is_some_and(|admission| { + admission.attempt_id == attempt_id + && admission.destination == destination + }) + { + return Ok(false); + } + let record = acquisitions.acquisitions()?.into_values().find(|record| { + record.demand.consumer == "hf.model" + && record.demand.operation == attempt_id + }); + let Some(record) = record else { + return Ok(false); + }; + if !matches!( + record.phase, + crate::acquisition::AcquisitionPhase::Using { .. } + ) { + return Ok(false); + } + Ok(persistence.read_hf_completion_receipt(record.id)?.is_none()) + }, + ) + .await + .map_err(|error| { + PumasError::Other(format!( + "Receiptless recovery custody observation failed: {error}" + )) + })? + } + async fn finalize_restored_download( &self, protected_context: &TaskContext, @@ -3183,7 +3447,7 @@ impl HuggingFaceClient { state.status, ) }; - let cancel_flag = Arc::new(AtomicBool::new(false)); + let cancel_flag = Arc::new(DownloadCancellation::new()); let pause_flag = Arc::new(AtomicBool::new(false)); let mut prepared_download = self .prepare_download_task( @@ -3369,7 +3633,7 @@ impl HuggingFaceClient { self.observe_finished_download_tasks().await; let download_id = uuid::Uuid::new_v4().to_string(); - let cancel_flag = Arc::new(AtomicBool::new(false)); + let cancel_flag = Arc::new(DownloadCancellation::new()); // Get file info let metadata_client = self.clone_for_invocation(); @@ -3534,7 +3798,7 @@ impl HuggingFaceClient { | DownloadStatus::Downloading | DownloadStatus::Pausing ) - && !state.cancel_flag.load(Ordering::Relaxed) + && !state.cancel_flag.is_cancelled() }) { return Ok(existing_id); } @@ -3588,7 +3852,7 @@ impl HuggingFaceClient { && state.repo_id == request.repo_id && state.revision == revision && same_files - && !state.cancel_flag.load(Ordering::Relaxed) + && !state.cancel_flag.is_cancelled() && matches!( state.status, DownloadStatus::Queued @@ -3884,7 +4148,7 @@ impl HuggingFaceClient { revision: DownloadRevision, files: Vec, destination: DownloadDestination, - cancel_flag: Arc, + cancel_flag: Arc, pause_flag: Arc, completion_callback: Option, aux_complete_callback: Option, @@ -3941,7 +4205,7 @@ impl HuggingFaceClient { repo_id: String, files: Vec, destination: DownloadDestination, - cancel_flag: Arc, + cancel_flag: Arc, pause_flag: Arc, completion_callback: Option, aux_complete_callback: Option, @@ -4137,7 +4401,7 @@ impl HuggingFaceClient { revision: &DownloadRevision, files: &[FileToDownload], destination: &DownloadDestination, - cancel_flag: Arc, + cancel_flag: Arc, pause_flag: Arc, persistence: Option>, terminal_cleanup_persistence: Option>, @@ -4174,7 +4438,7 @@ impl HuggingFaceClient { ) .ok() .and_then(|state| { - if state.cancel_flag.load(Ordering::Relaxed) { + if state.cancel_flag.is_cancelled() { return None; } if state.pause_flag.load(Ordering::Relaxed) { @@ -4526,7 +4790,7 @@ impl HuggingFaceClient { &[DownloadStatus::Downloading], ) .is_ok_and(|state| { - !state.cancel_flag.load(Ordering::Relaxed) + !state.cancel_flag.is_cancelled() && state.matches_destination(&destination.identity()) }) }; @@ -4710,12 +4974,38 @@ impl HuggingFaceClient { DownloadAdmissionDomain::Ambient }, task_context.held_root_execution_grant()?, + cancel_flag.clone(), )) }) .transpose()?, ) .await?; } + let managed_hf_completion = if download_importer.is_some() { + let expected = use_lease.record().clone(); + let acquisition_id = expected.id; + let persistence = terminal_cleanup_persistence + .as_ref() + .cloned() + .ok_or_else(|| PumasError::Config { + message: "Managed HF completion requires durable download persistence".into(), + })?; + let receipt = task_context + .run_fallible_blocking_named("read managed HF completion receipt", move || { + persistence.read_hf_completion_receipt(acquisition_id) + }) + .await + .map_err(|error| { + PumasError::Other(format!("Completion receipt observation failed: {error}")) + })?? + .ok_or_else(|| PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Managed HF import completed without a durable receipt".into(), + })?; + Some((expected, receipt)) + } else { + None + }; destination_guard = Some(destination_lock.clone().lock_owned().await); { let mut states = downloads.write().await; @@ -4743,53 +5033,97 @@ impl HuggingFaceClient { } } - acquisition.acknowledge(&task_context, use_lease).await?; - if !matches!(task_context.drain_blocking().await, Ok(0)) { - return Err(PumasError::Other( - "Acquisition adoption effects did not drain".into(), - )); - } - - // Persistence cleanup is part of successful completion. It is - // registered with the same task owner and must finish before the final - // drain, Completed projection, or recovery-capability release. let completion_admission = downloads.read().await.get(download_id).and_then(|state| { state .admission .as_ref() .map(|entry| entry.attempt_id.clone()) }); - if let Some(persistence) = terminal_cleanup_persistence - .as_ref() - .filter(|_| completion_admission.is_some() || !destination.is_recovery()) - { - let persistence = persistence.clone(); - let persisted_id = download_id.to_string(); - let attempt = completion_admission; - match task_context - .run_fallible_blocking_named("remove completed persisted download", move || { - if let Some(attempt) = attempt { - if persistence.settle_queue_admission(&persisted_id, &attempt)? { - Ok(()) + if let Some((expected, receipt)) = managed_hf_completion { + let importer = download_importer + .clone() + .ok_or_else(|| PumasError::Config { + message: "Managed HF completion importer is unavailable".into(), + })?; + let info = downloads + .read() + .await + .get(download_id) + .and_then(download_completion_info) + .ok_or_else(|| PumasError::Config { + message: "Managed HF completion metadata is unavailable".into(), + })?; + let settlement_revision = revision.clone(); + let settlement_context = task_context.clone(); + let settled = task_context + .run_fallible_async_named( + "validate and settle managed HF completion receipt", + move || async move { + importer + .settle_hf_completion_receipt( + &info, + &settlement_revision, + &expected, + &receipt, + &settlement_context, + ) + .await + }, + ) + .await + .map_err(|error| { + PumasError::Other(format!("Managed HF settlement owner failed: {error}")) + })??; + if !settled { + return Err(PumasError::Other( + "Managed HF completion settlement was not confirmed".into(), + )); + } + } else { + acquisition.acknowledge(&task_context, use_lease).await?; + if !matches!(task_context.drain_blocking().await, Ok(0)) { + return Err(PumasError::Other( + "Acquisition adoption effects did not drain".into(), + )); + } + } + + // Persistence cleanup is part of successful completion. Managed HF + // settlement already commits receipt, adoption, and queue release in + // one store publication; other consumers retain ordinary cleanup. + if download_importer.is_none() { + if let Some(persistence) = terminal_cleanup_persistence + .as_ref() + .filter(|_| completion_admission.is_some() || !destination.is_recovery()) + { + let persistence = persistence.clone(); + let persisted_id = download_id.to_string(); + let attempt = completion_admission; + match task_context + .run_fallible_blocking_named("remove completed persisted download", move || { + if let Some(attempt) = attempt { + if persistence.settle_queue_admission(&persisted_id, &attempt)? { + Ok(()) + } else { + Err(PumasError::Other( + "Completed download queue settlement was not confirmed".into(), + )) + } } else { - Err(PumasError::Other( - "Completed download queue settlement was not confirmed".into(), - )) + Err(PumasError::Config { + message: "Ordinary completion requires durable admission".into(), + }) } - } else { - Err(PumasError::Config { - message: "Ordinary completion requires durable admission".into(), - }) + }) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => return Err(error), + Err(error) => { + return Err(PumasError::Other(format!( + "failed to observe completed-download persistence cleanup: {error}" + ))); } - }) - .await - { - Ok(Ok(_)) => {} - Ok(Err(error)) => return Err(error), - Err(error) => { - return Err(PumasError::Other(format!( - "failed to observe completed-download persistence cleanup: {error}" - ))); } } } @@ -4959,7 +5293,10 @@ impl HuggingFaceClient { }); let protected_context = context.clone(); let configured_root = self.destination_root.clone(); - let transition = self.download_tasks.begin_cancel( + let Some(cancel_prepared) = state.cancel_flag.prepare_cancel() else { + return Ok(false); + }; + let transition = match self.download_tasks.begin_cancel( download_id, move |task_context, predecessor| async move { use futures::FutureExt; @@ -5162,23 +5499,44 @@ impl HuggingFaceClient { Err(payload) => std::panic::resume_unwind(payload), } }, - ); - let finalizer = match transition? { + ) { + Ok(transition) => transition, + Err(error) => { + state.cancel_flag.abort_cancel(cancel_prepared); + return Err(error); + } + }; + let finalizer = match transition { super::lifecycle::CancelTransition::Started(finalizer) | super::lifecycle::CancelTransition::Existing(finalizer) => finalizer, - super::lifecycle::CancelTransition::AlreadyRunning => return Ok(true), + super::lifecycle::CancelTransition::AlreadyRunning => { + if !state.cancel_flag.finish_cancel(cancel_prepared) { + return Ok(false); + } + return Ok(true); + } }; let reservation_bound = self.destination_executions.reserve( - cleanup_identity, + cleanup_identity.clone(), download_id.to_string(), cleanup_domain, finalizer.generation().clone(), ); if !reservation_bound { + state.cancel_flag.abort_cancel(cancel_prepared); + drop(finalizer); + None + } else if !state.cancel_flag.finish_cancel(cancel_prepared) { + let generation = finalizer.generation().clone(); + self.destination_executions.release( + &cleanup_identity, + download_id, + cleanup_domain, + &generation, + ); drop(finalizer); None } else { - state.cancel_flag.store(true, Ordering::Relaxed); state.status = DownloadStatus::Cancelling; state.speed = 0.0; state.task_registered = true; @@ -5409,7 +5767,7 @@ impl HuggingFaceClient { } }; - let cancel_flag = Arc::new(AtomicBool::new(false)); + let cancel_flag = Arc::new(DownloadCancellation::new()); let pause_flag = Arc::new(AtomicBool::new(false)); let prepared_download = self .prepare_download_task( @@ -6102,7 +6460,7 @@ impl HuggingFaceClient { }; if let Some((repo_id, revision, files, recovery_destination)) = recovery_resume { - let cancel_flag = Arc::new(AtomicBool::new(false)); + let cancel_flag = Arc::new(DownloadCancellation::new()); let pause_flag = Arc::new(AtomicBool::new(false)); let prepared_download = self .prepare_download_task( @@ -6229,7 +6587,7 @@ impl HuggingFaceClient { message: "Cannot resume a mutable source selection without a digest for every file; start a fresh selection to reacquire the current source".into(), }); } - let cancel_flag = Arc::new(AtomicBool::new(false)); + let cancel_flag = Arc::new(DownloadCancellation::new()); let pause_flag = Arc::new(AtomicBool::new(false)); let mut prepared_download = self .prepare_download_task( @@ -7903,7 +8261,7 @@ mod tests { downloaded_bytes: 2, total_bytes: Some(4), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -11288,6 +11646,410 @@ mod tests { (library, client, destination, request) } + #[tokio::test] + async fn receipt_reopen_settles_after_publication_failure_without_network_or_reimport() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + const COMMIT: &str = "0123456789abcdef0123456789abcdef01234567"; + let temp = TempDir::new().unwrap(); + let library = Arc::new( + crate::model_library::ModelLibrary::new(temp.path().join("library")) + .await + .unwrap(), + ); + let destination = library.build_model_path("vision", "acme", "model"); + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(destination.join("model.onnx"), b"data").unwrap(); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let requests = Arc::new(AtomicU64::new(0)); + let server_requests = requests.clone(); + let server = tokio::spawn(async move { + if let Ok((mut socket, _)) = listener.accept().await { + server_requests.fetch_add(1, Ordering::SeqCst); + let mut request = [0_u8; 2048]; + let _ = socket.read(&mut request).await; + let _ = socket + .write_all( + b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .await; + } + }); + + let publisher = Arc::new( + crate::model_library::download_store::FailAfterReceiptSettlementPublisher::new(), + ); + let persistence = Arc::new( + DownloadPersistence::new(temp.path()) + .with_receipt_settlement_failure_publisher_for_test(publisher.clone()), + ); + let mut client = HuggingFaceClient::new(temp.path().join("cache")).unwrap(); + client + .configure_download_destination_root(library.library_root()) + .unwrap(); + client.set_persistence(persistence.clone()); + let importer = importer_with_authority_for_test(library.clone(), &client).await; + client.set_download_importer(importer); + client.set_test_download_base_url(endpoint.clone()); + *client.auth_token.write().await = None; + + let revision = DownloadRevision::from_commit(COMMIT).unwrap(); + cache_pinned_repo_tree( + &client, + "acme/model", + &revision, + vec![LfsFileInfo { + filename: "model.onnx".into(), + size: 4, + sha256: "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7".into(), + }], + Vec::new(), + ); + let mut request = recovery_test_request("acme/model", &["model.onnx".into()]); + request.model_type = Some("vision".into()); + request.pipeline_tag = Some("image-classification".into()); + let download_id = client + .start_download_at_revision(&request, &destination, None, revision) + .await + .unwrap(); + + let terminal = tokio::time::timeout(Duration::from_secs(10), async { + loop { + client.observe_finished_download_tasks().await; + if matches!( + client.get_download_status(&download_id).await, + Some(DownloadStatus::Error | DownloadStatus::Completed) + ) && !client.download_tasks.contains(&download_id) + { + break; + } + tokio::task::yield_now().await; + } + }) + .await; + assert!(terminal.is_ok(), "unexpected wait state: status={:?}, error={:?}, publisher_triggered={}, network_requests={}", + client.get_download_status(&download_id).await, + client.downloads.read().await.get(&download_id).and_then(|state| state.error.clone()), + publisher.was_triggered(), + requests.load(Ordering::SeqCst)); + assert!(publisher.was_triggered()); + assert_eq!( + client.get_download_status(&download_id).await, + Some(DownloadStatus::Error) + ); + assert_eq!(requests.load(Ordering::SeqCst), 0); + + let records = client.acquisition.store().acquisitions().unwrap(); + let using = records + .values() + .find(|record| record.demand.consumer == "hf.model") + .unwrap() + .clone(); + assert!(matches!( + using.phase, + crate::acquisition::AcquisitionPhase::Using { .. } + )); + let receipt = persistence + .read_hf_completion_receipt(using.id) + .unwrap() + .expect("the import receipt must survive the settlement failure"); + assert!(persistence + .load_lifecycle_inventory_strict() + .unwrap() + .queue_admissions + .contains_key(&download_id)); + let pinned_tree_cache = client.get_cache_path( + &format!("acme/model@{}", using.manifest.source().revision().value()), + "files", + ); + drop(client); + std::fs::remove_file(&pinned_tree_cache).unwrap(); + assert!(!pinned_tree_cache.exists()); + + let mut reopened = HuggingFaceClient::new(temp.path().join("cache")).unwrap(); + reopened + .configure_download_destination_root(library.library_root()) + .unwrap(); + let reopened_persistence = Arc::new(DownloadPersistence::new(temp.path())); + reopened.set_persistence(reopened_persistence.clone()); + let importer = importer_with_authority_for_test(library.clone(), &reopened).await; + reopened.set_download_importer(importer); + reopened.set_test_download_base_url(endpoint); + let completed = reopened.restore_persisted_downloads().await.unwrap(); + assert!(completed.iter().any(|info| info.download_id == download_id)); + assert_eq!(requests.load(Ordering::SeqCst), 0); + assert_eq!( + reopened_persistence + .read_hf_completion_receipt(using.id) + .unwrap(), + Some(receipt) + ); + assert!(matches!( + reopened.acquisition.store().acquisitions().unwrap()[&using.id].phase, + crate::acquisition::AcquisitionPhase::Adopted { .. } + )); + assert!(!reopened_persistence + .load_lifecycle_inventory_strict() + .unwrap() + .queue_admissions + .contains_key(&download_id)); + assert_eq!( + std::fs::read(destination.join("model.onnx")).unwrap(), + b"data" + ); + server.abort(); + } + + #[tokio::test] + async fn restored_files_ready_settlement_is_atomic_and_receipt_reopen_does_not_reimport() { + const SHA256: &str = "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7"; + + let temp = TempDir::new().unwrap(); + let library_root = temp.path().join("library"); + let library = Arc::new( + crate::model_library::ModelLibrary::new(&library_root) + .await + .unwrap(), + ); + let destination = library.build_model_path("vision", "acme", "model"); + std::fs::create_dir_all(&destination).unwrap(); + std::fs::write(destination.join("model.onnx"), b"data").unwrap(); + + let publisher = Arc::new( + crate::model_library::download_store::FailAfterReceiptSettlementPublisher::new(), + ); + let persistence = Arc::new( + DownloadPersistence::new(temp.path()) + .with_receipt_settlement_failure_publisher_for_test(publisher.clone()), + ); + let revision = DownloadRevision::legacy_main(); + let files = vec![FileToDownload { + filename: "model.onnx".into(), + size: Some(4), + sha256: Some(SHA256.into()), + }]; + let mut request = recovery_test_request("acme/model", &["model.onnx".into()]); + request.model_type = Some("vision".into()); + request.pipeline_tag = Some("image-classification".into()); + let download_id = "files-ready-reopen".to_string(); + let snapshot = PersistedDownload { + download_id: download_id.clone(), + repo_id: request.repo_id.clone(), + filename: "model.onnx".into(), + filenames: vec!["model.onnx".into()], + dest_dir: destination.clone(), + total_bytes: Some(4), + status: DownloadStatus::Error, + download_request: request.clone(), + revision: None, + created_at: chrono::Utc::now().to_rfc3339(), + known_sha256: Some(SHA256.into()), + huggingface_evidence: None, + }; + let attempt = admit_snapshot_at_root(&persistence, &snapshot, &library_root); + std::fs::write( + destination.join(".pumas_download"), + serialize_download_marker(&request, vec!["model.onnx".into()], None, &revision) + .unwrap(), + ) + .unwrap(); + + let mut client = HuggingFaceClient::new(temp.path().join("cache")).unwrap(); + client + .configure_download_destination_root(&library_root) + .unwrap(); + client.set_persistence(persistence.clone()); + let importer = importer_with_authority_for_test(library.clone(), &client).await; + client.set_download_importer(importer); + + let setup_client = client.clone_for_invocation(); + let setup_destination = destination.clone(); + let setup_files = files.clone(); + let setup_attempt = attempt.clone(); + client + .run_download_invocation(move |context| async move { + let context = setup_client.protect_download_mutation(&context).await?; + let root = + setup_client + .destination_root + .clone() + .ok_or_else(|| PumasError::Config { + message: "FilesReady fixture requires a configured root".into(), + })?; + let path = setup_destination.clone(); + let destination = context + .run_fallible_blocking_named( + "resolve FilesReady fixture destination", + move || root.resolve(&path), + ) + .await + .map_err(|error| { + PumasError::Other(format!( + "FilesReady destination observation failed: {error}" + )) + })??; + let managed = DownloadDestination::Managed(destination); + managed.prepare(&context).await?; + let capability = managed.capability().clone(); + let execution_lease = context.held_execution_lease()?; + let workspace = context + .run_fallible_blocking_named( + "capture FilesReady fixture workspace", + move || capability.acquisition_workspace(execution_lease), + ) + .await + .map_err(|error| { + PumasError::Other(format!( + "FilesReady workspace observation failed: {error}" + )) + })??; + let manifest = crate::model_library::hf::acquisition_source::manifest_for_download( + "acme/model", + &revision, + &setup_files, + )?; + let demand = crate::acquisition::AcquisitionDemand { + consumer: "hf.model".into(), + operation: setup_attempt, + }; + let operation = setup_client + .acquisition + .begin( + &context, + demand.clone(), + manifest.clone(), + workspace.identity().clone(), + None, + ) + .await?; + drop(operation); + let store = setup_client.acquisition.store().clone(); + context + .run_fallible_blocking_named("persist restored FilesReady fixture", move || { + let verified = workspace.seal(&manifest)?; + let acquisition_id = store + .acquisitions()? + .into_values() + .find(|record| record.demand == demand) + .map(|record| record.id) + .ok_or_else(|| PumasError::Validation { + field: "acquisition.custody".into(), + message: "FilesReady fixture acquisition disappeared".into(), + })?; + store.update_acquisitions(|records| { + let record = records.get_mut(&acquisition_id).ok_or_else(|| { + PumasError::Validation { + field: "acquisition.custody".into(), + message: "FilesReady fixture acquisition disappeared".into(), + } + })?; + if !matches!( + record.phase, + crate::acquisition::AcquisitionPhase::Transferring + ) { + return Err(PumasError::Validation { + field: "acquisition.custody".into(), + message: "FilesReady fixture started from another phase".into(), + }); + } + record.files = verified; + record.phase = crate::acquisition::AcquisitionPhase::FilesReady; + Ok(()) + }) + }) + .await + .map_err(|error| { + PumasError::Other(format!("FilesReady fixture publication failed: {error}")) + })??; + Ok(()) + }) + .await + .unwrap(); + + let first_restore = client.restore_persisted_downloads().await.unwrap_err(); + assert!(publisher.was_triggered()); + assert!(first_restore + .to_string() + .contains("injected failure after HF receipt publication")); + let records = client.acquisition.store().acquisitions().unwrap(); + let using = records + .values() + .find(|record| record.demand.operation == attempt) + .unwrap() + .clone(); + assert!(matches!( + using.phase, + crate::acquisition::AcquisitionPhase::Using { .. } + )); + let receipt = persistence + .read_hf_completion_receipt(using.id) + .unwrap() + .expect("import completed before settlement publication failed"); + assert!(persistence + .load_lifecycle_inventory_strict() + .unwrap() + .queue_admissions + .contains_key(&download_id)); + drop(client); + + let mut reopened = HuggingFaceClient::new(temp.path().join("cache")).unwrap(); + reopened + .configure_download_destination_root(&library_root) + .unwrap(); + let reopened_persistence = Arc::new(DownloadPersistence::new(temp.path())); + reopened.set_persistence(reopened_persistence.clone()); + let reopened_library = Arc::new( + crate::model_library::ModelLibrary::new(&library_root) + .await + .unwrap(), + ); + reopened_library + .install_mutation_authority( + crate::api::RuntimeTasks::new(), + crate::model_library::download_recovery::DownloadDestinationRoot::open( + &library_root, + ) + .unwrap(), + reopened_persistence.clone(), + ) + .unwrap(); + let writes = Arc::new(AtomicU64::new(0)); + let observed_writes = writes.clone(); + reopened_library.set_metadata_write_notifier(Some(Arc::new(move |_| { + observed_writes.fetch_add(1, Ordering::SeqCst); + }))); + reopened.set_download_importer(Arc::new(crate::model_library::ModelImporter::new( + reopened_library.clone(), + ))); + + let completed = reopened.restore_persisted_downloads().await.unwrap(); + assert!(completed.iter().any(|info| info.download_id == download_id)); + assert_eq!(writes.load(Ordering::SeqCst), 0); + assert_eq!( + reopened_persistence + .read_hf_completion_receipt(using.id) + .unwrap(), + Some(receipt) + ); + assert!(matches!( + reopened.acquisition.store().acquisitions().unwrap()[&using.id].phase, + crate::acquisition::AcquisitionPhase::Adopted { .. } + )); + assert!(!reopened_persistence + .load_lifecycle_inventory_strict() + .unwrap() + .queue_admissions + .contains_key(&download_id)); + assert_eq!( + std::fs::read(destination.join("model.onnx")).unwrap(), + b"data" + ); + assert_eq!(writes.load(Ordering::SeqCst), 0); + reopened.shutdown_downloads().await.unwrap(); + } + #[tokio::test] async fn busy_root_refuses_mutation_but_preserves_idle_runtime_reads() { let temp = TempDir::new().unwrap(); @@ -11644,7 +12406,11 @@ mod tests { async move { client.shutdown_downloads().await }, )) } else { - assert!(client.cancel_download(&download_id).await.unwrap()); + assert_eq!( + client.cancel_download(&download_id).await.unwrap(), + !final_import, + "cancellation wins only before managed final import claims publication" + ); None }; if shutdown { @@ -11679,9 +12445,15 @@ mod tests { assert_eq!(outcome.is_err(), fail_import); } else { tokio::time::timeout(Duration::from_secs(3), async { - while client.get_download_status(&download_id).await - == Some(DownloadStatus::Cancelling) - { + while matches!( + client.get_download_status(&download_id).await, + Some( + DownloadStatus::Queued + | DownloadStatus::Downloading + | DownloadStatus::Pausing + | DownloadStatus::Cancelling + ) + ) { tokio::task::yield_now().await; } }) @@ -11698,7 +12470,7 @@ mod tests { assert!(!before.quarantines.contains_key(&download_id)); assert_eq!( status, - Some(if shutdown { + Some(if shutdown || final_import { DownloadStatus::Downloading } else { DownloadStatus::Cancelling @@ -11708,9 +12480,13 @@ mod tests { client.get_download_status(&download_id).await, Some(if shutdown || fail_import { DownloadStatus::Error + } else if final_import { + DownloadStatus::Completed } else { DownloadStatus::Cancelled - }) + }), + "wrong terminal status: final_import={final_import}, shutdown={shutdown}, fail_import={fail_import}, error={:?}", + client.downloads.read().await.get(&download_id).and_then(|state| state.error.clone()) ); if final_import { assert_eq!( @@ -16307,7 +17083,7 @@ mod tests { downloaded_bytes: 1, total_bytes: Some(2), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -16394,7 +17170,7 @@ mod tests { downloaded_bytes: 0, total_bytes: Some(1), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -17411,7 +18187,7 @@ mod tests { downloaded_bytes: 256, total_bytes: Some(1024), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 2, @@ -17491,7 +18267,7 @@ mod tests { downloaded_bytes: 256, total_bytes: Some(1024), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -17575,7 +18351,7 @@ mod tests { downloaded_bytes: 256, total_bytes: Some(1024), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -17683,7 +18459,7 @@ mod tests { downloaded_bytes: 512, total_bytes: Some(1024), speed: 1024.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -18332,7 +19108,7 @@ mod tests { downloaded_bytes: 256, total_bytes: Some(1024), speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new(DownloadCancellation::new()), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, @@ -18398,7 +19174,7 @@ mod tests { .get(&download_id) .unwrap() .cancel_flag - .load(Ordering::Relaxed); + .is_cancelled(); assert!(cancel_flag_set); client.observe_finished_download_tasks().await; assert!(!client.download_tasks.contains(&download_id)); diff --git a/rust/crates/pumas-core/src/model_library/hf/types.rs b/rust/crates/pumas-core/src/model_library/hf/types.rs index 3046a863..6f0a9623 100644 --- a/rust/crates/pumas-core/src/model_library/hf/types.rs +++ b/rust/crates/pumas-core/src/model_library/hf/types.rs @@ -135,7 +135,7 @@ pub(crate) struct DownloadState { /// Download speed (bytes/sec) pub speed: f64, /// Cancellation flag - pub cancel_flag: Arc, + pub cancel_flag: Arc, /// Pause flag -- signals graceful stop without deleting .part file pub pause_flag: Arc, /// Error message if failed @@ -305,7 +305,9 @@ impl DownloadState { downloaded_bytes, total_bytes: entry.total_bytes, speed: 0.0, - cancel_flag: Arc::new(AtomicBool::new(false)), + cancel_flag: Arc::new( + crate::model_library::mutation_authority::DownloadCancellation::new(), + ), pause_flag: Arc::new(AtomicBool::new(false)), error: None, retry_attempt: 0, diff --git a/rust/crates/pumas-core/src/model_library/importer.rs b/rust/crates/pumas-core/src/model_library/importer.rs index 3c61f01d..67146477 100644 --- a/rust/crates/pumas-core/src/model_library/importer.rs +++ b/rust/crates/pumas-core/src/model_library/importer.rs @@ -676,6 +676,7 @@ impl ModelImporter { .map_err(|error| { PumasError::Other(format!("Model import guard observation failed: {error}")) })??; + guard.claim_final_import_completion()?; let importer = Self { library: Arc::new(self.library.with_import_guard(guard)), }; @@ -684,6 +685,46 @@ impl ModelImporter { .await } + /// Validate a previously issued managed-HF completion receipt without + /// importing, repairing metadata, or resolving package facts. This is the + /// cold-reopen path after import effects may already have completed. + pub(crate) async fn settle_hf_completion_receipt( + &self, + info: &DownloadCompletionInfo, + revision: &DownloadRevision, + record: &crate::acquisition::AcquisitionRecord, + receipt: &crate::model_library::download_store::HfCompletionReceipt, + context: &crate::model_library::hf::DownloadInvocationContext, + ) -> Result { + receipt.validate_for_record(record)?; + + let mut selected = record + .manifest + .files() + .iter() + .map(|file| file.logical_path().to_string()) + .collect::>(); + let mut requested = info.filenames.clone(); + selected.sort(); + requested.sort(); + if info.download_id != receipt.download_id + || selected != requested + || record.manifest.source().provider() != "huggingface" + || record.manifest.source().source_id() != info.download_request.repo_id + || record.manifest.source().revision().value() != revision.as_str() + { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Completion receipt does not match the selected HF download".into(), + }); + } + + let grant = context.held_root_execution_grant()?; + self.library + .settle_hf_completion_receipt(&info.dest_dir, record, receipt, grant) + .await + } + async fn finalize_downloaded_directory_guarded( &self, info: &DownloadCompletionInfo, @@ -731,6 +772,9 @@ impl ModelImporter { // facts without generating or repairing them during status reads. self.library.resolve_model_package_facts(&model_id).await?; } + self.library + .issue_managed_hf_completion_receipt(&model_id, revision.as_persisted().is_some()) + .await?; Ok(result) } @@ -2676,6 +2720,10 @@ mod tests { "owner", DownloadAdmissionDomain::Ambient, grant.clone(), + Arc::new( + crate::model_library::mutation_authority::DownloadCancellation::new( + ), + ), ); assert!(wrong .validate_provenance("owner", "other/repo", "main", &info.filenames) @@ -2690,6 +2738,10 @@ mod tests { "owner", DownloadAdmissionDomain::Ambient, grant.clone(), + Arc::new( + crate::model_library::mutation_authority::DownloadCancellation::new( + ), + ), ); worker_downloads .acquisition_store() @@ -2727,6 +2779,10 @@ mod tests { "owner", DownloadAdmissionDomain::Ambient, grant.clone(), + Arc::new( + crate::model_library::mutation_authority::DownloadCancellation::new( + ), + ), ); importer .finalize_downloaded_directory_with_capability( @@ -2755,6 +2811,10 @@ mod tests { "owner", DownloadAdmissionDomain::Ambient, grant.clone(), + Arc::new( + crate::model_library::mutation_authority::DownloadCancellation::new( + ), + ), ); let before_hidden = std::fs::read(model.join("metadata.json"))?; assert!(matches!( @@ -2781,6 +2841,10 @@ mod tests { "owner", DownloadAdmissionDomain::Ambient, grant.clone(), + Arc::new( + crate::model_library::mutation_authority::DownloadCancellation::new( + ), + ), ); // Actual durable settlement invalidates an earlier unconsumed proof. worker_acquisition.acknowledge(&context, lease).await?; diff --git a/rust/crates/pumas-core/src/model_library/library.rs b/rust/crates/pumas-core/src/model_library/library.rs index 1e0d2191..ca24a4d7 100644 --- a/rust/crates/pumas-core/src/model_library/library.rs +++ b/rust/crates/pumas-core/src/model_library/library.rs @@ -20,6 +20,9 @@ use crate::metadata::{atomic_read_json, atomic_write_json}; use crate::model_library::artifact_load_target::{ library_unavailable_response, resolve_artifact_load_target_from_index, }; +use crate::model_library::download_store::{ + canonical_json_sha256, HfCompletionOutputProof, HfPackageFactsProof, +}; use crate::model_library::external_assets::{ get_diffusers_bundle_lookup_hints, is_diffusers_bundle, is_external_reference, refresh_external_metadata_validation, MODEL_EXECUTION_CONTRACT_VERSION, @@ -316,6 +319,76 @@ impl ModelLibrary { library } + pub(crate) async fn issue_managed_hf_completion_receipt( + &self, + model_id: &str, + require_package_facts: bool, + ) -> Result<()> { + let Some(guard) = self.import_guard.as_ref() else { + return Ok(()); + }; + if !guard.has_managed_final_import_proof() { + return Ok(()); + } + guard + .publish_hf_completion_receipt(self, model_id, require_package_facts) + .await?; + Ok(()) + } + + pub(crate) async fn validate_hf_completion_receipt( + &self, + model_dir: &Path, + receipt: &crate::model_library::download_store::HfCompletionReceipt, + record: &crate::acquisition::AcquisitionRecord, + grant: Arc, + ) -> Result<()> { + let authority = self.mutation_authority()?; + receipt.validate_for_record(record)?; + let model_dir = model_dir.to_path_buf(); + let receipt = receipt.clone(); + let require_package_facts = receipt.outputs.package_facts.is_some(); + let model_id = receipt.model_id.clone(); + let expected_outputs = receipt.outputs.clone(); + let destination = self + .run_import_blocking("validate HF completion destination", move || { + authority.validate_hf_completion_destination(&model_dir, &receipt, grant.as_ref()) + }) + .await??; + let outputs = self + .hf_completion_output_proof(&destination, &model_id, require_package_facts) + .await?; + if outputs != expected_outputs { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Current model outputs do not match the issued completion proof".into(), + }); + } + Ok(()) + } + + /// Revalidate both current model outputs and the exact receipt before + /// committing acquisition adoption and queue release through one store + /// transaction. Callers cannot bypass output proof validation at the + /// persistence settlement boundary. + pub(crate) async fn settle_hf_completion_receipt( + &self, + model_dir: &Path, + record: &crate::acquisition::AcquisitionRecord, + receipt: &crate::model_library::download_store::HfCompletionReceipt, + grant: Arc, + ) -> Result { + self.validate_hf_completion_receipt(model_dir, receipt, record, grant) + .await?; + let persistence = self.mutation_authority()?.downloads(); + let record = record.clone(); + let receipt = receipt.clone(); + self.run_import_blocking("settle validated HF completion receipt", move || { + persistence.settle_hf_completion(&record, &receipt) + }) + .await? + } + pub(crate) async fn run_import_blocking( &self, operation: &'static str, @@ -1270,7 +1343,7 @@ impl ModelLibrary { let Some(record) = self.index.get(model_id)? else { return Ok(false); }; - self.refresh_external_asset_state(&record).await + Box::pin(self.refresh_external_asset_state(&record)).await } fn indexed_model_dir(&self, record: &ModelRecord) -> Result { @@ -2713,6 +2786,107 @@ impl ModelLibrary { ) } + /// Read the final model outputs through the held destination and canonical + /// index. This path never repairs metadata, refreshes facts, or upserts an + /// index row; callers retain root custody through comparison/publication. + pub(crate) async fn hf_completion_output_proof( + &self, + destination: &crate::model_library::DownloadRecoveryDestination, + model_id: &str, + require_package_facts: bool, + ) -> Result { + let held_destination = destination.clone(); + let index = self.index.clone(); + let model_id_owned = model_id.to_string(); + let (metadata_value, metadata, record, package_facts) = self + .run_import_blocking("observe HF completion output projections", move || { + let metadata_value = + held_destination + .read_model_metadata_value()? + .ok_or_else(|| PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Receipt model metadata is missing".into(), + })?; + let metadata = serde_json::from_value::(metadata_value.clone())?; + if metadata.model_id.as_deref() != Some(model_id_owned.as_str()) { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Receipt metadata identifies another model".into(), + }); + } + let record = index + .get(&model_id_owned)? + .ok_or_else(|| PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Receipt model index projection is missing".into(), + })?; + if record.id != model_id_owned { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Receipt index projection identifies another model".into(), + }); + } + let package_facts = if require_package_facts { + Some( + index + .get_model_package_facts_cache( + &model_id_owned, + metadata.selected_artifact_id.as_deref(), + ModelPackageFactsCacheScope::Detail, + )? + .ok_or_else(|| PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Required pinned package-facts output is missing".into(), + })?, + ) + } else { + None + }; + Ok((metadata_value, metadata, record, package_facts)) + }) + .await??; + + let index_projection = serde_json::json!({ + "id": record.id, + "path": record.path, + "cleaned_name": record.cleaned_name, + "official_name": record.official_name, + "model_type": record.model_type, + "tags": record.tags, + "hashes": record.hashes, + "metadata": record.metadata, + }); + let package_facts = match package_facts { + Some(row) => { + if row.package_facts_contract_version != i64::from(PACKAGE_FACTS_CONTRACT_VERSION) + || !self + .cached_model_package_facts_are_current(&row, None, None) + .await? + { + return Err(PumasError::Validation { + field: "downloads.hf_completion_receipts".into(), + message: "Required pinned package-facts output is stale or unsupported" + .into(), + }); + } + let facts: Value = serde_json::from_str(&row.facts_json)?; + Some(HfPackageFactsProof { + contract_version: row.package_facts_contract_version, + content_sha256: canonical_json_sha256(&facts)?, + }) + } + None => None, + }; + let _ = metadata; + Ok(HfCompletionOutputProof { + metadata_sha256: canonical_json_sha256(&metadata_value)?, + // `updated_at` records index maintenance time, not the semantic + // imported model projection, so it is deliberately excluded. + index_sha256: canonical_json_sha256(&index_projection)?, + package_facts, + }) + } + /// Resolve versioned package facts for a model without selecting a runtime. pub async fn resolve_model_package_facts( &self, diff --git a/rust/crates/pumas-core/src/model_library/mutation_authority.rs b/rust/crates/pumas-core/src/model_library/mutation_authority.rs index da6bb709..9bda2337 100644 --- a/rust/crates/pumas-core/src/model_library/mutation_authority.rs +++ b/rust/crates/pumas-core/src/model_library/mutation_authority.rs @@ -6,13 +6,100 @@ use crate::index::{IntentDeletionClaimResult, ModelIndex}; use crate::model_library::download_recovery::{ DownloadDestinationRoot, DownloadRecoveryDestination, RootExecutionGrant, }; -use crate::model_library::download_store::DownloadAdmissionDomain; -use crate::model_library::download_store::{DownloadPersistence, PersistedDestinationIdentity}; +use crate::model_library::download_store::{ + DownloadAdmissionDomain, DownloadPersistence, HfCompletionReceipt, HfCompletionReceiptRequest, + PersistedDestinationIdentity, +}; use crate::{PumasError, Result}; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::Arc; use uuid::Uuid; +/// One operation-scoped decision between cancellation and durable completion. +/// It replaces the download's boolean cancellation flag so the receipt boundary +/// has a single atomic winner. +pub(crate) struct DownloadCancellation(AtomicU8); + +impl DownloadCancellation { + const ACTIVE: u8 = 0; + const CANCEL_PREPARING: u8 = 1; + const CANCELLED: u8 = 2; + const COMPLETING: u8 = 3; + + pub(crate) fn new() -> Self { + Self(AtomicU8::new(Self::ACTIVE)) + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.0.load(Ordering::Acquire) == Self::CANCELLED + } + + /// Reserve the cancellation decision before replacing the worker owner. + /// The returned boolean says whether this caller must commit or roll back + /// the reservation after installing the finalizer. + pub(crate) fn prepare_cancel(&self) -> Option { + loop { + match self.0.load(Ordering::Acquire) { + Self::ACTIVE => { + if self + .0 + .compare_exchange( + Self::ACTIVE, + Self::CANCEL_PREPARING, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + { + return Some(true); + } + } + Self::CANCELLED => return Some(false), + Self::CANCEL_PREPARING | Self::COMPLETING => return None, + _ => return None, + } + } + } + + pub(crate) fn finish_cancel(&self, prepared: bool) -> bool { + if !prepared { + return self.0.load(Ordering::Acquire) == Self::CANCELLED; + } + self.0 + .compare_exchange( + Self::CANCEL_PREPARING, + Self::CANCELLED, + Ordering::AcqRel, + Ordering::Acquire, + ) + .is_ok() + } + + pub(crate) fn abort_cancel(&self, prepared: bool) { + if prepared { + let _ = self.0.compare_exchange( + Self::CANCEL_PREPARING, + Self::ACTIVE, + Ordering::AcqRel, + Ordering::Acquire, + ); + } + } + + pub(crate) fn claim_completion(&self) -> bool { + match self.0.compare_exchange( + Self::ACTIVE, + Self::COMPLETING, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) | Err(Self::COMPLETING) => true, + Err(_) => false, + } + } +} + /// Model policy composes exact neutral proof with the current HF admission /// and its already-held native grant. Stage types expose no generic bypass. pub(crate) struct ModelFinalImportCapability(ModelImportProof); @@ -23,6 +110,8 @@ struct ModelImportProof { grant: Arc, download_id: String, domain: DownloadAdmissionDomain, + downloads: Option>, + completion_decision: Option>, } impl ModelFinalImportCapability { @@ -31,12 +120,15 @@ impl ModelFinalImportCapability { download_id: &str, domain: DownloadAdmissionDomain, grant: Arc, + completion_decision: Arc, ) -> Self { Self(ModelImportProof { acquisition: proof.into_proof(), grant, download_id: download_id.into(), domain, + downloads: None, + completion_decision: Some(completion_decision), }) } @@ -64,6 +156,8 @@ impl ModelPartialImportCapability { grant, download_id: download_id.into(), domain, + downloads: None, + completion_decision: None, }) } @@ -224,10 +318,60 @@ impl LibraryMutationAuthority { self.tasks.clone() } + pub(crate) fn downloads(&self) -> Arc { + self.downloads.clone() + } + pub(crate) fn root(&self) -> &DownloadDestinationRoot { &self.root } + /// Resolve the exact receipt destination under the caller's already-held + /// model-root grant. This observes metadata only and performs no repair. + pub(crate) fn validate_hf_completion_destination( + &self, + model_dir: &Path, + receipt: &HfCompletionReceipt, + grant: &RootExecutionGrant, + ) -> Result { + grant.validate_root(&self.root)?; + let destination = self.root.resolve(model_dir)?; + let identity = destination.persisted_identity()?; + if identity.library_root != receipt.workspace.root_identity + || identity.relative_target != receipt.workspace.relative_target + || identity != receipt.queue_admission.destination + { + return Err(import_invalid( + "Completion receipt does not identify the held model destination", + )); + } + let metadata = destination + .read_model_metadata()? + .ok_or_else(|| import_invalid("Completion receipt metadata is missing"))?; + if metadata.model_id.as_deref() != Some(receipt.model_id.as_str()) + || metadata.repo_id.as_deref() != Some(receipt.manifest.source().source_id()) + { + return Err(import_invalid( + "Completion receipt provenance does not match current model metadata", + )); + } + if receipt.manifest.source().provider() != "huggingface" { + return Err(import_invalid( + "Completion receipt source provider is not Hugging Face", + )); + } + if receipt.manifest.source().revision().strength() + == crate::acquisition::RevisionStrength::Immutable + && metadata.upstream_revision.as_deref() + != Some(receipt.manifest.source().revision().value()) + { + return Err(import_invalid( + "Completion receipt revision does not match current model metadata", + )); + } + Ok(destination) + } + /// Import admission is checked under native root exclusion, including /// idempotent imports whose indexing can still rewrite metadata. pub(crate) fn protect_import( @@ -273,10 +417,11 @@ impl LibraryMutationAuthority { fn protect_hf_import( &self, model_dir: &Path, - proof: ModelImportProof, + mut proof: ModelImportProof, partial: bool, context: crate::acquisition::task_custody::TaskContext, ) -> Result> { + proof.downloads = Some(self.downloads.clone()); let grant = proof.grant.clone(); grant.validate_root(&self.root)?; let destination = self.root.resolve(model_dir)?; @@ -446,6 +591,31 @@ enum ImportEffectContext { } impl LibraryImportGuard { + pub(crate) fn has_managed_final_import_proof(&self) -> bool { + !self.partial && self.proof.is_some() + } + + /// Resolve cancellation against managed import before any model publication + /// effects begin. Once this succeeds, cancellation cannot report success + /// while metadata/index effects are still being drained. + pub(crate) fn claim_final_import_completion(&self) -> Result<()> { + if self.partial { + return Err(import_invalid( + "Partial import authority cannot claim final completion", + )); + } + let completion_decision = self + .proof + .as_ref() + .and_then(|proof| proof.completion_decision.as_ref()) + .ok_or_else(|| import_invalid("Managed HF completion decision is unavailable"))?; + if completion_decision.claim_completion() { + Ok(()) + } else { + Err(PumasError::DownloadCancelled) + } + } + pub(crate) async fn run_blocking( self: &Arc, operation: &'static str, @@ -513,6 +683,81 @@ impl LibraryImportGuard { self.validate(path)?; self.destination.write_model_metadata(metadata) } + + pub(crate) async fn publish_hf_completion_receipt( + self: &Arc, + library: &crate::model_library::ModelLibrary, + model_id: &str, + require_package_facts: bool, + ) -> Result { + if self.partial { + return Err(import_invalid( + "Partial import authority cannot issue a completion receipt", + )); + } + let proof = self + .proof + .as_ref() + .ok_or_else(|| import_invalid("Ordinary import authority cannot issue a receipt"))?; + let destination = self.destination.clone(); + let guard = self.clone(); + let model_id_owned = model_id.to_string(); + self.run_blocking("publish durable HF completion metadata", move || { + guard.validate(destination.display_path())?; + let metadata = destination + .read_model_metadata()? + .ok_or_else(|| import_invalid("Completed import metadata is missing"))?; + if metadata.model_id.as_deref() != Some(model_id_owned.as_str()) { + return Err(import_invalid( + "Completed import metadata identifies another model", + )); + } + let metadata_value = destination + .read_model_metadata_value()? + .ok_or_else(|| import_invalid("Completed import metadata is missing"))?; + // Equality may have skipped a write during finalization. Re-publish + // the exact observed JSON and require its durable publication outcome. + destination.write_model_metadata_value(&metadata_value) + }) + .await??; + let outputs = library + .hf_completion_output_proof(&self.destination, model_id, require_package_facts) + .await?; + if outputs.package_facts.is_some() != require_package_facts { + return Err(import_invalid( + "Completed import package-facts projection is incomplete", + )); + } + let use_lease = match proof.acquisition.record().phase { + crate::acquisition::AcquisitionPhase::Using { lease } => lease, + _ => return Err(import_invalid("Completed import lease is no longer active")), + }; + let downloads = proof + .downloads + .as_ref() + .cloned() + .ok_or_else(|| import_invalid("Managed HF store authority is unavailable"))?; + let expected = proof.acquisition.record().clone(); + let download_id = proof.download_id.clone(); + let domain = proof.domain; + let destination = self.destination.clone(); + let guard = self.clone(); + let model_id = model_id.to_string(); + self.run_blocking("publish managed HF completion receipt", move || { + guard.validate(destination.display_path())?; + let destination_identity = destination.persisted_identity()?; + downloads.publish_hf_completion_receipt(HfCompletionReceiptRequest { + expected: &expected, + use_lease, + download_id: &download_id, + domain, + destination: &destination_identity, + model_id: &model_id, + outputs, + }) + }) + .await? + } } pub(crate) struct LibraryMutationGuard { diff --git a/rust/crates/pumas-core/src/network/github.rs b/rust/crates/pumas-core/src/network/github.rs index 5b41786e..5939c238 100644 --- a/rust/crates/pumas-core/src/network/github.rs +++ b/rust/crates/pumas-core/src/network/github.rs @@ -548,6 +548,35 @@ impl GitHubClient { Self::with_config(cache_dir, ttl, NetworkConfig::GITHUB_API_BASE.to_string()) } + /// Construct a real HTTP integration fixture against a literal loopback + /// address. This seam is excluded from ordinary product builds. + #[cfg(any(test, feature = "test-support"))] + pub fn with_loopback_api(cache_dir: PathBuf, ttl: Duration, api_base: String) -> Result { + let url = Url::parse(&api_base).map_err(|error| PumasError::Config { + message: format!("Invalid fixture API URL: {error}"), + })?; + let loopback = url + .host_str() + .and_then(|host| { + host.trim_matches(['[', ']']) + .parse::() + .ok() + }) + .is_some_and(|address| address.is_loopback()); + if !loopback + || !matches!(url.scheme(), "http" | "https") + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(PumasError::Config { + message: "Fixture API requires literal loopback HTTP(S) without credentials, query or fragment".into(), + }); + } + Self::with_config(cache_dir, ttl, api_base) + } + fn with_config(cache_dir: PathBuf, ttl: Duration, api_base: String) -> Result { let http = HttpClient::new()?; Ok(Self { @@ -1164,6 +1193,39 @@ mod tests { use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use tempfile::TempDir; + #[test] + fn fixture_api_seam_accepts_only_literal_loopback_http() { + let root = TempDir::new().unwrap(); + for endpoint in ["http://127.0.0.1:1234/api", "https://[::1]:1234/api"] { + assert!(GitHubClient::with_loopback_api( + root.path().into(), + Duration::from_secs(1), + endpoint.into() + ) + .is_ok()); + } + for endpoint in [ + "http://localhost:1234", + "http://192.0.2.1", + "https://github.com", + "file:///tmp/api", + "ftp://127.0.0.1", + "http://user@127.0.0.1", + "http://127.0.0.1?token=x", + "http://127.0.0.1#api", + ] { + assert!( + GitHubClient::with_loopback_api( + root.path().into(), + Duration::from_secs(1), + endpoint.into() + ) + .is_err(), + "{endpoint}" + ); + } + } + #[tokio::test] async fn coalesced_callers_share_one_owner_and_rate_limit_details() { let (client, _root) = create_test_client(); diff --git a/rust/crates/pumas-core/src/tests.rs b/rust/crates/pumas-core/src/tests.rs index 9141bd0f..407953fe 100644 --- a/rust/crates/pumas-core/src/tests.rs +++ b/rust/crates/pumas-core/src/tests.rs @@ -275,7 +275,7 @@ async fn ticket_recovery_refuses_busy_before_index_or_download_mutation() { } #[tokio::test] -async fn builder_retains_failed_download_import_and_retries_before_completion() { +async fn builder_retains_receiptless_download_custody_without_replaying_import() { use crate::model_library::download_store::{ DownloadAdmissionDomain, DownloadAdmissionRequest, DownloadPersistence, PersistedDestinationIdentity, PersistedDownload, @@ -385,6 +385,20 @@ async fn builder_retains_failed_download_import_and_retries_before_completion() payload ); assert!(api.model_library().index().list_all().unwrap().is_empty()); + let acquisitions = store.acquisition_store().acquisitions().unwrap(); + let using = acquisitions + .values() + .find(|record| record.demand.consumer == "hf.model") + .expect("failed import retains its exact managed acquisition"); + let acquisition_id = using.id; + assert!(matches!( + using.phase, + crate::acquisition::AcquisitionPhase::Using { .. } + )); + assert!(store + .read_hf_completion_receipt(acquisition_id) + .unwrap() + .is_none()); // Shutdown reports the retained importer failure, but must drain before // the fixture repairs the obstruction and opens a fresh owning instance. assert!(matches!( @@ -400,29 +414,35 @@ async fn builder_retains_failed_download_import_and_retries_before_completion() .with_process_manager(false) .build() .await - .unwrap(); - let metadata = api - .model_library() - .load_metadata(&destination) - .unwrap() - .unwrap(); + .expect("unrelated API services start with retained recovery custody"); + let downloads = api.list_hf_downloads().await.unwrap(); + assert_eq!(downloads.len(), 1); + assert_eq!(downloads[0].status, DownloadStatus::Error); + let inventory = store.load_lifecycle_inventory_strict().unwrap(); + assert_eq!(inventory.downloads.len(), 1); + assert!(inventory + .queue_admissions + .contains_key("builder-import-retry")); assert_eq!( - metadata.repo_id.as_deref(), - Some("IDEA-Research/grounding-dino-base") + serde_json::to_value(&inventory.queue_admissions["builder-import-retry"]).unwrap(), + original_admission, + "reopen must preserve exact queue custody without an issued receipt" ); - assert_eq!(metadata.match_source.as_deref(), Some("download")); - let model_id = metadata.model_id.as_ref().unwrap(); - assert!(api.model_library().index().get(model_id).unwrap().is_some()); - assert_eq!(api.model_library().index().count().unwrap(), 1); - assert!(api.list_hf_downloads().await.unwrap().is_empty()); - let inventory = store.load_lifecycle_inventory_strict().unwrap(); - assert!(inventory.downloads.is_empty()); - assert!(inventory.queue_admissions.is_empty()); + assert!(std::fs::read(destination.join("metadata.json")).is_err()); + assert!(api.model_library().index().list_all().unwrap().is_empty()); + assert!(store + .read_hf_completion_receipt(acquisition_id) + .unwrap() + .is_none()); + assert!(matches!( + store.acquisition_store().acquisitions().unwrap()[&acquisition_id].phase, + crate::acquisition::AcquisitionPhase::Using { .. } + )); assert_eq!( std::fs::read(destination.join("detector.onnx")).unwrap(), payload ); - api.shutdown_downloads().await.unwrap(); + let _ = api.shutdown_downloads().await; drop(api); } } diff --git a/rust/crates/pumas-core/tests/artifact_acquisition.rs b/rust/crates/pumas-core/tests/artifact_acquisition.rs index 2e496eb6..d3aead99 100644 --- a/rust/crates/pumas-core/tests/artifact_acquisition.rs +++ b/rust/crates/pumas-core/tests/artifact_acquisition.rs @@ -1,5 +1,18 @@ -//! Public construction/migration regression; all roots are local temporary fixtures. -use pumas_library::{model_library::DownloadPersistence, PumasApi, PumasError}; +//! Public construction/migration and shared-consumer regressions. All roots +//! and HTTP endpoints are local temporary fixtures. +use pumas_library::{ + acquisition::{ + AcquisitionDemand, AcquisitionHttpRequest, AcquisitionHttpSource, AcquisitionRetryPolicy, + AcquisitionService, AcquisitionStore, AcquisitionWorkspace, ArtifactFile, ArtifactManifest, + ArtifactRevisionEvidence, ArtifactSourceIdentity, FileVerificationRequirement, + RevisionStrength, + }, + model_library::DownloadPersistence, + network::RetryConfig, + PumasApi, PumasError, Result, +}; +use sha2::{Digest, Sha256}; +use std::{future::pending, io::Read, path::Path, sync::Arc, time::Duration}; #[tokio::test] async fn ordinary_builder_leaves_legacy_state_read_only_until_explicit_offline_migration() { @@ -36,8 +49,159 @@ async fn ordinary_builder_leaves_legacy_state_read_only_until_explicit_offline_m DownloadPersistence::migrate_legacy_offline(&data).unwrap(); let current: serde_json::Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); - assert_eq!(current["schema_version"], 6); + assert_eq!(current["schema_version"], 7); assert_eq!(current["acquisitions"], serde_json::json!({})); assert!(DownloadPersistence::new(&data).load_all().is_empty()); assert!(DownloadPersistence::migrate_legacy_offline(&data).is_err()); } + +struct LocalHttpHost; + +#[async_trait::async_trait] +impl pumas_library::acquisition::HttpAttemptHost for LocalHttpHost { + async fn pause_requested(&self) { + pending::<()>().await; + } + + fn pause_requested_now(&self) -> bool { + false + } + + fn cancel_requested(&self) -> bool { + false + } + + async fn record_progress(&mut self, _downloaded_for_file: u64) -> Result<()> { + Ok(()) + } +} + +#[async_trait::async_trait] +impl pumas_library::acquisition::AcquisitionHost for LocalHttpHost { + async fn retry( + &mut self, + _attempt: u32, + _delay: Option, + _error: Option<&str>, + ) -> Result<()> { + Ok(()) + } +} + +#[tokio::test] +async fn public_consumer_acquires_http_and_settles_its_receipt_under_shared_custody() { + let state = tempfile::TempDir::new().unwrap(); + let workspace_root = tempfile::TempDir::new().unwrap(); + std::fs::create_dir(workspace_root.path().join("install-stage")).unwrap(); + let workspace = AcquisitionWorkspace::from_reserved_directory( + workspace_root.path(), + Path::new("install-stage"), + Arc::new(()), + || Ok(()), + ) + .unwrap(); + + let bytes = b"verified native archive"; + let digest = format!("{:x}", Sha256::digest(bytes)); + let manifest = ArtifactManifest::new( + ArtifactSourceIdentity::new( + "fixture", + "native-release-asset", + ArtifactRevisionEvidence::new( + "fixture.release", + "immutable-asset-v1", + RevisionStrength::Immutable, + ) + .unwrap(), + ) + .unwrap(), + vec![ArtifactFile::new( + "runtime.tar.gz", + "runtime archive", + Some(bytes.len() as u64), + Some( + pumas_library::acquisition::Sha256Evidence::new("fixture.sha256", digest).unwrap(), + ), + FileVerificationRequirement::Sha256, + ) + .unwrap()], + ) + .unwrap(); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0u8; 1024]; + let _ = socket.read(&mut request).await.unwrap(); + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + bytes.len() + ) + .as_bytes(), + ) + .await + .unwrap(); + socket.write_all(bytes).await.unwrap(); + }); + + let store = Arc::new(AcquisitionStore::new(state.path())); + let service = Arc::new(AcquisitionService::new(store.clone())); + let consumer = service.open_consumer("runtime.llama.cpp").unwrap(); + let client = reqwest::Client::new(); + let result = consumer + .acquire_http( + AcquisitionHttpRequest { + demand: AcquisitionDemand { + consumer: "runtime.llama.cpp".into(), + operation: "llama.cpp:v1:linux-x86_64".into(), + }, + manifest, + workspace, + sources: vec![AcquisitionHttpSource { + url: format!("http://{address}/runtime.tar.gz"), + authorization: None, + }], + retry: AcquisitionRetryPolicy { + attempts: Some(1), + elapsed: Duration::ZERO, + backoff: RetryConfig::default().with_jitter(false), + }, + }, + client, + Box::new(LocalHttpHost), + move |use_set| async move { + let mut file = use_set.open_file(0).await?; + let mut observed = Vec::new(); + file.read_to_end(&mut observed)?; + assert_eq!(observed, bytes); + Ok((observed, serde_json::json!({"installed": true}))) + }, + move |observed, receipt| async move { + assert_eq!(receipt.payload, serde_json::json!({"installed": true})); + Ok(observed) + }, + ) + .await + .unwrap(); + assert_eq!(result, bytes); + server.await.unwrap(); + + let record = store.acquisitions().unwrap().into_values().next().unwrap(); + assert!(matches!( + record.phase, + pumas_library::acquisition::AcquisitionPhase::Adopted { .. } + )); + let receipt = service.consumer_receipt(record.id).unwrap().unwrap(); + assert_eq!(receipt.owner, "runtime.llama.cpp"); + assert_eq!(receipt.payload, serde_json::json!({"installed": true})); + let persisted: serde_json::Value = + serde_json::from_slice(&std::fs::read(state.path().join("downloads.json")).unwrap()) + .unwrap(); + assert!(persisted.get("downloads").is_none()); + consumer.shutdown().await.unwrap(); + service.shutdown().await.unwrap(); +} diff --git a/rust/crates/pumas-rpc/src/main.rs b/rust/crates/pumas-rpc/src/main.rs index 57e77e61..fb6c15a5 100644 --- a/rust/crates/pumas-rpc/src/main.rs +++ b/rust/crates/pumas-rpc/src/main.rs @@ -139,7 +139,7 @@ async fn run(args: Args, host: server::LoopbackHost) -> Result<()> { .await?; #[cfg(feature = "inference-plugins")] - let version_managers = initialize_version_managers(&launcher_root).await; + let version_managers = initialize_version_managers(&launcher_root, &api).await; #[cfg(feature = "inference-plugins")] info!("Initialized {} version manager(s)", version_managers.len()); @@ -205,11 +205,20 @@ async fn run(args: Args, host: server::LoopbackHost) -> Result<()> { } #[cfg(feature = "inference-plugins")] -async fn initialize_version_managers(launcher_root: &Path) -> HashMap { +async fn initialize_version_managers( + launcher_root: &Path, + api: &pumas_library::PumasApi, +) -> HashMap { let mut version_managers = HashMap::new(); for app_id in VERSION_MANAGED_APPS { - match VersionManager::new(launcher_root, *app_id).await { + let initialized = if *app_id == AppId::LlamaCpp { + VersionManager::new_with_acquisition(launcher_root, *app_id, api.acquisition().clone()) + .await + } else { + VersionManager::new(launcher_root, *app_id).await + }; + match initialized { Ok(manager) => { info!("{app_id} version manager initialized successfully"); version_managers.insert(app_id.as_str().to_string(), manager); diff --git a/rust/crates/pumas-rpc/src/server.rs b/rust/crates/pumas-rpc/src/server.rs index 399a4842..dfa33c6b 100644 --- a/rust/crates/pumas-rpc/src/server.rs +++ b/rust/crates/pumas-rpc/src/server.rs @@ -392,6 +392,17 @@ pub async fn start_server( Err(anyhow::anyhow!(summary.errors.join("; "))) } }); + // Consumer owners must finish draining before the shared supervisor + // closes admission. Always observe its settlement, including failures. + let acquisition_cleanup = state.api.shutdown_acquisition().await; + let installation_cleanup = match (installation_cleanup, acquisition_cleanup) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) => Err(error), + (Ok(()), Err(error)) => Err(anyhow::anyhow!("Acquisition cleanup: {error}")), + (Err(installation), Err(acquisition)) => Err(anyhow::anyhow!( + "{installation}; Acquisition cleanup: {acquisition}" + )), + }; let owners = match (owners, installation_cleanup) { (Ok(()), Ok(())) => Ok(()), (Err(error), Ok(())) => Err(error), @@ -886,9 +897,13 @@ mod tests { async fn real_server_shutdown_closes_native_installation_admission() { let root = TempDir::new().unwrap(); let api = crate::handlers::test_support::build_test_api_with_hf(root.path()).await; - let manager = VersionManager::new(root.path(), AppId::LlamaCpp) - .await - .unwrap(); + let manager = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); let managers = HashMap::from([("llama-cpp".into(), manager.clone())]); let sizes = SizeCalculator::new_with_cache(root.path().join("launcher-data/cache")).await; let plugins = PluginLoader::new_async(root.path().join("launcher-data/plugins")) From 4c58197103d266c4519ccdbf272a8db902f5d08a Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 30 Sep 2026 16:21:54 -0700 Subject: [PATCH 17/20] docs(release): refresh Q1 attribution input --- docs/plans/artifact-acquisition/execution-ledger.md | 6 ++++++ docs/plans/artifact-acquisition/reports/write-sets.md | 2 ++ docs/release-attribution/0.7.0/inventory.json | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index a92d5698..04af77e4 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -163,3 +163,9 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - The resumed Coding-Standards route used fresh snapshot `snapshot:v1:10b3e01e-6a86-4185-83e0-65ca7b0db870`, selected 44 standards with zero unresolved questions, and read focused `core`, persistence and concurrency policies in small batches. It records obligations, not implementation compliance or acceptance. See the usability record. - Passeur remains unavailable: current `passeur_status({})` returns `SERVICE_PROFILE_CONFLICT`, and valid `passeur_agents({ limit: 4 })` fails at `profile.open`. No task was submitted; implementation used the authorized GPT-6.1 Sol fallback. Reproduction/follow-up details remain in the separate Passeur usability record. - Exact source write set is enumerated in the current Q1 section of [write sets](reports/write-sets.md). The owning records updated in this resumed slice are `docs/contracts/artifact-acquisition.md`, the acquisition plan/issue/acceptance/dependency/write-set/ledger records, and both MCP usability reports. `docs/breif/future.md` remains untouched. AQ-HTTP stays not ready; live HF/GitHub/native acquisition, desktop workflow, deployed schema-6 inventory and old-writer exclusion, public compatibility, current-head hosted CI and cross-platform durability remain unqualified. No commit, push, PR update, merge or deploy has yet occurred at this ledger point. + +## 2026-09-30 — draft PR publication and release-attribution CI repair + +- Commit `e46fc9a47440b021539e1c2ab600c3ca4949854d` was created and pushed fast-forward to `work/acquisition-q1-http`. PR #7 remains open, draft, and targeted at `main`; its stale body was replaced with the current Q1 implementation/evidence/limits. No merge was attempted. +- Current-head Build run `36790377557` failed in `Workflow and release contracts`; actionlint, dependency ownership, and release-version steps passed, then `scripts/release/check-attribution.cjs` rejected a stale hash for `rust/crates/pumas-app-manager/Cargo.toml`. All other workflow jobs were skipped after that required contract job failed. This was caused by the new internal `pumas-library` test-support dev-dependency used to exercise native cancellation/reopen, not by a third-party dependency change. +- Followed the repository's attribution workflow with `python3 scripts/release/generate-notices.py`. It refreshed only `docs/release-attribution/0.7.0/inventory.json` input hash; `THIRD-PARTY-NOTICES.txt` did not change. `node scripts/release/check-attribution.cjs` passed and `node --test scripts/release/check-attribution.test.mjs` passed 1/1. The exact generated-file boundary is added to the current Q1 write set. A follow-up docs/metadata commit and current-head hosted rerun remain to be pushed/observed. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index c3f7be69..5ed041bf 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -82,6 +82,8 @@ Admitted source write set: `acquisition/{service.rs,store.rs}` for the schema-7 Exact current source paths are `rust/crates/pumas-core/src/acquisition/{http.rs,mod.rs,service.rs,store.rs,task_custody.rs,workspace.rs}`, `rust/crates/pumas-core/src/model_library/{download_recovery.rs,download_store.rs,importer.rs,library.rs,mutation_authority.rs}`, `rust/crates/pumas-core/src/model_library/hf/{download.rs,types.rs}`, `rust/crates/pumas-core/src/network/github.rs`, `rust/crates/pumas-core/src/tests.rs`, `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/Cargo.toml`, `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs}`, and `rust/crates/pumas-rpc/src/{main.rs,server.rs}`. These cover the shared transfer/store/custody owner, exact receipt and output projections, HF reopen reconciliation, native shared-service consumer and attempt withdrawal, RPC lifetime ordering, and direct regressions. This slice gives llama.cpp the existing shared acquisition service, binds a durable receipt to exact extracted output and metadata, reconciles committed publication after reopen, and withdraws only an unchanged unreceipted native `Using` lease after its attempt is revoked and workspace cleanup succeeds. The `library.rs` descriptor-refresh future is boxed because the exact Torch serving RPC regression overflowed the default worker stack in the composed candidate; the existing integration test now passes with the normal stack size. Temporary diagnostic logs/test edits were removed. This source and its local fixtures do not qualify live sources, desktop behavior, deployed schema-6 population/old-writer isolation, or cross-platform durability. Keep AQ-HTTP not ready until objective acceptance is complete. +The app-manager manifest is a hashed input to release attribution. The test-support dev-dependency required for the fresh-owner cancellation regression therefore also refreshes only `docs/release-attribution/0.7.0/inventory.json`; `THIRD-PARTY-NOTICES.txt` remains byte-for-byte unchanged because the dependency is the existing internal library and adds no third-party package. + ## Q2: exact package-file handoff Allowed: Q1 acquisition types/service only for demonstrated missing file-set/lease semantics; `torch-server/resolve_runtime.py`, retained preview/lock consumers in `rust/crates/pumas-app-manager/src/version_manager/{torch_preview.rs,installer/torch.rs}`, corresponding existing package/integrity/progress tests, and `artifact_acquisition_install.rs`. Keep package resolution/install semantics with those consumers. The exact managed-Python/provider files are first traced for a migrate-versus-retain traffic disposition; no speculative private integration is authorized. diff --git a/docs/release-attribution/0.7.0/inventory.json b/docs/release-attribution/0.7.0/inventory.json index d8722aed..01d9e95f 100644 --- a/docs/release-attribution/0.7.0/inventory.json +++ b/docs/release-attribution/0.7.0/inventory.json @@ -6697,7 +6697,7 @@ "pnpm-lock.yaml": "8ef64f92ef3734992ce825b3253a6edef6586cf87b0a22494096bb12b9c86304", "rust/Cargo.lock": "ec83a541f165ac6161d4f26b220152cebe422c594405a9dc982952ddef195927", "rust/Cargo.toml": "c81fc6cbbe7b8a371f7492cb08b8715cf567ba9ec5321f19ffc1a1b8df50fd28", - "rust/crates/pumas-app-manager/Cargo.toml": "2974043227b88401adef73f0706344c804f9aa1447f49ff8c7ed19ddf66e2a8e", + "rust/crates/pumas-app-manager/Cargo.toml": "1fda75a70c5a5e66dffa6fb1950bec89ac9e274e1e0253c80e0fba8c5b98ab39", "rust/crates/pumas-app-manager/src/version_manager/managed_python.rs": "e6a33e5febf57a21dee80f87281fe4e5ae34f8fd2a07956c0f55eab3038eb653", "rust/crates/pumas-core/Cargo.toml": "920dfa71dc7160c14c753b656d16f65ad2b3fcdc7cbe1c56525e195588897022", "rust/crates/pumas-rpc/Cargo.toml": "b51fc102e08bf9ba72cbec544cd8b933171031078b1ba273b289a6f4c5cf981a", From 1c228c3436e100b898d6fc2f05e471e88bee4162 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 30 Sep 2026 17:08:57 -0700 Subject: [PATCH 18/20] docs(acquisition): record final CI and Passeur state --- docs/plans/artifact-acquisition/execution-ledger.md | 10 ++++++++-- .../reports/passeur-mcp-usability.md | 13 +++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 04af77e4..9d445266 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -161,11 +161,17 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - **Final composed candidate test:** `cargo test --manifest-path rust/Cargo.toml --locked -p pumas-library -p pumas-app-manager -p pumas-rpc --all-targets -- --test-threads=1` exited 0 after the fix. Core unit tests: 1,506 passed, 6 ignored; API tests: 36 passed; `artifact_acquisition`: 2 passed; app-manager: 260 tests completed successfully; RPC unit tests: 265 passed; RPC integration: 17 passed, 10 ignored; intent integration: 2 passed, 2 ignored. Other library/RPC fixture targets also passed. The formerly failing Torch serving RPC case passed at the normal worker stack size. `cargo fmt --manifest-path rust/Cargo.toml --all -- --check` and `git diff --check` passed on this tree. Post-fix all-target Clippy with `-D warnings` and the pumas-library no-default-features check had also passed after the source edits. - Final independent source-only composed review checked exact current Rust diff SHA-256 `5e10716362784caa8a3146bb519b391a233960ed1a83fca4932bb1d577afe618` and found no substantiated P0–P3 findings. The reviewer did not run tests; test outcomes above are integrator-run. Review limitations remain: no injected crash/error at every revocation/cleanup/withdrawal boundary, no live source/desktop/deployed-state or cross-platform qualification. - The resumed Coding-Standards route used fresh snapshot `snapshot:v1:10b3e01e-6a86-4185-83e0-65ca7b0db870`, selected 44 standards with zero unresolved questions, and read focused `core`, persistence and concurrency policies in small batches. It records obligations, not implementation compliance or acceptance. See the usability record. -- Passeur remains unavailable: current `passeur_status({})` returns `SERVICE_PROFILE_CONFLICT`, and valid `passeur_agents({ limit: 4 })` fails at `profile.open`. No task was submitted; implementation used the authorized GPT-6.1 Sol fallback. Reproduction/follow-up details remain in the separate Passeur usability record. +- At that time, Passeur returned `SERVICE_PROFILE_CONFLICT` from status and `PATH_NOT_FOUND` from agent discovery. No task was submitted; implementation used the authorized GPT-6.1 Sol fallback. Later service recheck is recorded below. - Exact source write set is enumerated in the current Q1 section of [write sets](reports/write-sets.md). The owning records updated in this resumed slice are `docs/contracts/artifact-acquisition.md`, the acquisition plan/issue/acceptance/dependency/write-set/ledger records, and both MCP usability reports. `docs/breif/future.md` remains untouched. AQ-HTTP stays not ready; live HF/GitHub/native acquisition, desktop workflow, deployed schema-6 inventory and old-writer exclusion, public compatibility, current-head hosted CI and cross-platform durability remain unqualified. No commit, push, PR update, merge or deploy has yet occurred at this ledger point. ## 2026-09-30 — draft PR publication and release-attribution CI repair - Commit `e46fc9a47440b021539e1c2ab600c3ca4949854d` was created and pushed fast-forward to `work/acquisition-q1-http`. PR #7 remains open, draft, and targeted at `main`; its stale body was replaced with the current Q1 implementation/evidence/limits. No merge was attempted. - Current-head Build run `36790377557` failed in `Workflow and release contracts`; actionlint, dependency ownership, and release-version steps passed, then `scripts/release/check-attribution.cjs` rejected a stale hash for `rust/crates/pumas-app-manager/Cargo.toml`. All other workflow jobs were skipped after that required contract job failed. This was caused by the new internal `pumas-library` test-support dev-dependency used to exercise native cancellation/reopen, not by a third-party dependency change. -- Followed the repository's attribution workflow with `python3 scripts/release/generate-notices.py`. It refreshed only `docs/release-attribution/0.7.0/inventory.json` input hash; `THIRD-PARTY-NOTICES.txt` did not change. `node scripts/release/check-attribution.cjs` passed and `node --test scripts/release/check-attribution.test.mjs` passed 1/1. The exact generated-file boundary is added to the current Q1 write set. A follow-up docs/metadata commit and current-head hosted rerun remain to be pushed/observed. +- Followed the repository's attribution workflow with `python3 scripts/release/generate-notices.py`. It refreshed only `docs/release-attribution/0.7.0/inventory.json` input hash; `THIRD-PARTY-NOTICES.txt` did not change. `node scripts/release/check-attribution.cjs` passed and `node --test scripts/release/check-attribution.test.mjs` passed 1/1. The exact generated-file boundary is added to the current Q1 write set. A follow-up docs/metadata commit and current-head hosted rerun were pending at that observation; their outcome is recorded below. + +## 2026-09-30 — hosted candidate checks green; Passeur coordinator reconnected + +- Commit `4c58197103d266c4519ccdbf272a8db902f5d08a` was pushed to the existing branch and PR #7 remains open, draft, and based on `main`. Build #348 (`36790717885`) completed successfully on this exact head after retrying the only failed Windows job. Workflow/release contracts, Rust quality, no-inference library checks, frontend/desktop contracts, and Linux/macOS/Windows native QA passed. Rust release, desktop package, Linux release startup, headless archive, RPC-E2E and assembly jobs were skipped by workflow conditions. The first Windows attempt timed out waiting 10 seconds for `suspended_admission_assigns_descendants_to_job_and_drains_them` to create its child marker; rerun passed. The test source is byte-identical to accepted `main`, and the prior Windows native QA run `36661110500` also passed, so this is recorded as a transient fixture-start timeout rather than a candidate regression. +- A post-update `passeur_status({})` first reported `not_checked`; `passeur_prepare({})` then connected the repository service with open admission and ready coordination authority. Task listing returned zero tasks and coordination identity succeeded. `passeur_agents({ limit: 4 })` still failed with `PATH_NOT_FOUND` at `profile.open`; execution profile, provider and approval remained `not_checked`. No exact agent ID was available for the requested Muse Spark contributor, so no task was submitted. The coordinator is available for metadata, but the requested execution profile is not verifiably available. See the updated Passeur usability report. +- The PR body still needs to be refreshed with Build #348's terminal result after the final documentation push. AQ-HTTP remains not ready; all AC01–AC18 and the real HF/native source, desktop, deployed-store/old-writer and cross-platform evidence remain pending. No merge was attempted. diff --git a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md index 9cf09000..f3a612c4 100644 --- a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md @@ -82,3 +82,16 @@ not identify the unavailable profile. An initial request above the documented maximum (`limit: 50`) was rejected by validation, confirming the agents bound is 4. No task was submitted, and implementation remains on the user-authorized GPT-6.1 Sol fallback until the repository service becomes available. + +## 2026-09-30 — coordinator connected, contributor discovery unresolved + +A later status check changed from `unavailable` to `not_checked`. +`passeur_prepare({})` connected the repository service with open admission and a +ready coordination authority. Task listing succeeded and returned zero tasks; +the coordination identity read also succeeded. However, +`passeur_agents({ limit: 4 })` still failed at `profile.open` with +`PATH_NOT_FOUND`, and connected status reported execution profile/provider/ +approval as `not_checked`. No agent ID for the requested Muse Spark contributor +could be verified, so no implementation task was submitted or started. The +service can coordinate/read metadata, but this session still cannot safely +select the requested execution profile. From 07cf74dfb960bca005918f6923073cc4a5c82814 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 30 Sep 2026 18:47:15 -0700 Subject: [PATCH 19/20] feat(acquisition): bound shared task custody --- .../artifact-acquisition/execution-ledger.md | 10 + docs/plans/artifact-acquisition/issues.md | 8 +- docs/plans/artifact-acquisition/plan.md | 6 +- .../reports/architecture-review.md | 12 + .../reports/coding-standards-mcp-usability.md | 11 + .../reports/passeur-mcp-usability.md | 39 ++ .../src/version_manager/installer.rs | 375 ++++++++--- .../src/version_manager/mod.rs | 116 ++++ rust/crates/pumas-core/src/acquisition/mod.rs | 1 + .../pumas-core/src/acquisition/service.rs | 33 +- .../src/acquisition/task_custody.rs | 591 +++++++++++++++++- rust/crates/pumas-core/src/api/hf.rs | 61 +- rust/crates/pumas-core/src/error.rs | 5 + .../src/model_library/hf/download.rs | 202 ++---- .../src/model_library/hf/lifecycle.rs | 72 ++- 15 files changed, 1240 insertions(+), 302 deletions(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 9d445266..3cbd6c56 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -175,3 +175,13 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - Commit `4c58197103d266c4519ccdbf272a8db902f5d08a` was pushed to the existing branch and PR #7 remains open, draft, and based on `main`. Build #348 (`36790717885`) completed successfully on this exact head after retrying the only failed Windows job. Workflow/release contracts, Rust quality, no-inference library checks, frontend/desktop contracts, and Linux/macOS/Windows native QA passed. Rust release, desktop package, Linux release startup, headless archive, RPC-E2E and assembly jobs were skipped by workflow conditions. The first Windows attempt timed out waiting 10 seconds for `suspended_admission_assigns_descendants_to_job_and_drains_them` to create its child marker; rerun passed. The test source is byte-identical to accepted `main`, and the prior Windows native QA run `36661110500` also passed, so this is recorded as a transient fixture-start timeout rather than a candidate regression. - A post-update `passeur_status({})` first reported `not_checked`; `passeur_prepare({})` then connected the repository service with open admission and ready coordination authority. Task listing returned zero tasks and coordination identity succeeded. `passeur_agents({ limit: 4 })` still failed with `PATH_NOT_FOUND` at `profile.open`; execution profile, provider and approval remained `not_checked`. No exact agent ID was available for the requested Muse Spark contributor, so no task was submitted. The coordinator is available for metadata, but the requested execution profile is not verifiably available. See the updated Passeur usability report. - The PR body still needs to be refreshed with Build #348's terminal result after the final documentation push. AQ-HTTP remains not ready; all AC01–AC18 and the real HF/native source, desktop, deployed-store/old-writer and cross-platform evidence remain pending. No merge was attempted. + +## 2026-09-30 — shared acquisition capacity and native installer custody follow-up + +- The current local candidate adds configurable finite limits shared across all `AcquisitionService` scopes: 32 ordinary workers, 16 ordinary blocking jobs, 32 rescue workers, 4 rescue blocking jobs, and 256 live/draining scopes by default. Admission is fail-fast and reports typed `AcquisitionCapacityExhausted`; these counts are not measured thread, memory, throughput, queue, descriptor, or disk guarantees. Nested async effects share the worker reservation. AC10 remains open for representative resource measurements and remaining per-resource bounds. +- The existing public `VersionInstaller::with_acquisition` entry point uses the caller's existing acquisition service/store and reconciles retained native work before return. The ordinary llama.cpp install now registers workspace/log preparation and successful cleanup through that consumer. The old unconfigured direct constructor rejects before release lookup, progress initialization, or filesystem mutation. `VersionManager::new_with_acquisition` is the corresponding manager path; external caller population and cutover remain unverified under AC16. +- The initial final Sol High review found shutdown lock re-entry, direct recovery effects outside custody, and HF saturation error erasure. Each was fixed and covered by a deterministic regression or focused adapter test. A later integrator trace found ordinary native workspace setup/cleanup still outside shared custody; that path and pre-mutation refusal were fixed. The latest read-only Sol High review of the complete updated source found no new confirmed correctness issue and verified the ordinary install, constructor, recovery, shutdown and HF error paths. Reviewers ran no tests; all outcomes below are integrator-run. +- Exact composed verification on the current uncommitted candidate: `cargo test --manifest-path rust/Cargo.toml --locked --all-targets -p pumas-library -p pumas-app-manager -p pumas-rpc -- --test-threads=1` exited 0. Core unit tests: 1,515 passed, 6 ignored; API integration: 36 passed; `artifact_acquisition`: 2 passed; RPC unit tests: 265 passed; RPC integration: 17 passed, 10 ignored; intent integration: 2 passed, 2 ignored; other targets passed. The app-manager unit target, including direct setup refusal and native install/recovery cases, also completed successfully. `cargo clippy --manifest-path rust/Cargo.toml --locked -p pumas-library -p pumas-app-manager -p pumas-rpc --all-targets --all-features -- -D warnings`, `cargo check --manifest-path rust/Cargo.toml --locked -p pumas-library --no-default-features`, `cargo fmt --manifest-path rust/Cargo.toml --all -- --check`, and `git diff --check` passed. +- Passeur was rechecked after the frontend update. `passeur_status` reports connected/open service, held/ready coordination, and build `6a43daa6eec45ddceadea0ba2e2a623d7fc8e81e330237c7762a7f1c63c33f63`; provider/execution/approval remain `not_checked`. `passeur_agents({limit:4})` still fails with `PATH_NOT_FOUND` at `profile.open`, so no contributor ID exists and no task was submitted. Sol 6.1 remains the authorized implementation path until contributor execution is available. See the [Passeur usability report](reports/passeur-mcp-usability.md). +- The Coding-Standards route for this resumed slice used snapshot `snapshot:v1:28c08446-d4d8-4bab-bf4c-5122700bbd60`: 25 selected standards, zero unresolved routing facts, and 24 canonical policy reads. Routing/policy reads provided obligations only; they do not certify implementation or acceptance. See the [MCP usability report](reports/coding-standards-mcp-usability.md). +- Build #349 (`36794721238`) is green on prior head `1c228c3436e100b898d6fc2f05e471e88bee4162`, before this local source diff. It is not CI evidence for this candidate. The local branch has not yet been committed or pushed for current-head hosted checks; PR #7 remains open and draft. All AC01–AC18 and AQ-HTTP remain pending/not ready; AC10 resource qualification, AC16 external compatibility/cutover, live HF/native sources, desktop behavior, deployed-store/old-writer qualification, and supported-platform durability remain unproved. User-owned `docs/breif/future.md` remains untouched; no merge was attempted. diff --git a/docs/plans/artifact-acquisition/issues.md b/docs/plans/artifact-acquisition/issues.md index 315836cc..9b343fcc 100644 --- a/docs/plans/artifact-acquisition/issues.md +++ b/docs/plans/artifact-acquisition/issues.md @@ -7,7 +7,7 @@ Q1 is active; production repairs and acceptance claims remain pending until evid | AQ-I01 | High: model-specific destination authority cannot become the generic artifact root | Acquisition/core: fix in Q1 | AC01/AC04/AC09; re-plan if safe neutral capability extraction cannot preserve model custody | | AQ-I02 | High: retained model/HF records and recovery transitions need explicit migration | Core/recovery: fix in Q1 | AC04–AC06; actual source/deployment inventory before mutation | | AQ-I03 | High: cancellation/restart must retain worker and byte-use ownership | The local Q1 candidate closes the native pre-receipt cancellation gap: it durably revokes the exact attempt, drains/cleans its workspace under the native lock, and withdraws only the exact unchanged receipt-free `Using` lease. Cold reopen and same-tag retry are covered. Broader cancellation/crash-window acceptance remains pending. | AC05/AC06; retain exact-generation worker drainage and cleanup-failure/cold-reopen evidence; never equate caller cancellation with worker stop | -| AQ-I04 | High: acquisition/import/install completion and duplicate transfer owners | Current local Q1 candidate routes HF and llama.cpp through the shared transfer owner and gives both consumers exact completion receipts; composed source review found no remaining issue, while objective acceptance evidence remains pending | AC03/AC05/AC08/AC18; retain cold-reopen, cancellation, and no-duplicate-transfer proof; never infer import/install completion from path or metadata presence | +| AQ-I04 | High: acquisition/import/install completion and duplicate transfer owners | Current local Q1 candidate routes HF and llama.cpp through the shared transfer owner, gives both consumers exact completion receipts, and exposes `VersionInstaller::with_acquisition` for direct native callers. External caller inventory and migration acceptance remain open; objective acceptance evidence remains pending | AC03/AC05/AC08/AC16/AC18; retain cold-reopen, cancellation, public-consumer and no-duplicate-transfer proof; never infer import/install completion from path or metadata presence | | AQ-I05 | High: cross-plan duplicate authority or prerequisite cycle | Integrator: fixed in plan design; implementation evidence pending | Gate graph and Q1/Q2 old-consumer evidence; review after any milestone dependency change | | AQ-I06 | High: generic retrieval can be mistaken for package compatibility or executable trust | Runtime/package/security: Q2, with Q1 trust contract | AC07/AC11/AC12; keep origin and byte digest separate | | AQ-I07 | Medium: new generic module could accidentally require inference or overclaim feature isolation | Core/composition: Q1/Q3/Q4 | AC09/AC15/AC17; no unsupported minimal-build claim | @@ -16,6 +16,10 @@ Q1 is active; production repairs and acceptance claims remain pending until evid | AQ-I10 | High, fixed in local Q1 candidate `d0b71b5`: acquisition-document validation accepted duplicate demands and overlapping active workspace custody, allowing `begin` to select around a retained `Using` record | Acquisition store: fixed by cross-record `AcquisitionDocument::validate`; independent exact-candidate review found no new issue | Three persisted-document fixtures cover duplicate demand, active-workspace collision, and terminal-history/successor coexistence; retain these invariants through schema-7 receipt work | | AQ-I11 | High, fixed in local Q1 source tree `0c3e472c`: effectful in-place import and orphan adoption could act on a destination after an acquisition or hidden model admission retained custody | Model-library/importer: fixed by guard at the effect boundary with exact private partial/full HF stages; independent review found no P0–P3 issue | Worker reports focused real-store/filesystem/owner tests and cancellation/panic/replacement probes; reopened `Using` remains unresolved and AQ-HTTP is still not ready | | AQ-I12 | High: metadata/index presence and package-facts cache freshness alone do not prove which `Using` lease completed import | Current local Q1 candidate implements the schema-7 HF receipt and exact cold reconciliation; composed source review found no remaining issue, while objective acceptance evidence remains pending. Never replay import after ambiguous publication. | AC05/AC08; retain proof that the receipt binds acquisition/use identity and deterministic output content, survives every writer/migration, and cold-reopens without importer invocation | +| AQ-I13 | High: owner shutdown could drop the last scope handle while holding the custody lock after a prior scope shutdown, re-entering `TaskScope::drop` and deadlocking | Acquisition/task custody: fixed in the current Q1 candidate by retaining upgraded scope handles until after unlocking; deterministic regression passes and Sol High follow-up confirmed the lock-order repair | AC06/AC09; retain the prior-shutdown/last-handle ordering regression and keep no-runtime destruction limits explicit | +| AQ-I14 | High: direct llama.cpp constructor/recovery and ordinary workspace lifecycle performed blocking setup, verification or cleanup outside shared acquisition custody, so caller cancellation could outlive service shutdown | App-manager/acquisition: fixed in the current Q1 candidate; setup, retained lookups, workspace preparation, adopted-output verification, reconciled cleanup and post-publication cleanup use registered shared consumer work. The legacy unconfigured direct installer refuses before release lookup, progress mutation or filesystem changes. Cancellation/drain and no-mutation regressions pass; the latest Sol High read-only review found no new confirmed correctness issue | AC06/AC09/AC10; preserve exact shared service ownership and await effects before reporting shutdown complete | +| AQ-I15 | Medium: HF adapters converted shared blocking-capacity saturation into generic `Other`, erasing overload meaning from user-facing error projection | Core/HF: fixed in the current Q1 candidate with a shared typed error conversion across API, lifecycle and download adapters; exact reason-code and root-grant overload regressions pass | AC08/AC15; retain `acquisition_capacity_exhausted` through ordinary and recovery paths | +| AQ-I16 | Medium: finite task admission does not itself qualify a complete resource envelope; nested async effects share a worker reservation and queue, stream, buffer, hash, descriptor, peak-RAM, disk and network bounds lack representative evidence | Acquisition: pending workload/resource qualification under AC10; do not present configured task counts as memory or throughput guarantees | AC10; measure representative concurrent large artifacts and bound each required resource class before marking the claim satisfied | | AQ-D01 | Deferred: native Xet reconstruction | Acquisition/source owner | Revisit when a required HF source cannot use the supported existing file path or Xet transfer benefits are an admitted goal; evaluate maintained Rust implementation, no homemade protocol | | AQ-D02 | Deferred: peers and concurrent multi-source failover | Acquisition/distribution owner | Revisit with actual node/authorization contract and content-equivalence proof; no cross-origin ETag comparison | | AQ-D03 | Deferred: chunk CAS, reflink optimization and coalescing | Acquisition/storage owner | Revisit with measured duplication/network/storage need and explicit retention consumers; ordinary files and safe release are required now | @@ -25,4 +29,4 @@ Q1 is active; production repairs and acceptance claims remain pending until evid Pending cleanup replay and unrelated whole-runtime remediation are owned by the existing Rust/library plan. Preserve current refusal while migrating the selected transfer family. Do not mark those unrelated work items accepted from this plan's link/schema checks. -Q1 source preparation confirmed a public `DownloadManager` with no in-repository production caller, but did not establish its external compatibility population; removal or a compatibility disposition remains open. The supported download-store reader currently accepts schema 5 and upgrades schema 4, while deployment inventory and older-writer isolation are unavailable. Q1 must preserve both facts and keep live-root mutation blocked until that deployment evidence exists. The native llama.cpp cache currently uses size/filename admission and direct-final-directory extraction; these are admitted Q1 repair findings under AQ-I04/AQ-I07. +Q1 source preparation confirmed a public `DownloadManager` with no in-repository production caller, but did not establish its external compatibility population; removal or a compatibility disposition remains open. The direct native installer now has a public shared-service setup path, and `VersionManager::new` users must migrate to `new_with_acquisition` for llama.cpp installs; external consumer inventory and cutover acceptance remain pending under AC16. The supported download-store reader currently accepts schema 5 and upgrades schema 4, while deployment inventory and older-writer isolation are unavailable. Q1 must preserve both facts and keep live-root mutation blocked until that deployment evidence exists. The native llama.cpp cache currently uses size/filename admission and direct-final-directory extraction; these are admitted Q1 repair findings under AQ-I04/AQ-I07. diff --git a/docs/plans/artifact-acquisition/plan.md b/docs/plans/artifact-acquisition/plan.md index efdd9c4c..2b12d9d2 100644 --- a/docs/plans/artifact-acquisition/plan.md +++ b/docs/plans/artifact-acquisition/plan.md @@ -2,8 +2,8 @@ **Plan status:** `Active` — Q1 is admitted on the current accepted `main` base; AQ-HTTP remains not ready. **Objective acceptance status:** `pending`. -**Current phase:** The local Q1 candidate advances the shared `downloads.json` authority to schema 7 with exact HF completion receipts and cold `Using` reconciliation. The Hugging Face workflow imports through the shared verified-file owner, and the llama.cpp archive installer uses that same owner, extracts from its verified file handle, and records/reconciles a native installation receipt. Cancellation that wins before the native receipt now revokes the exact attempt, cleans its workspace, and withdraws the exact receipt-free lease; cold reopen and same-tag retry are covered locally. RPC composition injects the shared consumer and drains consumers before acquisition shutdown. Local disposable-state and loopback tests cover these paths; they do not qualify live Hugging Face behavior, desktop behavior, deployed migration safety, or cross-platform durability. Independent composed review of the cancellation repair found no substantiated source correctness issue. AQ-HTTP remains not ready; objective acceptance evidence, public consumer compatibility, current-candidate hosted CI, real-source/manual behavior, deployed schema-6 population and old-writer isolation, and supported-platform qualification remain pending. -**Exactly one next slice:** Collect **Q1 objective acceptance evidence** for this candidate. Record current-head hosted checks and affected consumer evidence. Exercise the real HF import and llama.cpp source/consumer workflows, desktop controls, public API compatibility dispositions, and supported-platform durability. Keep deployed retained-root mutation blocked until schema-6 population, overlapping old writers and rollback requirements are inventoried. Do not open Q2 or runtime R1 until AQ-HTTP's objective gate is accepted; local fixtures alone do not satisfy that gate. +**Current phase:** The local Q1 candidate advances the shared `downloads.json` authority to schema 7 with exact HF completion receipts and cold `Using` reconciliation. The Hugging Face workflow imports through the shared verified-file owner, and the llama.cpp archive installer uses that same owner, extracts from its verified file handle, and records/reconciles a native installation receipt. Cancellation that wins before the native receipt now revokes the exact attempt, cleans its workspace, and withdraws the exact receipt-free lease; cold reopen and same-tag retry are covered locally. RPC composition injects the shared consumer and drains consumers before acquisition shutdown. This resumed slice adds finite shared worker, blocking-job, rescue and live/draining-scope admission, typed overload errors, and public direct llama.cpp setup through the existing service with retained-work reconciliation before return. Ordinary direct installs now prepare and clean their workspaces through registered capacity; the legacy unconfigured installer refuses before filesystem or progress mutation. Configured counts remain admission defaults, not measured resource guarantees; nested async effects share a worker reservation, while queue, stream, buffer, hash, descriptor, peak-RAM, disk and network behavior remain unqualified under AC10. Initial Sol High findings on shutdown lock order, recovery-effect custody and HF error propagation were repaired with focused regressions. A final review after closing the ordinary-install custody gap found no new confirmed correctness issue. Public customer compatibility remains unresolved under AC16. Local disposable-state and loopback tests do not qualify live Hugging Face behavior, desktop behavior, deployed migration safety, or cross-platform durability. AQ-HTTP remains not ready; objective acceptance evidence, public consumer compatibility, exact-candidate hosted CI, real-source/manual behavior, deployed schema-6 population and old-writer isolation, and supported-platform qualification remain pending. +**Exactly one next slice:** Collect **Q1 objective acceptance evidence** for this reviewed candidate. Record current-head hosted checks and affected consumer evidence. Exercise the real HF import and llama.cpp source/consumer workflows, desktop controls, public API compatibility dispositions, and supported-platform durability. Keep deployed retained-root mutation blocked until schema-6 population, overlapping old writers and rollback requirements are inventoried. Do not open Q2 or runtime R1 until AQ-HTTP's objective gate is accepted; local fixtures alone do not satisfy that gate. **Canonical plan path:** `docs/plans/artifact-acquisition/plan.md`. **Owner:** Pumas acquisition integration. The repository owner assigns the implementation and integration roles when admitting source work. **Operation:** `start` this exact plan on `work/acquisition-q1-http`. @@ -137,7 +137,7 @@ Independent read-only review covers durable migration, credential/destination au ## Blockers, re-plan triggers and limits -No code is implemented in this delivery. The active recovery ownership and real retained-state population must be reconciled before their mutation; those are bounded admission conditions, not a requirement to finish unrelated remediation. Pending cleanup remains refused. Native/GPU/GUI/network credentials unavailable to an execution session block only the claims requiring them; all final production acceptance still waits for its required evidence. +The local implementation and read-only review fixes are present; focused source tests and formatting pass. Those results do not qualify AC10 resource behavior or satisfy production acceptance. The active recovery ownership and real retained-state population must be reconciled before their mutation; those are bounded admission conditions, not a requirement to finish unrelated remediation. Pending cleanup remains refused. Native/GPU/GUI/network credentials unavailable to an execution session block only the claims requiring them; all final production acceptance still waits for its required evidence. Re-plan when a new consumer changes authority, the proposed state store cannot preserve supported recovery, a file lease cannot survive required worker cleanup, a source cannot establish required identity, an S3 dependency cannot provide the selected API/target, or downstream semantics leak into acquisition. Adding a file within an already admitted owner only amends its concrete write set. diff --git a/docs/plans/artifact-acquisition/reports/architecture-review.md b/docs/plans/artifact-acquisition/reports/architecture-review.md index e176d88f..7f94ac15 100644 --- a/docs/plans/artifact-acquisition/reports/architecture-review.md +++ b/docs/plans/artifact-acquisition/reports/architecture-review.md @@ -101,3 +101,15 @@ The durable cutover must preserve the supported version-4 and version-5 populati The current preparatory multi-root change is reviewed as a physical-identity bookkeeping improvement only. It does not turn a model-root capability into runtime storage authority and is not evidence that a shared acquisition owner exists. The multi-root review found no blocker, while noting inactive root slots need pruning or bounded admission once dynamic multi-root use is introduced. A follow-up lifecycle review found that sharing the current `DownloadTaskOwner` unchanged is unsafe: task ID scans and finished/projection lookup are global, HF shutdown closes the entire owner, and the first shutdown callback supplies a single final projection. The selected supervisor therefore needs consumer-scoped handles for lookup, cancellation, projection, and close/drain while retaining every Tokio handle under one supervisor. Those scoped handles filter authority; they are not separate task owners. Current-generation checks remain scoped to the owning consumer operation. + +## Final capacity and direct-installer source review — 2026-09-30 + +The first final read-only Sol High review of the capacity and public direct-installer changes found three concrete defects: + +1. **High — shutdown lock re-entry:** owner shutdown could drop its last upgraded scope handle while holding the supervisor mutex when a scope already had a shutdown receipt; `TaskScope::drop` then tried to acquire that mutex again. The candidate now retains upgraded handles through mutex release. A deterministic prior-scope-shutdown/last-handle test covers the ordering. +2. **High — direct native recovery escaped custody:** public direct construction performed retained-output verification and cleanup in raw `spawn_blocking` jobs. Cancellation could leave filesystem work running after the consumer appeared dropped. Setup, retained-record/receipt lookup, workspace preparation, output verification and cleanup now run through `AcquisitionConsumer::run_blocking`, which registers worker and blocking effects in the shared owner. A gated cleanup test aborts constructor setup and proves shared shutdown remains pending until the actual effect finishes. +3. **Medium — HF overload meaning was erased:** API, root-grant and download adapters converted `BlockingTaskError::CapacityExhausted` to `PumasError::Other`. The candidate now uses a shared conversion that preserves typed capacity errors through those paths. Focused reason-code and root-grant saturation regressions cover the user-facing contract. + +A second read-only Sol High review inspected those repairs and found no new confirmed correctness issue. The integrator then found that ordinary native installs still prepared and cleaned workspaces outside the shared owner. That path now checks configuration before any filesystem mutation and registers workspace/log creation, preparation and successful cleanup through the shared consumer. A focused regression verifies the legacy unconfigured constructor leaves the native root untouched. The latest read-only Sol High follow-up inspected the complete updated source and found no new confirmed correctness issue; it verified ordinary install custody, early refusal, prior shutdown/recovery repairs and HF overload propagation. Review passes ran no tests and changed no files; the integration owner ran the tests recorded in the ledger. + +The candidate's finite worker, ordinary/rescue blocking and live/draining scope budgets establish admission counts only. Nested async effects still share the worker reservation without a distinct count budget, and queue, stream, buffer, aggregate hash, file-descriptor, peak-RAM, disk and network behavior are not measured. AC10 remains pending. The public `VersionInstaller::with_acquisition` path and `VersionManager::new_with_acquisition` migration are available, but supported external caller inventory and cutover acceptance remain open under AC16. These are acceptance limits, not findings that the follow-up review certified. diff --git a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md index a29af49d..9c90054e 100644 --- a/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/coding-standards-mcp-usability.md @@ -315,3 +315,14 @@ This is final-route usability evidence, not a Coding-Standards compliance or pro - **Recommendations:** Provide a safe snapshot-refresh path after session loss and retain the clear distinction between standards obligations and independent source/test evidence. This entry records the resumed candidate route only; implementation review and acceptance evidence remain separate. + +## Primary integrator — finite acquisition admission and direct-installer candidate + +- **Useful calls:** A fresh snapshot `snapshot:v1:28c08446-d4d8-4bab-bf4c-5122700bbd60` routed the library/launcher Rust implementation, concurrency, persistence, IPC error, compatibility, verification, documentation, and commit scope to 25 standards with zero unresolved fact categories. The 24 normative policy targets were read in one `read_many` call; the remaining route entry was the Router navigation index. Core and Rust profile obligations were available for the implementation review. +- **Confusing or redundant steps:** The first fact set over-reported cross-language binding, platform-target, and release detail; narrowing it to actual IPC/persistence, Rust API/async and resource-lifecycle concerns reduced the route from 35 to 25 while retaining zero unresolved facts. Adding `include_routing: true` to a policy `read_many` rejected the whole batch with `NAVIGATION.ROUTING_TARGET_INVALID` (“Routing definitions are available when reading Router”). Omitting that option allowed policy reads. Expanding all 24 full compact policy bodies still exceeded the visible output window even though the request stayed within the 2 MiB service limit; count limits alone do not bound what the caller can review. +- **Missing context:** The standards service did not know the current source diff, final Sol High review findings, Passeur agent lookup failure, or local command results; those came from source inspection and separate tools. +- **Where I left MCP:** The route supplies obligations, not code compliance or acceptance. A final source repair and review pass remain in progress; Q1 gates stay pending. +- **Smallest sufficient workflow:** Refresh routing facts, route the exact changed boundary, preserve the snapshot handle, read canonical policy bodies without `include_routing`, and focus output on the obligations relevant to the change. +- **Recommendations:** Separate Router navigation reads from canonical policy reads in the input contract; make `include_routing` invalidity specific to the affected target instead of rejecting a multi-policy batch; and add section-scoped or byte-budgeted reads so full route coverage can be reviewed without flooding the response. + +This is tool-usability evidence only and does not certify implementation or product acceptance. diff --git a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md index f3a612c4..53f7f435 100644 --- a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md @@ -95,3 +95,42 @@ approval as `not_checked`. No agent ID for the requested Muse Spark contributor could be verified, so no implementation task was submitted or started. The service can coordinate/read metadata, but this session still cannot safely select the requested execution profile. + +## 2026-09-30 — resumed check after the Passeur update + +The current installed build connected through `passeur_prepare({})` with open +admission and ready coordination authority. `passeur_status({})` reports the +execution profile, provider, and approval as `not_checked`; a valid +`passeur_tasks({ schema_version: 1, limit: 16, offset: 0 })` returned zero +tasks. `passeur_agents({ limit: 4, offset: 0 })` still fails with +`PATH_NOT_FOUND` at `profile.open`, without a contributor identity. No task +was submitted. Implementation therefore continues using the user's GPT-6.1 +Sol fallback until contributor discovery works. + +This confirms that metadata coordination is ready while execution-profile +discovery is not. The smallest useful recovery signal would identify the +unavailable profile lookup (or expose a safe configured-agent status) so a +caller can distinguish a missing profile from a service-wide failure without +guessing an agent ID. + +## 2026-09-30 — updated frontend with contributor lookup still unavailable + +The installed frontend changed build identity to +`6a43daa6eec45ddceadea0ba2e2a623d7fc8e81e330237c7762a7f1c63c33f63` and +attached to repository service generation `269decac-59b5-4028-9bac-8d09ee9a35c2`. +`passeur_prepare({})` reports open admission and ready coordination authority; +task listing succeeds with zero tasks. `passeur_status({})` still reports the +execution profile, provider, and approval as `not_checked`. + +A valid `passeur_agents({ limit: 4, offset: 0 })` request again fails at +`profile.open` with `PATH_NOT_FOUND`, without returning a configured agent ID. +No assignment was submitted. The requested contributor cannot be selected +reliably from metadata readiness alone. The already-authorized GPT-6.1 Sol +fallback completed the bounded implementation slice; final review then found +three source defects that are being corrected before verification. Passeur has +not yet contributed source work in this session. + +This update narrows the earlier feedback: coordination reads now work, but +contributor discovery still needs to expose which configured profile lookup +failed or provide a safe status/readiness result, and execution-profile, +provider, and approval readiness need explicit values before binding a task. diff --git a/rust/crates/pumas-app-manager/src/version_manager/installer.rs b/rust/crates/pumas-app-manager/src/version_manager/installer.rs index 5a2e1294..f52dbfed 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/installer.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/installer.rs @@ -569,39 +569,35 @@ impl pumas_library::acquisition::AcquisitionHost for LlamaCppHttpAttemptHost { } } -async fn open_native_acquisition_workspace( +fn prepare_native_acquisition_workspace( versions: PathBuf, tag: String, expected: Option, ) -> Result<(Arc, AcquisitionWorkspace)> { - tokio::task::spawn_blocking(move || { - let custody = Arc::new(NativeInstallWorkspace::create_for_attempt( - &versions, - &tag, - expected.as_deref(), - )?); - let relative = custody - .path() - .strip_prefix(&versions) - .map_err(|_| PumasError::InstallationFailed { - message: "Native acquisition stage escaped the versions root".into(), - })? - .to_path_buf(); - let execution_lease: Arc = Arc::new(custody._lock.clone()); - let validator_lease = execution_lease.clone(); - let workspace = AcquisitionWorkspace::from_reserved_directory( - &versions, - &relative, - execution_lease, - move || { - let _held = &validator_lease; - Ok(()) - }, - )?; - Ok((custody, workspace)) - }) - .await - .map_err(|error| PumasError::Other(format!("Failed to join native staging: {error}")))? + let custody = Arc::new(NativeInstallWorkspace::create_for_attempt( + &versions, + &tag, + expected.as_deref(), + )?); + let relative = custody + .path() + .strip_prefix(&versions) + .map_err(|_| PumasError::InstallationFailed { + message: "Native acquisition stage escaped the versions root".into(), + })? + .to_path_buf(); + let execution_lease: Arc = Arc::new(custody._lock.clone()); + let validator_lease = execution_lease.clone(); + let workspace = AcquisitionWorkspace::from_reserved_directory( + &versions, + &relative, + execution_lease, + move || { + let _held = &validator_lease; + Ok(()) + }, + )?; + Ok((custody, workspace)) } fn settled_result(outcome: Result, settlement: Result<()>) -> Result { @@ -981,6 +977,8 @@ pub struct VersionInstaller { #[cfg(test)] native_receipt_pause: Option>, #[cfg(test)] + native_recovery_pause: Option>, + #[cfg(test)] torch_stage_override: Option, #[cfg(test)] torch_publication_pause: Option>, @@ -988,9 +986,37 @@ pub struct VersionInstaller { torch_stage_pause: Option>, } +#[cfg(test)] +pub(crate) struct NativeRecoveryPause { + pub(crate) reached: tokio::sync::Notify, + resume: StdMutex>, +} + +#[cfg(test)] +impl NativeRecoveryPause { + pub(crate) fn new() -> (Self, std::sync::mpsc::Sender<()>) { + let (resume, receiver) = std::sync::mpsc::channel(); + ( + Self { + reached: tokio::sync::Notify::new(), + resume: StdMutex::new(receiver), + }, + resume, + ) + } + + fn block(&self) { + self.reached.notify_one(); + let _ = self.resume.lock().unwrap().recv(); + } +} + impl VersionInstaller { /// Create a new version installer. /// + /// For llama.cpp, configure the returned installer with + /// [`Self::with_acquisition`] before calling [`Self::install_version`]. + /// /// The supplied cancellation flag is a cooperative request observed at /// installer checkpoints; setting it directly does not report whether the /// request was accepted. Torch cancellation acknowledged by `VersionManager` @@ -1019,6 +1045,8 @@ impl VersionInstaller { #[cfg(test)] native_receipt_pause: None, #[cfg(test)] + native_recovery_pause: None, + #[cfg(test)] torch_stage_override: None, #[cfg(test)] torch_publication_pause: None, @@ -1027,6 +1055,58 @@ impl VersionInstaller { } } + /// Configure direct llama.cpp installation with the existing shared + /// acquisition capability. Initializes the GitHub metadata client from this + /// installer's cache; creates no acquisition store, service, or downloader. + /// Reads retained records from that same store and reconciles native uses + /// before returning an installer that can admit new work. Recovery failure + /// drains this consumer scope before returning the error; the caller retains + /// the shared service's shutdown responsibility. + /// External customer compatibility acceptance remains pending. + pub async fn with_acquisition( + mut self, + acquisition: Arc, + ) -> Result { + if self.app_id != AppId::LlamaCpp { + return Err(PumasError::Config { + message: "Shared artifact acquisition is currently supported for llama.cpp".into(), + }); + } + let cache = self + .launcher_root + .join("launcher-data") + .join(PathsConfig::CACHE_DIR_NAME); + let store = acquisition.store().clone(); + let consumer = Arc::new(acquisition.open_consumer("runtime.llama.cpp")?); + self.acquisition_consumer = Some(consumer.clone()); + let configured = async { + let (client, records) = consumer + .run_blocking("initialize native installer recovery", move || { + Ok((GitHubClient::new(cache)?, store.acquisitions()?)) + }) + .await?; + self.github_client = Some(Arc::new(client)); + self.reconcile_retained_llama_cpp(records.into_values().collect()) + .await + } + .await; + if let Err(error) = configured { + return Err(match consumer.shutdown().await { + Ok(()) => error, + Err(settlement) => PumasError::InstallationFailed { + message: format!("{error}; native recovery shutdown: {settlement}"), + }, + }); + } + Ok(self) + } + + #[cfg(test)] + pub(crate) fn with_native_recovery_pause(mut self, pause: Arc) -> Self { + self.native_recovery_pause = Some(pause); + self + } + #[cfg(test)] pub(crate) fn with_native_receipt_pause(mut self, pause: Arc) -> Self { self.native_receipt_pause = Some(pause); @@ -1093,8 +1173,14 @@ impl VersionInstaller { } let tag = tag.to_owned(); let versions = self.versions_dir(); - let current = - read_native_attempt(&versions, &tag)?.ok_or_else(|| PumasError::Validation { + let lookup_versions = versions.clone(); + let lookup_tag = tag.clone(); + let current = consumer + .run_blocking("read retained native attempt", move || { + read_native_attempt(&lookup_versions, &lookup_tag) + }) + .await? + .ok_or_else(|| PumasError::Validation { field: "acquisition.consumer_recovery_required".into(), message: "Retained native acquisition has no attempt identity".into(), })?; @@ -1107,19 +1193,18 @@ impl VersionInstaller { &record.phase, pumas_library::acquisition::AcquisitionPhase::Adopted { .. } ) { - let consumer = consumer.clone(); + let receipt_consumer = consumer.clone(); let retained = record.clone(); - let receipt = - tokio::task::spawn_blocking(move || consumer.completion_receipt(&retained)) - .await - .map_err(|error| { - PumasError::Other(format!("Native receipt lookup failed: {error}")) - })?? - .ok_or_else(|| PumasError::Validation { - field: "acquisition.consumer_recovery_required".into(), - message: "Adopted native installation has no exact completion receipt" - .into(), - })?; + let receipt = consumer + .run_blocking("read retained native completion receipt", move || { + receipt_consumer.completion_receipt(&retained) + }) + .await? + .ok_or_else(|| PumasError::Validation { + field: "acquisition.consumer_recovery_required".into(), + message: "Adopted native installation has no exact completion receipt" + .into(), + })?; let proof: LlamaCppInstallReceiptV1 = serde_json::from_value(receipt.payload) .map_err(|error| PumasError::Validation { field: "acquisition.consumer_recovery_required".into(), @@ -1131,9 +1216,12 @@ impl VersionInstaller { let expected_workspace = record.workspace.clone(); let manager = self.metadata_manager.clone(); let app_id = self.app_id; - let lock = NativeVersionsLock::acquire(versions.clone()).await?; - tokio::task::spawn_blocking(move || { - let _lock = lock; + #[cfg(test)] + let recovery_pause = self.native_recovery_pause.clone(); + consumer.run_blocking("verify adopted native output and clean workspace", move || { + #[cfg(test)] + if let Some(pause) = recovery_pause { pause.block(); } + let _lock = NativeVersionsLock::try_acquire(&versions_for_cleanup)?; let current = read_native_attempt(&versions_for_cleanup, &tag_for_cleanup)?; let Some(current) = current else { return Err(PumasError::Validation { @@ -1187,18 +1275,21 @@ impl VersionInstaller { } cleanup_native_workspace_if_present(&workspace, &versions_for_cleanup) }) - .await - .map_err(|error| { - PumasError::Other(format!("Native adopted cleanup task failed: {error}")) - })??; + .await?; continue; } - let (custody, workspace) = open_native_acquisition_workspace( - versions.clone(), - tag.clone(), - Some(attempt.to_owned()), - ) - .await?; + let recovery_versions = versions.clone(); + let recovery_tag = tag.clone(); + let recovery_attempt = attempt.to_owned(); + let (custody, workspace) = consumer + .run_blocking("open retained native workspace", move || { + prepare_native_acquisition_workspace( + recovery_versions, + recovery_tag, + Some(recovery_attempt), + ) + }) + .await?; let custody_for_reconcile = custody.clone(); let manager = self.metadata_manager.clone(); let app_id = self.app_id; @@ -1236,20 +1327,24 @@ impl VersionInstaller { field: "acquisition.consumer_recovery_required".into(), message: "Retained native use disappeared before reconciliation".into(), })?; - tokio::task::spawn_blocking(move || { - Arc::try_unwrap(custody) - .map_err(|workspace| PumasError::InstallationFailed { - message: format!( - "Reconciled native workspace remains in use: {}", - workspace.path().display() - ), - })? - .cleanup() - }) - .await - .map_err(|error| { - PumasError::Other(format!("Native recovery cleanup task failed: {error}")) - })??; + #[cfg(test)] + let recovery_pause = self.native_recovery_pause.clone(); + consumer + .run_blocking("clean reconciled native workspace", move || { + #[cfg(test)] + if let Some(pause) = recovery_pause { + pause.block(); + } + Arc::try_unwrap(custody) + .map_err(|workspace| PumasError::InstallationFailed { + message: format!( + "Reconciled native workspace remains in use: {}", + workspace.path().display() + ), + })? + .cleanup() + }) + .await?; } Ok(()) } @@ -1497,6 +1592,12 @@ impl VersionInstaller { release: &GitHubRelease, progress_tx: mpsc::Sender, ) -> Result<()> { + let consumer = self + .acquisition_consumer + .clone() + .ok_or_else(|| PumasError::Config { + message: "llama.cpp requires shared acquisition; configure with VersionInstaller::with_acquisition".into(), + })?; Self::validate_native_tag(tag)?; info!("Starting llama.cpp binary installation for {}", tag); @@ -1538,13 +1639,24 @@ impl VersionInstaller { ); let log_dir = self.logs_dir(); - fs::create_dir_all(&log_dir).await.ok(); let log_path = log_dir.join(format!( "install-llama-cpp-{}-{}.log", self.slugify_tag(tag), Utc::now().format("%Y%m%d-%H%M%S") )); + let versions = self.versions_dir(); + let workspace_versions = versions.clone(); + let workspace_tag = tag.to_owned(); + let (custody, workspace) = consumer + .run_blocking("prepare native installation workspace", move || { + std::fs::create_dir_all(&log_dir).ok(); + std::fs::create_dir_all(&workspace_versions) + .map_err(|error| PumasError::io_with_path(error, &workspace_versions))?; + prepare_native_acquisition_workspace(workspace_versions, workspace_tag, None) + }) + .await?; + { let mut tracker = self.progress_tracker.write().await; tracker.start_installation( @@ -1555,18 +1667,6 @@ impl VersionInstaller { ); } - let versions = self.versions_dir(); - fs::create_dir_all(&versions) - .await - .map_err(|e| PumasError::io_with_path(e, &versions))?; - let (custody, workspace) = - open_native_acquisition_workspace(versions.clone(), tag.to_owned(), None).await?; - let consumer = self - .acquisition_consumer - .clone() - .ok_or_else(|| PumasError::Config { - message: "llama.cpp requires the shared artifact acquisition consumer".into(), - })?; let demand = AcquisitionDemand { consumer: consumer.owner().to_owned(), operation: format!( @@ -1820,18 +1920,18 @@ impl VersionInstaller { }; let result = if result.is_ok() { - let cleanup = tokio::task::spawn_blocking(move || { - Arc::try_unwrap(custody) - .map_err(|workspace| PumasError::InstallationFailed { - message: format!( - "Native workspace remains in use: {}", - workspace.path().display() - ), - })? - .cleanup() - }) - .await - .map_err(|error| PumasError::Other(format!("Native cleanup join failed: {error}")))?; + let cleanup = consumer + .run_blocking("clean published native workspace", move || { + Arc::try_unwrap(custody) + .map_err(|workspace| PumasError::InstallationFailed { + message: format!( + "Native workspace remains in use: {}", + workspace.path().display() + ), + })? + .cleanup() + }) + .await; settled_result(result, cleanup) } else { result @@ -3175,6 +3275,91 @@ fn shell_single_quote(value: &str) -> String { mod tests { use super::*; + #[tokio::test] + async fn unconfigured_direct_llama_cpp_installer_rejects_before_native_mutation() { + let root = tempfile::tempdir().unwrap(); + let installer = crate::version_manager::VersionInstaller::new( + root.path().to_path_buf(), + AppId::LlamaCpp, + Arc::new(MetadataManager::new(root.path())), + Arc::new(RwLock::new(InstallationProgressTracker::new( + root.path().to_path_buf(), + ))), + Arc::new(AtomicBool::new(false)), + ); + // Refusal must precede even release validation/resolution, so no remote + // fixture or assets are needed to establish the construction contract. + let release = GitHubRelease { + tag_name: "b1234+cpu".into(), + name: "legacy direct fixture".into(), + published_at: "2026-09-30T00:00:00Z".into(), + body: None, + tarball_url: None, + zipball_url: None, + prerelease: false, + assets: Vec::new(), + html_url: "https://github.com/ggml-org/llama.cpp/releases/tag/b1234".into(), + total_size: None, + archive_size: None, + dependencies_size: None, + }; + let (progress, _updates) = mpsc::channel(1); + let error = installer + .install_version("b1234+cpu", &release, progress) + .await + .unwrap_err(); + assert!( + matches!(error, PumasError::Config { message } if message.contains("with_acquisition")) + ); + assert!(!installer.versions_dir().exists()); + assert!(!installer.logs_dir().exists()); + assert!(std::fs::read_dir(root.path()).unwrap().next().is_none()); + } + + #[tokio::test] + async fn public_direct_installer_configures_shared_llama_cpp_acquisition() { + use pumas_library::acquisition::{ + AcquisitionCapacity, AcquisitionService, AcquisitionStore, + }; + let root = tempfile::tempdir().unwrap(); + let store = Arc::new(AcquisitionStore::new(root.path())); + let acquisition = Arc::new( + AcquisitionService::with_capacity( + store.clone(), + AcquisitionCapacity { + scopes: 1, + ..AcquisitionCapacity::default() + }, + ) + .unwrap(), + ); + let installer = crate::version_manager::VersionInstaller::new( + root.path().to_path_buf(), + AppId::LlamaCpp, + Arc::new(MetadataManager::new(root.path())), + Arc::new(RwLock::new( + crate::version_manager::InstallationProgressTracker::new(root.path().to_path_buf()), + )), + Arc::new(AtomicBool::new(false)), + ) + .with_acquisition(acquisition.clone()) + .await + .unwrap(); + assert!(installer.github_client.is_some()); + assert_eq!( + installer.acquisition_consumer.as_ref().unwrap().owner(), + "runtime.llama.cpp" + ); + assert!(Arc::ptr_eq(acquisition.store(), &store)); + // The direct installer consumes this owner's only scope reservation. + assert!(matches!( + acquisition.open_consumer("second"), + Err(PumasError::AcquisitionCapacityExhausted { resource: "scopes" }) + )); + drop(installer); + acquisition.shutdown().await.unwrap(); + } + #[test] fn native_attempt_reopen_preserves_identity_and_remove_reinstall_renews_it() { let root = tempfile::tempdir().unwrap(); diff --git a/rust/crates/pumas-app-manager/src/version_manager/mod.rs b/rust/crates/pumas-app-manager/src/version_manager/mod.rs index debd35c7..0f2cdee2 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/mod.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/mod.rs @@ -255,6 +255,9 @@ impl VersionManager { /// Create a new version manager. /// + /// llama.cpp installation requires [`Self::new_with_acquisition`] with the + /// application's existing shared acquisition owner. + /// /// # Arguments /// /// * `launcher_root` - Path to the launcher root directory @@ -2028,6 +2031,105 @@ mod tests { reopened_api.shutdown_acquisition().await.unwrap(); } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn native_direct_recovery_cancellation_retains_effect_until_shared_shutdown() { + let root = TempDir::new().unwrap(); + let (server, observed, release, base_url) = native_archive_fixture(root.path()).await; + let api = pumas_library::PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() + .await + .unwrap(); + let mut manager = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + manager.github_client = Arc::new( + GitHubClient::with_loopback_api( + manager.cache_dir(), + Duration::from_secs(3600), + base_url, + ) + .unwrap(), + ); + let mut updates = manager.install_version("b1234+cpu").await.unwrap(); + observed.await.unwrap(); + release.send(true).unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + match updates.recv().await.unwrap() { + ProgressUpdate::Completed { success: true } => break, + ProgressUpdate::Error { message } => panic!("native fixture failed: {message}"), + _ => {} + } + } + }) + .await + .unwrap(); + server.await.unwrap(); + manager.shutdown_installations().await.unwrap(); + let record = api + .acquisition() + .store() + .acquisitions() + .unwrap() + .into_values() + .next() + .unwrap(); + let attempt = record.demand.operation.rsplit_once(':').unwrap().1; + use sha2::Digest; + let digest = format!("{:x}", sha2::Sha256::digest(b"b1234+cpu")); + let stale_workspace = manager + .versions_dir() + .join(format!(".llama-install-{}-{attempt}", &digest[..24])); + std::fs::create_dir(&stale_workspace).unwrap(); + std::fs::write( + stale_workspace.join("stale"), + b"registered recovery cleanup", + ) + .unwrap(); + let (pause, resume) = installer::NativeRecoveryPause::new(); + let pause = Arc::new(pause); + let direct = VersionInstaller::new( + root.path().to_path_buf(), + AppId::LlamaCpp, + manager.metadata_manager.clone(), + manager.progress_tracker.clone(), + Arc::new(AtomicBool::new(false)), + ) + .with_native_recovery_pause(pause.clone()); + let acquisition = api.acquisition().clone(); + let constructor = tokio::spawn(async move { direct.with_acquisition(acquisition).await }); + tokio::time::timeout(Duration::from_secs(5), pause.reached.notified()) + .await + .unwrap(); + constructor.abort(); + assert!(matches!(constructor.await, Err(error) if error.is_cancelled())); + let acquisition = api.acquisition().clone(); + let mut shutdown = tokio::spawn(async move { acquisition.shutdown().await }); + let early = tokio::time::timeout(Duration::from_millis(50), &mut shutdown).await; + let still_present = stale_workspace.exists(); + // Always release the real closure before checking outcomes so failures + // cannot strand a blocking thread during test-runtime shutdown. + resume.send(()).unwrap(); + assert!( + early.is_err(), + "shared shutdown must wait for the registered recovery effect" + ); + assert!(still_present); + tokio::time::timeout(Duration::from_secs(5), shutdown) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(!stale_workspace.exists()); + } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] #[tokio::test] async fn native_archive_publishes_complete_output_and_reopens_metadata() { @@ -2112,6 +2214,20 @@ mod tests { b"retry cleanup after adoption", ) .unwrap(); + // Direct public construction must finish the same retained adopted-use + // recovery before it returns an installer that can admit new work. + let direct = VersionInstaller::new( + root.path().to_path_buf(), + AppId::LlamaCpp, + manager.metadata_manager.clone(), + manager.progress_tracker.clone(), + Arc::new(AtomicBool::new(false)), + ) + .with_acquisition(api.acquisition().clone()) + .await + .unwrap(); + assert!(!stale_workspace.exists()); + drop(direct); let reopened = VersionManager::new_with_acquisition( root.path(), AppId::LlamaCpp, diff --git a/rust/crates/pumas-core/src/acquisition/mod.rs b/rust/crates/pumas-core/src/acquisition/mod.rs index 20138414..c2b8abeb 100644 --- a/rust/crates/pumas-core/src/acquisition/mod.rs +++ b/rust/crates/pumas-core/src/acquisition/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod task_custody; mod workspace; pub use http::HttpAttemptHost; +pub use task_custody::AcquisitionCapacity; pub(crate) use github_release::{select_github_release_asset, GitHubReleaseAssetMetadata}; pub use github_release::{ diff --git a/rust/crates/pumas-core/src/acquisition/service.rs b/rust/crates/pumas-core/src/acquisition/service.rs index e6f548b3..de55113d 100644 --- a/rust/crates/pumas-core/src/acquisition/service.rs +++ b/rust/crates/pumas-core/src/acquisition/service.rs @@ -411,6 +411,19 @@ impl AcquisitionService { } } + /// Construct the shared owner with explicit finite admission limits. + /// Every consumer and HF scope shares these budgets. Saturation fails + /// synchronously; terminal control work has separate blocking capacity. + pub fn with_capacity( + store: Arc, + capacity: super::AcquisitionCapacity, + ) -> Result { + Ok(Self { + store, + supervisor: Arc::new(TaskCustodyOwner::with_capacity(capacity)?), + }) + } + pub(crate) fn with_store(&self, store: Arc) -> Self { Self { store, @@ -1117,6 +1130,19 @@ impl AcquisitionConsumer { self.scope.shutdown().await } + /// Run consumer recovery or cleanup through this shared scope's bounded + /// task/effect custody. Dropping the result waiter cancels the outer task; + /// shutdown still joins a registered blocking closure until it truly ends. + pub async fn run_blocking( + &self, + name: &'static str, + work: impl FnOnce() -> Result + Send + 'static, + ) -> Result { + self.scope + .run_worker_invocation(move |context| async move { owned(&context, name, work).await }) + .await + } + /// Revalidate a retained consumer receipt under the supplied held /// workspace and exact source selection. The callback owns interpretation /// of its payload and must verify its durable output before returning. @@ -1377,8 +1403,11 @@ async fn owned( Ok(value) => Ok(Ok(value)), }) .await - .map_err(|error| { - PumasError::Other(format!("Acquisition effect observation failed: {error}")) + .map_err(|error| match error { + super::task_custody::BlockingTaskError::CapacityExhausted { resource } => { + PumasError::AcquisitionCapacityExhausted { resource } + } + error => PumasError::Other(format!("Acquisition effect observation failed: {error}")), })? .and_then(|result| result) } diff --git a/rust/crates/pumas-core/src/acquisition/task_custody.rs b/rust/crates/pumas-core/src/acquisition/task_custody.rs index 6cdcd807..038afb4e 100644 --- a/rust/crates/pumas-core/src/acquisition/task_custody.rs +++ b/rust/crates/pumas-core/src/acquisition/task_custody.rs @@ -17,12 +17,64 @@ use std::sync::MutexGuard; use std::sync::{Arc, Mutex, Weak}; use futures::FutureExt; -use tokio::sync::{oneshot, Notify}; +use tokio::sync::{oneshot, Notify, OwnedSemaphorePermit, Semaphore}; use tokio::task::JoinHandle; type FallibleBlockingReceiver = oneshot::Receiver, String>>; +/// Finite budgets shared by every scope of an acquisition service. +/// +/// Defaults are admission limits, not measured memory, thread, or throughput +/// guarantees. AC10 resource qualification remains pending. Scopes retain their +/// identities and shutdown receipts until the last scope handle is dropped and +/// its finalizer/effects drain. `scopes` bounds live and draining scopes. +#[derive(Clone, Copy, Debug)] +pub struct AcquisitionCapacity { + /// Ordinary prepared, installed, and draining tasks; default 32. + pub workers: usize, + /// Ordinary blocking jobs and their observers; default 16. + pub blocking: usize, + /// Cancellation and terminal tasks, independent of ordinary work; default 32. + pub rescue_workers: usize, + /// Blocking cleanup jobs and their observers; default 4. + pub rescue_blocking: usize, + /// Live scopes plus scopes draining after their last handle drops; default 256. + pub scopes: usize, +} + +impl Default for AcquisitionCapacity { + fn default() -> Self { + Self { + workers: 32, + blocking: 16, + rescue_workers: 32, + rescue_blocking: 4, + scopes: 256, + } + } +} + +impl AcquisitionCapacity { + fn validate(self) -> crate::Result { + if [ + self.workers, + self.blocking, + self.rescue_workers, + self.rescue_blocking, + self.scopes, + ] + .iter() + .any(|&limit| limit == 0 || limit > Semaphore::MAX_PERMITS) + { + return Err(crate::PumasError::Config { + message: "Acquisition capacities must be positive finite semaphore limits".into(), + }); + } + Ok(self) + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum TaskRole { Invocation, @@ -310,14 +362,30 @@ impl ProjectionCell { #[derive(Clone, Debug, Eq, PartialEq)] pub(crate) enum BlockingTaskError { StaleGeneration, + CapacityExhausted { resource: &'static str }, Join(String), ResultChannelClosed, } +impl BlockingTaskError { + /// Preserve shared admission refusal when an adapter adds observation context. + pub(crate) fn into_pumas_error(self, context: impl fmt::Display) -> crate::PumasError { + match self { + Self::CapacityExhausted { resource } => { + crate::PumasError::AcquisitionCapacityExhausted { resource } + } + error => crate::PumasError::Other(format!("{context}: {error}")), + } + } +} + impl fmt::Display for BlockingTaskError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::StaleGeneration => formatter.write_str("task generation is no longer current"), + Self::CapacityExhausted { resource } => { + write!(formatter, "acquisition {resource} capacity exhausted") + } Self::Join(detail) => write!(formatter, "blocking task failed: {detail}"), Self::ResultChannelClosed => formatter.write_str("blocking result channel closed"), } @@ -367,6 +435,7 @@ impl StartGate { } struct TaskEntry { + capacity: Option>, admission: Option<( Arc, tokio::sync::watch::Receiver, @@ -385,6 +454,7 @@ struct TaskEntry { } struct PreparedEntry { + capacity: Option>, download_id: String, generation: TaskGeneration, role: TaskRole, @@ -460,9 +530,21 @@ type ProjectionObserver = Arc; /// One owner for all consumer-scoped task and effect custody. A scope is an /// access handle, never a separately populated supervisor or task registry. -#[derive(Default)] pub(crate) struct TaskCustodyOwner { state: Mutex, + capacity: AcquisitionCapacity, + workers: Arc, + blocking: Arc, + rescue_workers: Arc, + rescue_blocking: Arc, + #[cfg(test)] + shutdown_keepalive_observer: Mutex>>, +} + +impl Default for TaskCustodyOwner { + fn default() -> Self { + Self::with_capacity(AcquisitionCapacity::default()).expect("valid default capacity") + } } #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] @@ -475,6 +557,7 @@ type ScopeFinalizer = struct SupervisorState { closed: bool, next_scope: u64, + closed_scope_failures: usize, scopes: HashMap, shutdown: Option, shutdown_driver: Option>, @@ -516,6 +599,33 @@ impl TaskCustodyOwner { Self::default() } + pub(crate) fn with_capacity(capacity: AcquisitionCapacity) -> crate::Result { + let capacity = capacity.validate()?; + Ok(Self { + state: Mutex::default(), + capacity, + workers: Arc::new(Semaphore::new(capacity.workers)), + blocking: Arc::new(Semaphore::new(capacity.blocking)), + rescue_workers: Arc::new(Semaphore::new(capacity.rescue_workers)), + rescue_blocking: Arc::new(Semaphore::new(capacity.rescue_blocking)), + #[cfg(test)] + shutdown_keepalive_observer: Mutex::default(), + }) + } + + fn acquire_worker(&self, rescue: bool) -> crate::Result> { + let (budget, resource) = if rescue { + (&self.rescue_workers, "rescue_workers") + } else { + (&self.workers, "workers") + }; + budget + .clone() + .try_acquire_owned() + .map(Arc::new) + .map_err(|_| crate::PumasError::AcquisitionCapacityExhausted { resource }) + } + /// Mint a scope and register its terminal projection before admitting work. /// Scope identities are never removed or reused during this owner's life. pub(crate) fn open_scope( @@ -533,6 +643,9 @@ impl TaskCustodyOwner { if state.closed { return Err(crate::PumasError::DownloadLifecycleClosed); } + if state.scopes.len() >= self.capacity.scopes { + return Err(crate::PumasError::AcquisitionCapacityExhausted { resource: "scopes" }); + } let identity = ScopeId(state.next_scope); state.next_scope = state.next_scope.checked_add(1).ok_or_else(|| { crate::PumasError::Other("Acquisition scope capacity exhausted".into()) @@ -591,25 +704,36 @@ impl TaskCustodyOwner { state.shutdown = Some(receipt.clone()); let mut receipts = Vec::with_capacity(state.scopes.len()); let mut starts = Vec::with_capacity(state.scopes.len()); + // Even the existing-receipt path can drop its supplied keepalive. + // Keep every upgraded scope alive until the custody lock is released. + let mut keepalives = Vec::with_capacity(state.scopes.len()); for scope in state.scopes.values_mut() { let keepalive: Arc = scope .handle .upgrade() - .map(|handle| handle as Arc) + .map(|handle| { + keepalives.push(handle.clone()); + handle as Arc + }) .unwrap_or_else(|| self.clone()); + #[cfg(test)] + if let Some(observer) = self.shutdown_keepalive_observer.lock().unwrap().as_ref() { + observer(); + } let (receipt, start) = begin_scope_shutdown(scope, keepalive); receipts.push(receipt); if let Some(start) = start { starts.push(start); } } + let closed_scope_failures = state.closed_scope_failures; match tokio::runtime::Handle::try_current() { Ok(runtime) => { let owner = self.clone(); let (start, started) = oneshot::channel(); state.shutdown_driver = Some(runtime.spawn(async move { let _ = started.await; - let mut failures = 0; + let mut failures = closed_scope_failures; for receipt in receipts { failures += receipt.failures().await; } @@ -623,6 +747,7 @@ impl TaskCustodyOwner { } } drop(state); + drop(keepalives); for start in starts { let _ = start.send(()); } @@ -666,6 +791,7 @@ fn begin_scope_shutdown( state.shutdown_driver = Some(runtime.spawn(async move { let _ = started.await; for (_, entry) in prepared { + let _capacity = entry.capacity; drop(entry.start); if entry.outer.await.is_err_and(|error| error.is_panic()) { failures += 1; @@ -746,10 +872,48 @@ struct ScopeState { retired_failures: usize, shutdown: Option, shutdown_driver: Option>, + cleanup_driver: Option>, finalizer: Option, handle: Weak, } +// Last-handle drop is a lifecycle boundary: preserve all effects and the +// registered finalizer, then remove the metadata only after their receipt. +impl Drop for TaskScope { + fn drop(&mut self) { + let mut state = self + .owner + .state + .lock() + .expect("acquisition task custody lock poisoned"); + let Some(scope) = state.scopes.get_mut(&self.identity) else { + return; + }; + let (receipt, start) = begin_scope_shutdown(scope, self.owner.clone()); + if let Ok(runtime) = tokio::runtime::Handle::try_current() { + let owner = self.owner.clone(); + let identity = self.identity; + scope.cleanup_driver = Some(runtime.spawn(async move { + let failures = receipt.failures().await; + let removed = { + let mut state = owner + .state + .lock() + .expect("acquisition task custody lock poisoned"); + // Preserve failed cleanup evidence for owner-wide shutdown. + state.closed_scope_failures += failures; + state.scopes.remove(&identity) + }; + drop(removed); + })); + } + drop(state); + if let Some(start) = start { + let _ = start.send(()); + } + } +} + struct InvocationWaiter { owner: Arc, id: String, @@ -1077,12 +1241,21 @@ impl TaskScope { } fn lock_state(&self) -> ScopeGuard<'_> { + let mut supervisor = self + .owner + .state + .lock() + .expect("acquisition task custody lock poisoned"); + // Completion evidence remains retained independently of admission. + for scope in supervisor.scopes.values_mut() { + for entry in scope.tasks.values_mut() { + if entry.finished() { + entry.capacity.take(); + } + } + } ScopeGuard { - supervisor: self - .owner - .state - .lock() - .expect("acquisition task custody lock poisoned"), + supervisor, identity: self.identity, } } @@ -1122,6 +1295,10 @@ impl TaskScope { if state.closed { return Err(crate::PumasError::DownloadLifecycleClosed); } + let capacity = Some(self.owner.acquire_worker(matches!( + role, + TaskRole::CancelFinalizer | TaskRole::TerminalProjection + ))?); let generation = TaskGeneration::new(); let context = TaskContext { owner: Arc::downgrade(self), @@ -1131,7 +1308,9 @@ impl TaskScope { effect_lease: None, }; let (start, started) = oneshot::channel(); + let outer_capacity = capacity.clone(); let outer = tokio::spawn(async move { + let _capacity = outer_capacity; if started.await.is_ok() { work(context).await; } @@ -1140,6 +1319,7 @@ impl TaskScope { state.prepared.insert( generation.key(), PreparedEntry { + capacity, download_id: download_id.clone(), generation: generation.clone(), role, @@ -1246,6 +1426,7 @@ impl TaskScope { tasks.insert( download_id.clone(), TaskEntry { + capacity: entry.capacity, admission: None, generation: generation.clone(), role: entry.role, @@ -1510,6 +1691,7 @@ impl TaskScope { ); } } + let capacity = Some(self.owner.acquire_worker(true)?); let mut current = tasks.remove(download_id); let outer_finished_before_replacement = current .as_ref() @@ -1549,15 +1731,23 @@ impl TaskScope { failed: AtomicBool::new(false), notify: Notify::new(), }); - let predecessor_observer = tokio::spawn(observe_cancellation_predecessor( - current, - outer_finished_before_replacement, - predecessor_started, - predecessor_completion.clone(), - predecessor_sender, - )); + let observer_capacity = capacity.clone(); + let observer_completion = predecessor_completion.clone(); + let predecessor_observer = tokio::spawn(async move { + let _capacity = observer_capacity; + observe_cancellation_predecessor( + current, + outer_finished_before_replacement, + predecessor_started, + observer_completion, + predecessor_sender, + ) + .await; + }); let start_state = Arc::new(AtomicU8::new(TaskStartState::Gated as u8)); + let outer_capacity = capacity.clone(); let outer = tokio::spawn(async move { + let _capacity = outer_capacity; if started.await.is_err() { return; } @@ -1571,6 +1761,7 @@ impl TaskScope { tasks.insert( download_id.to_string(), TaskEntry { + capacity, admission: None, generation: generation.clone(), role: TaskRole::CancelFinalizer, @@ -1699,6 +1890,7 @@ impl TaskScope { return Ok(ProjectionTransition::NotReady); } + let capacity = Some(self.owner.acquire_worker(true)?); let predecessor = tasks .remove(download_id) .expect("finished predecessor remained present"); @@ -1732,7 +1924,9 @@ impl TaskScope { }); let predecessor_completion_task = predecessor_completion.clone(); let (predecessor_start, predecessor_started) = oneshot::channel(); + let observer_capacity = capacity.clone(); let predecessor_observer = tokio::spawn(async move { + let _capacity = observer_capacity; if predecessor_started.await.is_err() { return; } @@ -1751,7 +1945,9 @@ impl TaskScope { let project_cell = cell.clone(); let (project_start, project_started) = oneshot::channel(); + let outer_capacity = capacity.clone(); let outer = tokio::spawn(async move { + let _capacity = outer_capacity; if project_started.await.is_err() { return; } @@ -1792,6 +1988,7 @@ impl TaskScope { tasks.insert( download_id.to_string(), TaskEntry { + capacity, admission: None, generation: generation.clone(), role: TaskRole::TerminalProjection, @@ -1950,6 +2147,7 @@ impl TaskScope { entry.outer.abort(); let (start, started) = oneshot::channel(); let observer = tokio::spawn(async move { + let _capacity = entry.capacity; let _ = started.await; drop(entry.start); let failures = @@ -2093,10 +2291,29 @@ impl TaskScope { let Some(entry) = tasks.get_mut(download_id) else { return Err(BlockingTaskError::StaleGeneration); }; - if !entry.generation.matches(generation) { + if !entry.generation.matches(generation) || entry.capacity.is_none() { return Err(BlockingTaskError::StaleGeneration); } entry.reap_completed_nested(); + let rescue = matches!( + entry.role, + TaskRole::CancelFinalizer | TaskRole::TerminalProjection + ); + let (budget, resource) = if rescue { + (&self.owner.rescue_blocking, "rescue_blocking") + } else { + (&self.owner.blocking, "blocking") + }; + // Reserve before either observer or blocking job is spawned. No waiter + // futures are allocated on saturation. The closure itself retains both + // permits if its result waiter or observer disappears. + let blocking_capacity = Arc::new( + budget + .clone() + .try_acquire_owned() + .map_err(|_| BlockingTaskError::CapacityExhausted { resource })?, + ); + let worker_capacity = entry.capacity.clone(); #[cfg(test)] let blocking_observer = self @@ -2120,9 +2337,13 @@ impl TaskScope { .clone(); let observer = tokio::spawn(async move { let closure_grant = effect_lease.clone(); + let closure_capacity = blocking_capacity.clone(); + let _observer_capacity = blocking_capacity; let result = if start_receiver.await.is_ok() { tokio::task::spawn_blocking(move || { let _grant = closure_grant; + let _blocking_capacity = closure_capacity; + let _worker_capacity = worker_capacity; #[cfg(test)] if let Some(observer) = blocking_observer { observer(operation); @@ -2374,7 +2595,9 @@ impl TaskContext { let entry = state .tasks .get_mut(&self.download_id) - .filter(|entry| entry.generation.matches(&self.generation)) + .filter(|entry| { + entry.generation.matches(&self.generation) && entry.capacity.is_some() + }) .ok_or(BlockingTaskError::StaleGeneration)?; entry.reap_completed_nested(); let (start, started) = oneshot::channel(); @@ -2386,7 +2609,9 @@ impl TaskContext { }); let observed = completion.clone(); let effect_lease = self.effect_lease.clone(); + let worker_capacity = entry.capacity.clone(); let handle = tokio::spawn(async move { + let _worker_capacity = worker_capacity; let _ = started.await; let result = AssertUnwindSafe(async move { function().await }) .catch_unwind() @@ -2788,6 +3013,336 @@ async fn wait_for_nested(completions: &[Arc]) -> usize { #[cfg(test)] mod tests { + fn bounded_owner(workers: usize, blocking: usize, scopes: usize) -> Arc { + Arc::new( + TaskCustodyOwner::with_capacity(AcquisitionCapacity { + workers, + blocking, + rescue_blocking: 1, + rescue_workers: 2, + scopes, + }) + .unwrap(), + ) + } + + fn assert_worker_full(scope: &Arc) { + assert!(matches!( + scope.prepare("rejected".into(), TaskRole::Worker, |_| async {}), + Err(crate::PumasError::AcquisitionCapacityExhausted { + resource: "workers" + }) + )); + } + + async fn wait_worker_slot(owner: &TaskCustodyOwner) { + tokio::time::timeout(Duration::from_secs(1), async { + while owner.workers.available_permits() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("drained worker must release admission"); + } + + #[tokio::test] + async fn global_shutdown_keeps_prior_closed_scope_alive_until_unlock() { + let owner = bounded_owner(1, 1, 1); + let scope = owner.open_scope(|| async { Ok(()) }).unwrap(); + scope.shutdown().await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while Arc::strong_count(&scope) != 1 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let (upgraded_tx, upgraded) = oneshot::channel(); + let upgraded_tx = Mutex::new(Some(upgraded_tx)); + let (resume_tx, resume) = std::sync::mpsc::channel(); + let resume = Mutex::new(resume); + *owner.shutdown_keepalive_observer.lock().unwrap() = Some(Arc::new(move || { + upgraded_tx + .lock() + .unwrap() + .take() + .unwrap() + .send(()) + .unwrap(); + resume.lock().unwrap().recv().unwrap(); + })); + let shutdown_owner = owner.clone(); + let runtime = tokio::runtime::Handle::current(); + let (receipt_tx, receipt) = oneshot::channel(); + // An independent thread makes a failed lock-order assertion time out + // without blocking the runtime responsible for the receipt. + let thread = std::thread::spawn(move || { + let _runtime = runtime.enter(); + assert!(receipt_tx.send(shutdown_owner.request_shutdown()).is_ok()); + }); + upgraded.await.unwrap(); + drop(scope); + resume_tx.send(()).unwrap(); + let receipt = tokio::time::timeout(Duration::from_secs(1), receipt) + .await + .expect("shutdown must release its lock before dropping the last scope handle") + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), receipt.wait()) + .await + .unwrap() + .unwrap(); + thread.join().unwrap(); + owner.request_shutdown().wait().await.unwrap(); + } + + #[tokio::test] + async fn capacity_is_shared_and_rejected_prepared_work_drains_before_reuse() { + let owner = bounded_owner(1, 1, 2); + let first = owner.open_scope(|| async { Ok(()) }).unwrap(); + let second = owner.open_scope(|| async { Ok(()) }).unwrap(); + let prepared = first + .prepare("held".into(), TaskRole::Worker, |_| async {}) + .unwrap(); + assert_worker_full(&second); + assert_eq!(second.prepared_count_for_test(), 0); + let rejected = second.install_gated(prepared).unwrap_err(); + drop(rejected); + assert_worker_full(&second); + first.rescue_abandoned(); + wait_worker_slot(&owner).await; + let prepared = second + .prepare("accepted".into(), TaskRole::Worker, |_| async {}) + .unwrap(); + second.install_gated(prepared).unwrap().start(); + while !second.snapshot("accepted").is_some_and(|s| s.finished) { + tokio::task::yield_now().await; + } + // Completed, unobserved entries preserve evidence without monopolizing capacity. + let prepared = first + .prepare("reused".into(), TaskRole::Worker, |_| async {}) + .unwrap(); + drop(prepared); + first.rescue_abandoned(); + owner.request_shutdown().wait().await.unwrap(); + assert_eq!(owner.workers.available_permits(), 1); + } + + #[tokio::test] + async fn capacity_survives_cancel_replacement_and_dropped_blocking_waiter() { + let owner = bounded_owner(1, 1, 2); + let first = owner.open_scope(|| async { Ok(()) }).unwrap(); + let second = owner.open_scope(|| async { Ok(()) }).unwrap(); + let prepared = first + .prepare("held".into(), TaskRole::Worker, |_| async { + std::future::pending::<()>().await; + }) + .unwrap(); + let installed = first.install_gated(prepared).unwrap(); + let generation = installed.generation().clone(); + installed.start(); + let (entered_tx, entered) = oneshot::channel(); + let (release_tx, release) = std::sync::mpsc::channel(); + let waiter = first + .register_blocking("held", &generation, "held effect", move || { + entered_tx.send(()).unwrap(); + release.recv().unwrap(); + }) + .unwrap(); + drop(waiter); + entered.await.unwrap(); + assert!(matches!( + first.register_blocking("held", &generation, "excess", || {}), + Err(BlockingTaskError::CapacityExhausted { + resource: "blocking" + }) + )); + assert_eq!(first.nested_count_for_test("held"), Some(1)); + let (cleanup_tx, cleanup) = oneshot::channel(); + let CancelTransition::Started(cancel) = first + .begin_cancel("held", move |context, _| async move { + context.run_blocking(|| {}).await.unwrap(); + cleanup_tx.send(()).unwrap(); + }) + .unwrap() + else { + panic!("worker requires cancellation"); + }; + cancel.start(); + assert_worker_full(&second); + assert_eq!(owner.blocking.available_permits(), 0); + release_tx.send(()).unwrap(); + cleanup.await.unwrap(); + wait_worker_slot(&owner).await; + assert_eq!(owner.blocking.available_permits(), 1); + owner.request_shutdown().wait().await.unwrap(); + assert_eq!(owner.rescue_blocking.available_permits(), 1); + } + + #[tokio::test] + async fn blocking_capacity_is_shared_and_rescue_work_remains_available() { + let owner = bounded_owner(2, 1, 2); + let first = owner.open_scope(|| async { Ok(()) }).unwrap(); + let second = owner.open_scope(|| async { Ok(()) }).unwrap(); + let mut generations = Vec::new(); + for scope in [&first, &second] { + let prepared = scope + .prepare("held".into(), TaskRole::Worker, |_| async { + std::future::pending::<()>().await; + }) + .unwrap(); + let installed = scope.install_gated(prepared).unwrap(); + generations.push(installed.generation().clone()); + installed.start(); + } + let (entered_tx, entered) = oneshot::channel(); + let (release_tx, release) = std::sync::mpsc::channel(); + let waiter = first + .register_blocking("held", &generations[0], "shared effect", move || { + entered_tx.send(()).unwrap(); + release.recv().unwrap(); + }) + .unwrap(); + entered.await.unwrap(); + assert!(matches!( + second.register_blocking("held", &generations[1], "excess", || {}), + Err(BlockingTaskError::CapacityExhausted { + resource: "blocking" + }) + )); + let (rescued_tx, rescued) = oneshot::channel(); + let CancelTransition::Started(cancel) = second + .begin_cancel("held", move |context, _| async move { + context.run_blocking(|| {}).await.unwrap(); + rescued_tx.send(()).unwrap(); + }) + .unwrap() + else { + panic!("worker requires cancellation"); + }; + cancel.start(); + tokio::time::timeout(Duration::from_secs(1), rescued) + .await + .unwrap() + .unwrap(); + assert_eq!(owner.blocking.available_permits(), 0); + let shutdown = owner.request_shutdown(); + assert_eq!(owner.workers.available_permits(), 1); + assert_eq!(owner.blocking.available_permits(), 0); + release_tx.send(()).unwrap(); + waiter.await.unwrap().unwrap(); + shutdown.wait().await.unwrap(); + assert_eq!(owner.workers.available_permits(), 2); + assert_eq!(owner.blocking.available_permits(), 1); + } + + #[tokio::test] + async fn worker_result_waiter_drop_retains_capacity_until_real_effect_finishes() { + let owner = bounded_owner(1, 1, 2); + let first = owner.open_scope(|| async { Ok(()) }).unwrap(); + let second = owner.open_scope(|| async { Ok(()) }).unwrap(); + let (entered_tx, entered) = oneshot::channel(); + let (release_tx, release) = std::sync::mpsc::channel(); + let invocation_scope = first.clone(); + let waiter = tokio::spawn(async move { + invocation_scope + .run_worker_invocation(move |context| async move { + context + .run_blocking(move || { + entered_tx.send(()).unwrap(); + release.recv().unwrap(); + }) + .await + .unwrap(); + Ok(()) + }) + .await + }); + entered.await.unwrap(); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + assert_worker_full(&second); + assert_eq!(owner.blocking.available_permits(), 0); + release_tx.send(()).unwrap(); + wait_worker_slot(&owner).await; + owner.request_shutdown().wait().await.unwrap(); + assert_eq!(owner.blocking.available_permits(), 1); + } + + #[tokio::test] + async fn rescue_worker_capacity_is_finite_and_independent_of_work_saturation() { + let owner = bounded_owner(1, 1, 1); + let scope = owner.open_scope(|| async { Ok(()) }).unwrap(); + let work = scope + .prepare("work".into(), TaskRole::Worker, |_| async {}) + .unwrap(); + assert_worker_full(&scope); + let control = scope + .prepare("control".into(), TaskRole::TerminalProjection, |_| async {}) + .unwrap(); + let CancelTransition::Started(cancel) = scope + .begin_cancel("absent", |_, _| async { + std::future::pending::<()>().await; + }) + .unwrap() + else { + panic!("state-only cancellation starts"); + }; + assert!(matches!( + scope.begin_cancel("another", |_, _| async {}), + Err(crate::PumasError::AcquisitionCapacityExhausted { + resource: "rescue_workers" + }) + )); + assert!(!scope.contains("another")); + drop(control); + drop(work); + drop(cancel); + scope.rescue_abandoned(); + owner.request_shutdown().wait().await.unwrap(); + assert_eq!(owner.rescue_workers.available_permits(), 2); + assert_eq!(owner.workers.available_permits(), 1); + } + + #[tokio::test] + async fn scope_capacity_releases_only_after_last_handle_and_finalizer_drain() { + let owner = bounded_owner(1, 1, 1); + let (entered_tx, entered) = oneshot::channel(); + let (release_tx, release) = oneshot::channel(); + let scope = owner + .open_scope(move || async move { + entered_tx.send(()).unwrap(); + let _ = release.await; + Ok(()) + }) + .unwrap(); + drop(scope); + entered.await.unwrap(); + assert!(matches!( + owner.open_scope(|| async { Ok(()) }), + Err(crate::PumasError::AcquisitionCapacityExhausted { resource: "scopes" }) + )); + release_tx.send(()).unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner.state.lock().unwrap().scopes.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + for _ in 0..4 { + let scope = owner.open_scope(|| async { Ok(()) }).unwrap(); + drop(scope); + tokio::time::timeout(Duration::from_secs(1), async { + while !owner.state.lock().unwrap().scopes.is_empty() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } + owner.request_shutdown().wait().await.unwrap(); + } + #[tokio::test] async fn shutdown_rejects_work_whose_start_gate_was_already_extracted() { let owner = TaskScope::new_test(); diff --git a/rust/crates/pumas-core/src/api/hf.rs b/rust/crates/pumas-core/src/api/hf.rs index 16224425..32c1e6bb 100644 --- a/rust/crates/pumas-core/src/api/hf.rs +++ b/rust/crates/pumas-core/src/api/hf.rs @@ -296,9 +296,7 @@ impl PumasApi { }) .await .map_err(|error| { - PumasError::Other(format!( - "Download metadata observation failed: {error}" - )) + error.into_pumas_error("Download metadata observation failed") })??; let mut huggingface_evidence = match snapshot { Ok((model, evidence)) => { @@ -344,9 +342,7 @@ impl PumasApi { ) .await .map_err(|error| { - PumasError::Other(format!( - "Download model type observation failed: {error}" - )) + error.into_pumas_error("Download model type observation failed") })??; (resolved.model_type != model_library::ModelType::Unknown) .then(|| resolved.model_type.as_str().to_string()) @@ -370,9 +366,9 @@ impl PumasApi { ) .await .map_err(|error| { - PumasError::Other(format!( - "Download repository observation failed: {error}" - )) + error.into_pumas_error( + "Download repository observation failed", + ) })??, ); } @@ -399,9 +395,7 @@ impl PumasApi { ) .await .map_err(|error| { - PumasError::Other(format!( - "Download model type observation failed: {error}" - )) + error.into_pumas_error("Download model type observation failed") })??; } } @@ -435,9 +429,7 @@ impl PumasApi { ) .await .map_err(|error| { - PumasError::Other(format!( - "Download classification observation failed: {error}" - )) + error.into_pumas_error("Download classification observation failed") })??; match classification { Ok(Some(bundle)) => { @@ -566,9 +558,7 @@ impl PumasApi { }) .await .map_err(|error| { - PumasError::Other(format!( - "Download destination preparation observation failed: {error}" - )) + error.into_pumas_error("Download destination preparation observation failed") })??; if prepared.model_type == "unknown" { warn!( @@ -728,9 +718,7 @@ impl PumasApi { validate_existing_local_directory_lookup_path(&dest_dir, "dest_dir").await }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery directory observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Recovery directory observation failed"))??; // Determine model_type from directory path relative to library root let library_root = library.library_root(); @@ -747,9 +735,7 @@ impl PumasApi { Ok::<_, PumasError>(library.load_metadata(&metadata_dest)?.unwrap_or_default()) }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery metadata observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Recovery metadata observation failed"))??; let recovery_filenames = metadata .selected_artifact_files .clone() @@ -814,9 +800,8 @@ impl PumasApi { library.get_model(&record_id).await }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery model observation failed: {error}")) - })? { + .map_err(|error| error.into_pumas_error("Recovery model observation failed"))? + { Ok(Some(record)) => record, Ok(None) => return Ok(partial_download_unavailable("model_not_found")), Err(error) => return Ok(partial_download_error(&error)), @@ -842,9 +827,7 @@ impl PumasApi { library.index_model_dir(&model_dir).await }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery index observation failed: {error}")) - })? + .map_err(|error| error.into_pumas_error("Recovery index observation failed"))? { return Ok(partial_download_error(&error)); } @@ -855,9 +838,8 @@ impl PumasApi { library.get_model(&record_id).await }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery model observation failed: {error}")) - })? { + .map_err(|error| error.into_pumas_error("Recovery model observation failed"))? + { Ok(Some(record)) => record, Ok(None) => return Ok(partial_download_unavailable("model_not_found")), Err(error) => return Ok(partial_download_error(&error)), @@ -999,7 +981,7 @@ impl PumasApi { ) .await .map_err(|error| { - PumasError::Other(format!("Partial directory observation failed: {error}")) + error.into_pumas_error("Partial directory observation failed") })? { Ok(dest) => dest, Err(PumasError::InvalidParams { .. } | PumasError::NotFound { .. }) => { @@ -1799,6 +1781,7 @@ pub(crate) fn partial_download_reason_code(err: &PumasError) -> &'static str { PumasError::ModelNotFound { .. } => "repo_not_found", PumasError::RateLimited { .. } => "rate_limited", PumasError::DownloadRootBusy => "download_root_busy", + PumasError::AcquisitionCapacityExhausted { .. } => "acquisition_capacity_exhausted", PumasError::PermissionDenied(_) => "permission_denied", PumasError::Network { message, .. } if message.contains("404 Not Found") => { "repo_not_found" @@ -2346,6 +2329,16 @@ pub(super) mod tests { use crate::models::HuggingFaceModel; use tempfile::TempDir; + #[test] + fn test_partial_download_reason_code_preserves_acquisition_capacity() { + assert_eq!( + partial_download_reason_code(&PumasError::AcquisitionCapacityExhausted { + resource: "workers", + }), + "acquisition_capacity_exhausted" + ); + } + #[test] fn test_partial_download_reason_code_preserves_root_contention() { assert_eq!( diff --git a/rust/crates/pumas-core/src/error.rs b/rust/crates/pumas-core/src/error.rs index f161cbdb..eab75388 100644 --- a/rust/crates/pumas-core/src/error.rs +++ b/rust/crates/pumas-core/src/error.rs @@ -138,6 +138,10 @@ pub enum PumasError { #[error("Download lifecycle is closed")] DownloadLifecycleClosed, + /// Shared acquisition admission failed without spawning or waiting. + #[error("Acquisition {resource} capacity exhausted")] + AcquisitionCapacityExhausted { resource: &'static str }, + /// Another cooperating download client owns mutation of this library root. #[error("Download library root is busy")] DownloadRootBusy, @@ -292,6 +296,7 @@ impl PumasError { match self { PumasError::Network { .. } | PumasError::DownloadLifecycleClosed + | PumasError::AcquisitionCapacityExhausted { .. } | PumasError::Timeout(_) | PumasError::RateLimited { .. } | PumasError::CircuitBreakerOpen { .. } => -32000, diff --git a/rust/crates/pumas-core/src/model_library/hf/download.rs b/rust/crates/pumas-core/src/model_library/hf/download.rs index 1ac5507f..16bb7ae6 100644 --- a/rust/crates/pumas-core/src/model_library/hf/download.rs +++ b/rust/crates/pumas-core/src/model_library/hf/download.rs @@ -778,9 +778,7 @@ async fn import_completed_download( .map(|_| ()) }) .await - .map_err(|error| { - PumasError::Other(format!("Download import observation failed: {error}")) - })? + .map_err(|error| error.into_pumas_error("Download import observation failed"))? } impl PreparedDownloadTask { @@ -812,11 +810,7 @@ impl PreparedDownloadTask { }, ) .await - .map_err(|error| { - PumasError::Other(format!( - "Pinned download evidence observation failed: {error}" - )) - })? + .map_err(|error| error.into_pumas_error("Pinned download evidence observation failed"))? .unwrap_or_else(|never| match never {}) } @@ -927,7 +921,7 @@ impl PreparedDownloadTask { ) .await .map_err(|error| { - PumasError::Other(format!("Restore publication observation failed: {error}")) + error.into_pumas_error("Restore publication observation failed") })??; } self.verify_pinned_final_files(context).await?; @@ -1000,7 +994,7 @@ impl PreparedDownloadTask { ) .await .map_err(|error| { - PumasError::Other(format!("Restore publication observation failed: {error}")) + error.into_pumas_error("Restore publication observation failed") })??; } Ok(true) @@ -1045,9 +1039,9 @@ impl PreparedDownloadTask { ) .await .map_err(|error| { - RestoredFinalizationError::Operation(PumasError::Other(format!( - "Download provenance observation failed: {error}" - ))) + RestoredFinalizationError::Operation( + error.into_pumas_error("Download provenance observation failed"), + ) })??; self.acquisition.require_schema(context).await?; let destination = self.destination.capability().clone(); @@ -1057,9 +1051,7 @@ impl PreparedDownloadTask { destination.acquisition_workspace(execution_lease) }) .await - .map_err(|error| { - PumasError::Other(format!("Restored workspace observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Restored workspace observation failed"))??; let demand = crate::acquisition::AcquisitionDemand { consumer: "hf.model".into(), operation: attempt.clone(), @@ -1088,9 +1080,7 @@ impl PreparedDownloadTask { persistence.read_hf_completion_receipt(acquisition_id) }) .await - .map_err(|error| { - PumasError::Other(format!("Restored receipt observation failed: {error}")) - })?? + .map_err(|error| error.into_pumas_error("Restored receipt observation failed"))?? .ok_or_else(|| PumasError::Validation { field: "downloads.hf_completion_receipts".into(), message: @@ -1146,9 +1136,7 @@ impl PreparedDownloadTask { }, ) .await - .map_err(|error| { - PumasError::Other(format!("Restored receipt settlement failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Restored receipt settlement failed"))??; if !matches!(context.drain_blocking().await, Ok(0)) { return Err(PumasError::Other( "Restored receipt validation effects did not drain".into(), @@ -1297,9 +1285,7 @@ impl PreparedDownloadTask { persistence.read_hf_completion_receipt(acquisition_id) }) .await - .map_err(|error| { - PumasError::Other(format!("Restored receipt observation failed: {error}")) - })?? + .map_err(|error| error.into_pumas_error("Restored receipt observation failed"))?? .ok_or_else(|| PumasError::Validation { field: "downloads.hf_completion_receipts".into(), message: "Restored managed HF import completed without a durable receipt" @@ -1344,9 +1330,7 @@ impl PreparedDownloadTask { }, ) .await - .map_err(|error| { - PumasError::Other(format!("Restore receipt settlement failed: {error}")) - })?? + .map_err(|error| error.into_pumas_error("Restore receipt settlement failed"))?? } else { self.acquisition.acknowledge(context, lease).await?; let persistence = self.persistence.clone().ok_or_else(|| PumasError::Config { @@ -1358,9 +1342,7 @@ impl PreparedDownloadTask { persistence.settle_queue_admission(&id, &attempt) }) .await - .map_err(|error| { - PumasError::Other(format!("Restore settlement owner failed: {error}")) - })?? + .map_err(|error| error.into_pumas_error("Restore settlement owner failed"))?? }; if !settled || !matches!(context.drain_blocking().await, Ok(0)) { return Err(PumasError::Other( @@ -1788,9 +1770,7 @@ impl PreparedDownloadTask { }) .await .map_err(|error| { - PumasError::Other(format!( - "Download execution validation observation failed: {error}" - )) + error.into_pumas_error("Download execution validation observation failed") })? } } @@ -1882,9 +1862,7 @@ impl DownloadDestination { }) .await .map_err(|error| { - PumasError::Other(format!( - "download integrity verification task failed: {error}" - )) + error.into_pumas_error("download integrity verification task failed") })? } } @@ -1996,9 +1974,7 @@ impl DownloadDestination { error, .. })) => Err(error), Ok(Err(failure)) => Err(failure.into_error()), - Err(error) => Err(PumasError::Other(format!( - "Download marker owner failed: {error}" - ))), + Err(error) => Err(error.into_pumas_error("Download marker owner failed")), } } Self::Recovery(_) => Err(PumasError::Other( @@ -2020,9 +1996,7 @@ impl DownloadFile { Ok::<_, std::io::Error>(file) }) .await - .map_err(|error| { - PumasError::Other(format!("Fixture write observation failed: {error}")) - })??, + .map_err(|error| error.into_pumas_error("Fixture write observation failed"))??, ); Ok(()) } @@ -2035,9 +2009,7 @@ impl DownloadFile { Ok::<_, std::io::Error>(file) }) .await - .map_err(|error| { - PumasError::Other(format!("Fixture sync observation failed: {error}")) - })??, + .map_err(|error| error.into_pumas_error("Fixture sync observation failed"))??, ); Ok(()) } @@ -2182,8 +2154,8 @@ where .run_fallible_blocking_named(operation, function) .await .map_err(|error| { - PumasError::Other(format!( - "download recovery filesystem capability task failed during {operation}: {error}" + error.into_pumas_error(format!( + "download recovery filesystem capability task failed during {operation}" )) })? .map_err(|error| { @@ -2205,11 +2177,7 @@ async fn assert_no_intent_deletion_claim( Ok::<_, std::convert::Infallible>(destination.assert_no_intent_deletion_claim()) }) .await - .map_err(|error| { - PumasError::Other(format!( - "Intent deletion custody observation failed: {error}" - )) - })? + .map_err(|error| error.into_pumas_error("Intent deletion custody observation failed"))? .expect("infallible custody observation envelope") } @@ -2672,9 +2640,7 @@ impl HuggingFaceClient { crate::model_library::canonical_managed_model_dir(&library_root, &record) }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery inspection owner failed: {error}")) - })? + .map_err(|error| error.into_pumas_error("Recovery inspection owner failed"))? }) .await } @@ -2695,9 +2661,7 @@ impl HuggingFaceClient { ) }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery verification owner failed: {error}")) - })? + .map_err(|error| error.into_pumas_error("Recovery verification owner failed"))? }) .await } @@ -3151,7 +3115,7 @@ impl HuggingFaceClient { }) .await .map_err(|error| { - PumasError::Other(format!("Download restore owner failed: {error}")) + error.into_pumas_error("Download restore owner failed") })? }) .await?; @@ -3202,7 +3166,7 @@ impl HuggingFaceClient { }) .await .map_err(|error| { - PumasError::Other(format!("Download restore authority owner failed: {error}")) + error.into_pumas_error("Download restore authority owner failed") })??; { if admission.domain != DownloadAdmissionDomain::Ambient { @@ -3244,9 +3208,7 @@ impl HuggingFaceClient { ) .await .map_err(|error| { - PumasError::Other(format!( - "Download restore authority owner failed: {error}" - )) + error.into_pumas_error("Download restore authority owner failed") })??; restored_entries.push(( entry, @@ -3389,9 +3351,7 @@ impl HuggingFaceClient { ) .await .map_err(|error| { - PumasError::Other(format!( - "Receiptless recovery custody observation failed: {error}" - )) + error.into_pumas_error("Receiptless recovery custody observation failed") })? } @@ -3615,9 +3575,7 @@ impl HuggingFaceClient { root.resolve(&requested_destination) }) .await - .map_err(|error| { - PumasError::Other(format!("Download authority resolution failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Download authority resolution failed"))??; let dest_dir = destination.display_path(); assert_no_intent_deletion_claim(context, &destination).await?; let provenance_destination = destination.clone(); @@ -3627,9 +3585,7 @@ impl HuggingFaceClient { validate_download_provenance_revision(&provenance_destination, &provenance_revision) }) .await - .map_err(|error| { - PumasError::Other(format!("Download provenance observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Download provenance observation failed"))??; self.observe_finished_download_tasks().await; let download_id = uuid::Uuid::new_v4().to_string(); @@ -3646,9 +3602,7 @@ impl HuggingFaceClient { .await }) .await - .map_err(|error| { - PumasError::Other(format!("Download metadata observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Download metadata observation failed"))??; // Resolve weight files to download. // Priority: filenames (explicit list) > filename (single) > quant (substring) > all. @@ -4025,9 +3979,7 @@ impl HuggingFaceClient { let confirmed = match outcome { Ok(Ok((transition, inventory, identity))) => transition.into_result().map(|_| (inventory, identity)), Ok(Err(error)) => Err(error), - Err(error) => Err(PumasError::Other(format!( - "Download admission owner failed: {error}" - ))), + Err(error) => Err(error.into_pumas_error("Download admission owner failed")), }; let (inventory, identity) = match confirmed { Ok(confirmed) => confirmed, @@ -4472,9 +4424,7 @@ impl HuggingFaceClient { validate_download_provenance_revision(&provenance_destination, &provenance_revision) }) .await - .map_err(|error| { - PumasError::Other(format!("Download provenance observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Download provenance observation failed"))??; publish_worker_snapshot_and_revalidate( &download_publications, &downloads, @@ -4495,9 +4445,7 @@ impl HuggingFaceClient { granted.acquisition_workspace(execution_lease) }) .await - .map_err(|error| { - PumasError::Other(format!("Workspace observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Workspace observation failed"))??; let operation_receipt = downloads .read() .await @@ -4585,7 +4533,7 @@ impl HuggingFaceClient { }) .await .map_err(|error| { - PumasError::Other(format!("Download resume marker owner failed: {error}")) + error.into_pumas_error("Download resume marker owner failed") })??; destination.write_marker(&task_context, marker).await?; } @@ -4613,9 +4561,9 @@ impl HuggingFaceClient { } Ok(Err(error)) => return Err(error), Err(error) => { - return Err(PumasError::Other(format!( - "failed to observe admitted download resume persistence: {error}" - ))); + return Err(error.into_pumas_error( + "failed to observe admitted download resume persistence", + )); } } } @@ -4756,9 +4704,7 @@ impl HuggingFaceClient { ) .await .map_err(|error| { - PumasError::Other(format!( - "Auxiliary metadata observation failed: {error}" - )) + error.into_pumas_error("Auxiliary metadata observation failed") })??; } let callback_outcome = if let Some(callback) = aux_complete_callback.clone() @@ -4805,9 +4751,9 @@ impl HuggingFaceClient { ))); } Err(error) => { - return Err(PumasError::Other(format!( - "failed to observe auxiliary-files-complete callback: {error}" - ))); + return Err(error.into_pumas_error( + "failed to observe auxiliary-files-complete callback", + )); } } } @@ -4995,9 +4941,7 @@ impl HuggingFaceClient { persistence.read_hf_completion_receipt(acquisition_id) }) .await - .map_err(|error| { - PumasError::Other(format!("Completion receipt observation failed: {error}")) - })?? + .map_err(|error| error.into_pumas_error("Completion receipt observation failed"))?? .ok_or_else(|| PumasError::Validation { field: "downloads.hf_completion_receipts".into(), message: "Managed HF import completed without a durable receipt".into(), @@ -5071,9 +5015,7 @@ impl HuggingFaceClient { }, ) .await - .map_err(|error| { - PumasError::Other(format!("Managed HF settlement owner failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Managed HF settlement owner failed"))??; if !settled { return Err(PumasError::Other( "Managed HF completion settlement was not confirmed".into(), @@ -5120,18 +5062,18 @@ impl HuggingFaceClient { Ok(Ok(_)) => {} Ok(Err(error)) => return Err(error), Err(error) => { - return Err(PumasError::Other(format!( - "failed to observe completed-download persistence cleanup: {error}" - ))); + return Err(error.into_pumas_error( + "failed to observe completed-download persistence cleanup", + )); } } } } let nested_failures = task_context.drain_blocking().await.map_err(|error| { - PumasError::Other(format!( - "failed to drain recovery filesystem operations before completion: {error}" - )) + error.into_pumas_error( + "failed to drain recovery filesystem operations before completion", + ) })?; if nested_failures > 0 { return Err(PumasError::Other(format!( @@ -5181,9 +5123,9 @@ impl HuggingFaceClient { .await { Ok(result) => result, - Err(error) => Err(PumasError::Other(format!( - "failed to observe persisted download status: {error}" - ))), + Err(error) => { + Err(error.into_pumas_error("failed to observe persisted download status")) + } } } @@ -5198,9 +5140,7 @@ impl HuggingFaceClient { }) .await .map_err(|error| { - PumasError::Other(format!( - "Failed to join persisted recovery authority check: {error}" - )) + error.into_pumas_error("Failed to join persisted recovery authority check") })? } @@ -5367,7 +5307,7 @@ impl HuggingFaceClient { let quarantine = if let Some(persistence) = cancellation_persistence.as_ref() { let persistence = persistence.clone(); task_context.run_fallible_blocking_named("quarantine download before cancellation", move || persistence.begin(unverified_lifecycle_failure || predecessor_failed)).await - .map_err(|error| PumasError::Other(format!("Cancellation quarantine owner failed: {error}"))).and_then(|result| result) + .map_err(|error| error.into_pumas_error("Cancellation quarantine owner failed")).and_then(|result| result) } else { Ok(None) }; let quarantine_failed = quarantine.is_err(); let quarantine = quarantine.ok().flatten(); @@ -5614,9 +5554,7 @@ impl HuggingFaceClient { root.resolve(&path) }) .await - .map_err(|error| { - PumasError::Other(format!("Download lookup owner failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Download lookup owner failed"))??; let downloads = client.downloads.read().await; Ok(downloads .values() @@ -5691,9 +5629,7 @@ impl HuggingFaceClient { metadata_client.get_repo_files(&repo_id).await }) .await - .map_err(|error| { - PumasError::Other(format!("Recovery metadata observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Recovery metadata observation failed"))??; let Some(files) = resolve_exact_recovery_files(&tree, &verified.files) else { return Ok(RecoveryDownloadAdmission::BoundFilesUnavailable); }; @@ -5838,9 +5774,7 @@ impl HuggingFaceClient { ) .await .map_err(|error| { - PumasError::Other(format!( - "download recovery persistence task failed: {error}" - )) + error.into_pumas_error("download recovery persistence task failed") })??; let (snapshot, admission_attempt) = { let mut states = downloads.write().await; @@ -5875,9 +5809,7 @@ impl HuggingFaceClient { ) .await .map_err(|error| { - PumasError::Other(format!( - "download recovery persistence task failed: {error}" - )) + error.into_pumas_error("download recovery persistence task failed") })??; } @@ -6258,9 +6190,7 @@ impl HuggingFaceClient { }, ) .await - .map_err(|error| { - PumasError::Other(format!("Explicit pause observation failed: {error}")) - })??; + .map_err(|error| error.into_pumas_error("Explicit pause observation failed"))??; if changed { self.publish_download_snapshot().await; } @@ -6377,9 +6307,7 @@ impl HuggingFaceClient { }) .await .map_err(|error| { - PumasError::Other(format!( - "Interrupted download authority observation failed: {error}" - )) + error.into_pumas_error("Interrupted download authority observation failed") })? .expect("infallible interrupted authority envelope") } @@ -11886,9 +11814,7 @@ mod tests { ) .await .map_err(|error| { - PumasError::Other(format!( - "FilesReady destination observation failed: {error}" - )) + error.into_pumas_error("FilesReady destination observation failed") })??; let managed = DownloadDestination::Managed(destination); managed.prepare(&context).await?; @@ -11901,9 +11827,7 @@ mod tests { ) .await .map_err(|error| { - PumasError::Other(format!( - "FilesReady workspace observation failed: {error}" - )) + error.into_pumas_error("FilesReady workspace observation failed") })??; let manifest = crate::model_library::hf::acquisition_source::manifest_for_download( "acme/model", @@ -11961,7 +11885,7 @@ mod tests { }) .await .map_err(|error| { - PumasError::Other(format!("FilesReady fixture publication failed: {error}")) + error.into_pumas_error("FilesReady fixture publication failed") })??; Ok(()) }) diff --git a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs index eca6f028..9fb5ffa8 100644 --- a/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs +++ b/rust/crates/pumas-core/src/model_library/hf/lifecycle.rs @@ -398,9 +398,7 @@ impl TaskContext { Ok::<_, std::convert::Infallible>(owner.acquire_root_grant(&context, root).await) }) .await - .map_err(|error| { - crate::PumasError::Other(format!("Download root grant observation failed: {error}")) - })? + .map_err(|error| error.into_pumas_error("Download root grant observation failed"))? .expect("infallible acquisition envelope")?; let mut scoped = self.clone(); scoped.inner = scoped.inner.with_effect_lease(Some(grant.clone())); @@ -660,9 +658,7 @@ impl DownloadTaskOwner { }) .await .map_err(|error| { - crate::PumasError::Other(format!( - "Download root validation observation failed: {error}" - )) + error.into_pumas_error("Download root validation observation failed") })?; } GrantAcquisition::Open => { @@ -675,9 +671,7 @@ impl DownloadTaskOwner { }) .await .map_err(|error| { - crate::PumasError::Other(format!( - "Download root acquisition observation failed: {error}" - )) + error.into_pumas_error("Download root acquisition observation failed") }) .and_then(|result| result); { @@ -707,6 +701,66 @@ mod tests { use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; use tokio::sync::oneshot; + #[tokio::test] + async fn ordinary_hf_root_grant_preserves_shared_blocking_saturation() { + let supervisor = Arc::new( + TaskCustodyOwner::with_capacity(crate::acquisition::AcquisitionCapacity { + workers: 2, + blocking: 1, + ..crate::acquisition::AcquisitionCapacity::default() + }) + .unwrap(), + ); + let hf = Arc::new( + DownloadTaskOwner::with_supervisor(supervisor.clone(), || async { Ok(()) }).unwrap(), + ); + let other = supervisor.open_scope(|| async { Ok(()) }).unwrap(); + let (entered_tx, entered) = oneshot::channel(); + let (release_tx, release) = std::sync::mpsc::channel(); + let held = tokio::spawn(async move { + other + .run_worker_invocation(move |context| async move { + context + .run_blocking(move || { + entered_tx.send(()).unwrap(); + let _ = release.recv(); + }) + .await + .unwrap(); + Ok(()) + }) + .await + }); + entered.await.unwrap(); + let root_dir = tempfile::TempDir::new().unwrap(); + let root = DownloadDestinationRoot::open(root_dir.path()).unwrap(); + let requested_root = root.clone(); + let refused = hf + .run_invocation(move |context| async move { + context.with_root_grant(requested_root).await.map(|_| ()) + }) + .await; + release_tx.send(()).unwrap(); + held.await.unwrap().unwrap(); + assert!(matches!( + refused, + Err(crate::PumasError::AcquisitionCapacityExhausted { + resource: "blocking" + }) + )); + // Refusal must clear the acquisition slot so a subsequent grant can proceed. + tokio::time::timeout( + Duration::from_secs(1), + hf.run_invocation(move |context| async move { + context.with_root_grant(root).await.map(|_| ()) + }), + ) + .await + .unwrap() + .unwrap(); + supervisor.request_shutdown().wait().await.unwrap(); + } + #[tokio::test] async fn admission_matching_uses_ordered_validated_selection_and_weak_legacy_revision() { use crate::model_library::artifact_identity::DownloadRevision; From eadfb6bb3abc82e3efe5c63c881696f26ab7619c Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 30 Sep 2026 18:59:08 -0700 Subject: [PATCH 20/20] fix(ffi): preserve acquisition capacity errors --- .../artifact-acquisition/execution-ledger.md | 7 ++++++ .../reports/passeur-mcp-usability.md | 24 +++++++++++++++++++ .../reports/write-sets.md | 2 +- rust/crates/pumas-uniffi/src/bindings.rs | 20 ++++++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) diff --git a/docs/plans/artifact-acquisition/execution-ledger.md b/docs/plans/artifact-acquisition/execution-ledger.md index 3cbd6c56..5d51447c 100644 --- a/docs/plans/artifact-acquisition/execution-ledger.md +++ b/docs/plans/artifact-acquisition/execution-ledger.md @@ -185,3 +185,10 @@ Created this acquisition plan, a proposed shared contract, gate record, source a - Passeur was rechecked after the frontend update. `passeur_status` reports connected/open service, held/ready coordination, and build `6a43daa6eec45ddceadea0ba2e2a623d7fc8e81e330237c7762a7f1c63c33f63`; provider/execution/approval remain `not_checked`. `passeur_agents({limit:4})` still fails with `PATH_NOT_FOUND` at `profile.open`, so no contributor ID exists and no task was submitted. Sol 6.1 remains the authorized implementation path until contributor execution is available. See the [Passeur usability report](reports/passeur-mcp-usability.md). - The Coding-Standards route for this resumed slice used snapshot `snapshot:v1:28c08446-d4d8-4bab-bf4c-5122700bbd60`: 25 selected standards, zero unresolved routing facts, and 24 canonical policy reads. Routing/policy reads provided obligations only; they do not certify implementation or acceptance. See the [MCP usability report](reports/coding-standards-mcp-usability.md). - Build #349 (`36794721238`) is green on prior head `1c228c3436e100b898d6fc2f05e471e88bee4162`, before this local source diff. It is not CI evidence for this candidate. The local branch has not yet been committed or pushed for current-head hosted checks; PR #7 remains open and draft. All AC01–AC18 and AQ-HTTP remain pending/not ready; AC10 resource qualification, AC16 external compatibility/cutover, live HF/native sources, desktop behavior, deployed-store/old-writer qualification, and supported-platform durability remain unproved. User-owned `docs/breif/future.md` remains untouched; no merge was attempted. + +## 2026-09-30 — downstream UniFFI compile gap and Passeur availability recheck + +- Commit `07cf74dfb960bca005918f6923073cc4a5c82814` was pushed fast-forward to PR #7. Build #350 (`36802790129`) is running on that head. `Workflow and release contracts`, `Headless without inference plugins`, `Frontend and desktop contracts`, and Linux/macOS native QA completed successfully; Rust quality failed, while Windows native QA was still running at this observation. The failure is an exhaustive match in `pumas-uniffi/src/bindings.rs` that omitted the new `PumasError::AcquisitionCapacityExhausted` variant. +- A local follow-up maps that variant to the existing `FfiError::Download` with `PumasError::to_string()`, preserving the capacity resource without changing the UniFFI enum or binding schema. A co-located test covers workers, blocking, rescue-workers, rescue-blocking, and scope labels. `cargo test -p pumas-uniffi acquisition_capacity_uses_existing_download_error_and_preserves_resource --lib` passed 1/1. All-target Clippy with `-D warnings` across library, app-manager, RPC, and UniFFI passed; formatting and `git diff --check` passed. A fresh read-only Sol High review found no compatibility issue or new confirmed finding. These changes were already authored through the Sol fallback before the user's next instruction to use Passeur; no Passeur task ran. +- After the user reported Passeur availability, `passeur_status` and `passeur_prepare` showed a connected service/open admission and ready/held metadata coordination; identity worked and the task list returned zero. But `passeur_agents({limit:4,offset:0})` again failed with `PATH_NOT_FOUND` at `profile.open`; the exact expected project profile file was absent, and `passeur_coordination` status returned `not_enabled`. Execution profile/provider/approval remain `not_checked`; no agent ID was returned and no task was submitted. The user was told that no Passeur subagent is currently working. The profile/readiness inconsistency is recorded in the [usability report](reports/passeur-mcp-usability.md). +- The current local UniFFI/write-set/Passeur-report follow-up has not yet been committed or pushed; the user-owned `docs/breif/future.md` remains untracked and untouched. PR #7 remains draft, no merge was attempted. The next Passeur code task must wait for a discoverable configured contributor; the currently required local integration fix can be verified and committed without claiming Passeur authored it. AC01–AC18 and AQ-HTTP remain pending/not ready. diff --git a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md index 53f7f435..fca01a2a 100644 --- a/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md +++ b/docs/plans/artifact-acquisition/reports/passeur-mcp-usability.md @@ -134,3 +134,27 @@ This update narrows the earlier feedback: coordination reads now work, but contributor discovery still needs to expose which configured profile lookup failed or provide a safe status/readiness result, and execution-profile, provider, and approval readiness need explicit values before binding a task. + +## 2026-09-30 — contributor availability rechecked after user update + +After the user reported that Passeur was available and requested Muse Spark 1.3 +Contributor implementation, this session ran `passeur_status`, +`passeur_prepare`, `passeur_agents({limit:4,offset:0})`, +`passeur_tasks({schema_version:1,limit:16,offset:0})`, and coordination identity +and status reads. The installed frontend remains build +`6a43daa6eec45ddceadea0ba2e2a623d7fc8e81e330237c7762a7f1c63c33f63`; the repo +service connects with open admission and `passeur_prepare` reports coordination +ready/held. Identity reads succeed and the task list is empty. However, +`passeur_agents` still returns `PATH_NOT_FOUND` at `profile.open`, and the +expected project profile path +`/home/jeremy/.config/muse-bridge/projects/6aaae9e5ae2b753918ac7478.json` does +not exist. Separately, `passeur_coordination` status reports `not_enabled`, +which conflicts with the `passeur_prepare` readiness report. Execution profile, +provider, and approval remain `not_checked`. No agent ID was returned and no +task was submitted or started. + +The readiness output should distinguish repository-service connection, +metadata-coordination enablement, project-profile existence, and contributor +execution/provider readiness. Agent-list failure should identify the missing +profile resource (or expose a safe status field) instead of returning only +`profile.open`; the current state is not sufficient to choose an agent safely. diff --git a/docs/plans/artifact-acquisition/reports/write-sets.md b/docs/plans/artifact-acquisition/reports/write-sets.md index 5ed041bf..a5c78dc1 100644 --- a/docs/plans/artifact-acquisition/reports/write-sets.md +++ b/docs/plans/artifact-acquisition/reports/write-sets.md @@ -80,7 +80,7 @@ Admitted source write set: `acquisition/{service.rs,store.rs}` for the schema-7 ### Current Q1 candidate slice — native receipt settlement and cancellation withdrawal -Exact current source paths are `rust/crates/pumas-core/src/acquisition/{http.rs,mod.rs,service.rs,store.rs,task_custody.rs,workspace.rs}`, `rust/crates/pumas-core/src/model_library/{download_recovery.rs,download_store.rs,importer.rs,library.rs,mutation_authority.rs}`, `rust/crates/pumas-core/src/model_library/hf/{download.rs,types.rs}`, `rust/crates/pumas-core/src/network/github.rs`, `rust/crates/pumas-core/src/tests.rs`, `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/Cargo.toml`, `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs}`, and `rust/crates/pumas-rpc/src/{main.rs,server.rs}`. These cover the shared transfer/store/custody owner, exact receipt and output projections, HF reopen reconciliation, native shared-service consumer and attempt withdrawal, RPC lifetime ordering, and direct regressions. This slice gives llama.cpp the existing shared acquisition service, binds a durable receipt to exact extracted output and metadata, reconciles committed publication after reopen, and withdraws only an unchanged unreceipted native `Using` lease after its attempt is revoked and workspace cleanup succeeds. The `library.rs` descriptor-refresh future is boxed because the exact Torch serving RPC regression overflowed the default worker stack in the composed candidate; the existing integration test now passes with the normal stack size. Temporary diagnostic logs/test edits were removed. This source and its local fixtures do not qualify live sources, desktop behavior, deployed schema-6 population/old-writer isolation, or cross-platform durability. Keep AQ-HTTP not ready until objective acceptance is complete. +Exact current source paths are `rust/crates/pumas-core/src/acquisition/{http.rs,mod.rs,service.rs,store.rs,task_custody.rs,workspace.rs}`, `rust/crates/pumas-core/src/model_library/{download_recovery.rs,download_store.rs,importer.rs,library.rs,mutation_authority.rs}`, `rust/crates/pumas-core/src/model_library/hf/{download.rs,types.rs}`, `rust/crates/pumas-core/src/network/github.rs`, `rust/crates/pumas-core/src/tests.rs`, `rust/crates/pumas-core/tests/artifact_acquisition.rs`, `rust/crates/pumas-app-manager/Cargo.toml`, `rust/crates/pumas-app-manager/src/version_manager/{installer.rs,mod.rs}`, `rust/crates/pumas-rpc/src/{main.rs,server.rs}`, and `rust/crates/pumas-uniffi/src/bindings.rs`. These cover the shared transfer/store/custody owner, exact receipt and output projections, HF reopen reconciliation, native shared-service consumer and attempt withdrawal, RPC lifetime ordering, the exhaustive UniFFI conversion for the new typed capacity error, and direct regressions. This slice gives llama.cpp the existing shared acquisition service, binds a durable receipt to exact extracted output and metadata, reconciles committed publication after reopen, and withdraws only an unchanged unreceipted native `Using` lease after its attempt is revoked and workspace cleanup succeeds. The `library.rs` descriptor-refresh future is boxed because the exact Torch serving RPC regression overflowed the default worker stack in the composed candidate; the existing integration test now passes with the normal stack size. Build #350 found that adding `PumasError::AcquisitionCapacityExhausted` left the UniFFI conversion match non-exhaustive. The admitted adjacent fix uses an existing FFI error shape and a co-located conversion regression; it adds no new public FFI enum variant. Temporary diagnostic logs/test edits were removed. This source and its local fixtures do not qualify live sources, desktop behavior, deployed schema-6 population/old-writer isolation, or cross-platform durability. Keep AQ-HTTP not ready until objective acceptance is complete. The app-manager manifest is a hashed input to release attribution. The test-support dev-dependency required for the fresh-owner cancellation regression therefore also refreshes only `docs/release-attribution/0.7.0/inventory.json`; `THIRD-PARTY-NOTICES.txt` remains byte-for-byte unchanged because the dependency is the existing internal library and adds no third-party package. diff --git a/rust/crates/pumas-uniffi/src/bindings.rs b/rust/crates/pumas-uniffi/src/bindings.rs index a7b59b94..0d4c5228 100644 --- a/rust/crates/pumas-uniffi/src/bindings.rs +++ b/rust/crates/pumas-uniffi/src/bindings.rs @@ -141,6 +141,9 @@ impl From for FfiError { PumasError::DownloadFailed { url, message } => FfiError::Download { message: format!("{}: {}", url, message), }, + error @ PumasError::AcquisitionCapacityExhausted { .. } => FfiError::Download { + message: error.to_string(), + }, PumasError::DownloadCancelled | PumasError::DownloadPaused => FfiError::Cancelled, PumasError::HashMismatch { expected, actual } => FfiError::Validation { message: format!("Hash mismatch: expected {}, got {}", expected, actual), @@ -368,6 +371,23 @@ mod tests { use std::path::PathBuf; use std::time::{SystemTime, UNIX_EPOCH}; + #[test] + fn acquisition_capacity_uses_existing_download_error_and_preserves_resource() { + for resource in [ + "workers", + "blocking", + "rescue_workers", + "rescue_blocking", + "scopes", + ] { + let expected = format!("Acquisition {resource} capacity exhausted"); + assert!(matches!( + FfiError::from(PumasError::AcquisitionCapacityExhausted { resource }), + FfiError::Download { message } if message == expected + )); + } + } + #[test] fn download_root_busy_uses_existing_host_config_error() { assert!(matches!(