From 56e2327a3da08275859f54682657289ed9b0cf27 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:10:06 -0600 Subject: [PATCH 01/15] feat(python): move nested downloads out of staging A nested job queued while staging is on now gets its own staging->final move once lftp no longer lists it. The move checks the row's remote size and leaves a stopped or partial download in staging for a re-queue to resume. Nested move failures show as MOVE_FAILED on the row, and their keys are pruned once the row is gone. A queue that overlaps an in-flight move is refused. Lifts the nested-queue refusal from #789. --- src/python/controller/command_pipeline.py | 33 ++- src/python/controller/model_builder.py | 8 + src/python/controller/model_updater.py | 49 +++++ src/python/controller/move/move_process.py | 29 ++- src/python/controller/pair_context.py | 3 + .../test_controller/test_command_pipeline.py | 158 +++++++++++++- .../test_controller/test_model_builder.py | 37 ++++ .../test_controller/test_model_updater.py | 206 +++++++++++++++++- .../test_controller/test_move_process.py | 100 +++++++++ 9 files changed, 610 insertions(+), 13 deletions(-) diff --git a/src/python/controller/command_pipeline.py b/src/python/controller/command_pipeline.py index 3fbd8c2b..a5a5403f 100644 --- a/src/python/controller/command_pipeline.py +++ b/src/python/controller/command_pipeline.py @@ -209,10 +209,9 @@ def _handle_queue( if file.remote_size is None: _notify_failure(command, f"File '{command.filename}' does not exist remotely") return False - # A nested job would land under the staging dir, and the staging move - # only handles top-level names, so the download would never arrive. - if file.parent is not None and pc.effective_local_path != pc.local_path: - _notify_failure(command, "Queuing a nested file is not supported while staging is enabled") + moving = self._find_overlapping_move(command.filename, pc) + if moving is not None: + _notify_failure(command, f"A staging move is in progress for '{moving}'") return False try: exclude = parse_exclude_patterns(self._context.config.general.exclude_patterns) @@ -220,8 +219,29 @@ def _handle_queue( except LftpError as e: _notify_failure(command, f"Lftp error: {e!s}") return False + if is_nested_name(command.filename): + if pc.effective_local_path != pc.local_path: + pc.pending_nested_moves.add(command.filename) + # A nested job's move has no requeue diff to reset it + # (ModelUpdater._handle_requeued_file sees top-level files only). + move_key = persist_key(pc.pair_id, command.filename) + self.moved_file_keys.discard(move_key) + if move_key in self._persist.move_failed_file_names: + self._persist.move_failed_file_names.discard(move_key) + self.sync_persist_callback() return True + def _find_overlapping_move(self, name: str, pc: PairContext) -> str | None: + """Name of an in-flight staging move for this pair that `name` lies inside + or contains, or None. lftp writing into a tree mid-move would race it.""" + for move_process in self.active_move_processes: + if move_process.pair_id != pc.pair_id: + continue + moving = move_process.file_name + if name == moving or name.startswith(moving + "/") or moving.startswith(name + "/"): + return moving + return None + @staticmethod def _iter_active_descendants(file: ModelFile) -> Iterator[ModelFile]: """BFS over `file`'s descendants, yielding each independently active @@ -782,13 +802,15 @@ def spawn_deferred_move(self, pair_id: str | None, file_name: str): return self.spawn_move_process(file_name, pc) - def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: + def spawn_move_process(self, file_name: str, pc: PairContext, expected_size: int | None = None) -> bool: """Spawn a MoveProcess to move a file from staging to the final local_path. Returns True if a move process was actually spawned, False if the call was a no-op (already moved / in flight, staging disabled, or nothing left in staging). A retry of a failed move uses this to tell a real retry from a resolved no-op so it can clear a stale MOVE_FAILED state (#536 follow-up). + + expected_size (see MoveProcess) is set for a nested job that just ended. """ pair_id = pc.pair_id move_key = persist_key(pair_id, file_name) @@ -815,6 +837,7 @@ def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: file_name=file_name, pair_id=pair_id, umask=self._context.args.umask, + expected_size=expected_size, ) process.set_mp_log_queue(self._mp_logger.queue, self._mp_logger.log_level) # Start before publishing bookkeeping: if start() raises we must not leave diff --git a/src/python/controller/model_builder.py b/src/python/controller/model_builder.py index d878a29c..bdfe3140 100644 --- a/src/python/controller/model_builder.py +++ b/src/python/controller/model_builder.py @@ -304,6 +304,14 @@ def _build_children( _child_model_file.state, _effective_transfer_state = ModelBuilder._determine_child_state( _is_dir, _child_transfer_state, _nested_status, _remote_child, _local_child, root_model_file ) + # A nested job whose staging->final move failed. Unlike a root + # (_determine_state) there's no post-download state to require: + # the file sits in staging, which the local scan doesn't cover. + if _child_model_file.full_path in self.__move_failed_files and _child_model_file.state not in ( + ModelFile.State.QUEUED, + ModelFile.State.DOWNLOADING, + ): + _child_model_file.state = ModelFile.State.MOVE_FAILED # fill the rest ModelBuilder._fill_model_file(_child_model_file, _remote_child, _local_child, _effective_transfer_state) diff --git a/src/python/controller/model_updater.py b/src/python/controller/model_updater.py index 47fc5f82..f34da143 100644 --- a/src/python/controller/model_updater.py +++ b/src/python/controller/model_updater.py @@ -327,6 +327,30 @@ def _prune_absent_persist_entries(self) -> None: for key in absent_keys: self._move_retry_counts.pop(key, None) self.sync_persist_to_all_builders() + self._prune_absent_nested_move_failures() + + def _prune_absent_nested_move_failures(self) -> None: + """Drop nested move_failed keys whose row is gone (remote and local). + + Nested keys never reach downloaded_file_names, which drives the prune + above, so they need their own pass. Called once all scans are in. + """ + absent_keys: set[str] = set() + for key in self._persist.move_failed_file_names: + for pc in self._pair_contexts: + bare_name = strip_persist_key(key, pc.pair_id) + if bare_name == key and pc.pair_id is not None: + continue + if is_nested_name(bare_name): + try: + self._registry.resolve_full_path(bare_name, pair_id=pc.pair_id) + except ModelError: + absent_keys.add(key) + break + if absent_keys: + self._logger.info(f"Persist cleanup (nested move failures gone): {absent_keys}") + self._persist.move_failed_file_names.difference_update(absent_keys) + self.sync_persist_to_all_builders() def _process_extraction_failures(self, latest_failed_extractions: list[ExtractFailedResult]) -> None: """Process extraction failures -- mark as failed immediately.""" @@ -382,6 +406,11 @@ def _retry_failed_moves(self) -> None: move_failed_file_names that is not already moving, bounded by a per-file budget so a permanently-failing move does not re-spawn forever (#536). """ + # A count outlives its failure when the key is cleared without a requeue + # diff (a nested re-queue, see CommandPipeline._handle_queue); drop it so + # a later failure starts a fresh budget. + for key in [k for k in self._move_retry_counts if k not in self._persist.move_failed_file_names]: + del self._move_retry_counts[key] cfg = self._context.config.controller if not (cfg.use_staging and cfg.staging_path): return @@ -552,6 +581,26 @@ def _detect_lftp_completions(self, pc: PairContext, lftp_statuses: list[LftpJobS # recursive local scan, just less often. pc.active_downloading_file_names = [name for name in current_downloading if not is_nested_name(name)] pc.prev_downloading_file_names = current_downloading + self._spawn_nested_moves(pc, {s.name for s in lftp_statuses}) + + def _spawn_nested_moves(self, pc: PairContext, lftp_job_names: set[str]) -> None: + """Move each nested job queued under staging from staging to local_path + once lftp no longer lists it. + + Top-level moves start from the DOWNLOADED diff; nested rows have no + diffs, so they start here. Driven by the queue rather than a + running->gone transition, which a job finishing between two status + polls (a small file, or one already complete in staging) never shows. + The job ending doesn't mean it finished (a Stop ends it too), so the + move gets the row's remote size to check. + """ + for name in pc.pending_nested_moves - lftp_job_names: + pc.pending_nested_moves.discard(name) + try: + expected_size = self._registry.resolve_full_path(name, pair_id=pc.pair_id).remote_size + except ModelError: + expected_size = None + self._pipeline.spawn_move_process(name, pc, expected_size=expected_size) def sync_persist_to_all_builders(self): """Push current persist state to all pair model builders, filtered by pair_id.""" diff --git a/src/python/controller/move/move_process.py b/src/python/controller/move/move_process.py index f0c13128..d1f21708 100644 --- a/src/python/controller/move/move_process.py +++ b/src/python/controller/move/move_process.py @@ -38,6 +38,7 @@ def __init__( file_name: str, pair_id: str | None = None, umask: int | None = None, + expected_size: int | None = None, ): super().__init__(name=self.__class__.__name__) self.__source_path = source_path @@ -47,6 +48,10 @@ def __init__( # When set, the moved tree is chmod'ed to 0666/0777 & ~umask here, in # this process, rather than on the controller thread (#811). self.__umask = umask + # Set for a nested job, whose move is triggered by its lftp job ending, + # which a Stop or an error also does. A smaller or unfinalized source + # was never completed, so it is left in staging for a re-queue to resume. + self.__expected_size = expected_size self.__failed_result_stream = PipeStream() @property @@ -122,6 +127,12 @@ def run_once(self): self._report_failure(f"Move failed: source does not exist: {src}") return + if self.__expected_size is not None and ( + self._has_lftp_temp_files(src) or self._get_total_size(src) < self.__expected_size + ): + self.logger.info(f"Move skipped for {self.__file_name}: download incomplete, left in staging") + return + # Defer while lftp is still finalizing the staging tree. lftp renames # ".lftp" -> "" on completion; copying (or renaming the # whole tree) while a rename is in flight races — copytree lists the @@ -137,14 +148,15 @@ def run_once(self): self.logger.info(f"Moving {src} -> {dst}") - # Ensure destination parent directory exists - os.makedirs(self.__dest_path, exist_ok=True) + # Ensure the destination's parent exists (several levels for a nested name) + os.makedirs(os.path.dirname(dst), exist_ok=True) # Try rename first (instant on same filesystem) try: os.rename(src, dst) self.logger.info(f"Move completed via rename: {self.__file_name}") self.__apply_modes(dst) + self.__prune_empty_staging_parents(src) return except OSError as e: if e.errno == errno.EXDEV: @@ -190,11 +202,24 @@ def run_once(self): self.logger.info(f"Move completed via copy+delete: {self.__file_name}") self.__apply_modes(dst) + self.__prune_empty_staging_parents(src) def __apply_modes(self, dst: str) -> None: if self.__umask is not None: apply_umask_modes(dst, self.__umask, self.logger) + def __prune_empty_staging_parents(self, src: str) -> None: + """Remove the staging dirs a nested name's move emptied, up to (not + including) source_path. Stops at the first one still holding something, + such as another nested job's download.""" + parent = os.path.dirname(src) + while os.path.normpath(parent) != os.path.normpath(self.__source_path): + try: + os.rmdir(parent) + except OSError: + return + parent = os.path.dirname(parent) + def pop_failed(self) -> list[MoveFailedResult]: """Process-safe method to retrieve any move failures.""" return self.__failed_result_stream.pop_all() diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index 218d764b..bac836db 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -51,6 +51,9 @@ class PairContext: active_extracting_file_names: list[str] = field(default_factory=list[str]) prev_downloading_file_names: set[str] = field(default_factory=set[str]) pending_completion: set[str] = field(default_factory=set[str]) + # Nested names queued while staging is on, awaiting their staging->final + # move once lftp no longer lists the job (see ModelUpdater._spawn_nested_moves) + pending_nested_moves: set[str] = field(default_factory=set[str]) remote_scan_received: bool = False local_scan_received: bool = False diff --git a/src/python/tests/unittests/test_controller/test_command_pipeline.py b/src/python/tests/unittests/test_controller/test_command_pipeline.py index c07cade4..8b167469 100644 --- a/src/python/tests/unittests/test_controller/test_command_pipeline.py +++ b/src/python/tests/unittests/test_controller/test_command_pipeline.py @@ -332,6 +332,36 @@ def test_spawn_move_process_start_failure_leaves_no_stale_state(self, mock_move_ self.assertNotIn(persist_key("pair-1", "file.txt"), pipeline.moved_file_keys) self.assertEqual([], pipeline.active_move_processes) + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.MoveProcess") + def test_spawn_move_process_passes_expected_size_to_move_process(self, mock_move_cls, _mock_exists): + """A nested job's just-ended expected_size must reach MoveProcess so it + can tell an incomplete source from a finished one.""" + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.local_path = "/local/pair-1" + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + + pipeline.spawn_move_process("Top/leaf.rar", pc, expected_size=12345) + + self.assertEqual(12345, mock_move_cls.call_args.kwargs["expected_size"]) + + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.MoveProcess") + def test_spawn_move_process_default_expected_size_is_none(self, mock_move_cls, _mock_exists): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.local_path = "/local/pair-1" + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + + pipeline.spawn_move_process("file.txt", pc) + + self.assertIsNone(mock_move_cls.call_args.kwargs["expected_size"]) + # --- propagate_exceptions: worker-fault isolation (#511) --- def test_propagate_exceptions_clean_when_no_worker_errors(self): @@ -686,6 +716,7 @@ def _make_pair_context(self, pair_id=None, effective_local_path="/local"): pc.pair_id = pair_id pc.local_path = "/local" pc.effective_local_path = effective_local_path + pc.pending_nested_moves = set() return pc @staticmethod @@ -1295,11 +1326,18 @@ def test_step_nested_queue_end_to_end(self): callback.on_success.assert_called_once() callback.on_failure.assert_not_called() - def test_nested_queue_refused_while_staging(self): - """A nested job would land in the staging dir, which only top-level - moves empty, so it's refused rather than orphaned.""" + def test_nested_queue_under_staging_clears_move_state(self): + """Nested queuing under staging is allowed. A successful queue has no + requeue diff to reset it, so it must directly clear any stale moved/ + move-failed bookkeeping for that nested name, syncing persist if it + cleared a failure.""" pc = self._make_pair_context("pair-1", effective_local_path="/staging/pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + move_key = persist_key("pair-1", "Top/leaf.rar") + pipeline.moved_file_keys.add(move_key) + pipeline._persist.move_failed_file_names = {move_key} + root = ModelFile("Top", True) leaf = ModelFile("leaf.rar", False) leaf.remote_size = 100 @@ -1307,11 +1345,121 @@ def test_nested_queue_refused_while_staging(self): command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") calls, notify = self._make_notify() - result = pipeline._dispatch_command(command, leaf, pc, [], notify) + result = pipeline._handle_queue(command, leaf, pc, notify) + + self.assertTrue(result) + self.assertEqual([], calls) + pc.lftp.queue.assert_called_once_with("Top/leaf.rar", False, exclude_patterns=[]) + self.assertNotIn(move_key, pipeline.moved_file_keys) + self.assertNotIn(move_key, pipeline._persist.move_failed_file_names) + pipeline.sync_persist_callback.assert_called_once() + self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + + def test_nested_queue_without_staging_adds_no_pending_move(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + leaf = ModelFile("leaf.rar", False) + leaf.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") + _calls, notify = self._make_notify() + + self.assertTrue(pipeline._handle_queue(command, leaf, pc, notify)) + self.assertEqual(set(), pc.pending_nested_moves) + + def test_top_level_queue_does_not_touch_move_state(self): + """The move-state clearing is nested-only (top-level files get a + requeue diff instead); a top-level queue must not touch moved_file_keys + or move_failed_file_names at all.""" + pc = self._make_pair_context("pair-1", effective_local_path="/staging/pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + move_key = persist_key("pair-1", "Top") + pipeline.moved_file_keys.add(move_key) + pipeline._persist.move_failed_file_names = {move_key} + + top = ModelFile("Top", True) + top.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top", pair_id="pair-1") + calls, notify = self._make_notify() + + result = pipeline._handle_queue(command, top, pc, notify) + + self.assertTrue(result) + self.assertEqual([], calls) + self.assertIn(move_key, pipeline.moved_file_keys) + self.assertIn(move_key, pipeline._persist.move_failed_file_names) + pipeline.sync_persist_callback.assert_not_called() + self.assertEqual(set(), pc.pending_nested_moves) + + # --- _find_overlapping_move --- + + @staticmethod + def _fake_move_process(pair_id, file_name): + move_process = MagicMock() + move_process.pair_id = pair_id + move_process.file_name = file_name + return move_process + + def test_find_overlapping_move_same_name(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top/Sub")) + + self.assertEqual("Top/Sub", pipeline._find_overlapping_move("Top/Sub", pc)) + + def test_find_overlapping_move_ancestor_moving_blocks_descendant_queue(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) + + self.assertEqual("Top", pipeline._find_overlapping_move("Top/Sub/f.bin", pc)) + + def test_find_overlapping_move_descendant_moving_blocks_ancestor_queue(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top/Sub/f.bin")) + + self.assertEqual("Top/Sub/f.bin", pipeline._find_overlapping_move("Top", pc)) + + def test_find_overlapping_move_unrelated_name_no_overlap(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Other")) + + self.assertIsNone(pipeline._find_overlapping_move("Top", pc)) + + def test_find_overlapping_move_different_pair_no_overlap(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-2", "Top")) + + self.assertIsNone(pipeline._find_overlapping_move("Top", pc)) + + def test_find_overlapping_move_prefix_name_not_overlap(self): + """'Top' vs 'TopX' must not be treated as overlapping - a string + prefix without the path separator is just a different, unrelated name.""" + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) + + self.assertIsNone(pipeline._find_overlapping_move("TopX", pc)) + + def test_handle_queue_refused_when_overlapping_move_in_progress(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) + + leaf = ModelFile("leaf.rar", False) + leaf.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") + calls, notify = self._make_notify() + + result = pipeline._handle_queue(command, leaf, pc, notify) self.assertFalse(result) self.assertEqual(1, len(calls)) - self.assertIn("staging", calls[0]) + self.assertIn("Top", calls[0]) pc.lftp.queue.assert_not_called() def test_top_level_queue_allowed_while_staging(self): diff --git a/src/python/tests/unittests/test_controller/test_model_builder.py b/src/python/tests/unittests/test_controller/test_model_builder.py index 611aba49..c0bd4757 100644 --- a/src/python/tests/unittests/test_controller/test_model_builder.py +++ b/src/python/tests/unittests/test_controller/test_model_builder.py @@ -2209,6 +2209,43 @@ def test_nested_dir_job_stays_downloading_when_incomplete(self): sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") self.assertEqual(ModelFile.State.DOWNLOADING, sub.state) + def test_nested_child_move_failed(self): + """A nested child whose staging->final move failed renders MOVE_FAILED + even though it sits in staging and never reaches a post-download local + state - unlike a root, there's no DOWNLOADED/EXTRACTED/etc. to require.""" + r_top = SystemFile("Top", 2000, True) + r_sub = SystemFile("Sub", 1000, True) + r_leaf = SystemFile("f.bin", 1000, False) + r_sub.add_child(r_leaf) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + self.model_builder.set_move_failed_files({"Top/Sub/f.bin"}) + + model = self.model_builder.build_model() + + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(ModelFile.State.MOVE_FAILED, leaf.state) + + def test_nested_child_move_failed_stays_downloading_with_active_job(self): + """An active standalone job on the same name overrides MOVE_FAILED - + e.g. a re-queue just restarted it.""" + r_top = SystemFile("Top", 2000, True) + r_sub = SystemFile("Sub", 1000, True) + r_leaf = SystemFile("f.bin", 1000, False) + r_sub.add_child(r_leaf) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + self.model_builder.set_move_failed_files({"Top/Sub/f.bin"}) + nested_status = LftpJobStatus(0, LftpJobStatus.Type.PGET, LftpJobStatus.State.RUNNING, "Top/Sub/f.bin", "") + self.model_builder.set_lftp_statuses([nested_status]) + + model = self.model_builder.build_model() + + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(ModelFile.State.DOWNLOADING, leaf.state) + def test_nested_status_never_creates_a_phantom_top_level_entry(self): """A '/'-containing status name is always routed to its nested child, even with nested navigation off (e.g. a nested job still running diff --git a/src/python/tests/unittests/test_controller/test_model_updater.py b/src/python/tests/unittests/test_controller/test_model_updater.py index a8f76d10..4cd1dbf7 100644 --- a/src/python/tests/unittests/test_controller/test_model_updater.py +++ b/src/python/tests/unittests/test_controller/test_model_updater.py @@ -8,7 +8,7 @@ from controller.persist_keys import KEY_SEP, persist_key from controller.scan.scanner_process import ScannerResult from lftp import LftpJobStatus -from model import ModelDiff, ModelFile +from model import ModelDiff, ModelError, ModelFile class TestSyncPersistToAllBuilders(unittest.TestCase): @@ -262,6 +262,184 @@ def test_top_level_completion_added_to_downloaded_file_names(self): self.assertEqual({"TopLevel"}, pc.pending_completion) +class TestSpawnNestedMoves(unittest.TestCase): + """A nested job's lftp job ending (completion or Stop) triggers its + staging->final move directly, since it has no DOWNLOADED diff to drive it.""" + + def _make_pair_context(self, pair_id=None): + pc = MagicMock() + pc.pair_id = pair_id + pc.prev_downloading_file_names = set() + pc.pending_completion = set() + pc.pending_nested_moves = set() + return pc + + def _make_updater(self, pair_contexts, persist=None, *, use_staging=True, staging_path="/tmp/staging"): + pipeline = MagicMock() + registry = MagicMock() + extract_process = MagicMock() + validate_process = MagicMock() + context = MagicMock() + context.config.controller.use_staging = use_staging + context.config.controller.staging_path = staging_path + logger = MagicMock() + + updater = ModelUpdater( + pair_contexts=pair_contexts, + persist=persist or MagicMock(), + pipeline=pipeline, + registry=registry, + extract_process=extract_process, + validate_process=validate_process, + context=context, + password=None, + logger=logger, + ) + return updater, pipeline, registry + + def _make_persist(self): + persist = MagicMock() + persist.downloaded_file_names = set() + return persist + + def _running(self, name): + return LftpJobStatus(0, LftpJobStatus.Type.PGET, LftpJobStatus.State.RUNNING, name, "") + + def test_pending_nested_move_spawned_once_lftp_no_longer_lists_it(self): + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + resolved = MagicMock() + resolved.remote_size = 12345 + registry.resolve_full_path.return_value = resolved + + updater._detect_lftp_completions(pc, []) + + registry.resolve_full_path.assert_called_once_with("Top/leaf.rar", pair_id="pair-1") + pipeline.spawn_move_process.assert_called_once_with("Top/leaf.rar", pc, expected_size=12345) + self.assertEqual(set(), pc.pending_nested_moves) + + def test_job_never_seen_running_still_spawns_move(self): + """A job that finishes between two status polls (a small file, or one + already complete in staging) never shows as running; the queue-driven + pending set still moves it.""" + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + pc.prev_downloading_file_names = set() + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + registry.resolve_full_path.return_value.remote_size = 10 + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_called_once_with("Top/leaf.rar", pc, expected_size=10) + + def test_pending_nested_move_waits_while_lftp_lists_it(self): + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + + updater._detect_lftp_completions(pc, [self._running("Top/leaf.rar")]) + + pipeline.spawn_move_process.assert_not_called() + self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + + def test_pending_nested_move_model_error_uses_none_expected_size(self): + """The row can be gone by the time this runs (e.g. a remote delete + raced the completion); fall back to expected_size=None rather than + failing to spawn the move at all.""" + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + registry.resolve_full_path.side_effect = ModelError("not found") + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_called_once_with("Top/leaf.rar", pc, expected_size=None) + + def test_completion_without_pending_entry_spawns_nothing(self): + """Only names queued under staging are moved: a nested job ending with + staging off, or any top-level job, never enters the pending set.""" + pc = self._make_pair_context("pair-1") + pc.prev_downloading_file_names = {"Top/leaf.rar", "TopLevel"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_not_called() + registry.resolve_full_path.assert_not_called() + + +class TestPruneAbsentNestedMoveFailures(unittest.TestCase): + """Nested move_failed keys never reach downloaded_file_names (the prune + that drives _prune_absent_persist_entries), so they get their own pass.""" + + def _make_pair_context(self, pair_id=None): + pc = MagicMock() + pc.pair_id = pair_id + return pc + + def _make_updater(self, pair_contexts, persist): + pipeline = MagicMock() + registry = MagicMock() + extract_process = MagicMock() + validate_process = MagicMock() + context = MagicMock() + logger = MagicMock() + + updater = ModelUpdater( + pair_contexts=pair_contexts, + persist=persist, + pipeline=pipeline, + registry=registry, + extract_process=extract_process, + validate_process=validate_process, + context=context, + password=None, + logger=logger, + ) + updater._persist_sync = MagicMock() + return updater, registry + + def _make_persist(self, move_failed): + persist = MagicMock() + persist.move_failed_file_names = move_failed + return persist + + def test_unresolvable_nested_key_removed(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "Top/leaf.rar") + persist = self._make_persist({fail_key}) + updater, registry = self._make_updater([pc], persist) + registry.resolve_full_path.side_effect = ModelError("gone") + + updater._prune_absent_nested_move_failures() + + registry.resolve_full_path.assert_called_once_with("Top/leaf.rar", pair_id="pair-1") + self.assertNotIn(fail_key, persist.move_failed_file_names) + + def test_resolvable_nested_key_kept(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "Top/leaf.rar") + persist = self._make_persist({fail_key}) + updater, registry = self._make_updater([pc], persist) + registry.resolve_full_path.return_value = MagicMock() + + updater._prune_absent_nested_move_failures() + + self.assertIn(fail_key, persist.move_failed_file_names) + + def test_top_level_key_untouched(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "TopLevel") + persist = self._make_persist({fail_key}) + updater, registry = self._make_updater([pc], persist) + + updater._prune_absent_nested_move_failures() + + registry.resolve_full_path.assert_not_called() + self.assertIn(fail_key, persist.move_failed_file_names) + + class TestRetryFailedMoves(unittest.TestCase): """In-session retry of failed staging->final moves (#536).""" @@ -342,6 +520,32 @@ def test_does_not_respawn_when_move_already_in_flight(self): pipeline.spawn_move_process.assert_not_called() + def test_retry_count_dropped_when_key_cleared_without_requeue_diff(self): + """A nested re-queue clears move_failed_file_names directly + (CommandPipeline._handle_queue), with no requeue diff to reset the + retry count, so the count must be dropped here instead - otherwise a + later failure on the same name would start part-way through the budget.""" + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "Top/leaf.rar") + persist = self._make_persist(move_failed=set()) # cleared by the re-queue + updater, pipeline = self._make_updater([pc], persist) + updater._move_retry_counts[fail_key] = ModelUpdater.MAX_MOVE_RETRIES + + updater._retry_failed_moves() + + self.assertNotIn(fail_key, updater._move_retry_counts) + + def test_retry_count_kept_while_key_still_failed(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "file.txt") + persist = self._make_persist(move_failed={fail_key}) + updater, pipeline = self._make_updater([pc], persist) + updater._move_retry_counts[fail_key] = 1 + + updater._retry_failed_moves() + + self.assertIn(fail_key, updater._move_retry_counts) + def test_retry_budget_caps_respawns(self): """A permanently-failing move must stop re-spawning after MAX_MOVE_RETRIES so it doesn't loop forever.""" diff --git a/src/python/tests/unittests/test_controller/test_move_process.py b/src/python/tests/unittests/test_controller/test_move_process.py index 664ae220..efe5e22c 100644 --- a/src/python/tests/unittests/test_controller/test_move_process.py +++ b/src/python/tests/unittests/test_controller/test_move_process.py @@ -315,6 +315,106 @@ def test_get_copied_size_ignores_preexisting_dest_files(self): self.assertEqual(5, MoveProcess._get_copied_size(src_subdir, dst_subdir)) + # --- Nested names: multi-level dest parents + staging prune --- + + def test_move_nested_name_creates_dest_parents_and_prunes_empty_staging_dirs(self): + """A nested name's move must create every missing dest parent level and, + once the source is gone, remove the staging dirs it emptied up to (not + including) source_path.""" + nested_dir = os.path.join(self.src_dir, "Top", "Sub") + os.makedirs(nested_dir) + src_file = os.path.join(nested_dir, "f.bin") + with open(src_file, "w") as f: + f.write("data") + + process = self._make_process(source_path=self.src_dir, dest_path=self.dst_dir, file_name="Top/Sub/f.bin") + self._run_process(process) + + dst_file = os.path.join(self.dst_dir, "Top", "Sub", "f.bin") + self.assertTrue(os.path.exists(dst_file)) + with open(dst_file) as f: + self.assertEqual("data", f.read()) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top", "Sub"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + self.assertTrue(os.path.exists(self.src_dir)) + self.assertEqual([], process.pop_failed()) + + def test_move_nested_prune_stops_at_nonempty_sibling(self): + """Pruning stops at the first staging dir still holding something, such + as a sibling file from another job - that dir (and its ancestors) must + survive.""" + nested_dir = os.path.join(self.src_dir, "Top", "Sub") + os.makedirs(nested_dir) + src_file = os.path.join(nested_dir, "f.bin") + with open(src_file, "w") as f: + f.write("data") + sibling_file = os.path.join(self.src_dir, "Top", "sibling.txt") + with open(sibling_file, "w") as f: + f.write("keepme") + + process = self._make_process(source_path=self.src_dir, dest_path=self.dst_dir, file_name="Top/Sub/f.bin") + self._run_process(process) + + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top", "Sub"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top"))) + self.assertTrue(os.path.exists(sibling_file)) + + def test_move_nested_via_copy_delete_prunes_empty_staging_dirs(self): + """The copy+delete fallback path must prune the same as the rename path.""" + nested_dir = os.path.join(self.src_dir, "Top", "Sub") + os.makedirs(nested_dir) + src_file = os.path.join(nested_dir, "f.bin") + with open(src_file, "w") as f: + f.write("data") + + process = self._make_process(source_path=self.src_dir, dest_path=self.dst_dir, file_name="Top/Sub/f.bin") + with mock.patch("controller.move.move_process.os.rename", side_effect=OSError(errno.EXDEV, "cross-device")): + self._run_process(process) + + dst_file = os.path.join(self.dst_dir, "Top", "Sub", "f.bin") + self.assertTrue(os.path.exists(dst_file)) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top", "Sub"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + self.assertEqual([], process.pop_failed()) + + # --- expected_size: skip (not fail) an in-flight nested download (#811) --- + + def test_expected_size_larger_than_source_skips_move_without_failure(self): + """A nested job's lftp process ending early (e.g. a Stop) leaves a + smaller-than-expected source; the move must be skipped, not failed, and + the source left untouched for a later re-queue to resume.""" + src_file = os.path.join(self.src_dir, "test.txt") + with open(src_file, "w") as f: + f.write("12345") # 5 bytes + + process = self._make_process( + source_path=self.src_dir, dest_path=self.dst_dir, file_name="test.txt", expected_size=100 + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(src_file)) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "test.txt"))) + + def test_expected_size_set_with_temp_files_skips_without_failure(self): + """With expected_size set, lftp temp files present also means skip (not + the recoverable-failure path used when expected_size is None).""" + src_subdir = os.path.join(self.src_dir, "mydir") + os.makedirs(src_subdir) + with open(os.path.join(src_subdir, "done.mkv"), "w") as f: + f.write("finished") + with open(os.path.join(src_subdir, "inprogress.mkv.lftp"), "w") as f: + f.write("partial") + + process = self._make_process( + source_path=self.src_dir, dest_path=self.dst_dir, file_name="mydir", expected_size=1000 + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(src_subdir)) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "mydir"))) + def test_move_applies_umask_modes_at_destination(self): """#811: lftp leaves 0664/0775 in staging; the moved copy gets 0666/0777 & ~umask.""" import stat From 3999e472efcb94a9fd7a2569220943543b824199 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:10:06 -0600 Subject: [PATCH 02/15] feat(angular): offer Queue on nested rows while staging is on The backend now moves nested downloads out of staging, so drop the staging block from the capability rules and the settings text. --- .../src/app/pages/settings/options-list.ts | 3 +-- .../files/view-file-capabilities.spec.ts | 10 ++------ .../services/files/view-file-capabilities.ts | 10 ++------ .../services/files/view-file.service.spec.ts | 15 ++--------- .../app/services/files/view-file.service.ts | 25 +++++++------------ 5 files changed, 16 insertions(+), 47 deletions(-) diff --git a/src/angular/src/app/pages/settings/options-list.ts b/src/angular/src/app/pages/settings/options-list.ts index 90962594..e474f06b 100644 --- a/src/angular/src/app/pages/settings/options-list.ts +++ b/src/angular/src/app/pages/settings/options-list.ts @@ -316,8 +316,7 @@ export const OPTIONS_CONTEXT_OTHER: IOptionsContext = { valuePath: ['controller', 'enable_nested_navigation'], description: 'Browse into subfolders of a tracked directory and queue, stop, or delete ' + - 'individual nested files and folders as independent jobs. Nested files ' + - "can't be queued while the staging directory is in use.", + 'individual nested files and folders as independent jobs.', }, ], }; diff --git a/src/angular/src/app/services/files/view-file-capabilities.spec.ts b/src/angular/src/app/services/files/view-file-capabilities.spec.ts index 2fc7947f..28eb1777 100644 --- a/src/angular/src/app/services/files/view-file-capabilities.spec.ts +++ b/src/angular/src/app/services/files/view-file-capabilities.spec.ts @@ -201,7 +201,7 @@ describe('view-file-capabilities', () => { }); }); describe('deriveCapabilities tree position', () => { - const base = { isTopLevel: true, ancestorActive: false, hasDownloadingDescendant: false, stagingEnabled: false }; + const base = { isTopLevel: true, ancestorActive: false, hasDownloadingDescendant: false }; it('disables queue, stop and deletes when an ancestor job is active', () => { const caps = deriveCapabilities(ViewFileStatus.DOWNLOADING, 100, 100, 100, 100, { @@ -233,15 +233,9 @@ describe('view-file-capabilities', () => { expect(caps.validateTooltip).toBeNull(); }); - it('blocks queuing a nested row only while staging is on', () => { + it('offers queue for a nested row', () => { const nested = { ...base, isTopLevel: false }; expect(deriveCapabilities(ViewFileStatus.DEFAULT, 0, 100, 0, 100, nested).isQueueable).toBe(true); - expect( - deriveCapabilities(ViewFileStatus.DEFAULT, 0, 100, 0, 100, { ...nested, stagingEnabled: true }).isQueueable, - ).toBe(false); - expect( - deriveCapabilities(ViewFileStatus.DEFAULT, 0, 100, 0, 100, { ...base, stagingEnabled: true }).isQueueable, - ).toBe(true); }); }); }); diff --git a/src/angular/src/app/services/files/view-file-capabilities.ts b/src/angular/src/app/services/files/view-file-capabilities.ts index a2f35bce..1b2637b5 100644 --- a/src/angular/src/app/services/files/view-file-capabilities.ts +++ b/src/angular/src/app/services/files/view-file-capabilities.ts @@ -88,15 +88,12 @@ export interface TreePosition { ancestorActive: boolean; /** A folder whose own job is idle but which has an independently active nested job. */ hasDownloadingDescendant: boolean; - /** Staging is on; nested jobs would land in staging, which only moves top-level names. */ - stagingEnabled: boolean; } const TOP_LEVEL: TreePosition = { isTopLevel: true, ancestorActive: false, hasDownloadingDescendant: false, - stagingEnabled: false, }; /** @@ -151,14 +148,11 @@ export function deriveCapabilities( // - hasDownloadingDescendant: a new job on the folder would race the nested // one, and deleting it would remove an in-flight download // (_find_active_job_conflict); Stop becomes "Stop Nested Downloads". - // - nested rows: EXTRACT/VALIDATE are top-level only, and so is QUEUE while - // staging is on. - const nestedQueueBlocked = !tree.isTopLevel && tree.stagingEnabled; + // - nested rows: EXTRACT/VALIDATE are top-level only. const isQueueable = ((QUEUEABLE_STATUSES.includes(status) && remoteSize > 0) || status === ViewFileStatus.CORRUPT) && !tree.ancestorActive && - !tree.hasDownloadingDescendant && - !nestedQueueBlocked; + !tree.hasDownloadingDescendant; const isStoppable = (STOPPABLE_STATUSES.includes(status) || tree.hasDownloadingDescendant) && !tree.ancestorActive; const isExtractable = LOCAL_ACTION_STATUSES.includes(status) && localSize > 0 && tree.isTopLevel; diff --git a/src/angular/src/app/services/files/view-file.service.spec.ts b/src/angular/src/app/services/files/view-file.service.spec.ts index bcbb2d53..c0d8120f 100644 --- a/src/angular/src/app/services/files/view-file.service.spec.ts +++ b/src/angular/src/app/services/files/view-file.service.spec.ts @@ -40,11 +40,8 @@ function makeModelFile( } interface TestConfig { - controller: { enable_nested_navigation: boolean; use_staging?: boolean; staging_path?: string }; + controller: { enable_nested_navigation: boolean }; } -const STAGING_ON: TestConfig = { - controller: { enable_nested_navigation: true, use_staging: true, staging_path: "/staging" }, -}; describe("ViewFileService", () => { let service: ViewFileService; @@ -596,17 +593,9 @@ describe("ViewFileService", () => { ]); } - it("blocks queuing nested rows while staging is on, but not the top-level row", () => { - configSubject.next(STAGING_ON); + it("offers queue for both the top-level row and a nested row", () => { emitQueueableTree(); expect(findByName(latestFiles(), "dir")!.isQueueable).toBe(true); - expect(findByName(latestFiles(), "child")!.isQueueable).toBe(false); - }); - - it("re-enables queuing nested rows when staging is turned off", () => { - configSubject.next(STAGING_ON); - emitQueueableTree(); - configSubject.next({ controller: { enable_nested_navigation: true, use_staging: false } }); expect(findByName(latestFiles(), "child")!.isQueueable).toBe(true); }); diff --git a/src/angular/src/app/services/files/view-file.service.ts b/src/angular/src/app/services/files/view-file.service.ts index 50d31546..e7f47a27 100644 --- a/src/angular/src/app/services/files/view-file.service.ts +++ b/src/angular/src/app/services/files/view-file.service.ts @@ -56,8 +56,6 @@ export class ViewFileService { // Nested rows exist in the view only while nested navigation is enabled, so // selection, select-all and bulk actions can never reach a hidden row. private nestedNavEnabled = false; - // Nested rows can't be queued while staging is on (see deriveCapabilities). - private stagingEnabled = false; // Resolve a view-file key to its backing ModelFile from the diffing-owned // snapshot. Threaded into ViewFileCommandService so command dispatch keeps the @@ -106,18 +104,14 @@ export class ViewFileService { } }); - // Toggling nested navigation adds or drops every nested row, and staging - // changes what nested rows offer, so rebuild the whole view; the rebuild - // also prunes checked keys of rows that went away. + // Toggling nested navigation adds or drops every nested row, so rebuild + // the whole view; the rebuild also prunes checked keys of rows that went + // away. this.configService.config$.pipe( - map((config) => ({ - nested: !!config?.controller.enable_nested_navigation, - staging: !!(config?.controller.use_staging && config.controller.staging_path), - })), - distinctUntilChanged((a, b) => a.nested === b.nested && a.staging === b.staging), - ).subscribe(({ nested, staging }) => { + map((config) => !!config?.controller.enable_nested_navigation), + distinctUntilChanged(), + ).subscribe((nested) => { this.nestedNavEnabled = nested; - this.stagingEnabled = staging; if (this.prevModelFiles.size > 0) { this.buildViewFromModelFiles(this.prevModelFiles, true); } @@ -392,7 +386,7 @@ export class ViewFileService { } private createTopLevelViewFile(modelFile: ModelFile): ViewFile { - return createViewFile(modelFile, this.pairNameMap, this.nestedNavEnabled, this.stagingEnabled); + return createViewFile(modelFile, this.pairNameMap, this.nestedNavEnabled); } private pushViewFiles(): void { @@ -553,7 +547,6 @@ function createViewFile( modelFile: ModelFile, pairNameMap: Map, includeChildren: boolean, - stagingEnabled: boolean, isTopLevel = true, ancestorActive = false, ): ViewFile { @@ -586,7 +579,7 @@ function createViewFile( remoteSize, modelFile.local_size, modelFile.remote_size, - { isTopLevel, ancestorActive, hasDownloadingDescendant, stagingEnabled }, + { isTopLevel, ancestorActive, hasDownloadingDescendant }, ); // Kept out of deriveCapabilities: it needs the ModelFile for is_dir and the @@ -603,7 +596,7 @@ function createViewFile( ancestorActive || modelFile.state === ModelFileState.QUEUED || modelFile.state === ModelFileState.DOWNLOADING; const children = includeChildren ? modelFile.children.map((child) => - createViewFile(child, pairNameMap, true, stagingEnabled, false, childAncestorActive), + createViewFile(child, pairNameMap, true, false, childAncestorActive), ) : []; From f979c95b25df90e1e87ca8cf5b4f7c3e9ac97868 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:47:54 -0600 Subject: [PATCH 03/15] fix(python): keep stopped nested partials out of top-level staging moves A top-level staging->final move took the whole staging/ tree, so a stopped nested download under it (pget writes under the final name) was carried into the final dir as if complete. The pipeline now tracks nested downloads queued into staging, and a top-level move leaves behind any of them that are still short of their remote size or hold temp files. Complete ones, e.g. resumed by a later mirror of Top, move with the tree. Also makes find_overlapping_move public for restart recovery. --- src/python/controller/command_pipeline.py | 29 ++- src/python/controller/move/move_process.py | 87 ++++++-- src/python/controller/pair_context.py | 4 + .../test_controller/test_command_pipeline.py | 124 ++++++++++- .../test_controller/test_move_process.py | 194 ++++++++++++++++++ 5 files changed, 413 insertions(+), 25 deletions(-) diff --git a/src/python/controller/command_pipeline.py b/src/python/controller/command_pipeline.py index a5a5403f..3057fe9f 100644 --- a/src/python/controller/command_pipeline.py +++ b/src/python/controller/command_pipeline.py @@ -209,7 +209,7 @@ def _handle_queue( if file.remote_size is None: _notify_failure(command, f"File '{command.filename}' does not exist remotely") return False - moving = self._find_overlapping_move(command.filename, pc) + moving = self.find_overlapping_move(command.filename, pc) if moving is not None: _notify_failure(command, f"A staging move is in progress for '{moving}'") return False @@ -222,6 +222,7 @@ def _handle_queue( if is_nested_name(command.filename): if pc.effective_local_path != pc.local_path: pc.pending_nested_moves.add(command.filename) + pc.staged_nested.add(command.filename) # A nested job's move has no requeue diff to reset it # (ModelUpdater._handle_requeued_file sees top-level files only). move_key = persist_key(pc.pair_id, command.filename) @@ -231,7 +232,7 @@ def _handle_queue( self.sync_persist_callback() return True - def _find_overlapping_move(self, name: str, pc: PairContext) -> str | None: + def find_overlapping_move(self, name: str, pc: PairContext) -> str | None: """Name of an in-flight staging move for this pair that `name` lies inside or contains, or None. lftp writing into a tree mid-move would race it.""" for move_process in self.active_move_processes: @@ -442,6 +443,7 @@ def post_callback(delete_path: str = delete_path, _pc: PairContext = pc) -> None _pc.local_scan_process.force_scan() if delete_path != _pc.local_path: _pc.active_scan_process.force_scan() + self._sweep_staged_nested(_pc) command_wrapper = CommandProcessWrapper(process=process, post_callback=post_callback, filename=command.filename) self.active_command_processes.append(command_wrapper) @@ -714,6 +716,7 @@ def _finalize_move_process(self, move_process: MoveProcess) -> None: # pair; fall back to all pairs only if the owner can't be located. owner = next((pc for pc in self._pair_contexts if pc.pair_id == move_process.pair_id), None) if owner is not None: + self._sweep_staged_nested(owner) owner.local_scan_process.force_scan() else: for pc in self._pair_contexts: @@ -838,6 +841,7 @@ def spawn_move_process(self, file_name: str, pc: PairContext, expected_size: int pair_id=pair_id, umask=self._context.args.umask, expected_size=expected_size, + nested_expected_sizes=self._staged_nested_sizes(file_name, pc), ) process.set_mp_log_queue(self._mp_logger.queue, self._mp_logger.log_level) # Start before publishing bookkeeping: if start() raises we must not leave @@ -850,6 +854,27 @@ def spawn_move_process(self, file_name: str, pc: PairContext, expected_size: int self._logger.info(f"Spawned move process for {file_name} (staging -> local)") return True + def _staged_nested_sizes(self, file_name: str, pc: PairContext) -> dict[str, int | None]: + """{path relative to file_name: remote size} for each staged nested + download under file_name (None when the row can't be resolved).""" + sizes: dict[str, int | None] = {} + prefix = file_name + "/" + for name in pc.staged_nested: + if name.startswith(prefix): + try: + sizes[name[len(prefix) :]] = self._registry.resolve_full_path(name, pair_id=pc.pair_id).remote_size + except ModelError: + sizes[name[len(prefix) :]] = None + return sizes + + def _sweep_staged_nested(self, pc: PairContext) -> None: + """Forget staged nested downloads that are no longer in staging (moved, + deleted, or carried along by an ancestor's move).""" + staging = self._pair_staging_dir(pc) + pc.staged_nested.difference_update( + [name for name in pc.staged_nested if staging is None or not os.path.exists(os.path.join(staging, name))] + ) + def _get_pair_context_for_command(self, command: Command) -> PairContext | None: """Find the pair context for a command based on pair_id.""" return self.find_pair_by_id(command.pair_id) diff --git a/src/python/controller/move/move_process.py b/src/python/controller/move/move_process.py index d1f21708..1725ee1b 100644 --- a/src/python/controller/move/move_process.py +++ b/src/python/controller/move/move_process.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import datetime import errno import os @@ -39,6 +40,7 @@ def __init__( pair_id: str | None = None, umask: int | None = None, expected_size: int | None = None, + nested_expected_sizes: dict[str, int | None] | None = None, ): super().__init__(name=self.__class__.__name__) self.__source_path = source_path @@ -52,6 +54,10 @@ def __init__( # which a Stop or an error also does. A smaller or unfinalized source # was never completed, so it is left in staging for a re-queue to resume. self.__expected_size = expected_size + # For a top-level move: nested downloads under it that sit in staging + # ({path relative to file_name: remote size}). Any still incomplete + # stays in staging for its own job; complete ones move with the tree. + self.__nested_expected_sizes = nested_expected_sizes or {} self.__failed_result_stream = PipeStream() @property @@ -95,8 +101,9 @@ def _get_copied_size(src: str, dst: str) -> int: return total @staticmethod - def _has_lftp_temp_files(src: str) -> bool: - """True if the source tree still contains lftp temp files (``*.lftp``). + def _has_lftp_temp_files(src: str, skip: frozenset[str] = frozenset()) -> bool: + """True if the source tree still contains lftp temp files (``*.lftp``), + not counting anything under the paths in ``skip``. lftp writes each download to ``.lftp`` and renames it to the final name on completion. A staging tree that still holds temp files has not @@ -104,8 +111,12 @@ def _has_lftp_temp_files(src: str) -> bool: """ suffix = Constants.LFTP_TEMP_FILE_SUFFIX if os.path.isfile(src): - return src.endswith(suffix) - return any(any(f.endswith(suffix) for f in filenames) for _root, _dirs, filenames in os.walk(src)) + return src.endswith(suffix) and src not in skip + for root, dirs, filenames in os.walk(src): + dirs[:] = [d for d in dirs if os.path.join(root, d) not in skip] + if any(f.endswith(suffix) and os.path.join(root, f) not in skip for f in filenames): + return True + return False def _report_failure(self, message: str) -> None: """Log and publish a move failure so the controller can surface it.""" @@ -133,13 +144,15 @@ def run_once(self): self.logger.info(f"Move skipped for {self.__file_name}: download incomplete, left in staging") return + incomplete = self._incomplete_nested(src) + # Defer while lftp is still finalizing the staging tree. lftp renames # ".lftp" -> "" on completion; copying (or renaming the # whole tree) while a rename is in flight races — copytree lists the # ".lftp" entry, lftp renames it, and the copy then fails with ENOENT # (which used to crash this process). A tree that still has temp files is # not ready, so report a recoverable failure and retry once lftp finishes. - if self._has_lftp_temp_files(src): + if self._has_lftp_temp_files(src, incomplete): self._report_failure( f"Move deferred for {self.__file_name}: download not finalized " "(lftp temp files still present in staging). Will retry." @@ -151,13 +164,56 @@ def run_once(self): # Ensure the destination's parent exists (several levels for a nested name) os.makedirs(os.path.dirname(dst), exist_ok=True) + if not incomplete: + if not self._move_item(src, dst): + return + else: + if not self._move_tree(src, dst, incomplete): + return + left = ", ".join(sorted(os.path.relpath(p, self.__source_path) for p in incomplete)) + self.logger.info(f"Left incomplete nested downloads in staging: {left}") + self.__apply_modes(dst) + self.__prune_empty_staging_parents(src) + + def _incomplete_nested(self, src: str) -> frozenset[str]: + """Absolute staging paths of the listed nested downloads under src that + aren't complete yet (temp files, short, or of unknown size).""" + incomplete: set[str] = set() + for rel, size in self.__nested_expected_sizes.items(): + path = os.path.join(src, rel) + if os.path.exists(path) and ( + size is None or self._has_lftp_temp_files(path) or self._get_total_size(path) < size + ): + incomplete.add(path) + return frozenset(incomplete) + + def _move_tree(self, src: str, dst: str, incomplete: frozenset[str]) -> bool: + """Move src to dst, leaving the paths in ``incomplete`` (and the dirs + holding them) in staging. Subtrees without one move whole.""" + if src in incomplete: + return True + if not any(p.startswith(src + os.sep) for p in incomplete): + # Parents are made only once something lands in them, so a dir + # holding nothing but incomplete downloads isn't created at dst. + os.makedirs(os.path.dirname(dst), exist_ok=True) + return self._move_item(src, dst) + for entry in os.listdir(src): + if not self._move_tree(os.path.join(src, entry), os.path.join(dst, entry), incomplete): + return False + with contextlib.suppress(OSError): # still holds an incomplete download + os.rmdir(src) + return True + + def _move_item(self, src: str, dst: str) -> bool: + """Move one file or tree: rename, or copy + verify + delete across + devices or onto a non-empty dir. Reports and returns False on failure.""" + name = os.path.relpath(src, self.__source_path) + # Try rename first (instant on same filesystem) try: os.rename(src, dst) - self.logger.info(f"Move completed via rename: {self.__file_name}") - self.__apply_modes(dst) - self.__prune_empty_staging_parents(src) - return + self.logger.info(f"Move completed via rename: {name}") + return True except OSError as e: if e.errno == errno.EXDEV: self.logger.debug("Cross-device move detected, falling back to copy+delete") @@ -181,18 +237,18 @@ def run_once(self): # narrow window the pre-check above doesn't fully close). Report a # recoverable failure and keep the source intact so the move is # retried — never crash the controller. - self._report_failure(f"Move copy failed for {self.__file_name}: {e}. Source NOT deleted.") - return + self._report_failure(f"Move copy failed for {name}: {e}. Source NOT deleted.") + return False # Verify only the copied subtree (ignore pre-existing files under dst) copied_size = self._get_copied_size(src, dst) if source_size != copied_size: self._report_failure( - f"Move size verification failed for {self.__file_name}: " + f"Move size verification failed for {name}: " f"source={source_size} copied={copied_size}. Source NOT deleted." ) - return + return False # Size verified, remove source if os.path.isfile(src): @@ -200,9 +256,8 @@ def run_once(self): else: shutil.rmtree(src) - self.logger.info(f"Move completed via copy+delete: {self.__file_name}") - self.__apply_modes(dst) - self.__prune_empty_staging_parents(src) + self.logger.info(f"Move completed via copy+delete: {name}") + return True def __apply_modes(self, dst: str) -> None: if self.__umask is not None: diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index bac836db..30152412 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -54,6 +54,10 @@ class PairContext: # Nested names queued while staging is on, awaiting their staging->final # move once lftp no longer lists the job (see ModelUpdater._spawn_nested_moves) pending_nested_moves: set[str] = field(default_factory=set[str]) + # Nested names queued while staging is on whose download still sits in + # staging (finished or not). A top-level move leaves the incomplete ones + # behind (see CommandPipeline.spawn_move_process). + staged_nested: set[str] = field(default_factory=set[str]) remote_scan_received: bool = False local_scan_received: bool = False diff --git a/src/python/tests/unittests/test_controller/test_command_pipeline.py b/src/python/tests/unittests/test_controller/test_command_pipeline.py index 8b167469..5d928732 100644 --- a/src/python/tests/unittests/test_controller/test_command_pipeline.py +++ b/src/python/tests/unittests/test_controller/test_command_pipeline.py @@ -299,6 +299,20 @@ def test_finalize_rescans_only_owning_pair(self): owner.local_scan_process.force_scan.assert_called_once() other.local_scan_process.force_scan.assert_not_called() + def test_finalize_sweeps_staged_nested_for_owner_pair(self): + """Finishing a move must sweep the owner's staged_nested so entries the + move carried along (or left behind) are reconciled.""" + owner = self._make_pair_context("pair-1") + other = self._make_pair_context("pair-2") + pipeline = self._make_pipeline([owner, other]) + move_process = self._make_move_process("pair-1", "Top", failed_results=[]) + pipeline.active_move_processes.append(move_process) + + with patch.object(pipeline, "_sweep_staged_nested") as mock_sweep: + pipeline.cleanup() + + mock_sweep.assert_called_once_with(owner) + def test_finalize_rescans_all_pairs_when_owner_not_found(self): """If the move's pair_id matches no context, fall back to rescanning all.""" pc1 = self._make_pair_context("pair-1") @@ -362,6 +376,99 @@ def test_spawn_move_process_default_expected_size_is_none(self, mock_move_cls, _ self.assertIsNone(mock_move_cls.call_args.kwargs["expected_size"]) + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.MoveProcess") + def test_spawn_move_process_passes_nested_expected_sizes_to_move_process(self, mock_move_cls, _mock_exists): + """A top-level move must carry the staged nested downloads under it to + MoveProcess so an incomplete one can be left behind.""" + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.local_path = "/local/pair-1" + pc.staged_nested = {"Top/Sub/big.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + resolved = MagicMock() + resolved.remote_size = 999 + pipeline._registry.resolve_full_path.return_value = resolved + + pipeline.spawn_move_process("Top", pc) + + self.assertEqual({"Sub/big.bin": 999}, mock_move_cls.call_args.kwargs["nested_expected_sizes"]) + + # --- _staged_nested_sizes --- + + def test_staged_nested_sizes_filters_by_prefix_and_resolves_remote_size(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/Sub/big.bin", "Top/a.bin", "TopX/other.bin"} + pipeline = self._make_pipeline([pc]) + + remote_sizes = {"Top/Sub/big.bin": 500, "Top/a.bin": 10} + + def resolve(name, pair_id): + file = MagicMock() + file.remote_size = remote_sizes[name] + return file + + pipeline._registry.resolve_full_path.side_effect = resolve + + result = pipeline._staged_nested_sizes("Top", pc) + + self.assertEqual({"Sub/big.bin": 500, "a.bin": 10}, result) + resolved_names = {call.args[0] for call in pipeline._registry.resolve_full_path.call_args_list} + self.assertNotIn("TopX/other.bin", resolved_names) + + def test_staged_nested_sizes_model_error_maps_to_none(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/gone.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._registry.resolve_full_path.side_effect = ModelError("not found") + + result = pipeline._staged_nested_sizes("Top", pc) + + self.assertEqual({"gone.bin": None}, result) + + def test_staged_nested_sizes_empty_when_no_entries_under_file_name(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Other/a.bin"} + pipeline = self._make_pipeline([pc]) + + result = pipeline._staged_nested_sizes("Top", pc) + + self.assertEqual({}, result) + pipeline._registry.resolve_full_path.assert_not_called() + + # --- _sweep_staged_nested --- + + def test_sweep_staged_nested_drops_entries_no_longer_in_staging(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/kept.bin", "Top/gone.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + kept_path = os.path.join("/tmp/staging", "pair-1", "Top/kept.bin") + + with patch("controller.command_pipeline.os.path.exists", side_effect=lambda p: p == kept_path): + pipeline._sweep_staged_nested(pc) + + self.assertEqual({"Top/kept.bin"}, pc.staged_nested) + + def test_sweep_staged_nested_clears_all_when_staging_disabled(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/a.bin", "Top/b.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = False + pipeline._context.config.controller.staging_path = None + + pipeline._sweep_staged_nested(pc) + + self.assertEqual(set(), pc.staged_nested) + # --- propagate_exceptions: worker-fault isolation (#511) --- def test_propagate_exceptions_clean_when_no_worker_errors(self): @@ -717,6 +824,7 @@ def _make_pair_context(self, pair_id=None, effective_local_path="/local"): pc.local_path = "/local" pc.effective_local_path = effective_local_path pc.pending_nested_moves = set() + pc.staged_nested = set() return pc @staticmethod @@ -1354,6 +1462,7 @@ def test_nested_queue_under_staging_clears_move_state(self): self.assertNotIn(move_key, pipeline._persist.move_failed_file_names) pipeline.sync_persist_callback.assert_called_once() self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + self.assertEqual({"Top/leaf.rar"}, pc.staged_nested) def test_nested_queue_without_staging_adds_no_pending_move(self): pc = self._make_pair_context("pair-1") @@ -1366,6 +1475,7 @@ def test_nested_queue_without_staging_adds_no_pending_move(self): self.assertTrue(pipeline._handle_queue(command, leaf, pc, notify)) self.assertEqual(set(), pc.pending_nested_moves) + self.assertEqual(set(), pc.staged_nested) def test_top_level_queue_does_not_touch_move_state(self): """The move-state clearing is nested-only (top-level files get a @@ -1392,7 +1502,7 @@ def test_top_level_queue_does_not_touch_move_state(self): pipeline.sync_persist_callback.assert_not_called() self.assertEqual(set(), pc.pending_nested_moves) - # --- _find_overlapping_move --- + # --- find_overlapping_move --- @staticmethod def _fake_move_process(pair_id, file_name): @@ -1406,35 +1516,35 @@ def test_find_overlapping_move_same_name(self): pipeline = self._make_pipeline([pc], nested_enabled=True) pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top/Sub")) - self.assertEqual("Top/Sub", pipeline._find_overlapping_move("Top/Sub", pc)) + self.assertEqual("Top/Sub", pipeline.find_overlapping_move("Top/Sub", pc)) def test_find_overlapping_move_ancestor_moving_blocks_descendant_queue(self): pc = self._make_pair_context("pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) - self.assertEqual("Top", pipeline._find_overlapping_move("Top/Sub/f.bin", pc)) + self.assertEqual("Top", pipeline.find_overlapping_move("Top/Sub/f.bin", pc)) def test_find_overlapping_move_descendant_moving_blocks_ancestor_queue(self): pc = self._make_pair_context("pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top/Sub/f.bin")) - self.assertEqual("Top/Sub/f.bin", pipeline._find_overlapping_move("Top", pc)) + self.assertEqual("Top/Sub/f.bin", pipeline.find_overlapping_move("Top", pc)) def test_find_overlapping_move_unrelated_name_no_overlap(self): pc = self._make_pair_context("pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Other")) - self.assertIsNone(pipeline._find_overlapping_move("Top", pc)) + self.assertIsNone(pipeline.find_overlapping_move("Top", pc)) def test_find_overlapping_move_different_pair_no_overlap(self): pc = self._make_pair_context("pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) pipeline.active_move_processes.append(self._fake_move_process("pair-2", "Top")) - self.assertIsNone(pipeline._find_overlapping_move("Top", pc)) + self.assertIsNone(pipeline.find_overlapping_move("Top", pc)) def test_find_overlapping_move_prefix_name_not_overlap(self): """'Top' vs 'TopX' must not be treated as overlapping - a string @@ -1443,7 +1553,7 @@ def test_find_overlapping_move_prefix_name_not_overlap(self): pipeline = self._make_pipeline([pc], nested_enabled=True) pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) - self.assertIsNone(pipeline._find_overlapping_move("TopX", pc)) + self.assertIsNone(pipeline.find_overlapping_move("TopX", pc)) def test_handle_queue_refused_when_overlapping_move_in_progress(self): pc = self._make_pair_context("pair-1") diff --git a/src/python/tests/unittests/test_controller/test_move_process.py b/src/python/tests/unittests/test_controller/test_move_process.py index efe5e22c..14f8d8e9 100644 --- a/src/python/tests/unittests/test_controller/test_move_process.py +++ b/src/python/tests/unittests/test_controller/test_move_process.py @@ -435,3 +435,197 @@ def test_move_applies_umask_modes_at_destination(self): dst_tree = os.path.join(self.dst_dir, "Show.S01") self.assertEqual(0o777, stat.S_IMODE(os.stat(dst_tree).st_mode)) self.assertEqual(0o666, stat.S_IMODE(os.stat(os.path.join(dst_tree, "ep1.mkv")).st_mode)) + + # --- nested_expected_sizes: leave a stopped nested job's partial in staging --- + + def test_top_level_move_leaves_incomplete_nested_download_in_staging(self): + """Repro: a top-level staging->final move used to carry a stopped nested + partial out of staging into the final dir. With nested_expected_sizes set, + the complete sibling must still move while the nested partial under it + stays in staging for its own job to resume.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin"), "w") as f: + f.write("short") # 5 bytes, far under expected + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) + + def test_top_level_move_moves_everything_when_nested_download_complete(self): + """A nested download whose size has caught up to expected moves along + with the rest of the tree; nothing is left behind.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin"), "w") as f: + f.write("12345") # 5 bytes, matches expected + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 5}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + + def test_nested_expected_size_none_leaves_download_in_staging(self): + """An unresolved nested row (ModelError during lookup in the pipeline) + maps to a None size here, which is always treated as incomplete.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin"), "w") as f: + f.write("anything") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": None}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) + + def test_nested_lftp_temp_file_in_listed_path_skipped_without_deferral(self): + """An lftp temp file inside a listed (incomplete) nested path must not + defer the whole move.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub2")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub2", "x.bin.lftp"), "w") as f: + f.write("partial") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub2": 999}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub2", "x.bin.lftp"))) + + def test_temp_file_outside_listed_nested_paths_still_defers(self): + """A temp file outside any listed nested path still defers the whole + move as a recoverable failure (existing behaviour, preserved alongside + the nested-skip list).""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(top_dir) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "stray.mkv.lftp"), "w") as f: + f.write("partial") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={}, + pair_id="pair-5", + ) + self._run_process(process) + + failures = process.pop_failed() + self.assertEqual(1, len(failures)) + self.assertIn("not finalized", failures[0].error_message) + self.assertTrue(os.path.exists(top_dir)) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top"))) + + def test_nested_expected_sizes_entry_missing_from_staging_is_ignored(self): + """A listed nested path that no longer exists in staging (already moved + or never downloaded) is ignored - the top-level move proceeds normally.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(top_dir) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + + def test_sibling_subtree_without_exclusions_moves_whole_via_rename(self): + """A subtree containing no incomplete nested path moves as one rename + of the whole subtree, not file-by-file.""" + top_dir = os.path.join(self.src_dir, "Top") + other_dir = os.path.join(top_dir, "Other") + os.makedirs(other_dir) + with open(os.path.join(other_dir, "c.bin"), "w") as f: + f.write("complete") + sub_dir = os.path.join(top_dir, "Sub") + os.makedirs(sub_dir) + with open(os.path.join(sub_dir, "big.bin"), "w") as f: + f.write("short") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + dst_other = os.path.join(self.dst_dir, "Top", "Other") + with mock.patch("controller.move.move_process.os.rename", side_effect=os.rename) as mock_rename: + self._run_process(process) + + self.assertTrue(os.path.exists(os.path.join(dst_other, "c.bin"))) + self.assertIn(mock.call(other_dir, dst_other), mock_rename.call_args_list) + self.assertEqual([], process.pop_failed()) + + def test_cross_device_move_tree_leaves_partial_and_moves_rest(self): + """EXDEV from os.rename inside the tree move must fall back to + copy+delete per subtree, still leaving the incomplete nested path + behind.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(top_dir) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + sub_dir = os.path.join(top_dir, "Sub") + os.makedirs(sub_dir) + with open(os.path.join(sub_dir, "big.bin"), "w") as f: + f.write("short") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + with mock.patch("controller.move.move_process.os.rename", side_effect=OSError(errno.EXDEV, "cross-device")): + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) From d354675bf68ea3559a70e06a01827211ed428d67 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:48:10 -0600 Subject: [PATCH 04/15] test(python): check no empty dest dir is made for a left-behind partial --- src/python/tests/unittests/test_controller/test_move_process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/tests/unittests/test_controller/test_move_process.py b/src/python/tests/unittests/test_controller/test_move_process.py index 14f8d8e9..0765c253 100644 --- a/src/python/tests/unittests/test_controller/test_move_process.py +++ b/src/python/tests/unittests/test_controller/test_move_process.py @@ -462,6 +462,8 @@ def test_top_level_move_leaves_incomplete_nested_download_in_staging(self): self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) + # Sub holds only the partial, so no empty Sub is created at the destination + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub"))) def test_top_level_move_moves_everything_when_nested_download_complete(self): """A nested download whose size has caught up to expected moves along From 6d215b3b056b75ff3cee083d1c7be0f1098d27f6 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:34:39 -0600 Subject: [PATCH 05/15] fix(python): leave a nested download in staging when its remote size is unknown The nested move fell back to no size check when the row couldn't be resolved, so a stopped partial could be moved as if finished. Skip the move with a warning instead, matching how a top-level move treats an unknown-size nested partial. MoveProcess comments now state its own contract rather than its caller's. --- src/python/controller/model_updater.py | 5 +++++ src/python/controller/move/move_process.py | 11 +++++------ .../test_controller/test_model_updater.py | 19 +++++++++++++++---- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/python/controller/model_updater.py b/src/python/controller/model_updater.py index f34da143..2b6f4f85 100644 --- a/src/python/controller/model_updater.py +++ b/src/python/controller/model_updater.py @@ -600,6 +600,11 @@ def _spawn_nested_moves(self, pc: PairContext, lftp_job_names: set[str]) -> None expected_size = self._registry.resolve_full_path(name, pair_id=pc.pair_id).remote_size except ModelError: expected_size = None + if expected_size is None: + # Without the remote size a Stop's partial can't be told from a + # finished download, so leave it in staging rather than move it. + self._logger.warning(f"Not moving '{name}' out of staging: its remote size is unknown") + continue self._pipeline.spawn_move_process(name, pc, expected_size=expected_size) def sync_persist_to_all_builders(self): diff --git a/src/python/controller/move/move_process.py b/src/python/controller/move/move_process.py index 1725ee1b..efd8676e 100644 --- a/src/python/controller/move/move_process.py +++ b/src/python/controller/move/move_process.py @@ -50,13 +50,12 @@ def __init__( # When set, the moved tree is chmod'ed to 0666/0777 & ~umask here, in # this process, rather than on the controller thread (#811). self.__umask = umask - # Set for a nested job, whose move is triggered by its lftp job ending, - # which a Stop or an error also does. A smaller or unfinalized source - # was never completed, so it is left in staging for a re-queue to resume. + # When set, a source smaller than this or still holding lftp temp files + # is left in place, with no failure reported. self.__expected_size = expected_size - # For a top-level move: nested downloads under it that sit in staging - # ({path relative to file_name: remote size}). Any still incomplete - # stays in staging for its own job; complete ones move with the tree. + # {path relative to file_name: expected size} of descendants to check + # the same way; an incomplete one (or one of unknown size) is left in + # place while the rest of the tree moves. self.__nested_expected_sizes = nested_expected_sizes or {} self.__failed_result_stream = PipeStream() diff --git a/src/python/tests/unittests/test_controller/test_model_updater.py b/src/python/tests/unittests/test_controller/test_model_updater.py index 4cd1dbf7..aa41baad 100644 --- a/src/python/tests/unittests/test_controller/test_model_updater.py +++ b/src/python/tests/unittests/test_controller/test_model_updater.py @@ -343,10 +343,10 @@ def test_pending_nested_move_waits_while_lftp_lists_it(self): pipeline.spawn_move_process.assert_not_called() self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) - def test_pending_nested_move_model_error_uses_none_expected_size(self): + def test_pending_nested_move_with_unresolvable_row_stays_in_staging(self): """The row can be gone by the time this runs (e.g. a remote delete - raced the completion); fall back to expected_size=None rather than - failing to spawn the move at all.""" + raced the completion). Without a remote size a stopped partial can't be + told from a finished download, so nothing is moved.""" pc = self._make_pair_context("pair-1") pc.pending_nested_moves = {"Top/leaf.rar"} updater, pipeline, registry = self._make_updater([pc], self._make_persist()) @@ -354,7 +354,18 @@ def test_pending_nested_move_model_error_uses_none_expected_size(self): updater._detect_lftp_completions(pc, []) - pipeline.spawn_move_process.assert_called_once_with("Top/leaf.rar", pc, expected_size=None) + pipeline.spawn_move_process.assert_not_called() + self.assertEqual(set(), pc.pending_nested_moves) + + def test_pending_nested_move_with_no_remote_size_stays_in_staging(self): + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + registry.resolve_full_path.return_value.remote_size = None + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_not_called() def test_completion_without_pending_entry_spawns_nothing(self): """Only names queued under staging are moved: a nested job ending with From 70504533ea85bc6cb692e852aa375dcf60dae52b Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 14:29:37 -0600 Subject: [PATCH 06/15] fix(python): apply UMASK modes to single-file moves (#811) apply_umask_modes_to joined base with a single file's "." entry into "file/.", so chmod failed with ENOTDIR and the warning was swallowed: a moved file never got 0666 & ~umask. That hit every top-level single-file download, and every file the nested-staging tree move moves on its own. --- src/python/system/file_modes.py | 3 +- .../test_controller/test_move_process.py | 32 +++++++++++++++++++ .../unittests/test_system/test_file_modes.py | 14 ++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/python/system/file_modes.py b/src/python/system/file_modes.py index 991b11e0..6cfe751d 100644 --- a/src/python/system/file_modes.py +++ b/src/python/system/file_modes.py @@ -53,7 +53,8 @@ def apply_umask_modes_to(base: str, entries: Iterable[tuple[str, bool]], umask: dir_mode = 0o777 & ~umask # Deepest first, for the same reason apply_umask_modes walks bottom-up. for rel, is_dir in sorted(entries, key=lambda e: e[0].count(os.sep), reverse=True): - _chmod(os.path.join(base, rel), dir_mode if is_dir else file_mode, logger) + # normpath: a single file's entry is "." and "file/." is ENOTDIR + _chmod(os.path.normpath(os.path.join(base, rel)), dir_mode if is_dir else file_mode, logger) def relative_entries(root: str) -> list[tuple[str, bool]]: diff --git a/src/python/tests/unittests/test_controller/test_move_process.py b/src/python/tests/unittests/test_controller/test_move_process.py index 1df0a648..826a9826 100644 --- a/src/python/tests/unittests/test_controller/test_move_process.py +++ b/src/python/tests/unittests/test_controller/test_move_process.py @@ -662,3 +662,35 @@ def test_merge_into_existing_destination_leaves_other_files_alone(self): self.assertEqual(0o600, stat.S_IMODE(os.stat(private).st_mode)) # The moved file gets the UMASK treatment self.assertEqual(0o666, stat.S_IMODE(os.stat(os.path.join(dst_tree, "ep2.mkv")).st_mode)) + + def test_partial_tree_move_applies_umask_only_to_what_it_moved(self): + """#811 with a nested partial left behind: dirs the tree move creates at + the destination and the files it moves get the UMASK modes; a destination + dir that was already there keeps its own.""" + import stat + + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "New", "Sub")) + with open(os.path.join(top_dir, "New", "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "New", "Sub", "big.bin"), "w") as f: + f.write("short") + os.chmod(os.path.join(top_dir, "New", "a.bin"), 0o664) + dst_top = os.path.join(self.dst_dir, "Top") + os.makedirs(dst_top) + os.chmod(dst_top, 0o700) + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + umask=0o000, + nested_expected_sizes={"New/Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "New", "Sub", "big.bin"))) + self.assertEqual(0o700, stat.S_IMODE(os.stat(dst_top).st_mode)) + self.assertEqual(0o777, stat.S_IMODE(os.stat(os.path.join(dst_top, "New")).st_mode)) + self.assertEqual(0o666, stat.S_IMODE(os.stat(os.path.join(dst_top, "New", "a.bin")).st_mode)) diff --git a/src/python/tests/unittests/test_system/test_file_modes.py b/src/python/tests/unittests/test_system/test_file_modes.py index 31cefff7..ef5f76ee 100644 --- a/src/python/tests/unittests/test_system/test_file_modes.py +++ b/src/python/tests/unittests/test_system/test_file_modes.py @@ -104,3 +104,17 @@ def test_apply_to_entries_only_touches_listed_paths(self): self.assertEqual(0o666, _mode(os.path.join(self.root, "ep1.mkv"))) self.assertEqual(0o777, _mode(os.path.join(self.root, "Sample"))) self.assertEqual(0o777, _mode(self.root)) + + def test_apply_to_entries_single_file(self): + """A single file's entries are [(".", False)]; "file/." must not be chmod'ed.""" + from system.file_modes import apply_umask_modes_to, relative_entries + + single = os.path.join(self.tmp.name, "single.mkv") + with open(single, "w"): + pass + os.chmod(single, 0o664) + + with self.assertNoLogs(self.logger): + apply_umask_modes_to(single, relative_entries(single), 0o000, self.logger) + + self.assertEqual(0o666, _mode(single)) From b3fd0c9d5d9bef4824f19e126ed183b1ee516b60 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:50:31 -0600 Subject: [PATCH 07/15] fix(python): show a nested download's size while it sits in staging A stopped nested partial left in staging showed local size 0: the local scan covers only local_path, and nested names were kept out of the active scanner. A top-level partial keeps showing its staged size through pending_completion. Feed staged_nested to the active scanner too, and have the model builder put each nested result on its child row instead of the top level. A root can now turn Downloaded while a nested download inside it is still in staging with its own move in flight, so the root's DOWNLOADED move is skipped while an overlapping move runs, rather than racing it. --- src/python/controller/model_builder.py | 15 +++-- src/python/controller/model_updater.py | 12 +++- .../test_controller/test_model_builder.py | 34 +++++++++++ .../test_controller/test_model_updater.py | 57 +++++++++++++++++++ 4 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/python/controller/model_builder.py b/src/python/controller/model_builder.py index bdfe3140..992d4a9d 100644 --- a/src/python/controller/model_builder.py +++ b/src/python/controller/model_builder.py @@ -30,6 +30,7 @@ def __init__(self, pair_id: str | None = None) -> None: self.__pair_id = pair_id self.__local_files: dict[str, SystemFile] = {} self.__active_files: dict[str, SystemFile] = {} + self.__staged_nested_files: dict[str, SystemFile] = {} self.__remote_files: dict[str, SystemFile] = {} self.__lftp_statuses: dict[str, LftpJobStatus] = {} self.__downloaded_files: set[str] = set() @@ -48,12 +49,15 @@ def set_base_logger(self, base_logger: logging.Logger): self.logger = base_logger.getChild("ModelBuilder") def set_active_files(self, active_files: list[SystemFile]): - prev_active_files = self.__active_files - self.__active_files = {file.name: file for file in active_files} + prev = (self.__active_files, self.__staged_nested_files) + # A nested name ("Top/Sub/f.bin") is a nested download in staging; it + # overrides that child's local side, not a top-level entry. + self.__active_files = {f.name: f for f in active_files if not is_nested_name(f.name)} + self.__staged_nested_files = {f.name: f for f in active_files if is_nested_name(f.name)} # Always invalidate when active files present (sizes change rapidly) # Also invalidate when active files change (e.g. files removed after # a stopped download is deleted locally) - if len(active_files) > 0 or self.__active_files != prev_active_files: + if len(active_files) > 0 or (self.__active_files, self.__staged_nested_files) != prev: self.__cached_model = None def set_local_files(self, local_files: list[SystemFile]): @@ -141,6 +145,7 @@ def set_auto_delete_remote(self, enabled: bool): def clear(self): self.__local_files.clear() self.__active_files.clear() + self.__staged_nested_files.clear() self.__remote_files.clear() self.__lftp_statuses.clear() self.__downloaded_files.clear() @@ -273,7 +278,9 @@ def _build_children( _all_children_names: set[str] = set[str]().union(_remote_children.keys(), _local_children.keys()) for _child_name in _all_children_names: _remote_child = _remote_children.get(_child_name) - _local_child = _local_children.get(_child_name) + _local_child = self.__staged_nested_files.get( + os.path.join(_model_file.full_path, _child_name), _local_children.get(_child_name) + ) _is_dir = _remote_child.is_dir if _remote_child else (_local_child.is_dir if _local_child else False) # sanity check is_dir if (_remote_child and _is_dir != _remote_child.is_dir) or ( diff --git a/src/python/controller/model_updater.py b/src/python/controller/model_updater.py index 2b6f4f85..5e6f989a 100644 --- a/src/python/controller/model_updater.py +++ b/src/python/controller/model_updater.py @@ -252,7 +252,13 @@ def _handle_downloaded_file(self, diff: ModelDiff, pc: PairContext | None) -> No if self._context.config.controller.use_staging and self._context.config.controller.staging_path: will_auto_extract = self._context.config.autoqueue.auto_extract and diff.new_file.is_extractable - if not will_auto_extract and not will_auto_validate: + # A nested download still in staging counts toward its root, so the + # root can turn Downloaded while that download's own move is in + # flight. Leave the files to that move rather than race it. + overlapping = self._pipeline.find_overlapping_move(diff.new_file.name, pc) + if overlapping is not None: + self._logger.info(f"Not moving '{diff.new_file.name}': '{overlapping}' is moving out of staging") + elif not will_auto_extract and not will_auto_validate: self._pipeline.spawn_move_process(diff.new_file.name, pc) def _handle_pending_completion(self, diff: ModelDiff, pc: PairContext | None) -> None: @@ -524,6 +530,10 @@ def _update_pair_model_state( active_files = pc.active_downloading_file_names + pc.active_extracting_file_names active_files += list(pc.pending_completion) + # Nested downloads in staging, finished or not: the local scan covers + # only local_path, so this is the only scan that sees them. The builder + # puts each on its nested row, not at the top level. + active_files += list(pc.staged_nested) pc.active_scanner.set_active_files(active_files) pc.model_builder.set_auto_delete_remote(bool(self._context.config.autoqueue.auto_delete_remote)) diff --git a/src/python/tests/unittests/test_controller/test_model_builder.py b/src/python/tests/unittests/test_controller/test_model_builder.py index c0bd4757..f8b18209 100644 --- a/src/python/tests/unittests/test_controller/test_model_builder.py +++ b/src/python/tests/unittests/test_controller/test_model_builder.py @@ -2246,6 +2246,40 @@ def test_nested_child_move_failed_stays_downloading_with_active_job(self): leaf = next(c for c in sub.get_children() if c.name == "f.bin") self.assertEqual(ModelFile.State.DOWNLOADING, leaf.state) + def test_staged_nested_file_shows_its_staging_size(self): + """A nested download sitting in staging (e.g. a stopped partial) shows + its staged size, like a top-level one does, and stays a nested row.""" + r_top = SystemFile("Top", 1000, True) + r_sub = SystemFile("Sub", 1000, True) + r_sub.add_child(SystemFile("f.bin", 1000, False)) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + self.model_builder.set_active_files([SystemFile("Top/Sub/f.bin", 400, False)]) + + model = self.model_builder.build_model() + + self.assertEqual({"Top"}, set(model.get_file_names())) + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(400, leaf.local_size) + + def test_staged_nested_dir_shows_its_staging_tree(self): + r_top = SystemFile("Top", 1000, True) + r_sub = SystemFile("Sub", 1000, True) + r_sub.add_child(SystemFile("f.bin", 1000, False)) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + s_sub = SystemFile("Top/Sub", 300, True) + s_sub.add_child(SystemFile("f.bin", 300, False)) + self.model_builder.set_active_files([s_sub]) + + model = self.model_builder.build_model() + + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(300, sub.local_size) + self.assertEqual(300, leaf.local_size) + def test_nested_status_never_creates_a_phantom_top_level_entry(self): """A '/'-containing status name is always routed to its nested child, even with nested navigation off (e.g. a nested job still running diff --git a/src/python/tests/unittests/test_controller/test_model_updater.py b/src/python/tests/unittests/test_controller/test_model_updater.py index 4cce5ebf..0aab5f04 100644 --- a/src/python/tests/unittests/test_controller/test_model_updater.py +++ b/src/python/tests/unittests/test_controller/test_model_updater.py @@ -231,6 +231,22 @@ def test_pending_completion_excludes_nested_names_on_job_completion(self): self.assertEqual({"TopLevel"}, pc.pending_completion) + def test_staged_nested_names_fed_to_active_scanner(self): + """The active scanner is rooted at staging when staging is on, so a + nested download still there (e.g. a stopped partial) is stat'd like a + top-level one in pending_completion, and its row shows its size.""" + pc = self._make_pair_context("pair-1") + pc.staged_nested = {"TopDir/nested_file.rar"} + pc.active_extracting_file_names = [] + for proc in (pc.remote_scan_process, pc.local_scan_process, pc.active_scan_process): + proc.pop_latest_result.return_value = None + pc.lftp.status.return_value = [] + updater = self._make_updater([pc]) + + updater._update_pair_model_state(pc, None, None) + + pc.active_scanner.set_active_files.assert_called_once_with(["TopDir/nested_file.rar"]) + def test_nested_completion_not_added_to_downloaded_file_names(self): """Nothing reads a nested persist key (model_builder's persist-authority check matches top-level names only) and nothing ever removes one, so @@ -775,3 +791,44 @@ def test_skips_when_umask_is_unset(self): apply, threads = self._run(None) apply.assert_not_called() self.assertEqual([], threads) + + +class TestHandleDownloadedFileOverlappingMove(unittest.TestCase): + """A root can turn Downloaded while a nested download inside it is still + in staging, its own move in flight. The root's move would race that one, + so it is left to the nested move.""" + + def _run(self, overlapping_move): + pc = MagicMock() + context = MagicMock() + context.args.umask = None + context.config.validate.enabled = False + context.config.controller.use_staging = True + context.config.controller.staging_path = "/staging" + context.config.autoqueue.auto_extract = False + pipeline = MagicMock() + pipeline.find_overlapping_move.return_value = overlapping_move + updater = ModelUpdater( + pair_contexts=[pc], + persist=MagicMock(), + pipeline=pipeline, + registry=MagicMock(), + extract_process=MagicMock(), + validate_process=MagicMock(), + context=context, + password=None, + logger=MagicMock(), + ) + new_file = ModelFile("Show", True) + new_file.state = ModelFile.State.DOWNLOADED + updater._handle_downloaded_file(ModelDiff(ModelDiff.Change.ADDED, old_file=None, new_file=new_file), pc) + return pipeline, pc + + def test_root_move_skipped_while_a_nested_move_inside_it_is_in_flight(self): + pipeline, pc = self._run("Show/Season 1") + pipeline.find_overlapping_move.assert_called_once_with("Show", pc) + pipeline.spawn_move_process.assert_not_called() + + def test_root_move_spawned_without_an_overlapping_move(self): + pipeline, pc = self._run(None) + pipeline.spawn_move_process.assert_called_once_with("Show", pc) From 04260b4f3040b4481772b46c321c3576c7015f48 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:08:28 -0600 Subject: [PATCH 08/15] fix(python): treat a temp-named nested partial as incomplete in staging moves With lftp temp files on, a stopped nested file sits in staging only as ".lftp". _incomplete_nested checked just the final name, so the partial wasn't left behind: its temp file deferred the whole top-level move, stranding complete siblings and retrying as MOVE_FAILED. The sweep also dropped the name from staged_nested. Both now count the temp file. --- src/python/controller/command_pipeline.py | 14 +++++++++-- src/python/controller/move/move_process.py | 6 ++++- .../test_controller/test_command_pipeline.py | 15 ++++++++++++ .../test_controller/test_move_process.py | 24 +++++++++++++++++++ 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/src/python/controller/command_pipeline.py b/src/python/controller/command_pipeline.py index 3057fe9f..dd7d132d 100644 --- a/src/python/controller/command_pipeline.py +++ b/src/python/controller/command_pipeline.py @@ -14,7 +14,7 @@ from collections.abc import Callable, Iterator from queue import Queue -from common import Context, MultiprocessingLogger, UnrecoverableError +from common import Constants, Context, MultiprocessingLogger, UnrecoverableError from lftp import LftpError from model import ModelError, ModelFile, is_nested_name @@ -871,8 +871,18 @@ def _sweep_staged_nested(self, pc: PairContext) -> None: """Forget staged nested downloads that are no longer in staging (moved, deleted, or carried along by an ancestor's move).""" staging = self._pair_staging_dir(pc) + if staging is None: + pc.staged_nested.clear() + return + # An unfinished file with lftp temp files on is ".lftp" + suffix = Constants.LFTP_TEMP_FILE_SUFFIX pc.staged_nested.difference_update( - [name for name in pc.staged_nested if staging is None or not os.path.exists(os.path.join(staging, name))] + [ + name + for name in pc.staged_nested + if not os.path.exists(os.path.join(staging, name)) + and not os.path.exists(os.path.join(staging, name + suffix)) + ] ) def _get_pair_context_for_command(self, command: Command) -> PairContext | None: diff --git a/src/python/controller/move/move_process.py b/src/python/controller/move/move_process.py index aa266289..965d1be9 100644 --- a/src/python/controller/move/move_process.py +++ b/src/python/controller/move/move_process.py @@ -180,7 +180,11 @@ def _incomplete_nested(self, src: str) -> frozenset[str]: incomplete: set[str] = set() for rel, size in self.__nested_expected_sizes.items(): path = os.path.join(src, rel) - if os.path.exists(path) and ( + temp_path = path + Constants.LFTP_TEMP_FILE_SUFFIX + if not os.path.exists(path) and os.path.isfile(temp_path): + # An unfinished file with lftp temp files on: ".lftp" until done + incomplete.add(temp_path) + elif os.path.exists(path) and ( size is None or self._has_lftp_temp_files(path) or self._get_total_size(path) < size ): incomplete.add(path) diff --git a/src/python/tests/unittests/test_controller/test_command_pipeline.py b/src/python/tests/unittests/test_controller/test_command_pipeline.py index 5d928732..e5745c54 100644 --- a/src/python/tests/unittests/test_controller/test_command_pipeline.py +++ b/src/python/tests/unittests/test_controller/test_command_pipeline.py @@ -457,6 +457,21 @@ def test_sweep_staged_nested_drops_entries_no_longer_in_staging(self): self.assertEqual({"Top/kept.bin"}, pc.staged_nested) + def test_sweep_staged_nested_keeps_entries_only_in_staging_as_temp_files(self): + """With lftp temp files on, an unfinished nested file is '.lftp'.""" + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/partial.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + temp_path = os.path.join("/tmp/staging", "pair-1", "Top/partial.bin.lftp") + + with patch("controller.command_pipeline.os.path.exists", side_effect=lambda p: p == temp_path): + pipeline._sweep_staged_nested(pc) + + self.assertEqual({"Top/partial.bin"}, pc.staged_nested) + def test_sweep_staged_nested_clears_all_when_staging_disabled(self): pc = self._make_pair_context("pair-1") pc.pair_id = "pair-1" diff --git a/src/python/tests/unittests/test_controller/test_move_process.py b/src/python/tests/unittests/test_controller/test_move_process.py index 826a9826..00b3cdfe 100644 --- a/src/python/tests/unittests/test_controller/test_move_process.py +++ b/src/python/tests/unittests/test_controller/test_move_process.py @@ -465,6 +465,30 @@ def test_top_level_move_leaves_incomplete_nested_download_in_staging(self): # Sub holds only the partial, so no empty Sub is created at the destination self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub"))) + def test_top_level_move_leaves_temp_named_nested_partial_in_staging(self): + """With lftp temp files on, a stopped nested file exists only as + '.lftp'. It is the incomplete entry: left behind, and it doesn't + defer the rest of the tree.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin.lftp"), "w") as f: + f.write("short") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin.lftp"))) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub"))) + def test_top_level_move_moves_everything_when_nested_download_complete(self): """A nested download whose size has caught up to expected moves along with the rest of the tree; nothing is left behind.""" From 6c8b979b834882648b9b0cc5b4f3b8f1859b25ec Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:08:28 -0600 Subject: [PATCH 09/15] fix(python): report a timed-out lftp status as unavailable A "jobs -v" timeout read back as empty output, which parses as no jobs. Callers then took every running job for finished: a pending nested move was dropped (and its file stranded in staging), and top-level downloads were logged and persisted as complete. status() now returns None, which callers already treat as status unavailable. --- src/python/lftp/lftp.py | 8 ++++++- .../unittests/test_lftp/test_lftp_spawn.py | 21 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/src/python/lftp/lftp.py b/src/python/lftp/lftp.py index cfa77731..9cdac8dd 100644 --- a/src/python/lftp/lftp.py +++ b/src/python/lftp/lftp.py @@ -113,6 +113,8 @@ def __init__( self.__timeout = 10 # in seconds self.__consecutive_status_errors = 0 self.__consecutive_timeouts = 0 + # Set by __run_command: its "" on a timeout reads the same as no output + self.__last_command_timed_out = False self.__settings_cache: dict[str, str] = {} self.__log_command_output = False @@ -289,6 +291,7 @@ def __run_command(self, command: str): # type: ignore[arg-type] self.__process.expect(self.__expect_pattern, timeout=self.__timeout) except pexpect.exceptions.TIMEOUT: timed_out = True + self.__last_command_timed_out = timed_out if timed_out: self.__consecutive_timeouts += 1 @@ -577,12 +580,15 @@ def xfer_verify_command(self, command: str): def status(self) -> list[LftpJobStatus] | None: """ Return a status list of queued and running jobs. - Returns None when the status output could not be parsed, so callers + Returns None when the status output timed out or could not be parsed, so callers can distinguish "no jobs" from "parse failed" and avoid false completion signals. :return: """ out = self.__run_command("jobs -v") # type: ignore[arg-type] + if self.__last_command_timed_out: + # Empty output here isn't "no jobs"; it would read as every job finished + return None try: statuses = self.__job_status_parser.parse(out) self.__consecutive_status_errors = 0 diff --git a/src/python/tests/unittests/test_lftp/test_lftp_spawn.py b/src/python/tests/unittests/test_lftp/test_lftp_spawn.py index 139d490e..44dd18e6 100644 --- a/src/python/tests/unittests/test_lftp/test_lftp_spawn.py +++ b/src/python/tests/unittests/test_lftp/test_lftp_spawn.py @@ -292,3 +292,24 @@ def test_existing_sftp_errors_still_detected(self): def test_clean_output_not_flagged(self): self.assertFalse(self._detect("`file' at 100 (50%) 1.2M/s eta:30s [Receiving data/TLS]")) + + +class TestLftpStatusTimeout(unittest.TestCase): + """A timed-out 'jobs -v' reads back as empty output, the same as no jobs. + status() must report it as unavailable (None), or callers take every + running job for finished.""" + + def test_status_is_none_when_jobs_command_times_out(self): + lftp, fake = make_lftp(address="host.example.com", port=22, user="bob", password=None) + with patch.object(fake, "expect", side_effect=lftp_mod.pexpect.exceptions.TIMEOUT("timed out")): + self.assertIsNone(lftp.status()) + + def test_status_is_empty_list_when_there_are_no_jobs(self): + lftp, _fake = make_lftp(address="host.example.com", port=22, user="bob", password=None) + self.assertEqual([], lftp.status()) + + def test_status_recovers_after_a_timeout(self): + lftp, fake = make_lftp(address="host.example.com", port=22, user="bob", password=None) + with patch.object(fake, "expect", side_effect=lftp_mod.pexpect.exceptions.TIMEOUT("timed out")): + lftp.status() + self.assertEqual([], lftp.status()) From 3b07792c03deee764269eddd5b9dc6e6a5edb3d1 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:14:50 -0600 Subject: [PATCH 10/15] refactor(python): state spawn_move_process and find_overlapping_move contracts in their docstrings Both described a caller (a nested job that just ended; lftp racing a queue) rather than what the method does; find_overlapping_move now also guards the root's DOWNLOADED move. --- src/python/controller/command_pipeline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/python/controller/command_pipeline.py b/src/python/controller/command_pipeline.py index dd7d132d..13435436 100644 --- a/src/python/controller/command_pipeline.py +++ b/src/python/controller/command_pipeline.py @@ -233,8 +233,8 @@ def _handle_queue( return True def find_overlapping_move(self, name: str, pc: PairContext) -> str | None: - """Name of an in-flight staging move for this pair that `name` lies inside - or contains, or None. lftp writing into a tree mid-move would race it.""" + """Name of an in-flight staging move for this pair that `name` is, lies + inside, or contains, or None.""" for move_process in self.active_move_processes: if move_process.pair_id != pc.pair_id: continue @@ -813,7 +813,8 @@ def spawn_move_process(self, file_name: str, pc: PairContext, expected_size: int staging). A retry of a failed move uses this to tell a real retry from a resolved no-op so it can clear a stale MOVE_FAILED state (#536 follow-up). - expected_size (see MoveProcess) is set for a nested job that just ended. + expected_size is passed to MoveProcess: a source short of it, or still + holding lftp temp files, is left in staging with no failure reported. """ pair_id = pc.pair_id move_key = persist_key(pair_id, file_name) From 0086c5a65d78fc37ed381457e88bdbae6069ea5b Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:14:50 -0600 Subject: [PATCH 11/15] test(python): cover the staged_nested sweep after a Delete Local in staging --- .../test_controller/test_command_pipeline.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/python/tests/unittests/test_controller/test_command_pipeline.py b/src/python/tests/unittests/test_controller/test_command_pipeline.py index e5745c54..660c05d8 100644 --- a/src/python/tests/unittests/test_controller/test_command_pipeline.py +++ b/src/python/tests/unittests/test_controller/test_command_pipeline.py @@ -1228,6 +1228,32 @@ def test_handle_delete_local_nested_uses_full_path(self, mock_delete_cls): self.assertTrue(result) mock_delete_cls.assert_called_once_with(local_path="/local", file_name="Top/leaf.rar") + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.DeleteLocalProcess") + def test_handle_delete_local_of_staged_nested_partial_sweeps_staged_nested(self, mock_delete_cls, _mock_exists): + """A stopped nested partial in staging is deleted there, and the sweep + afterwards forgets it so it isn't scanned or size-checked any more.""" + pc = self._make_pair_context("pair-1") + pc.local_path = "/local" + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + + leaf = ModelFile("leaf.rar", False) + leaf.state = ModelFile.State.DEFAULT + leaf.local_size = 40 + command = Command(Command.Action.DELETE_LOCAL, "Top/leaf.rar", pair_id="pair-1") + _calls, notify = self._make_notify() + + result = pipeline._handle_delete_local(command, leaf, pc, [], notify) + + self.assertTrue(result) + staging = os.path.join("/tmp/staging", "pair-1") + mock_delete_cls.assert_called_once_with(local_path=staging, file_name="Top/leaf.rar") + with patch.object(pipeline, "_sweep_staged_nested") as mock_sweep: + pipeline.active_command_processes[0].post_callback() + mock_sweep.assert_called_once_with(pc) + @patch("controller.command_pipeline.DeleteRemoteProcess") def test_handle_delete_remote_nested_uses_full_path(self, mock_delete_cls): pc = self._make_pair_context("pair-1") From 17fd924c5bde96f8a4d44f2741e08f5c70e9a7e7 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:35:43 -0600 Subject: [PATCH 12/15] fix(python): show and move top-level downloads left in staging after a restart pending_completion, which keeps the active scanner on a download in staging until it moves, is in memory only. After a restart the local scan (local_path only) couldn't see what was left in staging: a finished but unmoved download, recorded as downloaded with no local copy, showed as Deleted and was never moved; a stopped partial showed no size. Seed pending_completion from the pair's staging dir when its context is built (lftp's ".lftp" temp file counts as ), and hand the names to the active scanner then so its first scan covers them, ahead of the first remote scan. --- src/python/controller/controller.py | 11 +++++- src/python/controller/pair_context.py | 16 +++++++++ .../test_controller_lftp_wiring.py | 34 +++++++++++++++++++ .../test_controller/test_pair_context.py | 20 +++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/python/controller/controller.py b/src/python/controller/controller.py index 8a0c9de9..6c15f124 100644 --- a/src/python/controller/controller.py +++ b/src/python/controller/controller.py @@ -19,7 +19,7 @@ from .model_builder import ModelBuilder from .model_registry import ModelRegistry from .model_updater import ModelUpdater -from .pair_context import ControllerError, PairContext, configure_lftp, validate_config +from .pair_context import ControllerError, PairContext, configure_lftp, staged_top_level_names, validate_config from .persist_sync import PersistSync from .scan import ActiveScanner, LocalScanner, RemoteScanner, ScannerProcess from .validate import ValidateProcess @@ -212,6 +212,14 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, # (_LFTP_TUNING_KEYS), so it can change after these are built, and # temp files from an earlier interrupted download can outlive it. active_scanner = ActiveScanner(effective_local_path, lftp_temp_suffix=Constants.LFTP_TEMP_FILE_SUFFIX) + # pending_completion is in memory only: re-seed it with what a previous + # session left in staging (finished or stopped), so the active scanner + # shows it and a finished one still gets its move. The scanner gets the + # names now so its first scan, not the next one, covers them. + left_in_staging: set[str] = ( + staged_top_level_names(effective_local_path) if effective_local_path != local_path else set() + ) + active_scanner.set_active_files(sorted(left_in_staging)) local_scanner = LocalScanner(local_path=local_path) remote_scanner = RemoteScanner( remote_address=self.__context.config.lftp.remote_address, # type: ignore[arg-type] @@ -270,6 +278,7 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, local_scan_process=local_scan_process, remote_scan_process=remote_scan_process, model_builder=model_builder, + pending_completion=left_in_staging, ) def _configure_lftp(self, lftp: Lftp): diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index 218d764b..1f3b4142 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os from dataclasses import dataclass, field from common import AppError, Config, Constants, Context @@ -59,6 +60,21 @@ class PairContext: latest_local_scan: ScannerResult | None = None +def staged_top_level_names(staging_dir: str) -> set[str]: + """Names of the top-level downloads with something in staging_dir. + + lftp's ".lftp" temp file is listed as ; its + ".lftp-pget-status" progress file is skipped. A missing dir lists + nothing. + """ + try: + entries = os.listdir(staging_dir) + except FileNotFoundError: + return set() + suffix = Constants.LFTP_TEMP_FILE_SUFFIX + return {entry.removesuffix(suffix) for entry in entries if not entry.endswith(suffix + "-pget-status")} + + def validate_config(context: Context) -> None: """Validate that all required config fields are set (non-None) at startup. diff --git a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py index 0d99fa93..ac879733 100644 --- a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py +++ b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py @@ -4,6 +4,8 @@ protocol, port selection, and SSL params, while keeping the scanner on SSH.""" import logging +import os +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -129,6 +131,38 @@ def test_active_scanner_recognises_lftp_temp_files_with_use_temp_file_off(self): _, kwargs = mock_active_scanner.call_args self.assertEqual(Constants.LFTP_TEMP_FILE_SUFFIX, kwargs["lftp_temp_suffix"]) + def _pending_completion_for(self, ctx: Context): + controller = self._build_controller(ctx) + with ( + patch("controller.controller.Lftp"), + patch("controller.controller.RemoteScanner"), + patch("controller.controller.ActiveScanner") as mock_active_scanner, + patch("controller.controller.LocalScanner"), + patch("controller.controller.ScannerProcess"), + patch("controller.controller.ModelBuilder"), + patch("controller.controller.configure_lftp"), + ): + pc = controller._create_pair_context(pair_id="p1", name="pair", remote_path="/remote", local_path="/local") + # The scanner's first scan covers them too, ahead of the first update cycle + mock_active_scanner.return_value.set_active_files.assert_called_once_with(sorted(pc.pending_completion)) + return pc.pending_completion + + def test_downloads_left_in_staging_are_scanned_after_a_restart(self): + """pending_completion is in memory only, so a download that finished + or was stopped in staging before a restart would otherwise go unseen: + the local scan covers only local_path.""" + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + with tempfile.TemporaryDirectory() as staging: + ctx.config.controller.use_staging = True + ctx.config.controller.staging_path = staging + os.makedirs(os.path.join(staging, "p1", "Show.S01")) + + self.assertEqual({"Show.S01"}, self._pending_completion_for(ctx)) + + def test_nothing_seeded_with_staging_off(self): + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + self.assertEqual(set(), self._pending_completion_for(ctx)) + if __name__ == "__main__": unittest.main() diff --git a/src/python/tests/unittests/test_controller/test_pair_context.py b/src/python/tests/unittests/test_controller/test_pair_context.py index 98999aa0..002cd8c7 100644 --- a/src/python/tests/unittests/test_controller/test_pair_context.py +++ b/src/python/tests/unittests/test_controller/test_pair_context.py @@ -1,5 +1,7 @@ # Copyright 2017, Inderpreet Singh, All rights reserved. +import os +import tempfile import unittest from unittest.mock import MagicMock @@ -8,6 +10,7 @@ ControllerError, PairContext, configure_lftp, + staged_top_level_names, validate_config, ) @@ -341,3 +344,20 @@ def test_ftps_passes_persisted_values_through(self): self.assertEqual(lftp.num_connections_per_root_file, 20) self.assertEqual(lftp.num_connections_per_dir_file, 20) self.assertEqual(lftp.num_max_total_connections, 0) + + +class TestStagedTopLevelNames(unittest.TestCase): + def test_lists_downloads_in_staging_by_their_final_name(self): + """lftp's ".lftp" temp file is the download ; its + ".lftp-pget-status" progress file is not a download.""" + with tempfile.TemporaryDirectory() as staging: + os.makedirs(os.path.join(staging, "Show.S01")) + for name in ("movie.mkv", "partial.mkv.lftp", "movie.mkv.lftp-pget-status"): + with open(os.path.join(staging, name), "w") as f: + f.write("x") + + self.assertEqual({"Show.S01", "movie.mkv", "partial.mkv"}, staged_top_level_names(staging)) + + def test_missing_staging_dir_lists_nothing(self): + with tempfile.TemporaryDirectory() as parent: + self.assertEqual(set(), staged_top_level_names(os.path.join(parent, "nope"))) From 1482f3b010694b5160669f04122b0b24582575dc Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:41:06 -0600 Subject: [PATCH 13/15] fix(python): pick up nested downloads left in staging after a restart staged_nested and pending_nested_moves were in memory only, so after a restart a nested download still in staging was neither shown nor moved until it was queued again. Persist staged_nested (new optional "staged_nested" key; older files load an empty set and older versions ignore it), and at startup restore it, drop names no longer in staging, and queue each one's size-checked move. A top-level name holding a restored nested download is taken off the pending_completion seeded from staging: its staging copy holds only the nested downloads and would otherwise replace its local_path copy in the model. --- src/python/controller/command_pipeline.py | 44 +++++++--- src/python/controller/controller.py | 2 + src/python/controller/controller_persist.py | 6 +- .../test_controller/test_command_pipeline.py | 81 +++++++++++++++++++ .../test_controller_persist.py | 19 +++++ 5 files changed, 140 insertions(+), 12 deletions(-) diff --git a/src/python/controller/command_pipeline.py b/src/python/controller/command_pipeline.py index 13435436..1079039c 100644 --- a/src/python/controller/command_pipeline.py +++ b/src/python/controller/command_pipeline.py @@ -26,7 +26,7 @@ from .model_registry import ModelRegistry from .move import MoveProcess from .pair_context import PairContext -from .persist_keys import persist_key +from .persist_keys import persist_key, strip_persist_key from .validate import ValidateProcess, ValidateRequest from .worker_supervisor import WorkerSupervisor @@ -223,6 +223,7 @@ def _handle_queue( if pc.effective_local_path != pc.local_path: pc.pending_nested_moves.add(command.filename) pc.staged_nested.add(command.filename) + self._persist.staged_nested_file_names.add(persist_key(pc.pair_id, command.filename)) # A nested job's move has no requeue diff to reset it # (ModelUpdater._handle_requeued_file sees top-level files only). move_key = persist_key(pc.pair_id, command.filename) @@ -872,19 +873,40 @@ def _sweep_staged_nested(self, pc: PairContext) -> None: """Forget staged nested downloads that are no longer in staging (moved, deleted, or carried along by an ancestor's move).""" staging = self._pair_staging_dir(pc) - if staging is None: - pc.staged_nested.clear() - return # An unfinished file with lftp temp files on is ".lftp" suffix = Constants.LFTP_TEMP_FILE_SUFFIX - pc.staged_nested.difference_update( - [ - name - for name in pc.staged_nested - if not os.path.exists(os.path.join(staging, name)) + gone = [ + name + for name in pc.staged_nested + if staging is None + or ( + not os.path.exists(os.path.join(staging, name)) and not os.path.exists(os.path.join(staging, name + suffix)) - ] - ) + ) + ] + pc.staged_nested.difference_update(gone) + self._persist.staged_nested_file_names.difference_update(persist_key(pc.pair_id, name) for name in gone) + + def restore_staged_nested(self, pc: PairContext) -> None: + """Pick up the nested downloads a previous session left in staging. + + Restores pc.staged_nested from its persisted copy (dropping names no + longer in staging) and queues each one's move; the move checks its + size. A top-level name holding one is taken off pc.pending_completion: + the active scanner would report its staging copy, which holds only + the nested downloads, in place of its local_path copy. The active + scanner gets the corrected names before its first scan. + """ + if self._pair_staging_dir(pc) is None: + return + for key in self._persist.staged_nested_file_names: + name = strip_persist_key(key, pc.pair_id) + if name != key or pc.pair_id is None: + pc.staged_nested.add(name) + self._sweep_staged_nested(pc) + pc.pending_nested_moves.update(pc.staged_nested) + pc.pending_completion.difference_update(name.split("/", 1)[0] for name in pc.staged_nested) + pc.active_scanner.set_active_files(sorted(pc.pending_completion | pc.staged_nested)) def _get_pair_context_for_command(self, command: Command) -> PairContext | None: """Find the pair context for a command based on pair_id.""" diff --git a/src/python/controller/controller.py b/src/python/controller/controller.py index 6c15f124..9f7fe6cf 100644 --- a/src/python/controller/controller.py +++ b/src/python/controller/controller.py @@ -113,6 +113,8 @@ def __init__(self, context: Context, persist: ControllerPersist): # Seed each builder with filtered persist state self.__persist_sync.sync() + for pc in self.__pair_contexts: + self.__pipeline.restore_staged_nested(pc) # Flag for hot-reloading LFTP tuning settings (set from REST thread) self.__needs_lftp_reconfigure = threading.Event() diff --git a/src/python/controller/controller_persist.py b/src/python/controller/controller_persist.py index 50fdc0ef..38a505cf 100644 --- a/src/python/controller/controller_persist.py +++ b/src/python/controller/controller_persist.py @@ -29,6 +29,9 @@ class ControllerPersist(Persist): ("validated", False), ("corrupt", False), ("move_failed", False), + # Nested downloads still in staging (PairContext.staged_nested), so a + # restart can find them again + ("staged_nested", False), ) def __init__(self): @@ -38,9 +41,10 @@ def __init__(self): self.validated_file_names: set[str] = set() self.corrupt_file_names: set[str] = set() self.move_failed_file_names: set[str] = set() + self.staged_nested_file_names: set[str] = set() def all_sets(self) -> dict[str, set[str]]: - """The six persisted key sets, keyed by json key.""" + """The persisted key sets, keyed by json key.""" return {key: getattr(self, f"{key}_file_names") for key, _ in self._KEYS} @staticmethod diff --git a/src/python/tests/unittests/test_controller/test_command_pipeline.py b/src/python/tests/unittests/test_controller/test_command_pipeline.py index 660c05d8..fa53cfe2 100644 --- a/src/python/tests/unittests/test_controller/test_command_pipeline.py +++ b/src/python/tests/unittests/test_controller/test_command_pipeline.py @@ -1,6 +1,7 @@ # Copyright 2017, Inderpreet Singh, All rights reserved. import os +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -484,6 +485,72 @@ def test_sweep_staged_nested_clears_all_when_staging_disabled(self): self.assertEqual(set(), pc.staged_nested) + def test_sweep_staged_nested_drops_persisted_keys_too(self): + pc = self._make_pair_context("pair-1") + pc.staged_nested = {"Top/kept.bin", "Top/gone.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._persist.staged_nested_file_names = { + persist_key("pair-1", "Top/kept.bin"), + persist_key("pair-1", "Top/gone.bin"), + } + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + kept_path = os.path.join("/tmp/staging", "pair-1", "Top/kept.bin") + + with patch("controller.command_pipeline.os.path.exists", side_effect=lambda p: p == kept_path): + pipeline._sweep_staged_nested(pc) + + self.assertEqual({persist_key("pair-1", "Top/kept.bin")}, pipeline._persist.staged_nested_file_names) + + # --- restore_staged_nested --- + + def test_restore_staged_nested_picks_up_what_a_previous_session_left_in_staging(self): + """staged_nested is in memory; its persisted copy brings back the nested + downloads still in staging, queues their moves, and keeps their + top-level name off the active scan (its staging copy holds only them).""" + with tempfile.TemporaryDirectory() as staging_root: + os.makedirs(os.path.join(staging_root, "pair-1", "Top", "Sub")) + with open(os.path.join(staging_root, "pair-1", "Top", "Sub", "f.bin"), "w") as f: + f.write("x") + pc = self._make_pair_context("pair-1") + pc.staged_nested = set() + pc.pending_nested_moves = set() + pc.pending_completion = {"Top", "Movie.mkv"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = staging_root + other_pair_key = persist_key("pair-2", "X/y.bin") + pipeline._persist.staged_nested_file_names = { + persist_key("pair-1", "Top/Sub/f.bin"), + persist_key("pair-1", "Top/gone.bin"), + other_pair_key, + } + + pipeline.restore_staged_nested(pc) + + self.assertEqual({"Top/Sub/f.bin"}, pc.staged_nested) + self.assertEqual({"Top/Sub/f.bin"}, pc.pending_nested_moves) + self.assertEqual({"Movie.mkv"}, pc.pending_completion) + self.assertEqual( + {persist_key("pair-1", "Top/Sub/f.bin"), other_pair_key}, pipeline._persist.staged_nested_file_names + ) + pc.active_scanner.set_active_files.assert_called_once_with(["Movie.mkv", "Top/Sub/f.bin"]) + + def test_restore_staged_nested_does_nothing_with_staging_off(self): + pc = self._make_pair_context("pair-1") + pc.staged_nested = set() + pc.pending_nested_moves = set() + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = False + pipeline._context.config.controller.staging_path = None + pipeline._persist.staged_nested_file_names = {persist_key("pair-1", "Top/a.bin")} + + pipeline.restore_staged_nested(pc) + + self.assertEqual(set(), pc.staged_nested) + self.assertEqual(set(), pc.pending_nested_moves) + pc.active_scanner.set_active_files.assert_not_called() + # --- propagate_exceptions: worker-fault isolation (#511) --- def test_propagate_exceptions_clean_when_no_worker_errors(self): @@ -1505,6 +1572,20 @@ def test_nested_queue_under_staging_clears_move_state(self): self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) self.assertEqual({"Top/leaf.rar"}, pc.staged_nested) + def test_nested_queue_under_staging_persists_staged_nested(self): + """So a restart can find the download in staging again.""" + pc = self._make_pair_context("pair-1", effective_local_path="/staging/pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + pipeline._persist.staged_nested_file_names = set() + leaf = ModelFile("leaf.rar", False) + leaf.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") + _calls, notify = self._make_notify() + + self.assertTrue(pipeline._handle_queue(command, leaf, pc, notify)) + self.assertEqual({persist_key("pair-1", "Top/leaf.rar")}, pipeline._persist.staged_nested_file_names) + def test_nested_queue_without_staging_adds_no_pending_move(self): pc = self._make_pair_context("pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) diff --git a/src/python/tests/unittests/test_controller/test_controller_persist.py b/src/python/tests/unittests/test_controller/test_controller_persist.py index b09fbfd1..769044e2 100644 --- a/src/python/tests/unittests/test_controller/test_controller_persist.py +++ b/src/python/tests/unittests/test_controller/test_controller_persist.py @@ -219,6 +219,25 @@ def test_move_failed_legacy_keys_migrated(self): persist.move_failed_file_names, ) + def test_staged_nested_golden_round_trip(self): + """staged_nested: nested downloads left in staging, by persist key.""" + sep = "\x1f" + content = json.dumps( + { + "downloaded": [], + "extracted": [], + "staged_nested": ["Top/Sub/f.bin", f"pair-1{sep}Top/a.bin"], + } + ) + persist = ControllerPersist.from_str(content) + self.assertEqual({"Top/Sub/f.bin", f"pair-1{sep}Top/a.bin"}, persist.staged_nested_file_names) + self.assertEqual({"Top/Sub/f.bin", f"pair-1{sep}Top/a.bin"}, set(json.loads(persist.to_str())["staged_nested"])) + + def test_staged_nested_missing_key_defaults_empty(self): + """Persist files from before staged_nested load an empty set.""" + persist = ControllerPersist.from_str(json.dumps({"downloaded": ["a"], "extracted": []})) + self.assertEqual(set(), persist.staged_nested_file_names) + def test_validated_corrupt_legacy_keys_migrated(self): """Legacy colon-separated keys in validated/corrupt should be migrated.""" uuid1 = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" From 5a3a6da17bed5605303725dce28b1cf0094401a9 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:41:06 -0600 Subject: [PATCH 14/15] fix(python): hold nested moves until the remote scan is in the model A nested move needs its row's remote size. Pending moves restored at startup reached _spawn_nested_moves before the first remote scan, and in the cycle that scan arrives the registry still holds the previous model, so every row looked unresolvable and each move was dropped with a warning. Wait until a model built from a remote scan has been applied. --- src/python/controller/model_updater.py | 11 +++++++ src/python/controller/pair_context.py | 3 ++ .../test_controller/test_model_updater.py | 30 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/src/python/controller/model_updater.py b/src/python/controller/model_updater.py index 5e6f989a..6d5b0ac2 100644 --- a/src/python/controller/model_updater.py +++ b/src/python/controller/model_updater.py @@ -104,6 +104,7 @@ def update(self) -> None: if new_model is not None: model_diff = self._registry.apply_diff(new_model) self._process_model_diffs(model_diff) + self._note_remote_scans_in_model() self._prune_stale_persist() self._process_extraction_failures(latest_failed_extractions) @@ -604,6 +605,10 @@ def _spawn_nested_moves(self, pc: PairContext, lftp_job_names: set[str]) -> None The job ending doesn't mean it finished (a Stop ends it too), so the move gets the row's remote size to check. """ + if not pc.remote_scan_in_model: + # After a restart the pending moves come back before the first remote + # scan; without it in the model every row looks unresolvable. + return for name in pc.pending_nested_moves - lftp_job_names: pc.pending_nested_moves.discard(name) try: @@ -617,6 +622,12 @@ def _spawn_nested_moves(self, pc: PairContext, lftp_job_names: set[str]) -> None continue self._pipeline.spawn_move_process(name, pc, expected_size=expected_size) + def _note_remote_scans_in_model(self) -> None: + """Mark each pair whose remote scan has reached the registry.""" + for pc in self._pair_contexts: + if pc.remote_scan_received: + pc.remote_scan_in_model = True + def sync_persist_to_all_builders(self): """Push current persist state to all pair model builders, filtered by pair_id.""" self._persist_sync.sync() diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index bc50f03a..58dc95fa 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -60,6 +60,9 @@ class PairContext: # behind (see CommandPipeline.spawn_move_process). staged_nested: set[str] = field(default_factory=set[str]) remote_scan_received: bool = False + # True once a model built from a remote scan is in the registry, so a + # nested row's remote size can be resolved (see ModelUpdater._spawn_nested_moves) + remote_scan_in_model: bool = False local_scan_received: bool = False # Temporary storage for latest scan results (set during ModelUpdater._update_pair_model_state) diff --git a/src/python/tests/unittests/test_controller/test_model_updater.py b/src/python/tests/unittests/test_controller/test_model_updater.py index 0aab5f04..6de33892 100644 --- a/src/python/tests/unittests/test_controller/test_model_updater.py +++ b/src/python/tests/unittests/test_controller/test_model_updater.py @@ -288,6 +288,7 @@ def _make_pair_context(self, pair_id=None): pc.prev_downloading_file_names = set() pc.pending_completion = set() pc.pending_nested_moves = set() + pc.remote_scan_in_model = True return pc def _make_updater(self, pair_contexts, persist=None, *, use_staging=True, staging_path="/tmp/staging"): @@ -395,6 +396,35 @@ def test_completion_without_pending_entry_spawns_nothing(self): pipeline.spawn_move_process.assert_not_called() registry.resolve_full_path.assert_not_called() + def test_pending_nested_move_waits_until_the_remote_scan_is_in_the_model(self): + """After a restart, pending moves come back before the first remote + scan. Until a model built from it is in the registry the row can't be + resolved, and the move would be dropped for an unknown size.""" + pc = self._make_pair_context("pair-1") + pc.remote_scan_in_model = False + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + + updater._detect_lftp_completions(pc, []) + + registry.resolve_full_path.assert_not_called() + pipeline.spawn_move_process.assert_not_called() + self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + + def test_remote_scan_marked_in_model_once_a_model_is_built_with_it(self): + received = self._make_pair_context("pair-1") + received.remote_scan_received = True + received.remote_scan_in_model = False + waiting = self._make_pair_context("pair-2") + waiting.remote_scan_received = False + waiting.remote_scan_in_model = False + updater, _pipeline, _registry = self._make_updater([received, waiting], self._make_persist()) + + updater._note_remote_scans_in_model() + + self.assertTrue(received.remote_scan_in_model) + self.assertFalse(waiting.remote_scan_in_model) + class TestPruneAbsentNestedMoveFailures(unittest.TestCase): """Nested move_failed keys never reach downloaded_file_names (the prune From d0e57ce58ac6105c3f24875594eaca9a5d0a4e46 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:00:14 -0600 Subject: [PATCH 15/15] fix(python): don't apply persist authority before the first remote listing _check_persist_authority read a missing remote size as "deleted remotely" and marked a persisted name with any local copy Downloaded. Before the first remote scan the size is only unknown. With downloads left in staging now visible right after a restart, a stopped top-level partial (persisted when its job ended) read as Downloaded in that window and was moved to local_path as if complete. Skip persist authority until the builder has had a remote listing. --- src/python/controller/model_builder.py | 10 +++++++++- .../test_controller/test_model_builder.py | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/python/controller/model_builder.py b/src/python/controller/model_builder.py index d878a29c..8cb0e918 100644 --- a/src/python/controller/model_builder.py +++ b/src/python/controller/model_builder.py @@ -31,6 +31,9 @@ def __init__(self, pair_id: str | None = None) -> None: self.__local_files: dict[str, SystemFile] = {} self.__active_files: dict[str, SystemFile] = {} self.__remote_files: dict[str, SystemFile] = {} + # False until the first remote listing: until then a missing remote + # size means "not scanned yet", not "deleted remotely" + self.__remote_listing_received = False self.__lftp_statuses: dict[str, LftpJobStatus] = {} self.__downloaded_files: set[str] = set() self.__extract_statuses: dict[str, ExtractStatus] = {} @@ -67,8 +70,9 @@ def set_remote_files(self, remote_files: list[SystemFile]): prev_remote_files = self.__remote_files self.__remote_files = {file.name: file for file in remote_files} # Invalidate the cache - if self.__remote_files != prev_remote_files: + if self.__remote_files != prev_remote_files or not self.__remote_listing_received: self.__cached_model = None + self.__remote_listing_received = True def set_lftp_statuses(self, lftp_statuses: list[LftpJobStatus]): prev_lftp_statuses = self.__lftp_statuses @@ -142,6 +146,7 @@ def clear(self): self.__local_files.clear() self.__active_files.clear() self.__remote_files.clear() + self.__remote_listing_received = False self.__lftp_statuses.clear() self.__downloaded_files.clear() self.__extract_statuses.clear() @@ -596,6 +601,9 @@ def _check_persist_authority(self, model_file: ModelFile, incomplete_children: b and model_file.local_size < model_file.remote_size ): return # Stay DEFAULT for re-download + # A missing remote size only means "gone" once a remote listing is in + if model_file.remote_size is None and not self.__remote_listing_received: + return # Persist authority — applies when auto_delete_remote OFF, # or when remote is already gone (auto_delete_remote ON, delete succeeded) if not self.__auto_delete_remote or model_file.remote_size is None: diff --git a/src/python/tests/unittests/test_controller/test_model_builder.py b/src/python/tests/unittests/test_controller/test_model_builder.py index 611aba49..f8d797b0 100644 --- a/src/python/tests/unittests/test_controller/test_model_builder.py +++ b/src/python/tests/unittests/test_controller/test_model_builder.py @@ -356,6 +356,21 @@ def test_build_state(self): model = self.model_builder.build_model() self.assertEqual(ModelFile.State.DELETED, model.get_file("a").state) + def test_persist_authority_waits_for_a_remote_listing(self): + """Before the first remote scan a missing remote size means "not + scanned yet", not "deleted remotely". A stopped partial left in + staging, seen by the active scanner after a restart, must not read + as Downloaded (and be moved) in that window.""" + self.model_builder.clear() + self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_active_files([SystemFile("a", 30, False)]) + self.model_builder.set_downloaded_files({"a"}) + self.assertEqual(ModelFile.State.DEFAULT, self.model_builder.build_model().get_file("a").state) + + # Once a listing without "a" is in, persist authority applies + self.model_builder.set_remote_files([]) + self.assertEqual(ModelFile.State.DOWNLOADED, self.model_builder.build_model().get_file("a").state) + def test_build_state_persist_with_auto_delete_remote(self): """Test persist authority behavior under different auto_delete_remote settings""" @@ -370,6 +385,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -378,6 +394,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local absent, persist → DELETED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # Need at least one source for 'a' to appear in model — use lftp status self.model_builder.set_lftp_statuses( @@ -390,6 +407,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # Instead test: remote absent via local-only + persist self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # File must exist in some source to appear in model # If remote_size=None and local_size=None but file in lftp status: @@ -412,6 +430,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -426,6 +445,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote exists, local absent, persist → DEFAULT (re-downloadable) self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_remote_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model()