From 17fd924c5bde96f8a4d44f2741e08f5c70e9a7e7 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:35:43 -0600 Subject: [PATCH 1/2] fix(python): show and move top-level downloads left in staging after a restart pending_completion, which keeps the active scanner on a download in staging until it moves, is in memory only. After a restart the local scan (local_path only) couldn't see what was left in staging: a finished but unmoved download, recorded as downloaded with no local copy, showed as Deleted and was never moved; a stopped partial showed no size. Seed pending_completion from the pair's staging dir when its context is built (lftp's ".lftp" temp file counts as ), and hand the names to the active scanner then so its first scan covers them, ahead of the first remote scan. --- src/python/controller/controller.py | 11 +++++- src/python/controller/pair_context.py | 16 +++++++++ .../test_controller_lftp_wiring.py | 34 +++++++++++++++++++ .../test_controller/test_pair_context.py | 20 +++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) diff --git a/src/python/controller/controller.py b/src/python/controller/controller.py index 8a0c9de9..6c15f124 100644 --- a/src/python/controller/controller.py +++ b/src/python/controller/controller.py @@ -19,7 +19,7 @@ from .model_builder import ModelBuilder from .model_registry import ModelRegistry from .model_updater import ModelUpdater -from .pair_context import ControllerError, PairContext, configure_lftp, validate_config +from .pair_context import ControllerError, PairContext, configure_lftp, staged_top_level_names, validate_config from .persist_sync import PersistSync from .scan import ActiveScanner, LocalScanner, RemoteScanner, ScannerProcess from .validate import ValidateProcess @@ -212,6 +212,14 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, # (_LFTP_TUNING_KEYS), so it can change after these are built, and # temp files from an earlier interrupted download can outlive it. active_scanner = ActiveScanner(effective_local_path, lftp_temp_suffix=Constants.LFTP_TEMP_FILE_SUFFIX) + # pending_completion is in memory only: re-seed it with what a previous + # session left in staging (finished or stopped), so the active scanner + # shows it and a finished one still gets its move. The scanner gets the + # names now so its first scan, not the next one, covers them. + left_in_staging: set[str] = ( + staged_top_level_names(effective_local_path) if effective_local_path != local_path else set() + ) + active_scanner.set_active_files(sorted(left_in_staging)) local_scanner = LocalScanner(local_path=local_path) remote_scanner = RemoteScanner( remote_address=self.__context.config.lftp.remote_address, # type: ignore[arg-type] @@ -270,6 +278,7 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, local_scan_process=local_scan_process, remote_scan_process=remote_scan_process, model_builder=model_builder, + pending_completion=left_in_staging, ) def _configure_lftp(self, lftp: Lftp): diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index 218d764b..1f3b4142 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os from dataclasses import dataclass, field from common import AppError, Config, Constants, Context @@ -59,6 +60,21 @@ class PairContext: latest_local_scan: ScannerResult | None = None +def staged_top_level_names(staging_dir: str) -> set[str]: + """Names of the top-level downloads with something in staging_dir. + + lftp's ".lftp" temp file is listed as ; its + ".lftp-pget-status" progress file is skipped. A missing dir lists + nothing. + """ + try: + entries = os.listdir(staging_dir) + except FileNotFoundError: + return set() + suffix = Constants.LFTP_TEMP_FILE_SUFFIX + return {entry.removesuffix(suffix) for entry in entries if not entry.endswith(suffix + "-pget-status")} + + def validate_config(context: Context) -> None: """Validate that all required config fields are set (non-None) at startup. diff --git a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py index 0d99fa93..ac879733 100644 --- a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py +++ b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py @@ -4,6 +4,8 @@ protocol, port selection, and SSL params, while keeping the scanner on SSH.""" import logging +import os +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -129,6 +131,38 @@ def test_active_scanner_recognises_lftp_temp_files_with_use_temp_file_off(self): _, kwargs = mock_active_scanner.call_args self.assertEqual(Constants.LFTP_TEMP_FILE_SUFFIX, kwargs["lftp_temp_suffix"]) + def _pending_completion_for(self, ctx: Context): + controller = self._build_controller(ctx) + with ( + patch("controller.controller.Lftp"), + patch("controller.controller.RemoteScanner"), + patch("controller.controller.ActiveScanner") as mock_active_scanner, + patch("controller.controller.LocalScanner"), + patch("controller.controller.ScannerProcess"), + patch("controller.controller.ModelBuilder"), + patch("controller.controller.configure_lftp"), + ): + pc = controller._create_pair_context(pair_id="p1", name="pair", remote_path="/remote", local_path="/local") + # The scanner's first scan covers them too, ahead of the first update cycle + mock_active_scanner.return_value.set_active_files.assert_called_once_with(sorted(pc.pending_completion)) + return pc.pending_completion + + def test_downloads_left_in_staging_are_scanned_after_a_restart(self): + """pending_completion is in memory only, so a download that finished + or was stopped in staging before a restart would otherwise go unseen: + the local scan covers only local_path.""" + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + with tempfile.TemporaryDirectory() as staging: + ctx.config.controller.use_staging = True + ctx.config.controller.staging_path = staging + os.makedirs(os.path.join(staging, "p1", "Show.S01")) + + self.assertEqual({"Show.S01"}, self._pending_completion_for(ctx)) + + def test_nothing_seeded_with_staging_off(self): + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + self.assertEqual(set(), self._pending_completion_for(ctx)) + if __name__ == "__main__": unittest.main() diff --git a/src/python/tests/unittests/test_controller/test_pair_context.py b/src/python/tests/unittests/test_controller/test_pair_context.py index 98999aa0..002cd8c7 100644 --- a/src/python/tests/unittests/test_controller/test_pair_context.py +++ b/src/python/tests/unittests/test_controller/test_pair_context.py @@ -1,5 +1,7 @@ # Copyright 2017, Inderpreet Singh, All rights reserved. +import os +import tempfile import unittest from unittest.mock import MagicMock @@ -8,6 +10,7 @@ ControllerError, PairContext, configure_lftp, + staged_top_level_names, validate_config, ) @@ -341,3 +344,20 @@ def test_ftps_passes_persisted_values_through(self): self.assertEqual(lftp.num_connections_per_root_file, 20) self.assertEqual(lftp.num_connections_per_dir_file, 20) self.assertEqual(lftp.num_max_total_connections, 0) + + +class TestStagedTopLevelNames(unittest.TestCase): + def test_lists_downloads_in_staging_by_their_final_name(self): + """lftp's ".lftp" temp file is the download ; its + ".lftp-pget-status" progress file is not a download.""" + with tempfile.TemporaryDirectory() as staging: + os.makedirs(os.path.join(staging, "Show.S01")) + for name in ("movie.mkv", "partial.mkv.lftp", "movie.mkv.lftp-pget-status"): + with open(os.path.join(staging, name), "w") as f: + f.write("x") + + self.assertEqual({"Show.S01", "movie.mkv", "partial.mkv"}, staged_top_level_names(staging)) + + def test_missing_staging_dir_lists_nothing(self): + with tempfile.TemporaryDirectory() as parent: + self.assertEqual(set(), staged_top_level_names(os.path.join(parent, "nope"))) From d0e57ce58ac6105c3f24875594eaca9a5d0a4e46 Mon Sep 17 00:00:00 2001 From: JAAdventurer <96359266+JAAdventurer@users.noreply.github.com> Date: Mon, 16 Nov 2026 23:00:14 -0600 Subject: [PATCH 2/2] fix(python): don't apply persist authority before the first remote listing _check_persist_authority read a missing remote size as "deleted remotely" and marked a persisted name with any local copy Downloaded. Before the first remote scan the size is only unknown. With downloads left in staging now visible right after a restart, a stopped top-level partial (persisted when its job ended) read as Downloaded in that window and was moved to local_path as if complete. Skip persist authority until the builder has had a remote listing. --- src/python/controller/model_builder.py | 10 +++++++++- .../test_controller/test_model_builder.py | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/python/controller/model_builder.py b/src/python/controller/model_builder.py index d878a29c..8cb0e918 100644 --- a/src/python/controller/model_builder.py +++ b/src/python/controller/model_builder.py @@ -31,6 +31,9 @@ def __init__(self, pair_id: str | None = None) -> None: self.__local_files: dict[str, SystemFile] = {} self.__active_files: dict[str, SystemFile] = {} self.__remote_files: dict[str, SystemFile] = {} + # False until the first remote listing: until then a missing remote + # size means "not scanned yet", not "deleted remotely" + self.__remote_listing_received = False self.__lftp_statuses: dict[str, LftpJobStatus] = {} self.__downloaded_files: set[str] = set() self.__extract_statuses: dict[str, ExtractStatus] = {} @@ -67,8 +70,9 @@ def set_remote_files(self, remote_files: list[SystemFile]): prev_remote_files = self.__remote_files self.__remote_files = {file.name: file for file in remote_files} # Invalidate the cache - if self.__remote_files != prev_remote_files: + if self.__remote_files != prev_remote_files or not self.__remote_listing_received: self.__cached_model = None + self.__remote_listing_received = True def set_lftp_statuses(self, lftp_statuses: list[LftpJobStatus]): prev_lftp_statuses = self.__lftp_statuses @@ -142,6 +146,7 @@ def clear(self): self.__local_files.clear() self.__active_files.clear() self.__remote_files.clear() + self.__remote_listing_received = False self.__lftp_statuses.clear() self.__downloaded_files.clear() self.__extract_statuses.clear() @@ -596,6 +601,9 @@ def _check_persist_authority(self, model_file: ModelFile, incomplete_children: b and model_file.local_size < model_file.remote_size ): return # Stay DEFAULT for re-download + # A missing remote size only means "gone" once a remote listing is in + if model_file.remote_size is None and not self.__remote_listing_received: + return # Persist authority — applies when auto_delete_remote OFF, # or when remote is already gone (auto_delete_remote ON, delete succeeded) if not self.__auto_delete_remote or model_file.remote_size is None: diff --git a/src/python/tests/unittests/test_controller/test_model_builder.py b/src/python/tests/unittests/test_controller/test_model_builder.py index 611aba49..f8d797b0 100644 --- a/src/python/tests/unittests/test_controller/test_model_builder.py +++ b/src/python/tests/unittests/test_controller/test_model_builder.py @@ -356,6 +356,21 @@ def test_build_state(self): model = self.model_builder.build_model() self.assertEqual(ModelFile.State.DELETED, model.get_file("a").state) + def test_persist_authority_waits_for_a_remote_listing(self): + """Before the first remote scan a missing remote size means "not + scanned yet", not "deleted remotely". A stopped partial left in + staging, seen by the active scanner after a restart, must not read + as Downloaded (and be moved) in that window.""" + self.model_builder.clear() + self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_active_files([SystemFile("a", 30, False)]) + self.model_builder.set_downloaded_files({"a"}) + self.assertEqual(ModelFile.State.DEFAULT, self.model_builder.build_model().get_file("a").state) + + # Once a listing without "a" is in, persist authority applies + self.model_builder.set_remote_files([]) + self.assertEqual(ModelFile.State.DOWNLOADED, self.model_builder.build_model().get_file("a").state) + def test_build_state_persist_with_auto_delete_remote(self): """Test persist authority behavior under different auto_delete_remote settings""" @@ -370,6 +385,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -378,6 +394,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local absent, persist → DELETED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # Need at least one source for 'a' to appear in model — use lftp status self.model_builder.set_lftp_statuses( @@ -390,6 +407,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # Instead test: remote absent via local-only + persist self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # File must exist in some source to appear in model # If remote_size=None and local_size=None but file in lftp status: @@ -412,6 +430,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -426,6 +445,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote exists, local absent, persist → DEFAULT (re-downloadable) self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_remote_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model()