diff --git a/src/python/controller/controller.py b/src/python/controller/controller.py index 8a0c9de9..6c15f124 100644 --- a/src/python/controller/controller.py +++ b/src/python/controller/controller.py @@ -19,7 +19,7 @@ from .model_builder import ModelBuilder from .model_registry import ModelRegistry from .model_updater import ModelUpdater -from .pair_context import ControllerError, PairContext, configure_lftp, validate_config +from .pair_context import ControllerError, PairContext, configure_lftp, staged_top_level_names, validate_config from .persist_sync import PersistSync from .scan import ActiveScanner, LocalScanner, RemoteScanner, ScannerProcess from .validate import ValidateProcess @@ -212,6 +212,14 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, # (_LFTP_TUNING_KEYS), so it can change after these are built, and # temp files from an earlier interrupted download can outlive it. active_scanner = ActiveScanner(effective_local_path, lftp_temp_suffix=Constants.LFTP_TEMP_FILE_SUFFIX) + # pending_completion is in memory only: re-seed it with what a previous + # session left in staging (finished or stopped), so the active scanner + # shows it and a finished one still gets its move. The scanner gets the + # names now so its first scan, not the next one, covers them. + left_in_staging: set[str] = ( + staged_top_level_names(effective_local_path) if effective_local_path != local_path else set() + ) + active_scanner.set_active_files(sorted(left_in_staging)) local_scanner = LocalScanner(local_path=local_path) remote_scanner = RemoteScanner( remote_address=self.__context.config.lftp.remote_address, # type: ignore[arg-type] @@ -270,6 +278,7 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, local_scan_process=local_scan_process, remote_scan_process=remote_scan_process, model_builder=model_builder, + pending_completion=left_in_staging, ) def _configure_lftp(self, lftp: Lftp): diff --git a/src/python/controller/model_builder.py b/src/python/controller/model_builder.py index d878a29c..8cb0e918 100644 --- a/src/python/controller/model_builder.py +++ b/src/python/controller/model_builder.py @@ -31,6 +31,9 @@ def __init__(self, pair_id: str | None = None) -> None: self.__local_files: dict[str, SystemFile] = {} self.__active_files: dict[str, SystemFile] = {} self.__remote_files: dict[str, SystemFile] = {} + # False until the first remote listing: until then a missing remote + # size means "not scanned yet", not "deleted remotely" + self.__remote_listing_received = False self.__lftp_statuses: dict[str, LftpJobStatus] = {} self.__downloaded_files: set[str] = set() self.__extract_statuses: dict[str, ExtractStatus] = {} @@ -67,8 +70,9 @@ def set_remote_files(self, remote_files: list[SystemFile]): prev_remote_files = self.__remote_files self.__remote_files = {file.name: file for file in remote_files} # Invalidate the cache - if self.__remote_files != prev_remote_files: + if self.__remote_files != prev_remote_files or not self.__remote_listing_received: self.__cached_model = None + self.__remote_listing_received = True def set_lftp_statuses(self, lftp_statuses: list[LftpJobStatus]): prev_lftp_statuses = self.__lftp_statuses @@ -142,6 +146,7 @@ def clear(self): self.__local_files.clear() self.__active_files.clear() self.__remote_files.clear() + self.__remote_listing_received = False self.__lftp_statuses.clear() self.__downloaded_files.clear() self.__extract_statuses.clear() @@ -596,6 +601,9 @@ def _check_persist_authority(self, model_file: ModelFile, incomplete_children: b and model_file.local_size < model_file.remote_size ): return # Stay DEFAULT for re-download + # A missing remote size only means "gone" once a remote listing is in + if model_file.remote_size is None and not self.__remote_listing_received: + return # Persist authority — applies when auto_delete_remote OFF, # or when remote is already gone (auto_delete_remote ON, delete succeeded) if not self.__auto_delete_remote or model_file.remote_size is None: diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index 218d764b..1f3b4142 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os from dataclasses import dataclass, field from common import AppError, Config, Constants, Context @@ -59,6 +60,21 @@ class PairContext: latest_local_scan: ScannerResult | None = None +def staged_top_level_names(staging_dir: str) -> set[str]: + """Names of the top-level downloads with something in staging_dir. + + lftp's ".lftp" temp file is listed as ; its + ".lftp-pget-status" progress file is skipped. A missing dir lists + nothing. + """ + try: + entries = os.listdir(staging_dir) + except FileNotFoundError: + return set() + suffix = Constants.LFTP_TEMP_FILE_SUFFIX + return {entry.removesuffix(suffix) for entry in entries if not entry.endswith(suffix + "-pget-status")} + + def validate_config(context: Context) -> None: """Validate that all required config fields are set (non-None) at startup. diff --git a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py index 0d99fa93..ac879733 100644 --- a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py +++ b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py @@ -4,6 +4,8 @@ protocol, port selection, and SSL params, while keeping the scanner on SSH.""" import logging +import os +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -129,6 +131,38 @@ def test_active_scanner_recognises_lftp_temp_files_with_use_temp_file_off(self): _, kwargs = mock_active_scanner.call_args self.assertEqual(Constants.LFTP_TEMP_FILE_SUFFIX, kwargs["lftp_temp_suffix"]) + def _pending_completion_for(self, ctx: Context): + controller = self._build_controller(ctx) + with ( + patch("controller.controller.Lftp"), + patch("controller.controller.RemoteScanner"), + patch("controller.controller.ActiveScanner") as mock_active_scanner, + patch("controller.controller.LocalScanner"), + patch("controller.controller.ScannerProcess"), + patch("controller.controller.ModelBuilder"), + patch("controller.controller.configure_lftp"), + ): + pc = controller._create_pair_context(pair_id="p1", name="pair", remote_path="/remote", local_path="/local") + # The scanner's first scan covers them too, ahead of the first update cycle + mock_active_scanner.return_value.set_active_files.assert_called_once_with(sorted(pc.pending_completion)) + return pc.pending_completion + + def test_downloads_left_in_staging_are_scanned_after_a_restart(self): + """pending_completion is in memory only, so a download that finished + or was stopped in staging before a restart would otherwise go unseen: + the local scan covers only local_path.""" + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + with tempfile.TemporaryDirectory() as staging: + ctx.config.controller.use_staging = True + ctx.config.controller.staging_path = staging + os.makedirs(os.path.join(staging, "p1", "Show.S01")) + + self.assertEqual({"Show.S01"}, self._pending_completion_for(ctx)) + + def test_nothing_seeded_with_staging_off(self): + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + self.assertEqual(set(), self._pending_completion_for(ctx)) + if __name__ == "__main__": unittest.main() diff --git a/src/python/tests/unittests/test_controller/test_model_builder.py b/src/python/tests/unittests/test_controller/test_model_builder.py index 611aba49..f8d797b0 100644 --- a/src/python/tests/unittests/test_controller/test_model_builder.py +++ b/src/python/tests/unittests/test_controller/test_model_builder.py @@ -356,6 +356,21 @@ def test_build_state(self): model = self.model_builder.build_model() self.assertEqual(ModelFile.State.DELETED, model.get_file("a").state) + def test_persist_authority_waits_for_a_remote_listing(self): + """Before the first remote scan a missing remote size means "not + scanned yet", not "deleted remotely". A stopped partial left in + staging, seen by the active scanner after a restart, must not read + as Downloaded (and be moved) in that window.""" + self.model_builder.clear() + self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_active_files([SystemFile("a", 30, False)]) + self.model_builder.set_downloaded_files({"a"}) + self.assertEqual(ModelFile.State.DEFAULT, self.model_builder.build_model().get_file("a").state) + + # Once a listing without "a" is in, persist authority applies + self.model_builder.set_remote_files([]) + self.assertEqual(ModelFile.State.DOWNLOADED, self.model_builder.build_model().get_file("a").state) + def test_build_state_persist_with_auto_delete_remote(self): """Test persist authority behavior under different auto_delete_remote settings""" @@ -370,6 +385,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -378,6 +394,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local absent, persist → DELETED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # Need at least one source for 'a' to appear in model — use lftp status self.model_builder.set_lftp_statuses( @@ -390,6 +407,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # Instead test: remote absent via local-only + persist self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # File must exist in some source to appear in model # If remote_size=None and local_size=None but file in lftp status: @@ -412,6 +430,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -426,6 +445,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote exists, local absent, persist → DEFAULT (re-downloadable) self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_remote_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() diff --git a/src/python/tests/unittests/test_controller/test_pair_context.py b/src/python/tests/unittests/test_controller/test_pair_context.py index 98999aa0..002cd8c7 100644 --- a/src/python/tests/unittests/test_controller/test_pair_context.py +++ b/src/python/tests/unittests/test_controller/test_pair_context.py @@ -1,5 +1,7 @@ # Copyright 2017, Inderpreet Singh, All rights reserved. +import os +import tempfile import unittest from unittest.mock import MagicMock @@ -8,6 +10,7 @@ ControllerError, PairContext, configure_lftp, + staged_top_level_names, validate_config, ) @@ -341,3 +344,20 @@ def test_ftps_passes_persisted_values_through(self): self.assertEqual(lftp.num_connections_per_root_file, 20) self.assertEqual(lftp.num_connections_per_dir_file, 20) self.assertEqual(lftp.num_max_total_connections, 0) + + +class TestStagedTopLevelNames(unittest.TestCase): + def test_lists_downloads_in_staging_by_their_final_name(self): + """lftp's ".lftp" temp file is the download ; its + ".lftp-pget-status" progress file is not a download.""" + with tempfile.TemporaryDirectory() as staging: + os.makedirs(os.path.join(staging, "Show.S01")) + for name in ("movie.mkv", "partial.mkv.lftp", "movie.mkv.lftp-pget-status"): + with open(os.path.join(staging, name), "w") as f: + f.write("x") + + self.assertEqual({"Show.S01", "movie.mkv", "partial.mkv"}, staged_top_level_names(staging)) + + def test_missing_staging_dir_lists_nothing(self): + with tempfile.TemporaryDirectory() as parent: + self.assertEqual(set(), staged_top_level_names(os.path.join(parent, "nope")))