Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion src/python/controller/controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from .model_builder import ModelBuilder
from .model_registry import ModelRegistry
from .model_updater import ModelUpdater
from .pair_context import ControllerError, PairContext, configure_lftp, validate_config
from .pair_context import ControllerError, PairContext, configure_lftp, staged_top_level_names, validate_config
from .persist_sync import PersistSync
from .scan import ActiveScanner, LocalScanner, RemoteScanner, ScannerProcess
from .validate import ValidateProcess
Expand Down Expand Up @@ -212,6 +212,14 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str,
# (_LFTP_TUNING_KEYS), so it can change after these are built, and
# temp files from an earlier interrupted download can outlive it.
active_scanner = ActiveScanner(effective_local_path, lftp_temp_suffix=Constants.LFTP_TEMP_FILE_SUFFIX)
# pending_completion is in memory only: re-seed it with what a previous
# session left in staging (finished or stopped), so the active scanner
# shows it and a finished one still gets its move. The scanner gets the
# names now so its first scan, not the next one, covers them.
left_in_staging: set[str] = (
staged_top_level_names(effective_local_path) if effective_local_path != local_path else set()
)
active_scanner.set_active_files(sorted(left_in_staging))
local_scanner = LocalScanner(local_path=local_path)
remote_scanner = RemoteScanner(
remote_address=self.__context.config.lftp.remote_address, # type: ignore[arg-type]
Expand Down Expand Up @@ -270,6 +278,7 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str,
local_scan_process=local_scan_process,
remote_scan_process=remote_scan_process,
model_builder=model_builder,
pending_completion=left_in_staging,
)

def _configure_lftp(self, lftp: Lftp):
Expand Down
10 changes: 9 additions & 1 deletion src/python/controller/model_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ def __init__(self, pair_id: str | None = None) -> None:
self.__local_files: dict[str, SystemFile] = {}
self.__active_files: dict[str, SystemFile] = {}
self.__remote_files: dict[str, SystemFile] = {}
# False until the first remote listing: until then a missing remote
# size means "not scanned yet", not "deleted remotely"
self.__remote_listing_received = False
self.__lftp_statuses: dict[str, LftpJobStatus] = {}
self.__downloaded_files: set[str] = set()
self.__extract_statuses: dict[str, ExtractStatus] = {}
Expand Down Expand Up @@ -67,8 +70,9 @@ def set_remote_files(self, remote_files: list[SystemFile]):
prev_remote_files = self.__remote_files
self.__remote_files = {file.name: file for file in remote_files}
# Invalidate the cache
if self.__remote_files != prev_remote_files:
if self.__remote_files != prev_remote_files or not self.__remote_listing_received:
self.__cached_model = None
self.__remote_listing_received = True

def set_lftp_statuses(self, lftp_statuses: list[LftpJobStatus]):
prev_lftp_statuses = self.__lftp_statuses
Expand Down Expand Up @@ -142,6 +146,7 @@ def clear(self):
self.__local_files.clear()
self.__active_files.clear()
self.__remote_files.clear()
self.__remote_listing_received = False
self.__lftp_statuses.clear()
self.__downloaded_files.clear()
self.__extract_statuses.clear()
Expand Down Expand Up @@ -596,6 +601,9 @@ def _check_persist_authority(self, model_file: ModelFile, incomplete_children: b
and model_file.local_size < model_file.remote_size
):
return # Stay DEFAULT for re-download
# A missing remote size only means "gone" once a remote listing is in
if model_file.remote_size is None and not self.__remote_listing_received:
return
# Persist authority — applies when auto_delete_remote OFF,
# or when remote is already gone (auto_delete_remote ON, delete succeeded)
if not self.__auto_delete_remote or model_file.remote_size is None:
Expand Down
16 changes: 16 additions & 0 deletions src/python/controller/pair_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from __future__ import annotations

import os
from dataclasses import dataclass, field

from common import AppError, Config, Constants, Context
Expand Down Expand Up @@ -59,6 +60,21 @@ class PairContext:
latest_local_scan: ScannerResult | None = None


def staged_top_level_names(staging_dir: str) -> set[str]:
"""Names of the top-level downloads with something in staging_dir.

lftp's "<name>.lftp" temp file is listed as <name>; its
"<name>.lftp-pget-status" progress file is skipped. A missing dir lists
nothing.
"""
try:
entries = os.listdir(staging_dir)
except FileNotFoundError:
return set()
suffix = Constants.LFTP_TEMP_FILE_SUFFIX
return {entry.removesuffix(suffix) for entry in entries if not entry.endswith(suffix + "-pget-status")}


def validate_config(context: Context) -> None:
"""Validate that all required config fields are set (non-None) at startup.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
protocol, port selection, and SSL params, while keeping the scanner on SSH."""

import logging
import os
import tempfile
import unittest
from unittest.mock import MagicMock, patch

Expand Down Expand Up @@ -129,6 +131,38 @@ def test_active_scanner_recognises_lftp_temp_files_with_use_temp_file_off(self):
_, kwargs = mock_active_scanner.call_args
self.assertEqual(Constants.LFTP_TEMP_FILE_SUFFIX, kwargs["lftp_temp_suffix"])

def _pending_completion_for(self, ctx: Context):
controller = self._build_controller(ctx)
with (
patch("controller.controller.Lftp"),
patch("controller.controller.RemoteScanner"),
patch("controller.controller.ActiveScanner") as mock_active_scanner,
patch("controller.controller.LocalScanner"),
patch("controller.controller.ScannerProcess"),
patch("controller.controller.ModelBuilder"),
patch("controller.controller.configure_lftp"),
):
pc = controller._create_pair_context(pair_id="p1", name="pair", remote_path="/remote", local_path="/local")
# The scanner's first scan covers them too, ahead of the first update cycle
mock_active_scanner.return_value.set_active_files.assert_called_once_with(sorted(pc.pending_completion))
return pc.pending_completion

def test_downloads_left_in_staging_are_scanned_after_a_restart(self):
"""pending_completion is in memory only, so a download that finished
or was stopped in staging before a restart would otherwise go unseen:
the local scan covers only local_path."""
ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False)
with tempfile.TemporaryDirectory() as staging:
ctx.config.controller.use_staging = True
ctx.config.controller.staging_path = staging
os.makedirs(os.path.join(staging, "p1", "Show.S01"))

self.assertEqual({"Show.S01"}, self._pending_completion_for(ctx))

def test_nothing_seeded_with_staging_off(self):
ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False)
self.assertEqual(set(), self._pending_completion_for(ctx))


if __name__ == "__main__":
unittest.main()
20 changes: 20 additions & 0 deletions src/python/tests/unittests/test_controller/test_model_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,21 @@ def test_build_state(self):
model = self.model_builder.build_model()
self.assertEqual(ModelFile.State.DELETED, model.get_file("a").state)

def test_persist_authority_waits_for_a_remote_listing(self):
"""Before the first remote scan a missing remote size means "not
scanned yet", not "deleted remotely". A stopped partial left in
staging, seen by the active scanner after a restart, must not read
as Downloaded (and be moved) in that window."""
self.model_builder.clear()
self.model_builder.set_auto_delete_remote(False)
self.model_builder.set_active_files([SystemFile("a", 30, False)])
self.model_builder.set_downloaded_files({"a"})
self.assertEqual(ModelFile.State.DEFAULT, self.model_builder.build_model().get_file("a").state)

# Once a listing without "a" is in, persist authority applies
self.model_builder.set_remote_files([])
self.assertEqual(ModelFile.State.DOWNLOADED, self.model_builder.build_model().get_file("a").state)

def test_build_state_persist_with_auto_delete_remote(self):
"""Test persist authority behavior under different auto_delete_remote settings"""

Expand All @@ -370,6 +385,7 @@ def test_build_state_persist_with_auto_delete_remote(self):
# auto_delete_remote=False, remote absent, local exists, persist → DOWNLOADED
self.model_builder.clear()
self.model_builder.set_auto_delete_remote(False)
self.model_builder.set_remote_files([])
self.model_builder.set_local_files([SystemFile("a", 100, False)])
self.model_builder.set_downloaded_files({"a"})
model = self.model_builder.build_model()
Expand All @@ -378,6 +394,7 @@ def test_build_state_persist_with_auto_delete_remote(self):
# auto_delete_remote=False, remote absent, local absent, persist → DELETED
self.model_builder.clear()
self.model_builder.set_auto_delete_remote(False)
self.model_builder.set_remote_files([])
self.model_builder.set_downloaded_files({"a"})
# Need at least one source for 'a' to appear in model — use lftp status
self.model_builder.set_lftp_statuses(
Expand All @@ -390,6 +407,7 @@ def test_build_state_persist_with_auto_delete_remote(self):
# Instead test: remote absent via local-only + persist
self.model_builder.clear()
self.model_builder.set_auto_delete_remote(False)
self.model_builder.set_remote_files([])
self.model_builder.set_downloaded_files({"a"})
# File must exist in some source to appear in model
# If remote_size=None and local_size=None but file in lftp status:
Expand All @@ -412,6 +430,7 @@ def test_build_state_persist_with_auto_delete_remote(self):
# auto_delete_remote=True, remote absent, local exists, persist → DOWNLOADED
self.model_builder.clear()
self.model_builder.set_auto_delete_remote(True)
self.model_builder.set_remote_files([])
self.model_builder.set_local_files([SystemFile("a", 100, False)])
self.model_builder.set_downloaded_files({"a"})
model = self.model_builder.build_model()
Expand All @@ -426,6 +445,7 @@ def test_build_state_persist_with_auto_delete_remote(self):
# auto_delete_remote=True, remote exists, local absent, persist → DEFAULT (re-downloadable)
self.model_builder.clear()
self.model_builder.set_auto_delete_remote(True)
self.model_builder.set_remote_files([])
self.model_builder.set_remote_files([SystemFile("a", 100, False)])
self.model_builder.set_downloaded_files({"a"})
model = self.model_builder.build_model()
Expand Down
20 changes: 20 additions & 0 deletions src/python/tests/unittests/test_controller/test_pair_context.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# Copyright 2017, Inderpreet Singh, All rights reserved.

import os
import tempfile
import unittest
from unittest.mock import MagicMock

Expand All @@ -8,6 +10,7 @@
ControllerError,
PairContext,
configure_lftp,
staged_top_level_names,
validate_config,
)

Expand Down Expand Up @@ -341,3 +344,20 @@ def test_ftps_passes_persisted_values_through(self):
self.assertEqual(lftp.num_connections_per_root_file, 20)
self.assertEqual(lftp.num_connections_per_dir_file, 20)
self.assertEqual(lftp.num_max_total_connections, 0)


class TestStagedTopLevelNames(unittest.TestCase):
def test_lists_downloads_in_staging_by_their_final_name(self):
"""lftp's "<name>.lftp" temp file is the download <name>; its
"<name>.lftp-pget-status" progress file is not a download."""
with tempfile.TemporaryDirectory() as staging:
os.makedirs(os.path.join(staging, "Show.S01"))
for name in ("movie.mkv", "partial.mkv.lftp", "movie.mkv.lftp-pget-status"):
with open(os.path.join(staging, name), "w") as f:
f.write("x")

self.assertEqual({"Show.S01", "movie.mkv", "partial.mkv"}, staged_top_level_names(staging))

def test_missing_staging_dir_lists_nothing(self):
with tempfile.TemporaryDirectory() as parent:
self.assertEqual(set(), staged_top_level_names(os.path.join(parent, "nope")))
Loading