diff --git a/src/angular/src/app/pages/settings/options-list.ts b/src/angular/src/app/pages/settings/options-list.ts index 90962594..e474f06b 100644 --- a/src/angular/src/app/pages/settings/options-list.ts +++ b/src/angular/src/app/pages/settings/options-list.ts @@ -316,8 +316,7 @@ export const OPTIONS_CONTEXT_OTHER: IOptionsContext = { valuePath: ['controller', 'enable_nested_navigation'], description: 'Browse into subfolders of a tracked directory and queue, stop, or delete ' + - 'individual nested files and folders as independent jobs. Nested files ' + - "can't be queued while the staging directory is in use.", + 'individual nested files and folders as independent jobs.', }, ], }; diff --git a/src/angular/src/app/services/files/view-file-capabilities.spec.ts b/src/angular/src/app/services/files/view-file-capabilities.spec.ts index 2fc7947f..28eb1777 100644 --- a/src/angular/src/app/services/files/view-file-capabilities.spec.ts +++ b/src/angular/src/app/services/files/view-file-capabilities.spec.ts @@ -201,7 +201,7 @@ describe('view-file-capabilities', () => { }); }); describe('deriveCapabilities tree position', () => { - const base = { isTopLevel: true, ancestorActive: false, hasDownloadingDescendant: false, stagingEnabled: false }; + const base = { isTopLevel: true, ancestorActive: false, hasDownloadingDescendant: false }; it('disables queue, stop and deletes when an ancestor job is active', () => { const caps = deriveCapabilities(ViewFileStatus.DOWNLOADING, 100, 100, 100, 100, { @@ -233,15 +233,9 @@ describe('view-file-capabilities', () => { expect(caps.validateTooltip).toBeNull(); }); - it('blocks queuing a nested row only while staging is on', () => { + it('offers queue for a nested row', () => { const nested = { ...base, isTopLevel: false }; expect(deriveCapabilities(ViewFileStatus.DEFAULT, 0, 100, 0, 100, nested).isQueueable).toBe(true); - expect( - deriveCapabilities(ViewFileStatus.DEFAULT, 0, 100, 0, 100, { ...nested, stagingEnabled: true }).isQueueable, - ).toBe(false); - expect( - deriveCapabilities(ViewFileStatus.DEFAULT, 0, 100, 0, 100, { ...base, stagingEnabled: true }).isQueueable, - ).toBe(true); }); }); }); diff --git a/src/angular/src/app/services/files/view-file-capabilities.ts b/src/angular/src/app/services/files/view-file-capabilities.ts index a2f35bce..1b2637b5 100644 --- a/src/angular/src/app/services/files/view-file-capabilities.ts +++ b/src/angular/src/app/services/files/view-file-capabilities.ts @@ -88,15 +88,12 @@ export interface TreePosition { ancestorActive: boolean; /** A folder whose own job is idle but which has an independently active nested job. */ hasDownloadingDescendant: boolean; - /** Staging is on; nested jobs would land in staging, which only moves top-level names. */ - stagingEnabled: boolean; } const TOP_LEVEL: TreePosition = { isTopLevel: true, ancestorActive: false, hasDownloadingDescendant: false, - stagingEnabled: false, }; /** @@ -151,14 +148,11 @@ export function deriveCapabilities( // - hasDownloadingDescendant: a new job on the folder would race the nested // one, and deleting it would remove an in-flight download // (_find_active_job_conflict); Stop becomes "Stop Nested Downloads". - // - nested rows: EXTRACT/VALIDATE are top-level only, and so is QUEUE while - // staging is on. - const nestedQueueBlocked = !tree.isTopLevel && tree.stagingEnabled; + // - nested rows: EXTRACT/VALIDATE are top-level only. const isQueueable = ((QUEUEABLE_STATUSES.includes(status) && remoteSize > 0) || status === ViewFileStatus.CORRUPT) && !tree.ancestorActive && - !tree.hasDownloadingDescendant && - !nestedQueueBlocked; + !tree.hasDownloadingDescendant; const isStoppable = (STOPPABLE_STATUSES.includes(status) || tree.hasDownloadingDescendant) && !tree.ancestorActive; const isExtractable = LOCAL_ACTION_STATUSES.includes(status) && localSize > 0 && tree.isTopLevel; diff --git a/src/angular/src/app/services/files/view-file.service.spec.ts b/src/angular/src/app/services/files/view-file.service.spec.ts index bcbb2d53..c0d8120f 100644 --- a/src/angular/src/app/services/files/view-file.service.spec.ts +++ b/src/angular/src/app/services/files/view-file.service.spec.ts @@ -40,11 +40,8 @@ function makeModelFile( } interface TestConfig { - controller: { enable_nested_navigation: boolean; use_staging?: boolean; staging_path?: string }; + controller: { enable_nested_navigation: boolean }; } -const STAGING_ON: TestConfig = { - controller: { enable_nested_navigation: true, use_staging: true, staging_path: "/staging" }, -}; describe("ViewFileService", () => { let service: ViewFileService; @@ -596,17 +593,9 @@ describe("ViewFileService", () => { ]); } - it("blocks queuing nested rows while staging is on, but not the top-level row", () => { - configSubject.next(STAGING_ON); + it("offers queue for both the top-level row and a nested row", () => { emitQueueableTree(); expect(findByName(latestFiles(), "dir")!.isQueueable).toBe(true); - expect(findByName(latestFiles(), "child")!.isQueueable).toBe(false); - }); - - it("re-enables queuing nested rows when staging is turned off", () => { - configSubject.next(STAGING_ON); - emitQueueableTree(); - configSubject.next({ controller: { enable_nested_navigation: true, use_staging: false } }); expect(findByName(latestFiles(), "child")!.isQueueable).toBe(true); }); diff --git a/src/angular/src/app/services/files/view-file.service.ts b/src/angular/src/app/services/files/view-file.service.ts index 50d31546..e7f47a27 100644 --- a/src/angular/src/app/services/files/view-file.service.ts +++ b/src/angular/src/app/services/files/view-file.service.ts @@ -56,8 +56,6 @@ export class ViewFileService { // Nested rows exist in the view only while nested navigation is enabled, so // selection, select-all and bulk actions can never reach a hidden row. private nestedNavEnabled = false; - // Nested rows can't be queued while staging is on (see deriveCapabilities). - private stagingEnabled = false; // Resolve a view-file key to its backing ModelFile from the diffing-owned // snapshot. Threaded into ViewFileCommandService so command dispatch keeps the @@ -106,18 +104,14 @@ export class ViewFileService { } }); - // Toggling nested navigation adds or drops every nested row, and staging - // changes what nested rows offer, so rebuild the whole view; the rebuild - // also prunes checked keys of rows that went away. + // Toggling nested navigation adds or drops every nested row, so rebuild + // the whole view; the rebuild also prunes checked keys of rows that went + // away. this.configService.config$.pipe( - map((config) => ({ - nested: !!config?.controller.enable_nested_navigation, - staging: !!(config?.controller.use_staging && config.controller.staging_path), - })), - distinctUntilChanged((a, b) => a.nested === b.nested && a.staging === b.staging), - ).subscribe(({ nested, staging }) => { + map((config) => !!config?.controller.enable_nested_navigation), + distinctUntilChanged(), + ).subscribe((nested) => { this.nestedNavEnabled = nested; - this.stagingEnabled = staging; if (this.prevModelFiles.size > 0) { this.buildViewFromModelFiles(this.prevModelFiles, true); } @@ -392,7 +386,7 @@ export class ViewFileService { } private createTopLevelViewFile(modelFile: ModelFile): ViewFile { - return createViewFile(modelFile, this.pairNameMap, this.nestedNavEnabled, this.stagingEnabled); + return createViewFile(modelFile, this.pairNameMap, this.nestedNavEnabled); } private pushViewFiles(): void { @@ -553,7 +547,6 @@ function createViewFile( modelFile: ModelFile, pairNameMap: Map, includeChildren: boolean, - stagingEnabled: boolean, isTopLevel = true, ancestorActive = false, ): ViewFile { @@ -586,7 +579,7 @@ function createViewFile( remoteSize, modelFile.local_size, modelFile.remote_size, - { isTopLevel, ancestorActive, hasDownloadingDescendant, stagingEnabled }, + { isTopLevel, ancestorActive, hasDownloadingDescendant }, ); // Kept out of deriveCapabilities: it needs the ModelFile for is_dir and the @@ -603,7 +596,7 @@ function createViewFile( ancestorActive || modelFile.state === ModelFileState.QUEUED || modelFile.state === ModelFileState.DOWNLOADING; const children = includeChildren ? modelFile.children.map((child) => - createViewFile(child, pairNameMap, true, stagingEnabled, false, childAncestorActive), + createViewFile(child, pairNameMap, true, false, childAncestorActive), ) : []; diff --git a/src/python/controller/command_pipeline.py b/src/python/controller/command_pipeline.py index 3fbd8c2b..1079039c 100644 --- a/src/python/controller/command_pipeline.py +++ b/src/python/controller/command_pipeline.py @@ -14,7 +14,7 @@ from collections.abc import Callable, Iterator from queue import Queue -from common import Context, MultiprocessingLogger, UnrecoverableError +from common import Constants, Context, MultiprocessingLogger, UnrecoverableError from lftp import LftpError from model import ModelError, ModelFile, is_nested_name @@ -26,7 +26,7 @@ from .model_registry import ModelRegistry from .move import MoveProcess from .pair_context import PairContext -from .persist_keys import persist_key +from .persist_keys import persist_key, strip_persist_key from .validate import ValidateProcess, ValidateRequest from .worker_supervisor import WorkerSupervisor @@ -209,10 +209,9 @@ def _handle_queue( if file.remote_size is None: _notify_failure(command, f"File '{command.filename}' does not exist remotely") return False - # A nested job would land under the staging dir, and the staging move - # only handles top-level names, so the download would never arrive. - if file.parent is not None and pc.effective_local_path != pc.local_path: - _notify_failure(command, "Queuing a nested file is not supported while staging is enabled") + moving = self.find_overlapping_move(command.filename, pc) + if moving is not None: + _notify_failure(command, f"A staging move is in progress for '{moving}'") return False try: exclude = parse_exclude_patterns(self._context.config.general.exclude_patterns) @@ -220,8 +219,31 @@ def _handle_queue( except LftpError as e: _notify_failure(command, f"Lftp error: {e!s}") return False + if is_nested_name(command.filename): + if pc.effective_local_path != pc.local_path: + pc.pending_nested_moves.add(command.filename) + pc.staged_nested.add(command.filename) + self._persist.staged_nested_file_names.add(persist_key(pc.pair_id, command.filename)) + # A nested job's move has no requeue diff to reset it + # (ModelUpdater._handle_requeued_file sees top-level files only). + move_key = persist_key(pc.pair_id, command.filename) + self.moved_file_keys.discard(move_key) + if move_key in self._persist.move_failed_file_names: + self._persist.move_failed_file_names.discard(move_key) + self.sync_persist_callback() return True + def find_overlapping_move(self, name: str, pc: PairContext) -> str | None: + """Name of an in-flight staging move for this pair that `name` is, lies + inside, or contains, or None.""" + for move_process in self.active_move_processes: + if move_process.pair_id != pc.pair_id: + continue + moving = move_process.file_name + if name == moving or name.startswith(moving + "/") or moving.startswith(name + "/"): + return moving + return None + @staticmethod def _iter_active_descendants(file: ModelFile) -> Iterator[ModelFile]: """BFS over `file`'s descendants, yielding each independently active @@ -422,6 +444,7 @@ def post_callback(delete_path: str = delete_path, _pc: PairContext = pc) -> None _pc.local_scan_process.force_scan() if delete_path != _pc.local_path: _pc.active_scan_process.force_scan() + self._sweep_staged_nested(_pc) command_wrapper = CommandProcessWrapper(process=process, post_callback=post_callback, filename=command.filename) self.active_command_processes.append(command_wrapper) @@ -694,6 +717,7 @@ def _finalize_move_process(self, move_process: MoveProcess) -> None: # pair; fall back to all pairs only if the owner can't be located. owner = next((pc for pc in self._pair_contexts if pc.pair_id == move_process.pair_id), None) if owner is not None: + self._sweep_staged_nested(owner) owner.local_scan_process.force_scan() else: for pc in self._pair_contexts: @@ -782,13 +806,16 @@ def spawn_deferred_move(self, pair_id: str | None, file_name: str): return self.spawn_move_process(file_name, pc) - def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: + def spawn_move_process(self, file_name: str, pc: PairContext, expected_size: int | None = None) -> bool: """Spawn a MoveProcess to move a file from staging to the final local_path. Returns True if a move process was actually spawned, False if the call was a no-op (already moved / in flight, staging disabled, or nothing left in staging). A retry of a failed move uses this to tell a real retry from a resolved no-op so it can clear a stale MOVE_FAILED state (#536 follow-up). + + expected_size is passed to MoveProcess: a source short of it, or still + holding lftp temp files, is left in staging with no failure reported. """ pair_id = pc.pair_id move_key = persist_key(pair_id, file_name) @@ -815,6 +842,8 @@ def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: file_name=file_name, pair_id=pair_id, umask=self._context.args.umask, + expected_size=expected_size, + nested_expected_sizes=self._staged_nested_sizes(file_name, pc), ) process.set_mp_log_queue(self._mp_logger.queue, self._mp_logger.log_level) # Start before publishing bookkeeping: if start() raises we must not leave @@ -827,6 +856,58 @@ def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: self._logger.info(f"Spawned move process for {file_name} (staging -> local)") return True + def _staged_nested_sizes(self, file_name: str, pc: PairContext) -> dict[str, int | None]: + """{path relative to file_name: remote size} for each staged nested + download under file_name (None when the row can't be resolved).""" + sizes: dict[str, int | None] = {} + prefix = file_name + "/" + for name in pc.staged_nested: + if name.startswith(prefix): + try: + sizes[name[len(prefix) :]] = self._registry.resolve_full_path(name, pair_id=pc.pair_id).remote_size + except ModelError: + sizes[name[len(prefix) :]] = None + return sizes + + def _sweep_staged_nested(self, pc: PairContext) -> None: + """Forget staged nested downloads that are no longer in staging (moved, + deleted, or carried along by an ancestor's move).""" + staging = self._pair_staging_dir(pc) + # An unfinished file with lftp temp files on is ".lftp" + suffix = Constants.LFTP_TEMP_FILE_SUFFIX + gone = [ + name + for name in pc.staged_nested + if staging is None + or ( + not os.path.exists(os.path.join(staging, name)) + and not os.path.exists(os.path.join(staging, name + suffix)) + ) + ] + pc.staged_nested.difference_update(gone) + self._persist.staged_nested_file_names.difference_update(persist_key(pc.pair_id, name) for name in gone) + + def restore_staged_nested(self, pc: PairContext) -> None: + """Pick up the nested downloads a previous session left in staging. + + Restores pc.staged_nested from its persisted copy (dropping names no + longer in staging) and queues each one's move; the move checks its + size. A top-level name holding one is taken off pc.pending_completion: + the active scanner would report its staging copy, which holds only + the nested downloads, in place of its local_path copy. The active + scanner gets the corrected names before its first scan. + """ + if self._pair_staging_dir(pc) is None: + return + for key in self._persist.staged_nested_file_names: + name = strip_persist_key(key, pc.pair_id) + if name != key or pc.pair_id is None: + pc.staged_nested.add(name) + self._sweep_staged_nested(pc) + pc.pending_nested_moves.update(pc.staged_nested) + pc.pending_completion.difference_update(name.split("/", 1)[0] for name in pc.staged_nested) + pc.active_scanner.set_active_files(sorted(pc.pending_completion | pc.staged_nested)) + def _get_pair_context_for_command(self, command: Command) -> PairContext | None: """Find the pair context for a command based on pair_id.""" return self.find_pair_by_id(command.pair_id) diff --git a/src/python/controller/controller.py b/src/python/controller/controller.py index 8a0c9de9..9f7fe6cf 100644 --- a/src/python/controller/controller.py +++ b/src/python/controller/controller.py @@ -19,7 +19,7 @@ from .model_builder import ModelBuilder from .model_registry import ModelRegistry from .model_updater import ModelUpdater -from .pair_context import ControllerError, PairContext, configure_lftp, validate_config +from .pair_context import ControllerError, PairContext, configure_lftp, staged_top_level_names, validate_config from .persist_sync import PersistSync from .scan import ActiveScanner, LocalScanner, RemoteScanner, ScannerProcess from .validate import ValidateProcess @@ -113,6 +113,8 @@ def __init__(self, context: Context, persist: ControllerPersist): # Seed each builder with filtered persist state self.__persist_sync.sync() + for pc in self.__pair_contexts: + self.__pipeline.restore_staged_nested(pc) # Flag for hot-reloading LFTP tuning settings (set from REST thread) self.__needs_lftp_reconfigure = threading.Event() @@ -212,6 +214,14 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, # (_LFTP_TUNING_KEYS), so it can change after these are built, and # temp files from an earlier interrupted download can outlive it. active_scanner = ActiveScanner(effective_local_path, lftp_temp_suffix=Constants.LFTP_TEMP_FILE_SUFFIX) + # pending_completion is in memory only: re-seed it with what a previous + # session left in staging (finished or stopped), so the active scanner + # shows it and a finished one still gets its move. The scanner gets the + # names now so its first scan, not the next one, covers them. + left_in_staging: set[str] = ( + staged_top_level_names(effective_local_path) if effective_local_path != local_path else set() + ) + active_scanner.set_active_files(sorted(left_in_staging)) local_scanner = LocalScanner(local_path=local_path) remote_scanner = RemoteScanner( remote_address=self.__context.config.lftp.remote_address, # type: ignore[arg-type] @@ -270,6 +280,7 @@ def _create_pair_context(self, pair_id: str | None, name: str, remote_path: str, local_scan_process=local_scan_process, remote_scan_process=remote_scan_process, model_builder=model_builder, + pending_completion=left_in_staging, ) def _configure_lftp(self, lftp: Lftp): diff --git a/src/python/controller/controller_persist.py b/src/python/controller/controller_persist.py index 50fdc0ef..38a505cf 100644 --- a/src/python/controller/controller_persist.py +++ b/src/python/controller/controller_persist.py @@ -29,6 +29,9 @@ class ControllerPersist(Persist): ("validated", False), ("corrupt", False), ("move_failed", False), + # Nested downloads still in staging (PairContext.staged_nested), so a + # restart can find them again + ("staged_nested", False), ) def __init__(self): @@ -38,9 +41,10 @@ def __init__(self): self.validated_file_names: set[str] = set() self.corrupt_file_names: set[str] = set() self.move_failed_file_names: set[str] = set() + self.staged_nested_file_names: set[str] = set() def all_sets(self) -> dict[str, set[str]]: - """The six persisted key sets, keyed by json key.""" + """The persisted key sets, keyed by json key.""" return {key: getattr(self, f"{key}_file_names") for key, _ in self._KEYS} @staticmethod diff --git a/src/python/controller/model_builder.py b/src/python/controller/model_builder.py index d878a29c..62bd0e7e 100644 --- a/src/python/controller/model_builder.py +++ b/src/python/controller/model_builder.py @@ -30,7 +30,11 @@ def __init__(self, pair_id: str | None = None) -> None: self.__pair_id = pair_id self.__local_files: dict[str, SystemFile] = {} self.__active_files: dict[str, SystemFile] = {} + self.__staged_nested_files: dict[str, SystemFile] = {} self.__remote_files: dict[str, SystemFile] = {} + # False until the first remote listing: until then a missing remote + # size means "not scanned yet", not "deleted remotely" + self.__remote_listing_received = False self.__lftp_statuses: dict[str, LftpJobStatus] = {} self.__downloaded_files: set[str] = set() self.__extract_statuses: dict[str, ExtractStatus] = {} @@ -48,12 +52,15 @@ def set_base_logger(self, base_logger: logging.Logger): self.logger = base_logger.getChild("ModelBuilder") def set_active_files(self, active_files: list[SystemFile]): - prev_active_files = self.__active_files - self.__active_files = {file.name: file for file in active_files} + prev = (self.__active_files, self.__staged_nested_files) + # A nested name ("Top/Sub/f.bin") is a nested download in staging; it + # overrides that child's local side, not a top-level entry. + self.__active_files = {f.name: f for f in active_files if not is_nested_name(f.name)} + self.__staged_nested_files = {f.name: f for f in active_files if is_nested_name(f.name)} # Always invalidate when active files present (sizes change rapidly) # Also invalidate when active files change (e.g. files removed after # a stopped download is deleted locally) - if len(active_files) > 0 or self.__active_files != prev_active_files: + if len(active_files) > 0 or (self.__active_files, self.__staged_nested_files) != prev: self.__cached_model = None def set_local_files(self, local_files: list[SystemFile]): @@ -67,8 +74,9 @@ def set_remote_files(self, remote_files: list[SystemFile]): prev_remote_files = self.__remote_files self.__remote_files = {file.name: file for file in remote_files} # Invalidate the cache - if self.__remote_files != prev_remote_files: + if self.__remote_files != prev_remote_files or not self.__remote_listing_received: self.__cached_model = None + self.__remote_listing_received = True def set_lftp_statuses(self, lftp_statuses: list[LftpJobStatus]): prev_lftp_statuses = self.__lftp_statuses @@ -141,7 +149,9 @@ def set_auto_delete_remote(self, enabled: bool): def clear(self): self.__local_files.clear() self.__active_files.clear() + self.__staged_nested_files.clear() self.__remote_files.clear() + self.__remote_listing_received = False self.__lftp_statuses.clear() self.__downloaded_files.clear() self.__extract_statuses.clear() @@ -273,7 +283,9 @@ def _build_children( _all_children_names: set[str] = set[str]().union(_remote_children.keys(), _local_children.keys()) for _child_name in _all_children_names: _remote_child = _remote_children.get(_child_name) - _local_child = _local_children.get(_child_name) + _local_child = self.__staged_nested_files.get( + os.path.join(_model_file.full_path, _child_name), _local_children.get(_child_name) + ) _is_dir = _remote_child.is_dir if _remote_child else (_local_child.is_dir if _local_child else False) # sanity check is_dir if (_remote_child and _is_dir != _remote_child.is_dir) or ( @@ -304,6 +316,14 @@ def _build_children( _child_model_file.state, _effective_transfer_state = ModelBuilder._determine_child_state( _is_dir, _child_transfer_state, _nested_status, _remote_child, _local_child, root_model_file ) + # A nested job whose staging->final move failed. Unlike a root + # (_determine_state) there's no post-download state to require: + # the file sits in staging, which the local scan doesn't cover. + if _child_model_file.full_path in self.__move_failed_files and _child_model_file.state not in ( + ModelFile.State.QUEUED, + ModelFile.State.DOWNLOADING, + ): + _child_model_file.state = ModelFile.State.MOVE_FAILED # fill the rest ModelBuilder._fill_model_file(_child_model_file, _remote_child, _local_child, _effective_transfer_state) @@ -596,6 +616,9 @@ def _check_persist_authority(self, model_file: ModelFile, incomplete_children: b and model_file.local_size < model_file.remote_size ): return # Stay DEFAULT for re-download + # A missing remote size only means "gone" once a remote listing is in + if model_file.remote_size is None and not self.__remote_listing_received: + return # Persist authority — applies when auto_delete_remote OFF, # or when remote is already gone (auto_delete_remote ON, delete succeeded) if not self.__auto_delete_remote or model_file.remote_size is None: diff --git a/src/python/controller/model_updater.py b/src/python/controller/model_updater.py index 47fc5f82..6d5b0ac2 100644 --- a/src/python/controller/model_updater.py +++ b/src/python/controller/model_updater.py @@ -104,6 +104,7 @@ def update(self) -> None: if new_model is not None: model_diff = self._registry.apply_diff(new_model) self._process_model_diffs(model_diff) + self._note_remote_scans_in_model() self._prune_stale_persist() self._process_extraction_failures(latest_failed_extractions) @@ -252,7 +253,13 @@ def _handle_downloaded_file(self, diff: ModelDiff, pc: PairContext | None) -> No if self._context.config.controller.use_staging and self._context.config.controller.staging_path: will_auto_extract = self._context.config.autoqueue.auto_extract and diff.new_file.is_extractable - if not will_auto_extract and not will_auto_validate: + # A nested download still in staging counts toward its root, so the + # root can turn Downloaded while that download's own move is in + # flight. Leave the files to that move rather than race it. + overlapping = self._pipeline.find_overlapping_move(diff.new_file.name, pc) + if overlapping is not None: + self._logger.info(f"Not moving '{diff.new_file.name}': '{overlapping}' is moving out of staging") + elif not will_auto_extract and not will_auto_validate: self._pipeline.spawn_move_process(diff.new_file.name, pc) def _handle_pending_completion(self, diff: ModelDiff, pc: PairContext | None) -> None: @@ -327,6 +334,30 @@ def _prune_absent_persist_entries(self) -> None: for key in absent_keys: self._move_retry_counts.pop(key, None) self.sync_persist_to_all_builders() + self._prune_absent_nested_move_failures() + + def _prune_absent_nested_move_failures(self) -> None: + """Drop nested move_failed keys whose row is gone (remote and local). + + Nested keys never reach downloaded_file_names, which drives the prune + above, so they need their own pass. Called once all scans are in. + """ + absent_keys: set[str] = set() + for key in self._persist.move_failed_file_names: + for pc in self._pair_contexts: + bare_name = strip_persist_key(key, pc.pair_id) + if bare_name == key and pc.pair_id is not None: + continue + if is_nested_name(bare_name): + try: + self._registry.resolve_full_path(bare_name, pair_id=pc.pair_id) + except ModelError: + absent_keys.add(key) + break + if absent_keys: + self._logger.info(f"Persist cleanup (nested move failures gone): {absent_keys}") + self._persist.move_failed_file_names.difference_update(absent_keys) + self.sync_persist_to_all_builders() def _process_extraction_failures(self, latest_failed_extractions: list[ExtractFailedResult]) -> None: """Process extraction failures -- mark as failed immediately.""" @@ -382,6 +413,11 @@ def _retry_failed_moves(self) -> None: move_failed_file_names that is not already moving, bounded by a per-file budget so a permanently-failing move does not re-spawn forever (#536). """ + # A count outlives its failure when the key is cleared without a requeue + # diff (a nested re-queue, see CommandPipeline._handle_queue); drop it so + # a later failure starts a fresh budget. + for key in [k for k in self._move_retry_counts if k not in self._persist.move_failed_file_names]: + del self._move_retry_counts[key] cfg = self._context.config.controller if not (cfg.use_staging and cfg.staging_path): return @@ -495,6 +531,10 @@ def _update_pair_model_state( active_files = pc.active_downloading_file_names + pc.active_extracting_file_names active_files += list(pc.pending_completion) + # Nested downloads in staging, finished or not: the local scan covers + # only local_path, so this is the only scan that sees them. The builder + # puts each on its nested row, not at the top level. + active_files += list(pc.staged_nested) pc.active_scanner.set_active_files(active_files) pc.model_builder.set_auto_delete_remote(bool(self._context.config.autoqueue.auto_delete_remote)) @@ -552,6 +592,41 @@ def _detect_lftp_completions(self, pc: PairContext, lftp_statuses: list[LftpJobS # recursive local scan, just less often. pc.active_downloading_file_names = [name for name in current_downloading if not is_nested_name(name)] pc.prev_downloading_file_names = current_downloading + self._spawn_nested_moves(pc, {s.name for s in lftp_statuses}) + + def _spawn_nested_moves(self, pc: PairContext, lftp_job_names: set[str]) -> None: + """Move each nested job queued under staging from staging to local_path + once lftp no longer lists it. + + Top-level moves start from the DOWNLOADED diff; nested rows have no + diffs, so they start here. Driven by the queue rather than a + running->gone transition, which a job finishing between two status + polls (a small file, or one already complete in staging) never shows. + The job ending doesn't mean it finished (a Stop ends it too), so the + move gets the row's remote size to check. + """ + if not pc.remote_scan_in_model: + # After a restart the pending moves come back before the first remote + # scan; without it in the model every row looks unresolvable. + return + for name in pc.pending_nested_moves - lftp_job_names: + pc.pending_nested_moves.discard(name) + try: + expected_size = self._registry.resolve_full_path(name, pair_id=pc.pair_id).remote_size + except ModelError: + expected_size = None + if expected_size is None: + # Without the remote size a Stop's partial can't be told from a + # finished download, so leave it in staging rather than move it. + self._logger.warning(f"Not moving '{name}' out of staging: its remote size is unknown") + continue + self._pipeline.spawn_move_process(name, pc, expected_size=expected_size) + + def _note_remote_scans_in_model(self) -> None: + """Mark each pair whose remote scan has reached the registry.""" + for pc in self._pair_contexts: + if pc.remote_scan_received: + pc.remote_scan_in_model = True def sync_persist_to_all_builders(self): """Push current persist state to all pair model builders, filtered by pair_id.""" diff --git a/src/python/controller/move/move_process.py b/src/python/controller/move/move_process.py index 578a7972..965d1be9 100644 --- a/src/python/controller/move/move_process.py +++ b/src/python/controller/move/move_process.py @@ -2,6 +2,7 @@ from __future__ import annotations +import contextlib import datetime import errno import os @@ -39,6 +40,8 @@ def __init__( file_name: str, pair_id: str | None = None, umask: int | None = None, + expected_size: int | None = None, + nested_expected_sizes: dict[str, int | None] | None = None, ): super().__init__(name=self.__class__.__name__) self.__source_path = source_path @@ -48,6 +51,13 @@ def __init__( # When set, the moved tree is chmod'ed to 0666/0777 & ~umask here, in # this process, rather than on the controller thread (#811). self.__umask = umask + # When set, a source smaller than this or still holding lftp temp files + # is left in place, with no failure reported. + self.__expected_size = expected_size + # {path relative to file_name: expected size} of descendants to check + # the same way; an incomplete one (or one of unknown size) is left in + # place while the rest of the tree moves. + self.__nested_expected_sizes = nested_expected_sizes or {} self.__failed_result_stream = PipeStream() @property @@ -91,8 +101,9 @@ def _get_copied_size(src: str, dst: str) -> int: return total @staticmethod - def _has_lftp_temp_files(src: str) -> bool: - """True if the source tree still contains lftp temp files (``*.lftp``). + def _has_lftp_temp_files(src: str, skip: frozenset[str] = frozenset()) -> bool: + """True if the source tree still contains lftp temp files (``*.lftp``), + not counting anything under the paths in ``skip``. lftp writes each download to ``.lftp`` and renames it to the final name on completion. A staging tree that still holds temp files has not @@ -100,8 +111,12 @@ def _has_lftp_temp_files(src: str) -> bool: """ suffix = Constants.LFTP_TEMP_FILE_SUFFIX if os.path.isfile(src): - return src.endswith(suffix) - return any(any(f.endswith(suffix) for f in filenames) for _root, _dirs, filenames in os.walk(src)) + return src.endswith(suffix) and src not in skip + for root, dirs, filenames in os.walk(src): + dirs[:] = [d for d in dirs if os.path.join(root, d) not in skip] + if any(f.endswith(suffix) and os.path.join(root, f) not in skip for f in filenames): + return True + return False def _report_failure(self, message: str) -> None: """Log and publish a move failure so the controller can surface it.""" @@ -123,13 +138,21 @@ def run_once(self): self._report_failure(f"Move failed: source does not exist: {src}") return + if self.__expected_size is not None and ( + self._has_lftp_temp_files(src) or self._get_total_size(src) < self.__expected_size + ): + self.logger.info(f"Move skipped for {self.__file_name}: download incomplete, left in staging") + return + + incomplete = self._incomplete_nested(src) + # Defer while lftp is still finalizing the staging tree. lftp renames # ".lftp" -> "" on completion; copying (or renaming the # whole tree) while a rename is in flight races — copytree lists the # ".lftp" entry, lftp renames it, and the copy then fails with ENOENT # (which used to crash this process). A tree that still has temp files is # not ready, so report a recoverable failure and retry once lftp finishes. - if self._has_lftp_temp_files(src): + if self._has_lftp_temp_files(src, incomplete): self._report_failure( f"Move deferred for {self.__file_name}: download not finalized " "(lftp temp files still present in staging). Will retry." @@ -138,6 +161,61 @@ def run_once(self): self.logger.info(f"Moving {src} -> {dst}") + # Ensure the destination's parent exists (several levels for a nested name) + os.makedirs(os.path.dirname(dst), exist_ok=True) + + if not incomplete: + if not self._move_item(src, dst): + return + else: + if not self._move_tree(src, dst, incomplete): + return + left = ", ".join(sorted(os.path.relpath(p, self.__source_path) for p in incomplete)) + self.logger.info(f"Left incomplete nested downloads in staging: {left}") + self.__prune_empty_staging_parents(src) + + def _incomplete_nested(self, src: str) -> frozenset[str]: + """Absolute staging paths of the listed nested downloads under src that + aren't complete yet (temp files, short, or of unknown size).""" + incomplete: set[str] = set() + for rel, size in self.__nested_expected_sizes.items(): + path = os.path.join(src, rel) + temp_path = path + Constants.LFTP_TEMP_FILE_SUFFIX + if not os.path.exists(path) and os.path.isfile(temp_path): + # An unfinished file with lftp temp files on: ".lftp" until done + incomplete.add(temp_path) + elif os.path.exists(path) and ( + size is None or self._has_lftp_temp_files(path) or self._get_total_size(path) < size + ): + incomplete.add(path) + return frozenset(incomplete) + + def _move_tree(self, src: str, dst: str, incomplete: frozenset[str]) -> bool: + """Move src to dst, leaving the paths in ``incomplete`` (and the dirs + holding them) in staging. Subtrees without one move whole.""" + if src in incomplete: + return True + if not any(p.startswith(src + os.sep) for p in incomplete): + # Parents are made only once something lands in them, so a dir + # holding nothing but incomplete downloads isn't created at dst. + os.makedirs(os.path.dirname(dst), exist_ok=True) + return self._move_item(src, dst) + created = not os.path.exists(dst) + for entry in os.listdir(src): + if not self._move_tree(os.path.join(src, entry), os.path.join(dst, entry), incomplete): + return False + if created and os.path.isdir(dst): + # Made here by a child's makedirs; a dir that was already there keeps its mode + self.__apply_modes(dst, [(".", True)]) + with contextlib.suppress(OSError): # still holds an incomplete download + os.rmdir(src) + return True + + def _move_item(self, src: str, dst: str) -> bool: + """Move one file or tree: rename, or copy + verify + delete across + devices or onto a non-empty dir. Reports and returns False on failure.""" + name = os.path.relpath(src, self.__source_path) + # What we are about to move, so the UMASK pass can be limited to it: a # copy that merges into an existing destination must leave the files # already there alone (#811). Directories that already exist at the @@ -146,15 +224,12 @@ def run_once(self): moved_entries = relative_entries(src) if self.__umask is not None else [] existing_dir_modes = self._existing_dir_modes(dst) - # Ensure destination parent directory exists - os.makedirs(self.__dest_path, exist_ok=True) - # Try rename first (instant on same filesystem) try: os.rename(src, dst) - self.logger.info(f"Move completed via rename: {self.__file_name}") + self.logger.info(f"Move completed via rename: {name}") self.__apply_modes(dst, moved_entries) - return + return True except OSError as e: if e.errno == errno.EXDEV: self.logger.debug("Cross-device move detected, falling back to copy+delete") @@ -178,18 +253,18 @@ def run_once(self): # narrow window the pre-check above doesn't fully close). Report a # recoverable failure and keep the source intact so the move is # retried — never crash the controller. - self._report_failure(f"Move copy failed for {self.__file_name}: {e}. Source NOT deleted.") - return + self._report_failure(f"Move copy failed for {name}: {e}. Source NOT deleted.") + return False # Verify only the copied subtree (ignore pre-existing files under dst) copied_size = self._get_copied_size(src, dst) if source_size != copied_size: self._report_failure( - f"Move size verification failed for {self.__file_name}: " + f"Move size verification failed for {name}: " f"source={source_size} copied={copied_size}. Source NOT deleted." ) - return + return False # Size verified, remove source if os.path.isfile(src): @@ -197,10 +272,11 @@ def run_once(self): else: shutil.rmtree(src) - self.logger.info(f"Move completed via copy+delete: {self.__file_name}") + self.logger.info(f"Move completed via copy+delete: {name}") for rel, mode in existing_dir_modes.items(): os.chmod(os.path.join(dst, rel), mode) self.__apply_modes(dst, [e for e in moved_entries if e[0] not in existing_dir_modes]) + return True @staticmethod def _existing_dir_modes(dst: str) -> dict[str, int]: @@ -219,6 +295,18 @@ def __apply_modes(self, dst: str, moved_entries: list[tuple[str, bool]]) -> None if self.__umask is not None: apply_umask_modes_to(dst, moved_entries, self.__umask, self.logger) + def __prune_empty_staging_parents(self, src: str) -> None: + """Remove the staging dirs a nested name's move emptied, up to (not + including) source_path. Stops at the first one still holding something, + such as another nested job's download.""" + parent = os.path.dirname(src) + while os.path.normpath(parent) != os.path.normpath(self.__source_path): + try: + os.rmdir(parent) + except OSError: + return + parent = os.path.dirname(parent) + def pop_failed(self) -> list[MoveFailedResult]: """Process-safe method to retrieve any move failures.""" return self.__failed_result_stream.pop_all() diff --git a/src/python/controller/pair_context.py b/src/python/controller/pair_context.py index 218d764b..58dc95fa 100644 --- a/src/python/controller/pair_context.py +++ b/src/python/controller/pair_context.py @@ -8,6 +8,7 @@ from __future__ import annotations +import os from dataclasses import dataclass, field from common import AppError, Config, Constants, Context @@ -51,7 +52,17 @@ class PairContext: active_extracting_file_names: list[str] = field(default_factory=list[str]) prev_downloading_file_names: set[str] = field(default_factory=set[str]) pending_completion: set[str] = field(default_factory=set[str]) + # Nested names queued while staging is on, awaiting their staging->final + # move once lftp no longer lists the job (see ModelUpdater._spawn_nested_moves) + pending_nested_moves: set[str] = field(default_factory=set[str]) + # Nested names queued while staging is on whose download still sits in + # staging (finished or not). A top-level move leaves the incomplete ones + # behind (see CommandPipeline.spawn_move_process). + staged_nested: set[str] = field(default_factory=set[str]) remote_scan_received: bool = False + # True once a model built from a remote scan is in the registry, so a + # nested row's remote size can be resolved (see ModelUpdater._spawn_nested_moves) + remote_scan_in_model: bool = False local_scan_received: bool = False # Temporary storage for latest scan results (set during ModelUpdater._update_pair_model_state) @@ -59,6 +70,21 @@ class PairContext: latest_local_scan: ScannerResult | None = None +def staged_top_level_names(staging_dir: str) -> set[str]: + """Names of the top-level downloads with something in staging_dir. + + lftp's ".lftp" temp file is listed as ; its + ".lftp-pget-status" progress file is skipped. A missing dir lists + nothing. + """ + try: + entries = os.listdir(staging_dir) + except FileNotFoundError: + return set() + suffix = Constants.LFTP_TEMP_FILE_SUFFIX + return {entry.removesuffix(suffix) for entry in entries if not entry.endswith(suffix + "-pget-status")} + + def validate_config(context: Context) -> None: """Validate that all required config fields are set (non-None) at startup. diff --git a/src/python/lftp/lftp.py b/src/python/lftp/lftp.py index cfa77731..9cdac8dd 100644 --- a/src/python/lftp/lftp.py +++ b/src/python/lftp/lftp.py @@ -113,6 +113,8 @@ def __init__( self.__timeout = 10 # in seconds self.__consecutive_status_errors = 0 self.__consecutive_timeouts = 0 + # Set by __run_command: its "" on a timeout reads the same as no output + self.__last_command_timed_out = False self.__settings_cache: dict[str, str] = {} self.__log_command_output = False @@ -289,6 +291,7 @@ def __run_command(self, command: str): # type: ignore[arg-type] self.__process.expect(self.__expect_pattern, timeout=self.__timeout) except pexpect.exceptions.TIMEOUT: timed_out = True + self.__last_command_timed_out = timed_out if timed_out: self.__consecutive_timeouts += 1 @@ -577,12 +580,15 @@ def xfer_verify_command(self, command: str): def status(self) -> list[LftpJobStatus] | None: """ Return a status list of queued and running jobs. - Returns None when the status output could not be parsed, so callers + Returns None when the status output timed out or could not be parsed, so callers can distinguish "no jobs" from "parse failed" and avoid false completion signals. :return: """ out = self.__run_command("jobs -v") # type: ignore[arg-type] + if self.__last_command_timed_out: + # Empty output here isn't "no jobs"; it would read as every job finished + return None try: statuses = self.__job_status_parser.parse(out) self.__consecutive_status_errors = 0 diff --git a/src/python/system/file_modes.py b/src/python/system/file_modes.py index 991b11e0..6cfe751d 100644 --- a/src/python/system/file_modes.py +++ b/src/python/system/file_modes.py @@ -53,7 +53,8 @@ def apply_umask_modes_to(base: str, entries: Iterable[tuple[str, bool]], umask: dir_mode = 0o777 & ~umask # Deepest first, for the same reason apply_umask_modes walks bottom-up. for rel, is_dir in sorted(entries, key=lambda e: e[0].count(os.sep), reverse=True): - _chmod(os.path.join(base, rel), dir_mode if is_dir else file_mode, logger) + # normpath: a single file's entry is "." and "file/." is ENOTDIR + _chmod(os.path.normpath(os.path.join(base, rel)), dir_mode if is_dir else file_mode, logger) def relative_entries(root: str) -> list[tuple[str, bool]]: diff --git a/src/python/tests/unittests/test_controller/test_command_pipeline.py b/src/python/tests/unittests/test_controller/test_command_pipeline.py index c07cade4..fa53cfe2 100644 --- a/src/python/tests/unittests/test_controller/test_command_pipeline.py +++ b/src/python/tests/unittests/test_controller/test_command_pipeline.py @@ -1,6 +1,7 @@ # Copyright 2017, Inderpreet Singh, All rights reserved. import os +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -299,6 +300,20 @@ def test_finalize_rescans_only_owning_pair(self): owner.local_scan_process.force_scan.assert_called_once() other.local_scan_process.force_scan.assert_not_called() + def test_finalize_sweeps_staged_nested_for_owner_pair(self): + """Finishing a move must sweep the owner's staged_nested so entries the + move carried along (or left behind) are reconciled.""" + owner = self._make_pair_context("pair-1") + other = self._make_pair_context("pair-2") + pipeline = self._make_pipeline([owner, other]) + move_process = self._make_move_process("pair-1", "Top", failed_results=[]) + pipeline.active_move_processes.append(move_process) + + with patch.object(pipeline, "_sweep_staged_nested") as mock_sweep: + pipeline.cleanup() + + mock_sweep.assert_called_once_with(owner) + def test_finalize_rescans_all_pairs_when_owner_not_found(self): """If the move's pair_id matches no context, fall back to rescanning all.""" pc1 = self._make_pair_context("pair-1") @@ -332,6 +347,210 @@ def test_spawn_move_process_start_failure_leaves_no_stale_state(self, mock_move_ self.assertNotIn(persist_key("pair-1", "file.txt"), pipeline.moved_file_keys) self.assertEqual([], pipeline.active_move_processes) + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.MoveProcess") + def test_spawn_move_process_passes_expected_size_to_move_process(self, mock_move_cls, _mock_exists): + """A nested job's just-ended expected_size must reach MoveProcess so it + can tell an incomplete source from a finished one.""" + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.local_path = "/local/pair-1" + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + + pipeline.spawn_move_process("Top/leaf.rar", pc, expected_size=12345) + + self.assertEqual(12345, mock_move_cls.call_args.kwargs["expected_size"]) + + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.MoveProcess") + def test_spawn_move_process_default_expected_size_is_none(self, mock_move_cls, _mock_exists): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.local_path = "/local/pair-1" + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + + pipeline.spawn_move_process("file.txt", pc) + + self.assertIsNone(mock_move_cls.call_args.kwargs["expected_size"]) + + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.MoveProcess") + def test_spawn_move_process_passes_nested_expected_sizes_to_move_process(self, mock_move_cls, _mock_exists): + """A top-level move must carry the staged nested downloads under it to + MoveProcess so an incomplete one can be left behind.""" + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.local_path = "/local/pair-1" + pc.staged_nested = {"Top/Sub/big.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + resolved = MagicMock() + resolved.remote_size = 999 + pipeline._registry.resolve_full_path.return_value = resolved + + pipeline.spawn_move_process("Top", pc) + + self.assertEqual({"Sub/big.bin": 999}, mock_move_cls.call_args.kwargs["nested_expected_sizes"]) + + # --- _staged_nested_sizes --- + + def test_staged_nested_sizes_filters_by_prefix_and_resolves_remote_size(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/Sub/big.bin", "Top/a.bin", "TopX/other.bin"} + pipeline = self._make_pipeline([pc]) + + remote_sizes = {"Top/Sub/big.bin": 500, "Top/a.bin": 10} + + def resolve(name, pair_id): + file = MagicMock() + file.remote_size = remote_sizes[name] + return file + + pipeline._registry.resolve_full_path.side_effect = resolve + + result = pipeline._staged_nested_sizes("Top", pc) + + self.assertEqual({"Sub/big.bin": 500, "a.bin": 10}, result) + resolved_names = {call.args[0] for call in pipeline._registry.resolve_full_path.call_args_list} + self.assertNotIn("TopX/other.bin", resolved_names) + + def test_staged_nested_sizes_model_error_maps_to_none(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/gone.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._registry.resolve_full_path.side_effect = ModelError("not found") + + result = pipeline._staged_nested_sizes("Top", pc) + + self.assertEqual({"gone.bin": None}, result) + + def test_staged_nested_sizes_empty_when_no_entries_under_file_name(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Other/a.bin"} + pipeline = self._make_pipeline([pc]) + + result = pipeline._staged_nested_sizes("Top", pc) + + self.assertEqual({}, result) + pipeline._registry.resolve_full_path.assert_not_called() + + # --- _sweep_staged_nested --- + + def test_sweep_staged_nested_drops_entries_no_longer_in_staging(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/kept.bin", "Top/gone.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + kept_path = os.path.join("/tmp/staging", "pair-1", "Top/kept.bin") + + with patch("controller.command_pipeline.os.path.exists", side_effect=lambda p: p == kept_path): + pipeline._sweep_staged_nested(pc) + + self.assertEqual({"Top/kept.bin"}, pc.staged_nested) + + def test_sweep_staged_nested_keeps_entries_only_in_staging_as_temp_files(self): + """With lftp temp files on, an unfinished nested file is '.lftp'.""" + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/partial.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + temp_path = os.path.join("/tmp/staging", "pair-1", "Top/partial.bin.lftp") + + with patch("controller.command_pipeline.os.path.exists", side_effect=lambda p: p == temp_path): + pipeline._sweep_staged_nested(pc) + + self.assertEqual({"Top/partial.bin"}, pc.staged_nested) + + def test_sweep_staged_nested_clears_all_when_staging_disabled(self): + pc = self._make_pair_context("pair-1") + pc.pair_id = "pair-1" + pc.staged_nested = {"Top/a.bin", "Top/b.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = False + pipeline._context.config.controller.staging_path = None + + pipeline._sweep_staged_nested(pc) + + self.assertEqual(set(), pc.staged_nested) + + def test_sweep_staged_nested_drops_persisted_keys_too(self): + pc = self._make_pair_context("pair-1") + pc.staged_nested = {"Top/kept.bin", "Top/gone.bin"} + pipeline = self._make_pipeline([pc]) + pipeline._persist.staged_nested_file_names = { + persist_key("pair-1", "Top/kept.bin"), + persist_key("pair-1", "Top/gone.bin"), + } + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + kept_path = os.path.join("/tmp/staging", "pair-1", "Top/kept.bin") + + with patch("controller.command_pipeline.os.path.exists", side_effect=lambda p: p == kept_path): + pipeline._sweep_staged_nested(pc) + + self.assertEqual({persist_key("pair-1", "Top/kept.bin")}, pipeline._persist.staged_nested_file_names) + + # --- restore_staged_nested --- + + def test_restore_staged_nested_picks_up_what_a_previous_session_left_in_staging(self): + """staged_nested is in memory; its persisted copy brings back the nested + downloads still in staging, queues their moves, and keeps their + top-level name off the active scan (its staging copy holds only them).""" + with tempfile.TemporaryDirectory() as staging_root: + os.makedirs(os.path.join(staging_root, "pair-1", "Top", "Sub")) + with open(os.path.join(staging_root, "pair-1", "Top", "Sub", "f.bin"), "w") as f: + f.write("x") + pc = self._make_pair_context("pair-1") + pc.staged_nested = set() + pc.pending_nested_moves = set() + pc.pending_completion = {"Top", "Movie.mkv"} + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = staging_root + other_pair_key = persist_key("pair-2", "X/y.bin") + pipeline._persist.staged_nested_file_names = { + persist_key("pair-1", "Top/Sub/f.bin"), + persist_key("pair-1", "Top/gone.bin"), + other_pair_key, + } + + pipeline.restore_staged_nested(pc) + + self.assertEqual({"Top/Sub/f.bin"}, pc.staged_nested) + self.assertEqual({"Top/Sub/f.bin"}, pc.pending_nested_moves) + self.assertEqual({"Movie.mkv"}, pc.pending_completion) + self.assertEqual( + {persist_key("pair-1", "Top/Sub/f.bin"), other_pair_key}, pipeline._persist.staged_nested_file_names + ) + pc.active_scanner.set_active_files.assert_called_once_with(["Movie.mkv", "Top/Sub/f.bin"]) + + def test_restore_staged_nested_does_nothing_with_staging_off(self): + pc = self._make_pair_context("pair-1") + pc.staged_nested = set() + pc.pending_nested_moves = set() + pipeline = self._make_pipeline([pc]) + pipeline._context.config.controller.use_staging = False + pipeline._context.config.controller.staging_path = None + pipeline._persist.staged_nested_file_names = {persist_key("pair-1", "Top/a.bin")} + + pipeline.restore_staged_nested(pc) + + self.assertEqual(set(), pc.staged_nested) + self.assertEqual(set(), pc.pending_nested_moves) + pc.active_scanner.set_active_files.assert_not_called() + # --- propagate_exceptions: worker-fault isolation (#511) --- def test_propagate_exceptions_clean_when_no_worker_errors(self): @@ -686,6 +905,8 @@ def _make_pair_context(self, pair_id=None, effective_local_path="/local"): pc.pair_id = pair_id pc.local_path = "/local" pc.effective_local_path = effective_local_path + pc.pending_nested_moves = set() + pc.staged_nested = set() return pc @staticmethod @@ -1074,6 +1295,32 @@ def test_handle_delete_local_nested_uses_full_path(self, mock_delete_cls): self.assertTrue(result) mock_delete_cls.assert_called_once_with(local_path="/local", file_name="Top/leaf.rar") + @patch("controller.command_pipeline.os.path.exists", return_value=True) + @patch("controller.command_pipeline.DeleteLocalProcess") + def test_handle_delete_local_of_staged_nested_partial_sweeps_staged_nested(self, mock_delete_cls, _mock_exists): + """A stopped nested partial in staging is deleted there, and the sweep + afterwards forgets it so it isn't scanned or size-checked any more.""" + pc = self._make_pair_context("pair-1") + pc.local_path = "/local" + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.controller.use_staging = True + pipeline._context.config.controller.staging_path = "/tmp/staging" + + leaf = ModelFile("leaf.rar", False) + leaf.state = ModelFile.State.DEFAULT + leaf.local_size = 40 + command = Command(Command.Action.DELETE_LOCAL, "Top/leaf.rar", pair_id="pair-1") + _calls, notify = self._make_notify() + + result = pipeline._handle_delete_local(command, leaf, pc, [], notify) + + self.assertTrue(result) + staging = os.path.join("/tmp/staging", "pair-1") + mock_delete_cls.assert_called_once_with(local_path=staging, file_name="Top/leaf.rar") + with patch.object(pipeline, "_sweep_staged_nested") as mock_sweep: + pipeline.active_command_processes[0].post_callback() + mock_sweep.assert_called_once_with(pc) + @patch("controller.command_pipeline.DeleteRemoteProcess") def test_handle_delete_remote_nested_uses_full_path(self, mock_delete_cls): pc = self._make_pair_context("pair-1") @@ -1295,11 +1542,18 @@ def test_step_nested_queue_end_to_end(self): callback.on_success.assert_called_once() callback.on_failure.assert_not_called() - def test_nested_queue_refused_while_staging(self): - """A nested job would land in the staging dir, which only top-level - moves empty, so it's refused rather than orphaned.""" + def test_nested_queue_under_staging_clears_move_state(self): + """Nested queuing under staging is allowed. A successful queue has no + requeue diff to reset it, so it must directly clear any stale moved/ + move-failed bookkeeping for that nested name, syncing persist if it + cleared a failure.""" pc = self._make_pair_context("pair-1", effective_local_path="/staging/pair-1") pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + move_key = persist_key("pair-1", "Top/leaf.rar") + pipeline.moved_file_keys.add(move_key) + pipeline._persist.move_failed_file_names = {move_key} + root = ModelFile("Top", True) leaf = ModelFile("leaf.rar", False) leaf.remote_size = 100 @@ -1307,11 +1561,137 @@ def test_nested_queue_refused_while_staging(self): command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") calls, notify = self._make_notify() - result = pipeline._dispatch_command(command, leaf, pc, [], notify) + result = pipeline._handle_queue(command, leaf, pc, notify) + + self.assertTrue(result) + self.assertEqual([], calls) + pc.lftp.queue.assert_called_once_with("Top/leaf.rar", False, exclude_patterns=[]) + self.assertNotIn(move_key, pipeline.moved_file_keys) + self.assertNotIn(move_key, pipeline._persist.move_failed_file_names) + pipeline.sync_persist_callback.assert_called_once() + self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + self.assertEqual({"Top/leaf.rar"}, pc.staged_nested) + + def test_nested_queue_under_staging_persists_staged_nested(self): + """So a restart can find the download in staging again.""" + pc = self._make_pair_context("pair-1", effective_local_path="/staging/pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + pipeline._persist.staged_nested_file_names = set() + leaf = ModelFile("leaf.rar", False) + leaf.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") + _calls, notify = self._make_notify() + + self.assertTrue(pipeline._handle_queue(command, leaf, pc, notify)) + self.assertEqual({persist_key("pair-1", "Top/leaf.rar")}, pipeline._persist.staged_nested_file_names) + + def test_nested_queue_without_staging_adds_no_pending_move(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + leaf = ModelFile("leaf.rar", False) + leaf.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") + _calls, notify = self._make_notify() + + self.assertTrue(pipeline._handle_queue(command, leaf, pc, notify)) + self.assertEqual(set(), pc.pending_nested_moves) + self.assertEqual(set(), pc.staged_nested) + + def test_top_level_queue_does_not_touch_move_state(self): + """The move-state clearing is nested-only (top-level files get a + requeue diff instead); a top-level queue must not touch moved_file_keys + or move_failed_file_names at all.""" + pc = self._make_pair_context("pair-1", effective_local_path="/staging/pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline._context.config.general.exclude_patterns = "" + move_key = persist_key("pair-1", "Top") + pipeline.moved_file_keys.add(move_key) + pipeline._persist.move_failed_file_names = {move_key} + + top = ModelFile("Top", True) + top.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top", pair_id="pair-1") + calls, notify = self._make_notify() + + result = pipeline._handle_queue(command, top, pc, notify) + + self.assertTrue(result) + self.assertEqual([], calls) + self.assertIn(move_key, pipeline.moved_file_keys) + self.assertIn(move_key, pipeline._persist.move_failed_file_names) + pipeline.sync_persist_callback.assert_not_called() + self.assertEqual(set(), pc.pending_nested_moves) + + # --- find_overlapping_move --- + + @staticmethod + def _fake_move_process(pair_id, file_name): + move_process = MagicMock() + move_process.pair_id = pair_id + move_process.file_name = file_name + return move_process + + def test_find_overlapping_move_same_name(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top/Sub")) + + self.assertEqual("Top/Sub", pipeline.find_overlapping_move("Top/Sub", pc)) + + def test_find_overlapping_move_ancestor_moving_blocks_descendant_queue(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) + + self.assertEqual("Top", pipeline.find_overlapping_move("Top/Sub/f.bin", pc)) + + def test_find_overlapping_move_descendant_moving_blocks_ancestor_queue(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top/Sub/f.bin")) + + self.assertEqual("Top/Sub/f.bin", pipeline.find_overlapping_move("Top", pc)) + + def test_find_overlapping_move_unrelated_name_no_overlap(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Other")) + + self.assertIsNone(pipeline.find_overlapping_move("Top", pc)) + + def test_find_overlapping_move_different_pair_no_overlap(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-2", "Top")) + + self.assertIsNone(pipeline.find_overlapping_move("Top", pc)) + + def test_find_overlapping_move_prefix_name_not_overlap(self): + """'Top' vs 'TopX' must not be treated as overlapping - a string + prefix without the path separator is just a different, unrelated name.""" + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) + + self.assertIsNone(pipeline.find_overlapping_move("TopX", pc)) + + def test_handle_queue_refused_when_overlapping_move_in_progress(self): + pc = self._make_pair_context("pair-1") + pipeline = self._make_pipeline([pc], nested_enabled=True) + pipeline.active_move_processes.append(self._fake_move_process("pair-1", "Top")) + + leaf = ModelFile("leaf.rar", False) + leaf.remote_size = 100 + command = Command(Command.Action.QUEUE, "Top/leaf.rar", pair_id="pair-1") + calls, notify = self._make_notify() + + result = pipeline._handle_queue(command, leaf, pc, notify) self.assertFalse(result) self.assertEqual(1, len(calls)) - self.assertIn("staging", calls[0]) + self.assertIn("Top", calls[0]) pc.lftp.queue.assert_not_called() def test_top_level_queue_allowed_while_staging(self): diff --git a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py index 0d99fa93..ac879733 100644 --- a/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py +++ b/src/python/tests/unittests/test_controller/test_controller_lftp_wiring.py @@ -4,6 +4,8 @@ protocol, port selection, and SSL params, while keeping the scanner on SSH.""" import logging +import os +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -129,6 +131,38 @@ def test_active_scanner_recognises_lftp_temp_files_with_use_temp_file_off(self): _, kwargs = mock_active_scanner.call_args self.assertEqual(Constants.LFTP_TEMP_FILE_SUFFIX, kwargs["lftp_temp_suffix"]) + def _pending_completion_for(self, ctx: Context): + controller = self._build_controller(ctx) + with ( + patch("controller.controller.Lftp"), + patch("controller.controller.RemoteScanner"), + patch("controller.controller.ActiveScanner") as mock_active_scanner, + patch("controller.controller.LocalScanner"), + patch("controller.controller.ScannerProcess"), + patch("controller.controller.ModelBuilder"), + patch("controller.controller.configure_lftp"), + ): + pc = controller._create_pair_context(pair_id="p1", name="pair", remote_path="/remote", local_path="/local") + # The scanner's first scan covers them too, ahead of the first update cycle + mock_active_scanner.return_value.set_active_files.assert_called_once_with(sorted(pc.pending_completion)) + return pc.pending_completion + + def test_downloads_left_in_staging_are_scanned_after_a_restart(self): + """pending_completion is in memory only, so a download that finished + or was stopped in staging before a restart would otherwise go unseen: + the local scan covers only local_path.""" + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + with tempfile.TemporaryDirectory() as staging: + ctx.config.controller.use_staging = True + ctx.config.controller.staging_path = staging + os.makedirs(os.path.join(staging, "p1", "Show.S01")) + + self.assertEqual({"Show.S01"}, self._pending_completion_for(ctx)) + + def test_nothing_seeded_with_staging_off(self): + ctx = _make_context(protocol="sftp", remote_port=22, remote_ftp_port=21, verify_cert=False) + self.assertEqual(set(), self._pending_completion_for(ctx)) + if __name__ == "__main__": unittest.main() diff --git a/src/python/tests/unittests/test_controller/test_controller_persist.py b/src/python/tests/unittests/test_controller/test_controller_persist.py index b09fbfd1..769044e2 100644 --- a/src/python/tests/unittests/test_controller/test_controller_persist.py +++ b/src/python/tests/unittests/test_controller/test_controller_persist.py @@ -219,6 +219,25 @@ def test_move_failed_legacy_keys_migrated(self): persist.move_failed_file_names, ) + def test_staged_nested_golden_round_trip(self): + """staged_nested: nested downloads left in staging, by persist key.""" + sep = "\x1f" + content = json.dumps( + { + "downloaded": [], + "extracted": [], + "staged_nested": ["Top/Sub/f.bin", f"pair-1{sep}Top/a.bin"], + } + ) + persist = ControllerPersist.from_str(content) + self.assertEqual({"Top/Sub/f.bin", f"pair-1{sep}Top/a.bin"}, persist.staged_nested_file_names) + self.assertEqual({"Top/Sub/f.bin", f"pair-1{sep}Top/a.bin"}, set(json.loads(persist.to_str())["staged_nested"])) + + def test_staged_nested_missing_key_defaults_empty(self): + """Persist files from before staged_nested load an empty set.""" + persist = ControllerPersist.from_str(json.dumps({"downloaded": ["a"], "extracted": []})) + self.assertEqual(set(), persist.staged_nested_file_names) + def test_validated_corrupt_legacy_keys_migrated(self): """Legacy colon-separated keys in validated/corrupt should be migrated.""" uuid1 = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" diff --git a/src/python/tests/unittests/test_controller/test_model_builder.py b/src/python/tests/unittests/test_controller/test_model_builder.py index 611aba49..3accae95 100644 --- a/src/python/tests/unittests/test_controller/test_model_builder.py +++ b/src/python/tests/unittests/test_controller/test_model_builder.py @@ -356,6 +356,21 @@ def test_build_state(self): model = self.model_builder.build_model() self.assertEqual(ModelFile.State.DELETED, model.get_file("a").state) + def test_persist_authority_waits_for_a_remote_listing(self): + """Before the first remote scan a missing remote size means "not + scanned yet", not "deleted remotely". A stopped partial left in + staging, seen by the active scanner after a restart, must not read + as Downloaded (and be moved) in that window.""" + self.model_builder.clear() + self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_active_files([SystemFile("a", 30, False)]) + self.model_builder.set_downloaded_files({"a"}) + self.assertEqual(ModelFile.State.DEFAULT, self.model_builder.build_model().get_file("a").state) + + # Once a listing without "a" is in, persist authority applies + self.model_builder.set_remote_files([]) + self.assertEqual(ModelFile.State.DOWNLOADED, self.model_builder.build_model().get_file("a").state) + def test_build_state_persist_with_auto_delete_remote(self): """Test persist authority behavior under different auto_delete_remote settings""" @@ -370,6 +385,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -378,6 +394,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=False, remote absent, local absent, persist → DELETED self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # Need at least one source for 'a' to appear in model — use lftp status self.model_builder.set_lftp_statuses( @@ -390,6 +407,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # Instead test: remote absent via local-only + persist self.model_builder.clear() self.model_builder.set_auto_delete_remote(False) + self.model_builder.set_remote_files([]) self.model_builder.set_downloaded_files({"a"}) # File must exist in some source to appear in model # If remote_size=None and local_size=None but file in lftp status: @@ -412,6 +430,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote absent, local exists, persist → DOWNLOADED self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_local_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -426,6 +445,7 @@ def test_build_state_persist_with_auto_delete_remote(self): # auto_delete_remote=True, remote exists, local absent, persist → DEFAULT (re-downloadable) self.model_builder.clear() self.model_builder.set_auto_delete_remote(True) + self.model_builder.set_remote_files([]) self.model_builder.set_remote_files([SystemFile("a", 100, False)]) self.model_builder.set_downloaded_files({"a"}) model = self.model_builder.build_model() @@ -2209,6 +2229,77 @@ def test_nested_dir_job_stays_downloading_when_incomplete(self): sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") self.assertEqual(ModelFile.State.DOWNLOADING, sub.state) + def test_nested_child_move_failed(self): + """A nested child whose staging->final move failed renders MOVE_FAILED + even though it sits in staging and never reaches a post-download local + state - unlike a root, there's no DOWNLOADED/EXTRACTED/etc. to require.""" + r_top = SystemFile("Top", 2000, True) + r_sub = SystemFile("Sub", 1000, True) + r_leaf = SystemFile("f.bin", 1000, False) + r_sub.add_child(r_leaf) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + self.model_builder.set_move_failed_files({"Top/Sub/f.bin"}) + + model = self.model_builder.build_model() + + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(ModelFile.State.MOVE_FAILED, leaf.state) + + def test_nested_child_move_failed_stays_downloading_with_active_job(self): + """An active standalone job on the same name overrides MOVE_FAILED - + e.g. a re-queue just restarted it.""" + r_top = SystemFile("Top", 2000, True) + r_sub = SystemFile("Sub", 1000, True) + r_leaf = SystemFile("f.bin", 1000, False) + r_sub.add_child(r_leaf) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + self.model_builder.set_move_failed_files({"Top/Sub/f.bin"}) + nested_status = LftpJobStatus(0, LftpJobStatus.Type.PGET, LftpJobStatus.State.RUNNING, "Top/Sub/f.bin", "") + self.model_builder.set_lftp_statuses([nested_status]) + + model = self.model_builder.build_model() + + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(ModelFile.State.DOWNLOADING, leaf.state) + + def test_staged_nested_file_shows_its_staging_size(self): + """A nested download sitting in staging (e.g. a stopped partial) shows + its staged size, like a top-level one does, and stays a nested row.""" + r_top = SystemFile("Top", 1000, True) + r_sub = SystemFile("Sub", 1000, True) + r_sub.add_child(SystemFile("f.bin", 1000, False)) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + self.model_builder.set_active_files([SystemFile("Top/Sub/f.bin", 400, False)]) + + model = self.model_builder.build_model() + + self.assertEqual({"Top"}, set(model.get_file_names())) + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(400, leaf.local_size) + + def test_staged_nested_dir_shows_its_staging_tree(self): + r_top = SystemFile("Top", 1000, True) + r_sub = SystemFile("Sub", 1000, True) + r_sub.add_child(SystemFile("f.bin", 1000, False)) + r_top.add_child(r_sub) + self.model_builder.set_remote_files([r_top]) + s_sub = SystemFile("Top/Sub", 300, True) + s_sub.add_child(SystemFile("f.bin", 300, False)) + self.model_builder.set_active_files([s_sub]) + + model = self.model_builder.build_model() + + sub = next(c for c in model.get_file("Top").get_children() if c.name == "Sub") + leaf = next(c for c in sub.get_children() if c.name == "f.bin") + self.assertEqual(300, sub.local_size) + self.assertEqual(300, leaf.local_size) + def test_nested_status_never_creates_a_phantom_top_level_entry(self): """A '/'-containing status name is always routed to its nested child, even with nested navigation off (e.g. a nested job still running diff --git a/src/python/tests/unittests/test_controller/test_model_updater.py b/src/python/tests/unittests/test_controller/test_model_updater.py index 4ee2cda1..6de33892 100644 --- a/src/python/tests/unittests/test_controller/test_model_updater.py +++ b/src/python/tests/unittests/test_controller/test_model_updater.py @@ -8,7 +8,7 @@ from controller.persist_keys import KEY_SEP, persist_key from controller.scan.scanner_process import ScannerResult from lftp import LftpJobStatus -from model import ModelDiff, ModelFile +from model import ModelDiff, ModelError, ModelFile class TestSyncPersistToAllBuilders(unittest.TestCase): @@ -231,6 +231,22 @@ def test_pending_completion_excludes_nested_names_on_job_completion(self): self.assertEqual({"TopLevel"}, pc.pending_completion) + def test_staged_nested_names_fed_to_active_scanner(self): + """The active scanner is rooted at staging when staging is on, so a + nested download still there (e.g. a stopped partial) is stat'd like a + top-level one in pending_completion, and its row shows its size.""" + pc = self._make_pair_context("pair-1") + pc.staged_nested = {"TopDir/nested_file.rar"} + pc.active_extracting_file_names = [] + for proc in (pc.remote_scan_process, pc.local_scan_process, pc.active_scan_process): + proc.pop_latest_result.return_value = None + pc.lftp.status.return_value = [] + updater = self._make_updater([pc]) + + updater._update_pair_model_state(pc, None, None) + + pc.active_scanner.set_active_files.assert_called_once_with(["TopDir/nested_file.rar"]) + def test_nested_completion_not_added_to_downloaded_file_names(self): """Nothing reads a nested persist key (model_builder's persist-authority check matches top-level names only) and nothing ever removes one, so @@ -262,6 +278,225 @@ def test_top_level_completion_added_to_downloaded_file_names(self): self.assertEqual({"TopLevel"}, pc.pending_completion) +class TestSpawnNestedMoves(unittest.TestCase): + """A nested job's lftp job ending (completion or Stop) triggers its + staging->final move directly, since it has no DOWNLOADED diff to drive it.""" + + def _make_pair_context(self, pair_id=None): + pc = MagicMock() + pc.pair_id = pair_id + pc.prev_downloading_file_names = set() + pc.pending_completion = set() + pc.pending_nested_moves = set() + pc.remote_scan_in_model = True + return pc + + def _make_updater(self, pair_contexts, persist=None, *, use_staging=True, staging_path="/tmp/staging"): + pipeline = MagicMock() + registry = MagicMock() + extract_process = MagicMock() + validate_process = MagicMock() + context = MagicMock() + context.config.controller.use_staging = use_staging + context.config.controller.staging_path = staging_path + logger = MagicMock() + + updater = ModelUpdater( + pair_contexts=pair_contexts, + persist=persist or MagicMock(), + pipeline=pipeline, + registry=registry, + extract_process=extract_process, + validate_process=validate_process, + context=context, + password=None, + logger=logger, + ) + return updater, pipeline, registry + + def _make_persist(self): + persist = MagicMock() + persist.downloaded_file_names = set() + return persist + + def _running(self, name): + return LftpJobStatus(0, LftpJobStatus.Type.PGET, LftpJobStatus.State.RUNNING, name, "") + + def test_pending_nested_move_spawned_once_lftp_no_longer_lists_it(self): + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + resolved = MagicMock() + resolved.remote_size = 12345 + registry.resolve_full_path.return_value = resolved + + updater._detect_lftp_completions(pc, []) + + registry.resolve_full_path.assert_called_once_with("Top/leaf.rar", pair_id="pair-1") + pipeline.spawn_move_process.assert_called_once_with("Top/leaf.rar", pc, expected_size=12345) + self.assertEqual(set(), pc.pending_nested_moves) + + def test_job_never_seen_running_still_spawns_move(self): + """A job that finishes between two status polls (a small file, or one + already complete in staging) never shows as running; the queue-driven + pending set still moves it.""" + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + pc.prev_downloading_file_names = set() + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + registry.resolve_full_path.return_value.remote_size = 10 + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_called_once_with("Top/leaf.rar", pc, expected_size=10) + + def test_pending_nested_move_waits_while_lftp_lists_it(self): + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + + updater._detect_lftp_completions(pc, [self._running("Top/leaf.rar")]) + + pipeline.spawn_move_process.assert_not_called() + self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + + def test_pending_nested_move_with_unresolvable_row_stays_in_staging(self): + """The row can be gone by the time this runs (e.g. a remote delete + raced the completion). Without a remote size a stopped partial can't be + told from a finished download, so nothing is moved.""" + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + registry.resolve_full_path.side_effect = ModelError("not found") + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_not_called() + self.assertEqual(set(), pc.pending_nested_moves) + + def test_pending_nested_move_with_no_remote_size_stays_in_staging(self): + pc = self._make_pair_context("pair-1") + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + registry.resolve_full_path.return_value.remote_size = None + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_not_called() + + def test_completion_without_pending_entry_spawns_nothing(self): + """Only names queued under staging are moved: a nested job ending with + staging off, or any top-level job, never enters the pending set.""" + pc = self._make_pair_context("pair-1") + pc.prev_downloading_file_names = {"Top/leaf.rar", "TopLevel"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + + updater._detect_lftp_completions(pc, []) + + pipeline.spawn_move_process.assert_not_called() + registry.resolve_full_path.assert_not_called() + + def test_pending_nested_move_waits_until_the_remote_scan_is_in_the_model(self): + """After a restart, pending moves come back before the first remote + scan. Until a model built from it is in the registry the row can't be + resolved, and the move would be dropped for an unknown size.""" + pc = self._make_pair_context("pair-1") + pc.remote_scan_in_model = False + pc.pending_nested_moves = {"Top/leaf.rar"} + updater, pipeline, registry = self._make_updater([pc], self._make_persist()) + + updater._detect_lftp_completions(pc, []) + + registry.resolve_full_path.assert_not_called() + pipeline.spawn_move_process.assert_not_called() + self.assertEqual({"Top/leaf.rar"}, pc.pending_nested_moves) + + def test_remote_scan_marked_in_model_once_a_model_is_built_with_it(self): + received = self._make_pair_context("pair-1") + received.remote_scan_received = True + received.remote_scan_in_model = False + waiting = self._make_pair_context("pair-2") + waiting.remote_scan_received = False + waiting.remote_scan_in_model = False + updater, _pipeline, _registry = self._make_updater([received, waiting], self._make_persist()) + + updater._note_remote_scans_in_model() + + self.assertTrue(received.remote_scan_in_model) + self.assertFalse(waiting.remote_scan_in_model) + + +class TestPruneAbsentNestedMoveFailures(unittest.TestCase): + """Nested move_failed keys never reach downloaded_file_names (the prune + that drives _prune_absent_persist_entries), so they get their own pass.""" + + def _make_pair_context(self, pair_id=None): + pc = MagicMock() + pc.pair_id = pair_id + return pc + + def _make_updater(self, pair_contexts, persist): + pipeline = MagicMock() + registry = MagicMock() + extract_process = MagicMock() + validate_process = MagicMock() + context = MagicMock() + logger = MagicMock() + + updater = ModelUpdater( + pair_contexts=pair_contexts, + persist=persist, + pipeline=pipeline, + registry=registry, + extract_process=extract_process, + validate_process=validate_process, + context=context, + password=None, + logger=logger, + ) + updater._persist_sync = MagicMock() + return updater, registry + + def _make_persist(self, move_failed): + persist = MagicMock() + persist.move_failed_file_names = move_failed + return persist + + def test_unresolvable_nested_key_removed(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "Top/leaf.rar") + persist = self._make_persist({fail_key}) + updater, registry = self._make_updater([pc], persist) + registry.resolve_full_path.side_effect = ModelError("gone") + + updater._prune_absent_nested_move_failures() + + registry.resolve_full_path.assert_called_once_with("Top/leaf.rar", pair_id="pair-1") + self.assertNotIn(fail_key, persist.move_failed_file_names) + + def test_resolvable_nested_key_kept(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "Top/leaf.rar") + persist = self._make_persist({fail_key}) + updater, registry = self._make_updater([pc], persist) + registry.resolve_full_path.return_value = MagicMock() + + updater._prune_absent_nested_move_failures() + + self.assertIn(fail_key, persist.move_failed_file_names) + + def test_top_level_key_untouched(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "TopLevel") + persist = self._make_persist({fail_key}) + updater, registry = self._make_updater([pc], persist) + + updater._prune_absent_nested_move_failures() + + registry.resolve_full_path.assert_not_called() + self.assertIn(fail_key, persist.move_failed_file_names) + + class TestRetryFailedMoves(unittest.TestCase): """In-session retry of failed staging->final moves (#536).""" @@ -342,6 +577,32 @@ def test_does_not_respawn_when_move_already_in_flight(self): pipeline.spawn_move_process.assert_not_called() + def test_retry_count_dropped_when_key_cleared_without_requeue_diff(self): + """A nested re-queue clears move_failed_file_names directly + (CommandPipeline._handle_queue), with no requeue diff to reset the + retry count, so the count must be dropped here instead - otherwise a + later failure on the same name would start part-way through the budget.""" + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "Top/leaf.rar") + persist = self._make_persist(move_failed=set()) # cleared by the re-queue + updater, pipeline = self._make_updater([pc], persist) + updater._move_retry_counts[fail_key] = ModelUpdater.MAX_MOVE_RETRIES + + updater._retry_failed_moves() + + self.assertNotIn(fail_key, updater._move_retry_counts) + + def test_retry_count_kept_while_key_still_failed(self): + pc = self._make_pair_context("pair-1") + fail_key = persist_key("pair-1", "file.txt") + persist = self._make_persist(move_failed={fail_key}) + updater, pipeline = self._make_updater([pc], persist) + updater._move_retry_counts[fail_key] = 1 + + updater._retry_failed_moves() + + self.assertIn(fail_key, updater._move_retry_counts) + def test_retry_budget_caps_respawns(self): """A permanently-failing move must stop re-spawning after MAX_MOVE_RETRIES so it doesn't loop forever.""" @@ -560,3 +821,44 @@ def test_skips_when_umask_is_unset(self): apply, threads = self._run(None) apply.assert_not_called() self.assertEqual([], threads) + + +class TestHandleDownloadedFileOverlappingMove(unittest.TestCase): + """A root can turn Downloaded while a nested download inside it is still + in staging, its own move in flight. The root's move would race that one, + so it is left to the nested move.""" + + def _run(self, overlapping_move): + pc = MagicMock() + context = MagicMock() + context.args.umask = None + context.config.validate.enabled = False + context.config.controller.use_staging = True + context.config.controller.staging_path = "/staging" + context.config.autoqueue.auto_extract = False + pipeline = MagicMock() + pipeline.find_overlapping_move.return_value = overlapping_move + updater = ModelUpdater( + pair_contexts=[pc], + persist=MagicMock(), + pipeline=pipeline, + registry=MagicMock(), + extract_process=MagicMock(), + validate_process=MagicMock(), + context=context, + password=None, + logger=MagicMock(), + ) + new_file = ModelFile("Show", True) + new_file.state = ModelFile.State.DOWNLOADED + updater._handle_downloaded_file(ModelDiff(ModelDiff.Change.ADDED, old_file=None, new_file=new_file), pc) + return pipeline, pc + + def test_root_move_skipped_while_a_nested_move_inside_it_is_in_flight(self): + pipeline, pc = self._run("Show/Season 1") + pipeline.find_overlapping_move.assert_called_once_with("Show", pc) + pipeline.spawn_move_process.assert_not_called() + + def test_root_move_spawned_without_an_overlapping_move(self): + pipeline, pc = self._run(None) + pipeline.spawn_move_process.assert_called_once_with("Show", pc) diff --git a/src/python/tests/unittests/test_controller/test_move_process.py b/src/python/tests/unittests/test_controller/test_move_process.py index 48b8b7f5..00b3cdfe 100644 --- a/src/python/tests/unittests/test_controller/test_move_process.py +++ b/src/python/tests/unittests/test_controller/test_move_process.py @@ -315,6 +315,106 @@ def test_get_copied_size_ignores_preexisting_dest_files(self): self.assertEqual(5, MoveProcess._get_copied_size(src_subdir, dst_subdir)) + # --- Nested names: multi-level dest parents + staging prune --- + + def test_move_nested_name_creates_dest_parents_and_prunes_empty_staging_dirs(self): + """A nested name's move must create every missing dest parent level and, + once the source is gone, remove the staging dirs it emptied up to (not + including) source_path.""" + nested_dir = os.path.join(self.src_dir, "Top", "Sub") + os.makedirs(nested_dir) + src_file = os.path.join(nested_dir, "f.bin") + with open(src_file, "w") as f: + f.write("data") + + process = self._make_process(source_path=self.src_dir, dest_path=self.dst_dir, file_name="Top/Sub/f.bin") + self._run_process(process) + + dst_file = os.path.join(self.dst_dir, "Top", "Sub", "f.bin") + self.assertTrue(os.path.exists(dst_file)) + with open(dst_file) as f: + self.assertEqual("data", f.read()) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top", "Sub"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + self.assertTrue(os.path.exists(self.src_dir)) + self.assertEqual([], process.pop_failed()) + + def test_move_nested_prune_stops_at_nonempty_sibling(self): + """Pruning stops at the first staging dir still holding something, such + as a sibling file from another job - that dir (and its ancestors) must + survive.""" + nested_dir = os.path.join(self.src_dir, "Top", "Sub") + os.makedirs(nested_dir) + src_file = os.path.join(nested_dir, "f.bin") + with open(src_file, "w") as f: + f.write("data") + sibling_file = os.path.join(self.src_dir, "Top", "sibling.txt") + with open(sibling_file, "w") as f: + f.write("keepme") + + process = self._make_process(source_path=self.src_dir, dest_path=self.dst_dir, file_name="Top/Sub/f.bin") + self._run_process(process) + + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top", "Sub"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top"))) + self.assertTrue(os.path.exists(sibling_file)) + + def test_move_nested_via_copy_delete_prunes_empty_staging_dirs(self): + """The copy+delete fallback path must prune the same as the rename path.""" + nested_dir = os.path.join(self.src_dir, "Top", "Sub") + os.makedirs(nested_dir) + src_file = os.path.join(nested_dir, "f.bin") + with open(src_file, "w") as f: + f.write("data") + + process = self._make_process(source_path=self.src_dir, dest_path=self.dst_dir, file_name="Top/Sub/f.bin") + with mock.patch("controller.move.move_process.os.rename", side_effect=OSError(errno.EXDEV, "cross-device")): + self._run_process(process) + + dst_file = os.path.join(self.dst_dir, "Top", "Sub", "f.bin") + self.assertTrue(os.path.exists(dst_file)) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top", "Sub"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + self.assertEqual([], process.pop_failed()) + + # --- expected_size: skip (not fail) an in-flight nested download (#811) --- + + def test_expected_size_larger_than_source_skips_move_without_failure(self): + """A nested job's lftp process ending early (e.g. a Stop) leaves a + smaller-than-expected source; the move must be skipped, not failed, and + the source left untouched for a later re-queue to resume.""" + src_file = os.path.join(self.src_dir, "test.txt") + with open(src_file, "w") as f: + f.write("12345") # 5 bytes + + process = self._make_process( + source_path=self.src_dir, dest_path=self.dst_dir, file_name="test.txt", expected_size=100 + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(src_file)) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "test.txt"))) + + def test_expected_size_set_with_temp_files_skips_without_failure(self): + """With expected_size set, lftp temp files present also means skip (not + the recoverable-failure path used when expected_size is None).""" + src_subdir = os.path.join(self.src_dir, "mydir") + os.makedirs(src_subdir) + with open(os.path.join(src_subdir, "done.mkv"), "w") as f: + f.write("finished") + with open(os.path.join(src_subdir, "inprogress.mkv.lftp"), "w") as f: + f.write("partial") + + process = self._make_process( + source_path=self.src_dir, dest_path=self.dst_dir, file_name="mydir", expected_size=1000 + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(src_subdir)) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "mydir"))) + def test_move_applies_umask_modes_at_destination(self): """#811: lftp leaves 0664/0775 in staging; the moved copy gets 0666/0777 & ~umask.""" import stat @@ -336,6 +436,226 @@ def test_move_applies_umask_modes_at_destination(self): self.assertEqual(0o777, stat.S_IMODE(os.stat(dst_tree).st_mode)) self.assertEqual(0o666, stat.S_IMODE(os.stat(os.path.join(dst_tree, "ep1.mkv")).st_mode)) + # --- nested_expected_sizes: leave a stopped nested job's partial in staging --- + + def test_top_level_move_leaves_incomplete_nested_download_in_staging(self): + """Repro: a top-level staging->final move used to carry a stopped nested + partial out of staging into the final dir. With nested_expected_sizes set, + the complete sibling must still move while the nested partial under it + stays in staging for its own job to resume.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin"), "w") as f: + f.write("short") # 5 bytes, far under expected + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) + # Sub holds only the partial, so no empty Sub is created at the destination + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub"))) + + def test_top_level_move_leaves_temp_named_nested_partial_in_staging(self): + """With lftp temp files on, a stopped nested file exists only as + '.lftp'. It is the incomplete entry: left behind, and it doesn't + defer the rest of the tree.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin.lftp"), "w") as f: + f.write("short") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin.lftp"))) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub"))) + + def test_top_level_move_moves_everything_when_nested_download_complete(self): + """A nested download whose size has caught up to expected moves along + with the rest of the tree; nothing is left behind.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin"), "w") as f: + f.write("12345") # 5 bytes, matches expected + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 5}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + + def test_nested_expected_size_none_leaves_download_in_staging(self): + """An unresolved nested row (ModelError during lookup in the pipeline) + maps to a None size here, which is always treated as incomplete.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub", "big.bin"), "w") as f: + f.write("anything") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": None}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) + + def test_nested_lftp_temp_file_in_listed_path_skipped_without_deferral(self): + """An lftp temp file inside a listed (incomplete) nested path must not + defer the whole move.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "Sub2")) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "Sub2", "x.bin.lftp"), "w") as f: + f.write("partial") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub2": 999}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub2", "x.bin.lftp"))) + + def test_temp_file_outside_listed_nested_paths_still_defers(self): + """A temp file outside any listed nested path still defers the whole + move as a recoverable failure (existing behaviour, preserved alongside + the nested-skip list).""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(top_dir) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "stray.mkv.lftp"), "w") as f: + f.write("partial") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={}, + pair_id="pair-5", + ) + self._run_process(process) + + failures = process.pop_failed() + self.assertEqual(1, len(failures)) + self.assertIn("not finalized", failures[0].error_message) + self.assertTrue(os.path.exists(top_dir)) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top"))) + + def test_nested_expected_sizes_entry_missing_from_staging_is_ignored(self): + """A listed nested path that no longer exists in staging (already moved + or never downloaded) is ignored - the top-level move proceeds normally.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(top_dir) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.src_dir, "Top"))) + + def test_sibling_subtree_without_exclusions_moves_whole_via_rename(self): + """A subtree containing no incomplete nested path moves as one rename + of the whole subtree, not file-by-file.""" + top_dir = os.path.join(self.src_dir, "Top") + other_dir = os.path.join(top_dir, "Other") + os.makedirs(other_dir) + with open(os.path.join(other_dir, "c.bin"), "w") as f: + f.write("complete") + sub_dir = os.path.join(top_dir, "Sub") + os.makedirs(sub_dir) + with open(os.path.join(sub_dir, "big.bin"), "w") as f: + f.write("short") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + dst_other = os.path.join(self.dst_dir, "Top", "Other") + with mock.patch("controller.move.move_process.os.rename", side_effect=os.rename) as mock_rename: + self._run_process(process) + + self.assertTrue(os.path.exists(os.path.join(dst_other, "c.bin"))) + self.assertIn(mock.call(other_dir, dst_other), mock_rename.call_args_list) + self.assertEqual([], process.pop_failed()) + + def test_cross_device_move_tree_leaves_partial_and_moves_rest(self): + """EXDEV from os.rename inside the tree move must fall back to + copy+delete per subtree, still leaving the incomplete nested path + behind.""" + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(top_dir) + with open(os.path.join(top_dir, "a.bin"), "w") as f: + f.write("complete") + sub_dir = os.path.join(top_dir, "Sub") + os.makedirs(sub_dir) + with open(os.path.join(sub_dir, "big.bin"), "w") as f: + f.write("short") + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + nested_expected_sizes={"Sub/big.bin": 1000}, + ) + with mock.patch("controller.move.move_process.os.rename", side_effect=OSError(errno.EXDEV, "cross-device")): + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.dst_dir, "Top", "a.bin"))) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "Sub", "big.bin"))) + self.assertFalse(os.path.exists(os.path.join(self.dst_dir, "Top", "Sub", "big.bin"))) + def test_merge_into_existing_destination_leaves_other_files_alone(self): """#811: a staged tree merged into an existing folder must not widen the modes of files that were already there.""" @@ -366,3 +686,35 @@ def test_merge_into_existing_destination_leaves_other_files_alone(self): self.assertEqual(0o600, stat.S_IMODE(os.stat(private).st_mode)) # The moved file gets the UMASK treatment self.assertEqual(0o666, stat.S_IMODE(os.stat(os.path.join(dst_tree, "ep2.mkv")).st_mode)) + + def test_partial_tree_move_applies_umask_only_to_what_it_moved(self): + """#811 with a nested partial left behind: dirs the tree move creates at + the destination and the files it moves get the UMASK modes; a destination + dir that was already there keeps its own.""" + import stat + + top_dir = os.path.join(self.src_dir, "Top") + os.makedirs(os.path.join(top_dir, "New", "Sub")) + with open(os.path.join(top_dir, "New", "a.bin"), "w") as f: + f.write("complete") + with open(os.path.join(top_dir, "New", "Sub", "big.bin"), "w") as f: + f.write("short") + os.chmod(os.path.join(top_dir, "New", "a.bin"), 0o664) + dst_top = os.path.join(self.dst_dir, "Top") + os.makedirs(dst_top) + os.chmod(dst_top, 0o700) + + process = self._make_process( + source_path=self.src_dir, + dest_path=self.dst_dir, + file_name="Top", + umask=0o000, + nested_expected_sizes={"New/Sub/big.bin": 1000}, + ) + self._run_process(process) + + self.assertEqual([], process.pop_failed()) + self.assertTrue(os.path.exists(os.path.join(self.src_dir, "Top", "New", "Sub", "big.bin"))) + self.assertEqual(0o700, stat.S_IMODE(os.stat(dst_top).st_mode)) + self.assertEqual(0o777, stat.S_IMODE(os.stat(os.path.join(dst_top, "New")).st_mode)) + self.assertEqual(0o666, stat.S_IMODE(os.stat(os.path.join(dst_top, "New", "a.bin")).st_mode)) diff --git a/src/python/tests/unittests/test_controller/test_pair_context.py b/src/python/tests/unittests/test_controller/test_pair_context.py index 98999aa0..002cd8c7 100644 --- a/src/python/tests/unittests/test_controller/test_pair_context.py +++ b/src/python/tests/unittests/test_controller/test_pair_context.py @@ -1,5 +1,7 @@ # Copyright 2017, Inderpreet Singh, All rights reserved. +import os +import tempfile import unittest from unittest.mock import MagicMock @@ -8,6 +10,7 @@ ControllerError, PairContext, configure_lftp, + staged_top_level_names, validate_config, ) @@ -341,3 +344,20 @@ def test_ftps_passes_persisted_values_through(self): self.assertEqual(lftp.num_connections_per_root_file, 20) self.assertEqual(lftp.num_connections_per_dir_file, 20) self.assertEqual(lftp.num_max_total_connections, 0) + + +class TestStagedTopLevelNames(unittest.TestCase): + def test_lists_downloads_in_staging_by_their_final_name(self): + """lftp's ".lftp" temp file is the download ; its + ".lftp-pget-status" progress file is not a download.""" + with tempfile.TemporaryDirectory() as staging: + os.makedirs(os.path.join(staging, "Show.S01")) + for name in ("movie.mkv", "partial.mkv.lftp", "movie.mkv.lftp-pget-status"): + with open(os.path.join(staging, name), "w") as f: + f.write("x") + + self.assertEqual({"Show.S01", "movie.mkv", "partial.mkv"}, staged_top_level_names(staging)) + + def test_missing_staging_dir_lists_nothing(self): + with tempfile.TemporaryDirectory() as parent: + self.assertEqual(set(), staged_top_level_names(os.path.join(parent, "nope"))) diff --git a/src/python/tests/unittests/test_lftp/test_lftp_spawn.py b/src/python/tests/unittests/test_lftp/test_lftp_spawn.py index 139d490e..44dd18e6 100644 --- a/src/python/tests/unittests/test_lftp/test_lftp_spawn.py +++ b/src/python/tests/unittests/test_lftp/test_lftp_spawn.py @@ -292,3 +292,24 @@ def test_existing_sftp_errors_still_detected(self): def test_clean_output_not_flagged(self): self.assertFalse(self._detect("`file' at 100 (50%) 1.2M/s eta:30s [Receiving data/TLS]")) + + +class TestLftpStatusTimeout(unittest.TestCase): + """A timed-out 'jobs -v' reads back as empty output, the same as no jobs. + status() must report it as unavailable (None), or callers take every + running job for finished.""" + + def test_status_is_none_when_jobs_command_times_out(self): + lftp, fake = make_lftp(address="host.example.com", port=22, user="bob", password=None) + with patch.object(fake, "expect", side_effect=lftp_mod.pexpect.exceptions.TIMEOUT("timed out")): + self.assertIsNone(lftp.status()) + + def test_status_is_empty_list_when_there_are_no_jobs(self): + lftp, _fake = make_lftp(address="host.example.com", port=22, user="bob", password=None) + self.assertEqual([], lftp.status()) + + def test_status_recovers_after_a_timeout(self): + lftp, fake = make_lftp(address="host.example.com", port=22, user="bob", password=None) + with patch.object(fake, "expect", side_effect=lftp_mod.pexpect.exceptions.TIMEOUT("timed out")): + lftp.status() + self.assertEqual([], lftp.status()) diff --git a/src/python/tests/unittests/test_system/test_file_modes.py b/src/python/tests/unittests/test_system/test_file_modes.py index 31cefff7..ef5f76ee 100644 --- a/src/python/tests/unittests/test_system/test_file_modes.py +++ b/src/python/tests/unittests/test_system/test_file_modes.py @@ -104,3 +104,17 @@ def test_apply_to_entries_only_touches_listed_paths(self): self.assertEqual(0o666, _mode(os.path.join(self.root, "ep1.mkv"))) self.assertEqual(0o777, _mode(os.path.join(self.root, "Sample"))) self.assertEqual(0o777, _mode(self.root)) + + def test_apply_to_entries_single_file(self): + """A single file's entries are [(".", False)]; "file/." must not be chmod'ed.""" + from system.file_modes import apply_umask_modes_to, relative_entries + + single = os.path.join(self.tmp.name, "single.mkv") + with open(single, "w"): + pass + os.chmod(single, 0o664) + + with self.assertNoLogs(self.logger): + apply_umask_modes_to(single, relative_entries(single), 0o000, self.logger) + + self.assertEqual(0o666, _mode(single))