Repository navigation
feat: move nested downloads out of staging #824
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
56e2327
3999e47
f979c95
d354675
6d215b3
9da4cfc
7050453
b3fd0c9
04260b4
6c8b979
3b07792
0086c5a
17fd924
47ecddd
1482f3b
5a3a6da
d0e57ce
a989ada
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,7 +14,7 @@ | |
| from collections.abc import Callable, Iterator | ||
| from queue import Queue | ||
|
|
||
| from common import Context, MultiprocessingLogger, UnrecoverableError | ||
| from common import Constants, Context, MultiprocessingLogger, UnrecoverableError | ||
| from lftp import LftpError | ||
| from model import ModelError, ModelFile, is_nested_name | ||
|
|
||
|
|
@@ -26,7 +26,7 @@ | |
| from .model_registry import ModelRegistry | ||
| from .move import MoveProcess | ||
| from .pair_context import PairContext | ||
| from .persist_keys import persist_key | ||
| from .persist_keys import persist_key, strip_persist_key | ||
| from .validate import ValidateProcess, ValidateRequest | ||
| from .worker_supervisor import WorkerSupervisor | ||
|
|
||
|
|
@@ -209,19 +209,41 @@ def _handle_queue( | |
| if file.remote_size is None: | ||
| _notify_failure(command, f"File '{command.filename}' does not exist remotely") | ||
| return False | ||
| # A nested job would land under the staging dir, and the staging move | ||
| # only handles top-level names, so the download would never arrive. | ||
| if file.parent is not None and pc.effective_local_path != pc.local_path: | ||
| _notify_failure(command, "Queuing a nested file is not supported while staging is enabled") | ||
| moving = self.find_overlapping_move(command.filename, pc) | ||
| if moving is not None: | ||
| _notify_failure(command, f"A staging move is in progress for '{moving}'") | ||
| return False | ||
| try: | ||
| exclude = parse_exclude_patterns(self._context.config.general.exclude_patterns) | ||
| pc.lftp.queue(command.filename, file.is_dir, exclude_patterns=exclude) | ||
| except LftpError as e: | ||
| _notify_failure(command, f"Lftp error: {e!s}") | ||
| return False | ||
| if is_nested_name(command.filename): | ||
| if pc.effective_local_path != pc.local_path: | ||
| pc.pending_nested_moves.add(command.filename) | ||
| pc.staged_nested.add(command.filename) | ||
| self._persist.staged_nested_file_names.add(persist_key(pc.pair_id, command.filename)) | ||
| # A nested job's move has no requeue diff to reset it | ||
| # (ModelUpdater._handle_requeued_file sees top-level files only). | ||
| move_key = persist_key(pc.pair_id, command.filename) | ||
| self.moved_file_keys.discard(move_key) | ||
| if move_key in self._persist.move_failed_file_names: | ||
| self._persist.move_failed_file_names.discard(move_key) | ||
| self.sync_persist_callback() | ||
| return True | ||
|
|
||
| def find_overlapping_move(self, name: str, pc: PairContext) -> str | None: | ||
| """Name of an in-flight staging move for this pair that `name` is, lies | ||
| inside, or contains, or None.""" | ||
| for move_process in self.active_move_processes: | ||
| if move_process.pair_id != pc.pair_id: | ||
| continue | ||
| moving = move_process.file_name | ||
| if name == moving or name.startswith(moving + "/") or moving.startswith(name + "/"): | ||
| return moving | ||
| return None | ||
|
|
||
| @staticmethod | ||
| def _iter_active_descendants(file: ModelFile) -> Iterator[ModelFile]: | ||
| """BFS over `file`'s descendants, yielding each independently active | ||
|
|
@@ -422,6 +444,7 @@ def post_callback(delete_path: str = delete_path, _pc: PairContext = pc) -> None | |
| _pc.local_scan_process.force_scan() | ||
| if delete_path != _pc.local_path: | ||
| _pc.active_scan_process.force_scan() | ||
| self._sweep_staged_nested(_pc) | ||
|
|
||
| command_wrapper = CommandProcessWrapper(process=process, post_callback=post_callback, filename=command.filename) | ||
| self.active_command_processes.append(command_wrapper) | ||
|
|
@@ -694,6 +717,7 @@ def _finalize_move_process(self, move_process: MoveProcess) -> None: | |
| # pair; fall back to all pairs only if the owner can't be located. | ||
| owner = next((pc for pc in self._pair_contexts if pc.pair_id == move_process.pair_id), None) | ||
| if owner is not None: | ||
| self._sweep_staged_nested(owner) | ||
| owner.local_scan_process.force_scan() | ||
| else: | ||
| for pc in self._pair_contexts: | ||
|
|
@@ -782,13 +806,16 @@ def spawn_deferred_move(self, pair_id: str | None, file_name: str): | |
| return | ||
| self.spawn_move_process(file_name, pc) | ||
|
|
||
| def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: | ||
| def spawn_move_process(self, file_name: str, pc: PairContext, expected_size: int | None = None) -> bool: | ||
| """Spawn a MoveProcess to move a file from staging to the final local_path. | ||
|
|
||
| Returns True if a move process was actually spawned, False if the call was | ||
| a no-op (already moved / in flight, staging disabled, or nothing left in | ||
| staging). A retry of a failed move uses this to tell a real retry from a | ||
| resolved no-op so it can clear a stale MOVE_FAILED state (#536 follow-up). | ||
|
|
||
| expected_size is passed to MoveProcess: a source short of it, or still | ||
| holding lftp temp files, is left in staging with no failure reported. | ||
| """ | ||
| pair_id = pc.pair_id | ||
| move_key = persist_key(pair_id, file_name) | ||
|
|
@@ -815,6 +842,8 @@ def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: | |
| file_name=file_name, | ||
| pair_id=pair_id, | ||
| umask=self._context.args.umask, | ||
| expected_size=expected_size, | ||
| nested_expected_sizes=self._staged_nested_sizes(file_name, pc), | ||
| ) | ||
| process.set_mp_log_queue(self._mp_logger.queue, self._mp_logger.log_level) | ||
| # Start before publishing bookkeeping: if start() raises we must not leave | ||
|
|
@@ -827,6 +856,58 @@ def spawn_move_process(self, file_name: str, pc: PairContext) -> bool: | |
| self._logger.info(f"Spawned move process for {file_name} (staging -> local)") | ||
| return True | ||
|
|
||
| def _staged_nested_sizes(self, file_name: str, pc: PairContext) -> dict[str, int | None]: | ||
| """{path relative to file_name: remote size} for each staged nested | ||
| download under file_name (None when the row can't be resolved).""" | ||
| sizes: dict[str, int | None] = {} | ||
| prefix = file_name + "/" | ||
| for name in pc.staged_nested: | ||
| if name.startswith(prefix): | ||
| try: | ||
| sizes[name[len(prefix) :]] = self._registry.resolve_full_path(name, pair_id=pc.pair_id).remote_size | ||
| except ModelError: | ||
| sizes[name[len(prefix) :]] = None | ||
| return sizes | ||
|
|
||
| def _sweep_staged_nested(self, pc: PairContext) -> None: | ||
| """Forget staged nested downloads that are no longer in staging (moved, | ||
| deleted, or carried along by an ancestor's move).""" | ||
| staging = self._pair_staging_dir(pc) | ||
| # An unfinished file with lftp temp files on is "<name>.lftp" | ||
| suffix = Constants.LFTP_TEMP_FILE_SUFFIX | ||
| gone = [ | ||
| name | ||
| for name in pc.staged_nested | ||
| if staging is None | ||
| or ( | ||
| not os.path.exists(os.path.join(staging, name)) | ||
| and not os.path.exists(os.path.join(staging, name + suffix)) | ||
|
Comment on lines
+883
to
+884
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Keep queued nested downloads in the staging tracking set. If When
|
||
| ) | ||
| ] | ||
| pc.staged_nested.difference_update(gone) | ||
| self._persist.staged_nested_file_names.difference_update(persist_key(pc.pair_id, name) for name in gone) | ||
|
|
||
| def restore_staged_nested(self, pc: PairContext) -> None: | ||
| """Pick up the nested downloads a previous session left in staging. | ||
|
|
||
| Restores pc.staged_nested from its persisted copy (dropping names no | ||
| longer in staging) and queues each one's move; the move checks its | ||
| size. A top-level name holding one is taken off pc.pending_completion: | ||
| the active scanner would report its staging copy, which holds only | ||
| the nested downloads, in place of its local_path copy. The active | ||
| scanner gets the corrected names before its first scan. | ||
| """ | ||
| if self._pair_staging_dir(pc) is None: | ||
| return | ||
| for key in self._persist.staged_nested_file_names: | ||
| name = strip_persist_key(key, pc.pair_id) | ||
| if name != key or pc.pair_id is None: | ||
| pc.staged_nested.add(name) | ||
| self._sweep_staged_nested(pc) | ||
| pc.pending_nested_moves.update(pc.staged_nested) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Coalesce overlapping restored nested moves. Persisted entries can contain both Suppose the complete directory contains a 10-byte item and another 20-byte file. If the item moves first, the directory worker sees 20 bytes against its expected 30 bytes and skips without failure. Its pending entry is already removed, and its moved key prevents another attempt. The remaining complete file stays in staging. Process overlapping entries as one owned move, or order and reconcile them so a child move cannot invalidate the parent's completeness check.
|
||
| pc.pending_completion.difference_update(name.split("/", 1)[0] for name in pc.staged_nested) | ||
| pc.active_scanner.set_active_files(sorted(pc.pending_completion | pc.staged_nested)) | ||
|
|
||
| def _get_pair_context_for_command(self, command: Command) -> PairContext | None: | ||
| """Find the pair context for a command based on pair_id.""" | ||
| return self.find_pair_by_id(command.pair_id) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.