From e699602a32ca68e1b9d9c6d63b4a48375a24bb75 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 00:16:43 +0100 Subject: [PATCH 1/9] feat(orchestrator): cacheMode, move-dir default Ports game-ci/orchestrator#44 (the stranded content of #41) into plugins/orchestrator, and stages cross-volume copies so a failed copy never leaves a partial cache. --- plugins/orchestrator/action.yml | 9 +- .../cli-plugin/build-parameters-adapter.ts | 4 +- .../src/cli-plugin/cli-plugin.test.ts | 15 +++ .../cli-plugin/orchestrator-options-plugin.ts | 12 +- .../services/cache/child-workspace-service.ts | 30 ++++- .../cache/local-cache-service.test.ts | 103 +++++++++++++++++- .../services/cache/local-cache-service.ts | 45 ++++++-- .../services/cache/move-directory.test.ts | 101 +++++++++++++++++ .../services/cache/move-directory.ts | 62 +++++++++++ plugins/orchestrator/src/plugin-lifecycle.ts | 27 ++++- 10 files changed, 383 insertions(+), 25 deletions(-) create mode 100644 plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts create mode 100644 plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts diff --git a/plugins/orchestrator/action.yml b/plugins/orchestrator/action.yml index 824f78a6..3e8486e8 100644 --- a/plugins/orchestrator/action.yml +++ b/plugins/orchestrator/action.yml @@ -485,10 +485,14 @@ inputs: required: false default: '' description: 'Comma-separated explicit local cache fallback keys to try after the exact key.' + cacheMode: + required: false + default: '' + description: 'Library cache mode: move-directory (default, O(1) rename, falls back to copy across filesystem volumes), copy-directory, or tar (opt-in — needed when distributing cache entries via the rclone/S3 built-in container hooks). Renamed from localCacheMode, which still works as a deprecated alias — see game-ci/roadmap#11. NOTE: canonical-overlay has been removed — setting it now throws a clear error instead of running.' localCacheMode: required: false - default: 'tar' - description: 'Local Library cache mode: tar, copy-directory, or move-directory. move-directory is intended for same-volume Windows self-hosted runners. NOTE: canonical-overlay has been removed (see game-ci/roadmap#11) — setting it now throws a clear error instead of running.' + default: '' + description: 'Deprecated alias for cacheMode — "local" no longer accurately describes this input now that distributed rclone/S3 caching is a supported mode too. Still works, but prefer cacheMode. Will be removed in a future major version.' upmOfflineEnabled: required: false default: 'false' @@ -894,6 +898,7 @@ runs: INPUT_LOCALCACHELFS: ${{ inputs.localCacheLfs }} INPUT_LOCALCACHEFALLBACK: ${{ inputs.localCacheFallback }} INPUT_LOCALCACHEFALLBACKKEYS: ${{ inputs.localCacheFallbackKeys }} + INPUT_CACHEMODE: ${{ inputs.cacheMode }} INPUT_LOCALCACHEMODE: ${{ inputs.localCacheMode }} INPUT_UPMOFFLINEENABLED: ${{ inputs.upmOfflineEnabled }} INPUT_BACKGROUNDCACHESAVE: ${{ inputs.backgroundCacheSave }} diff --git a/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts b/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts index 5405af89..e41d4c6d 100644 --- a/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts +++ b/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts @@ -119,7 +119,9 @@ export function createBuildParametersFromCliOptions(options: Record bp.localCacheFallback = options.localCacheFallback === true || options.localCacheFallback === 'true'; bp.localCacheFallbackKeys = options.localCacheFallbackKeys || ''; - bp.localCacheMode = options.localCacheMode || 'tar'; + // cacheMode replaces localCacheMode, which stays as a deprecated alias. move-directory is an O(1) + // rename (falling back to a staged copy across volumes); tar stays opt-in, e.g. for rclone/S3 hooks. + bp.localCacheMode = options.cacheMode || options.localCacheMode || 'move-directory'; bp.localCacheSaveOnFailure = options.localCacheSaveOnFailure === true || options.localCacheSaveOnFailure === 'true'; bp.localCacheFloorCorruptionCategories = options.localCacheFloorCorruptionCategories || ''; diff --git a/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts b/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts index 35b8825e..f3805caf 100644 --- a/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts +++ b/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts @@ -103,6 +103,21 @@ describe('CLI Plugin Adapter', () => { expect(bp.engine).toBe('unity'); }); + it('defaults the cache mode to move-directory', () => { + expect(createBuildParametersFromCliOptions({}).localCacheMode).toBe('move-directory'); + }); + + it('prefers cacheMode, keeping localCacheMode as a deprecated alias', () => { + expect(createBuildParametersFromCliOptions({ cacheMode: 'tar' }).localCacheMode).toBe('tar'); + expect( + createBuildParametersFromCliOptions({ localCacheMode: 'copy-directory' }).localCacheMode, + ).toBe('copy-directory'); + expect( + createBuildParametersFromCliOptions({ cacheMode: 'tar', localCacheMode: 'copy-directory' }) + .localCacheMode, + ).toBe('tar'); + }); + it('applies defaults for missing options', () => { const bp = createBuildParametersFromCliOptions({}); diff --git a/plugins/orchestrator/src/cli-plugin/orchestrator-options-plugin.ts b/plugins/orchestrator/src/cli-plugin/orchestrator-options-plugin.ts index d053686d..59c031d4 100644 --- a/plugins/orchestrator/src/cli-plugin/orchestrator-options-plugin.ts +++ b/plugins/orchestrator/src/cli-plugin/orchestrator-options-plugin.ts @@ -513,10 +513,18 @@ export function configureOrchestratorOptions(yargs: any): void { default: '', }); + yargs.option('cacheMode', { + description: + 'Cache save/restore mode: move-directory (default; falls back to a copy across volumes), copy-directory, or tar', + type: 'string', + default: '', + }); + yargs.option('localCacheMode', { - description: 'Local cache save/restore mode: tar, move-directory, or copy-directory', + description: 'Deprecated alias for --cacheMode', type: 'string', - default: 'tar', + default: '', + deprecated: 'use --cacheMode', }); yargs.option('gcTimeoutMinutes', { diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/child-workspace-service.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/child-workspace-service.ts index f561e95d..41833a6b 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/child-workspace-service.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/child-workspace-service.ts @@ -1,6 +1,7 @@ import fs from 'node:fs'; import path from 'node:path'; import OrchestratorLogger from '../core/orchestrator-logger'; +import { moveDirectory } from './move-directory'; import { getEngine } from '../../../engine'; export interface ChildWorkspaceConfig { @@ -83,7 +84,7 @@ export class ChildWorkspaceService { OrchestratorLogger.log( `[ChildWorkspace] Restoring workspace: ${cachedWorkspacePath} -> ${projectPath}`, ); - fs.renameSync(cachedWorkspacePath, projectPath); + ChildWorkspaceService.renameOrCopy(cachedWorkspacePath, projectPath); OrchestratorLogger.log(`[ChildWorkspace] Workspace restored via atomic move`); // Restore Library cache separately if configured @@ -151,7 +152,7 @@ export class ChildWorkspaceService { OrchestratorLogger.log( `[ChildWorkspace] Saving workspace: ${projectPath} -> ${cachedWorkspacePath}`, ); - fs.renameSync(projectPath, cachedWorkspacePath); + ChildWorkspaceService.renameOrCopy(projectPath, cachedWorkspacePath); OrchestratorLogger.log(`[ChildWorkspace] Workspace saved via atomic move`); } catch (error: any) { OrchestratorLogger.logWarning(`[ChildWorkspace] Workspace save failed: ${error.message}`); @@ -194,7 +195,7 @@ export class ChildWorkspaceService { OrchestratorLogger.log( `[ChildWorkspace] Restoring ${folder} cache: ${backupPath} -> ${destination}`, ); - fs.renameSync(backupPath, destination); + ChildWorkspaceService.renameOrCopy(backupPath, destination); OrchestratorLogger.log(`[ChildWorkspace] ${folder} cache restored`); restored = true; } catch (error: any) { @@ -246,7 +247,7 @@ export class ChildWorkspaceService { OrchestratorLogger.log( `[ChildWorkspace] Caching ${folder}: ${folderPath} -> ${backupPath}`, ); - fs.renameSync(folderPath, backupPath); + ChildWorkspaceService.renameOrCopy(folderPath, backupPath); OrchestratorLogger.log(`[ChildWorkspace] ${folder} cached separately`); } catch (error: any) { OrchestratorLogger.logWarning( @@ -409,4 +410,25 @@ export class ChildWorkspaceService { return `${value.toFixed(2)} ${units[index]}`; } + + /** + * Moves a directory, degrading to a staged copy when parentCacheRoot sits on + * another volume than the workspace: slower, but the cache survives instead + * of falling through to "start fresh". + */ + private static renameOrCopy(source: string, destination: string): void { + const move = moveDirectory(source, destination); + if (move.method === 'rename') return; + + OrchestratorLogger.logWarning( + `[ChildWorkspace] rename failed (${move.renameError}) from ${source} to ${destination}; ` + + `copied instead. This is slower - consider configuring parentCacheRoot on ` + + `the same volume as the workspace.`, + ); + if (move.sourceCleanupError) { + OrchestratorLogger.logWarning( + `[ChildWorkspace] copied ${source} to ${destination} but could not remove the source: ${move.sourceCleanupError}`, + ); + } + } } diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts index 91074550..8402e5f7 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts @@ -142,7 +142,9 @@ describe('LocalCacheService', () => { expect(result).toBe(false); }); - it('should restore from the latest tar file on cache hit', async () => { + it('should restore from the latest tar file on cache hit (explicit tar mode)', async () => { + // move-directory is the default since #40; tar remains available as an explicit opt-in + // (e.g. for distributing cache entries via the rclone/S3 built-in container hooks). (mockFs.existsSync as vi.Mock).mockReturnValue(true); (mockFs.readdirSync as vi.Mock).mockReturnValue(['lib-1000.tar', 'lib-2000.tar']); (mockFs.statSync as vi.Mock).mockImplementation((filePath: string) => ({ @@ -150,7 +152,9 @@ describe('LocalCacheService', () => { })); (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); - const result = await LocalCacheService.restoreLibraryCache('/project', '/cache', 'key1'); + const result = await LocalCacheService.restoreEngineCache('/project', '/cache', 'key1', { + restoreMode: 'tar', + }); expect(result).toBe(true); expect(OrchestratorSystem.Run).toHaveBeenCalledWith( @@ -193,6 +197,49 @@ describe('LocalCacheService', () => { }); expect(mockFs.renameSync).not.toHaveBeenCalled(); }); + + it('should default to move-directory (rename) when no restoreMode is specified', async () => { + const cachePath = path.join('/cache', 'key1', 'Library'); + const destPath = path.join('/project', 'Library'); + + (mockFs.existsSync as vi.Mock).mockReturnValue(true); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); + (mockFs.renameSync as vi.Mock).mockReturnValue(undefined); + + const result = await LocalCacheService.restoreEngineCache('/project', '/cache', 'key1'); + + expect(result).toBe(true); + expect(mockFs.renameSync).toHaveBeenCalledWith(cachePath, destPath); + expect(mockFs.cpSync).not.toHaveBeenCalled(); + }); + + it('should fall back to copy when rename fails with EXDEV (cross-volume)', async () => { + const cachePath = path.join('/cache', 'key1', 'Library'); + const destPath = path.join('/project', 'Library'); + + (mockFs.existsSync as vi.Mock).mockReturnValue(true); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); + (mockFs.renameSync as vi.Mock).mockImplementationOnce(() => { + const error: any = new Error('cross-device link'); + error.code = 'EXDEV'; + throw error; + }); + (mockFs.cpSync as vi.Mock).mockReturnValue(undefined); + (mockFs.rmSync as vi.Mock).mockReturnValue(undefined); + + const result = await LocalCacheService.restoreEngineCache('/project', '/cache', 'key1'); + + expect(result).toBe(true); + expect(mockFs.renameSync).toHaveBeenCalledWith(cachePath, destPath); + const staging = expect.stringContaining(path.join('/project', '.Library.partial-')); + expect(mockFs.cpSync).toHaveBeenCalledWith(cachePath, staging, { recursive: true }); + expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, destPath); + expect(mockFs.rmSync).toHaveBeenCalledWith(cachePath, { recursive: true, force: true }); + }); }); describe('saveLibraryCache', () => { @@ -209,7 +256,8 @@ describe('LocalCacheService', () => { expect(mockFs.mkdirSync).not.toHaveBeenCalled(); }); - it('should create cache directory and save tar', async () => { + it('should create cache directory and save tar (explicit tar mode)', async () => { + // move-directory is the default since #40; tar remains available as an explicit opt-in. (mockFs.existsSync as vi.Mock).mockReturnValue(true); (mockFs.readdirSync as vi.Mock).mockImplementation((dirPath: string) => { if (String(dirPath).includes('Library') && !String(dirPath).includes('cache')) { @@ -222,12 +270,58 @@ describe('LocalCacheService', () => { OrchestratorSystem.Run.mockResolvedValue(''); - await LocalCacheService.saveLibraryCache('/project', '/cache', 'key1'); + await LocalCacheService.saveEngineCache('/project', '/cache', 'key1', { saveMode: 'tar' }); expect(mockFs.mkdirSync).toHaveBeenCalledWith(path.join('/cache', 'key1', 'Library'), { recursive: true, }); expect(OrchestratorSystem.Run).toHaveBeenCalledWith(expect.stringContaining('tar -cf'), true); }); + + it('should default to move-directory (rename) when no saveMode is specified', async () => { + const folderPath = path.join('/project', 'Library'); + const cachePath = path.join('/cache', 'key1', 'Library'); + + (mockFs.existsSync as vi.Mock).mockImplementation( + (filePath: string) => filePath === folderPath, + ); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); + (mockFs.renameSync as vi.Mock).mockReturnValue(undefined); + + await LocalCacheService.saveEngineCache('/project', '/cache', 'key1'); + + expect(mockFs.renameSync).toHaveBeenCalledWith(folderPath, cachePath); + expect(mockFs.cpSync).not.toHaveBeenCalled(); + expect(OrchestratorSystem.Run).not.toHaveBeenCalled(); + }); + + it('should fall back to copy when save rename fails with EXDEV (cross-volume)', async () => { + const folderPath = path.join('/project', 'Library'); + const cachePath = path.join('/cache', 'key1', 'Library'); + + (mockFs.existsSync as vi.Mock).mockImplementation( + (filePath: string) => filePath === folderPath, + ); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); + (mockFs.renameSync as vi.Mock).mockImplementationOnce(() => { + const error: any = new Error('cross-device link'); + error.code = 'EXDEV'; + throw error; + }); + (mockFs.cpSync as vi.Mock).mockReturnValue(undefined); + (mockFs.rmSync as vi.Mock).mockReturnValue(undefined); + + await LocalCacheService.saveEngineCache('/project', '/cache', 'key1'); + + expect(mockFs.renameSync).toHaveBeenCalledWith(folderPath, cachePath); + const staging = expect.stringContaining(path.join('/cache', 'key1', '.Library.partial-')); + expect(mockFs.cpSync).toHaveBeenCalledWith(folderPath, staging, { recursive: true }); + expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, cachePath); + expect(mockFs.rmSync).toHaveBeenCalledWith(folderPath, { recursive: true, force: true }); + }); }); describe('saveEngineCache gating (skipOnCorruptionEvidence / skipOnCrashEvidence)', () => { @@ -271,6 +365,7 @@ describe('LocalCacheService', () => { OrchestratorSystem.Run.mockResolvedValue(''); await LocalCacheService.saveEngineCache('/project', '/cache', 'key1', { + saveMode: 'tar', skipOnCrashEvidence: true, diagnostics: { importCompleted: true, crashEvidenceFound: true }, }); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts index c86aa418..3db9f521 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts @@ -3,6 +3,7 @@ import path from 'node:path'; import { execSync, spawn } from 'node:child_process'; import { OrchestratorSystem } from '../core/orchestrator-system'; import OrchestratorLogger from '../core/orchestrator-logger'; +import { moveDirectory } from './move-directory'; import { getEngine } from '../../../engine'; export type LocalCacheMode = 'tar' | 'move-directory' | 'copy-directory'; @@ -246,8 +247,13 @@ export class LocalCacheService { return false; } - if (options.restoreMode && options.restoreMode !== 'tar') { - return LocalCacheService.restoreDirectoryCache(projectPath, cachePath, folder, options); + // move-directory is the default mode (O(1) rename); tar is opt-in for cases that + // need it (e.g. distributing cache entries via the rclone/S3 built-in container hooks). + if (options.restoreMode !== 'tar') { + return LocalCacheService.restoreDirectoryCache(projectPath, cachePath, folder, { + ...options, + restoreMode: options.restoreMode ?? 'move-directory', + }); } const files = fs.readdirSync(cachePath).filter((f) => f.endsWith('.tar')); @@ -364,11 +370,15 @@ export class LocalCacheService { const cachePath = path.join(cacheRoot, cacheKey, folder); fs.mkdirSync(cachePath, { recursive: true }); - if (options.saveMode && options.saveMode !== 'tar') { + // move-directory is the default mode (O(1) rename); tar is opt-in for cases that + // need it (e.g. distributing cache entries via the rclone/S3 built-in container hooks). + if (options.saveMode !== 'tar') { + const directorySaveMode: 'move-directory' | 'copy-directory' = + options.saveMode === 'copy-directory' ? 'copy-directory' : 'move-directory'; LocalCacheService.saveDirectoryCache( folderPath, cachePath, - options.saveMode, + directorySaveMode, options.backgroundSave, ); @@ -414,7 +424,7 @@ export class LocalCacheService { const entries = fs.readdirSync(cachePath); if (entries.length > 0 && entries.every((entry) => entry.endsWith('.tar'))) { OrchestratorLogger.logWarning( - `[LocalCache] ${folder} cache at ${cachePath} contains tar archives; use localCacheMode=tar to restore it`, + `[LocalCache] ${folder} cache at ${cachePath} contains tar archives; use cacheMode=tar to restore it`, ); return false; @@ -428,7 +438,7 @@ export class LocalCacheService { if (options.restoreMode === 'copy-directory') { fs.cpSync(cachePath, dest, { recursive: true }); } else { - fs.renameSync(cachePath, dest); + LocalCacheService.renameOrCopy(cachePath, dest); } if ( @@ -874,7 +884,7 @@ export class LocalCacheService { if (saveMode === 'copy-directory') { fs.cpSync(folderPath, cachePath, { recursive: true }); } else { - fs.renameSync(folderPath, cachePath); + LocalCacheService.renameOrCopy(folderPath, cachePath); } OrchestratorLogger.log( @@ -882,6 +892,27 @@ export class LocalCacheService { ); } + /** + * Moves a directory, degrading to a staged copy when localCacheRoot sits on + * another volume than the workspace: slower, but the build keeps working + * instead of failing every restore and save. + */ + private static renameOrCopy(source: string, destination: string): void { + const move = moveDirectory(source, destination); + if (move.method === 'rename') return; + + OrchestratorLogger.logWarning( + `[LocalCache] move-directory rename failed (${move.renameError}) from ${source} to ` + + `${destination}; copied instead. This is slower - consider configuring ` + + `localCacheRoot on the same volume as the workspace.`, + ); + if (move.sourceCleanupError) { + OrchestratorLogger.logWarning( + `[LocalCache] copied ${source} to ${destination} but could not remove the source: ${move.sourceCleanupError}`, + ); + } + } + /** * Spawn a detached background process to copy the cache directory. * Writes a lock file with PID before starting; removes it on completion. diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts new file mode 100644 index 00000000..28440934 --- /dev/null +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts @@ -0,0 +1,101 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { moveDirectory } from './move-directory'; + +function renameError(code: string): NodeJS.ErrnoException { + return Object.assign(new Error(`${code}: rename refused`), { code }); +} + +describe('moveDirectory', () => { + let root: string; + let source: string; + let destination: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'move-directory-')); + source = path.join(root, 'Library'); + destination = path.join(root, 'cache', 'Library'); + fs.mkdirSync(path.join(source, 'nested'), { recursive: true }); + fs.writeFileSync(path.join(source, 'a.asset'), 'a'); + fs.writeFileSync(path.join(source, 'nested', 'b.asset'), 'b'); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + /** Fails only the first rename (source -> destination); later renames run for real. */ + function refuseFirstRename(code: string) { + const realRename = fs.renameSync; + let refused = false; + + return vi.spyOn(fs, 'renameSync').mockImplementation((from, to) => { + if (!refused) { + refused = true; + throw renameError(code); + } + realRename(from, to); + }); + } + + it('moves with a rename when the OS allows it', () => { + expect(moveDirectory(source, destination)).toEqual({ method: 'rename' }); + expect(fs.readFileSync(path.join(destination, 'nested', 'b.asset'), 'utf8')).toBe('b'); + expect(fs.existsSync(source)).toBe(false); + }); + + it.each(['EXDEV', 'EPERM'])( + 'copies into place and removes the source when rename fails with %s', + (code) => { + refuseFirstRename(code); + + expect(moveDirectory(source, destination)).toEqual({ method: 'copy', renameError: code }); + expect(fs.readFileSync(path.join(destination, 'a.asset'), 'utf8')).toBe('a'); + expect(fs.readFileSync(path.join(destination, 'nested', 'b.asset'), 'utf8')).toBe('b'); + expect(fs.existsSync(source)).toBe(false); + expect(fs.readdirSync(path.dirname(destination))).toEqual(['Library']); + }, + ); + + it.each(['EXDEV', 'EPERM'])( + 'leaves no partial destination and keeps the source when the %s copy fails', + (code) => { + refuseFirstRename(code); + const realCp = fs.cpSync; + vi.spyOn(fs, 'cpSync').mockImplementation((from, to, options) => { + realCp(path.join(String(from), 'a.asset'), path.join(String(to), 'a.asset'), options); + throw new Error('ENOSPC: no space left on device'); + }); + + expect(() => moveDirectory(source, destination)).toThrow('ENOSPC'); + expect(fs.existsSync(destination)).toBe(false); + expect(fs.readdirSync(path.dirname(destination))).toEqual([]); + expect(fs.readFileSync(path.join(source, 'nested', 'b.asset'), 'utf8')).toBe('b'); + }, + ); + + it('reports, rather than throws, a source that cannot be removed after a complete copy', () => { + refuseFirstRename('EXDEV'); + vi.spyOn(fs, 'rmSync').mockImplementationOnce(() => { + throw new Error('EBUSY: resource busy'); + }); + + expect(moveDirectory(source, destination)).toEqual({ + method: 'copy', + renameError: 'EXDEV', + sourceCleanupError: 'EBUSY: resource busy', + }); + expect(fs.readFileSync(path.join(destination, 'a.asset'), 'utf8')).toBe('a'); + }); + + it('rethrows rename failures a copy cannot fix', () => { + refuseFirstRename('ENOENT'); + + expect(() => moveDirectory(source, destination)).toThrow('ENOENT'); + expect(fs.existsSync(destination)).toBe(false); + }); +}); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts new file mode 100644 index 00000000..89b0b60a --- /dev/null +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts @@ -0,0 +1,62 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +/** Rename failures that a copy can work around: another volume, or a rename the OS refuses. */ +const copyableRenameErrors = new Set(['EXDEV', 'EPERM']); + +export type DirectoryMove = + | { method: 'rename' } + | { + method: 'copy'; + /** Why the rename was refused (EXDEV or EPERM). */ + renameError: string; + /** Set when the copy is complete but the source could not be removed afterwards. */ + sourceCleanupError?: string; + }; + +/** + * Moves a directory with an atomic rename, falling back to a copy when the + * rename crosses filesystems (EXDEV) or is refused (EPERM), for instance a + * cache root configured on another volume than the workspace. + * + * The copy is staged in a sibling directory and renamed into place only once + * complete, so a failed copy never leaves a partial destination behind for a + * later restore to mistake for a valid cache. The source is removed only after + * the destination is in place. + */ +export function moveDirectory(source: string, destination: string): DirectoryMove { + try { + fs.renameSync(source, destination); + return { method: 'rename' }; + } catch (error: any) { + if (!copyableRenameErrors.has(error?.code)) throw error; + + copyIntoPlace(source, destination); + + try { + fs.rmSync(source, { recursive: true, force: true }); + return { method: 'copy', renameError: error.code }; + } catch (cleanupError: any) { + return { + method: 'copy', + renameError: error.code, + sourceCleanupError: cleanupError?.message ?? String(cleanupError), + }; + } + } +} + +function copyIntoPlace(source: string, destination: string): void { + const staging = path.join( + path.dirname(destination), + `.${path.basename(destination)}.partial-${process.pid}-${Date.now().toString(36)}`, + ); + + try { + fs.cpSync(source, staging, { recursive: true }); + fs.renameSync(staging, destination); + } catch (error) { + fs.rmSync(staging, { recursive: true, force: true }); + throw error; + } +} diff --git a/plugins/orchestrator/src/plugin-lifecycle.ts b/plugins/orchestrator/src/plugin-lifecycle.ts index 0e6b03a4..cd171050 100644 --- a/plugins/orchestrator/src/plugin-lifecycle.ts +++ b/plugins/orchestrator/src/plugin-lifecycle.ts @@ -85,6 +85,8 @@ function getNumber(name: string, defaultValue: number): number { // Lazy getters — values are read from env/inputs at access time, // so they pick up whatever the composite action or user has set. +let hasWarnedDeprecatedCacheMode = false; + const config = { // Provider get providerStrategy() { @@ -271,11 +273,22 @@ const config = { get localCacheFallbackKeys() { return getInput('localCacheFallbackKeys'); }, - get localCacheMode() { - const mode = getInput('localCacheMode') || 'tar'; + get cacheMode() { + const explicitCacheMode = getInput('cacheMode'); + const legacyLocalCacheMode = getInput('localCacheMode'); + if (!explicitCacheMode && legacyLocalCacheMode && !hasWarnedDeprecatedCacheMode) { + hasWarnedDeprecatedCacheMode = true; + core.warning( + "[plugin-lifecycle] Input 'localCacheMode' is deprecated — use 'cacheMode' instead. " + + "'local' no longer accurately describes this input now that distributed rclone/S3 " + + 'caching is a supported mode too. localCacheMode will keep working for now, but ' + + 'prefer cacheMode going forward — see game-ci/roadmap#11.', + ); + } + const mode = explicitCacheMode || legacyLocalCacheMode || 'move-directory'; if (mode === 'canonical-overlay') { throw new Error( - '[plugin-lifecycle] localCacheMode: canonical-overlay has been removed. ' + + '[plugin-lifecycle] cacheMode: canonical-overlay has been removed. ' + "Use 'move-directory' (recommended for retained runners/build farms) or 'tar' " + '(recommended for remote/ephemeral caches, or combine with rclone/S3 built-in ' + 'container hooks — see the caching docs for details).', @@ -283,6 +296,10 @@ const config = { } return mode; }, + /** @deprecated Use `cacheMode` instead — kept only so internal call sites reading the old property name don't need to change too. */ + get localCacheMode() { + return config.cacheMode; + }, get maxCacheEntries() { return Number(getInput('maxCacheEntries')) || 2; }, @@ -616,7 +633,7 @@ export function createPlugin(): OrchestratorPlugin { : []; await LocalCacheService.restoreEngineCache(projectFullPath, cacheRoot, cacheKey, { fallbackKeys, - restoreMode: config.localCacheMode as any, + restoreMode: config.cacheMode as any, }); } } @@ -802,7 +819,7 @@ export function createPlugin(): OrchestratorPlugin { if (config.localCacheLibrary) { const projectFullPath = path.join(ws, coreParams.projectPath); await LocalCacheService.saveEngineCache(projectFullPath, cacheRoot, cacheKey, { - saveMode: config.localCacheMode as any, + saveMode: config.cacheMode as any, skipOnLfsPointerPoisoning: true, backgroundSave: config.backgroundCacheSave, maxCacheEntries: config.maxCacheEntries, From 905714ab6a852e89f64218c22e7a855b761b48d4 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 02:32:25 +0100 Subject: [PATCH 2/9] fix(orchestrator): shared cacheMode, safe fallback --- .../cli-plugin/build-parameters-adapter.ts | 3 +- .../services/cache/cache-mode.test.ts | 23 ++++++++++++ .../orchestrator/services/cache/cache-mode.ts | 20 +++++++++++ .../cache/local-cache-service.test.ts | 35 +++++++++++++++++-- .../services/cache/local-cache-service.ts | 11 +++--- .../services/cache/move-directory.test.ts | 10 ++++++ .../services/cache/move-directory.ts | 3 +- plugins/orchestrator/src/plugin-lifecycle.ts | 12 ++----- 8 files changed, 99 insertions(+), 18 deletions(-) create mode 100644 plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts create mode 100644 plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts diff --git a/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts b/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts index e41d4c6d..441dc4b3 100644 --- a/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts +++ b/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts @@ -2,6 +2,7 @@ import BuildParameters from '../model/build-parameters'; import { customAlphabet } from 'nanoid'; import OrchestratorConstants from '../model/orchestrator/options/orchestrator-constants'; import { initEngine } from '../model/engine'; +import { resolveCacheMode } from '../model/orchestrator/services/cache/cache-mode'; /** * Maps CLI yargs options (flat key-value pairs) into a BuildParameters object. @@ -121,7 +122,7 @@ export function createBuildParametersFromCliOptions(options: Record bp.localCacheFallbackKeys = options.localCacheFallbackKeys || ''; // cacheMode replaces localCacheMode, which stays as a deprecated alias. move-directory is an O(1) // rename (falling back to a staged copy across volumes); tar stays opt-in, e.g. for rclone/S3 hooks. - bp.localCacheMode = options.cacheMode || options.localCacheMode || 'move-directory'; + bp.localCacheMode = resolveCacheMode(options.cacheMode, options.localCacheMode); bp.localCacheSaveOnFailure = options.localCacheSaveOnFailure === true || options.localCacheSaveOnFailure === 'true'; bp.localCacheFloorCorruptionCategories = options.localCacheFloorCorruptionCategories || ''; diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts new file mode 100644 index 00000000..3b961bf2 --- /dev/null +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { resolveCacheMode } from './cache-mode'; + +describe('resolveCacheMode', () => { + it('defaults to move-directory', () => { + expect(resolveCacheMode()).toBe('move-directory'); + expect(resolveCacheMode('', '')).toBe('move-directory'); + }); + + it('prefers cacheMode over the deprecated localCacheMode', () => { + expect(resolveCacheMode('tar', 'copy-directory')).toBe('tar'); + expect(resolveCacheMode('', 'copy-directory')).toBe('copy-directory'); + }); + + it('rejects the removed canonical-overlay mode, also through the deprecated alias', () => { + expect(() => resolveCacheMode('canonical-overlay')).toThrow( + 'canonical-overlay has been removed', + ); + expect(() => resolveCacheMode('', 'canonical-overlay')).toThrow( + 'canonical-overlay has been removed', + ); + }); +}); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts new file mode 100644 index 00000000..65211d39 --- /dev/null +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts @@ -0,0 +1,20 @@ +/** + * Resolves the cache mode both entry points (the GitHub Action and + * `game-ci orchestrate`) run with: `cacheMode` first, then its deprecated alias + * `localCacheMode`, then `move-directory`. The removed `canonical-overlay` mode + * fails loudly rather than silently running as a directory move. + */ +export function resolveCacheMode(cacheMode?: string, legacyLocalCacheMode?: string): string { + const mode = cacheMode || legacyLocalCacheMode || 'move-directory'; + + if (mode === 'canonical-overlay') { + throw new Error( + 'cacheMode: canonical-overlay has been removed. ' + + "Use 'move-directory' (recommended for retained runners/build farms) or 'tar' " + + '(recommended for remote/ephemeral caches, or combine with rclone/S3 built-in ' + + 'container hooks - see the caching docs for details).', + ); + } + + return mode; +} diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts index 8402e5f7..8224c433 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts @@ -198,6 +198,31 @@ describe('LocalCacheService', () => { expect(mockFs.renameSync).not.toHaveBeenCalled(); }); + it('should copy, not move, a fallback directory cache when restoreMode is omitted', async () => { + const exactPath = path.join('/cache', 'exact', 'Library'); + const fallbackPath = path.join('/cache', 'fallback', 'Library'); + + (mockFs.existsSync as vi.Mock).mockImplementation( + (filePath: string) => filePath !== exactPath, + ); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); + (mockFs.cpSync as vi.Mock).mockReturnValue(undefined); + (mockFs.rmSync as vi.Mock).mockReturnValue(undefined); + + const result = await LocalCacheService.restoreEngineCache('/project', '/cache', 'exact', { + fallbackKeys: ['fallback'], + }); + + expect(result).toBe(true); + expect(mockFs.cpSync).toHaveBeenCalledWith(fallbackPath, path.join('/project', 'Library'), { + recursive: true, + }); + expect(mockFs.renameSync).not.toHaveBeenCalled(); + expect(mockFs.rmSync).not.toHaveBeenCalledWith(fallbackPath, expect.anything()); + }); + it('should default to move-directory (rename) when no restoreMode is specified', async () => { const cachePath = path.join('/cache', 'key1', 'Library'); const destPath = path.join('/project', 'Library'); @@ -236,7 +261,10 @@ describe('LocalCacheService', () => { expect(result).toBe(true); expect(mockFs.renameSync).toHaveBeenCalledWith(cachePath, destPath); const staging = expect.stringContaining(path.join('/project', '.Library.partial-')); - expect(mockFs.cpSync).toHaveBeenCalledWith(cachePath, staging, { recursive: true }); + expect(mockFs.cpSync).toHaveBeenCalledWith(cachePath, staging, { + recursive: true, + verbatimSymlinks: true, + }); expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, destPath); expect(mockFs.rmSync).toHaveBeenCalledWith(cachePath, { recursive: true, force: true }); }); @@ -318,7 +346,10 @@ describe('LocalCacheService', () => { expect(mockFs.renameSync).toHaveBeenCalledWith(folderPath, cachePath); const staging = expect.stringContaining(path.join('/cache', 'key1', '.Library.partial-')); - expect(mockFs.cpSync).toHaveBeenCalledWith(folderPath, staging, { recursive: true }); + expect(mockFs.cpSync).toHaveBeenCalledWith(folderPath, staging, { + recursive: true, + verbatimSymlinks: true, + }); expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, cachePath); expect(mockFs.rmSync).toHaveBeenCalledWith(folderPath, { recursive: true, force: true }); }); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts index 3db9f521..c7d9f0c4 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts @@ -199,6 +199,7 @@ export class LocalCacheService { folder: string, options: LocalCacheRestoreOptions = {}, ): Promise { + const restoreMode: LocalCacheMode = options.restoreMode ?? 'move-directory'; const candidates = [ cacheKey, ...(options.fallbackKeys || []).filter((key) => key !== cacheKey), @@ -206,10 +207,12 @@ export class LocalCacheService { for (const candidateKey of candidates) { const isFallback = candidateKey !== cacheKey; - const candidateOptions = - isFallback && options.restoreMode === 'move-directory' - ? { ...options, restoreMode: 'copy-directory' as const } - : options; + // A fallback entry belongs to another key: copy it rather than move it, so it stays reusable. + const candidateOptions: LocalCacheRestoreOptions = { + ...options, + restoreMode: + isFallback && restoreMode === 'move-directory' ? 'copy-directory' : restoreMode, + }; if ( await LocalCacheService.restoreCacheFolderCandidate( diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts index 28440934..af7628ef 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts @@ -98,4 +98,14 @@ describe('moveDirectory', () => { expect(() => moveDirectory(source, destination)).toThrow('ENOENT'); expect(fs.existsSync(destination)).toBe(false); }); + + it('keeps relative symlinks working after a copy fallback removes the source', () => { + fs.symlinkSync('a.asset', path.join(source, 'link.asset')); + refuseFirstRename('EXDEV'); + + moveDirectory(source, destination); + + expect(fs.readlinkSync(path.join(destination, 'link.asset'))).toBe('a.asset'); + expect(fs.readFileSync(path.join(destination, 'link.asset'), 'utf8')).toBe('a'); + }); }); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts index 89b0b60a..7925d707 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts @@ -53,7 +53,8 @@ function copyIntoPlace(source: string, destination: string): void { ); try { - fs.cpSync(source, staging, { recursive: true }); + // verbatimSymlinks keeps relative links relative; resolved, they would point into the removed source. + fs.cpSync(source, staging, { recursive: true, verbatimSymlinks: true }); fs.renameSync(staging, destination); } catch (error) { fs.rmSync(staging, { recursive: true, force: true }); diff --git a/plugins/orchestrator/src/plugin-lifecycle.ts b/plugins/orchestrator/src/plugin-lifecycle.ts index cd171050..261e64fe 100644 --- a/plugins/orchestrator/src/plugin-lifecycle.ts +++ b/plugins/orchestrator/src/plugin-lifecycle.ts @@ -45,6 +45,7 @@ import { OutputService } from './model/orchestrator/services/output/output-servi import { OutputTypeRegistry } from './model/orchestrator/services/output/output-type-registry'; import { ArtifactUploadHandler } from './model/orchestrator/services/output/artifact-upload-handler'; import { IncrementalSyncService } from './model/orchestrator/services/sync'; +import { resolveCacheMode } from './model/orchestrator/services/cache/cache-mode'; // ── Input helpers ──────────────────────────────────────────────────── @@ -285,16 +286,7 @@ const config = { 'prefer cacheMode going forward — see game-ci/roadmap#11.', ); } - const mode = explicitCacheMode || legacyLocalCacheMode || 'move-directory'; - if (mode === 'canonical-overlay') { - throw new Error( - '[plugin-lifecycle] cacheMode: canonical-overlay has been removed. ' + - "Use 'move-directory' (recommended for retained runners/build farms) or 'tar' " + - '(recommended for remote/ephemeral caches, or combine with rclone/S3 built-in ' + - 'container hooks — see the caching docs for details).', - ); - } - return mode; + return resolveCacheMode(explicitCacheMode, legacyLocalCacheMode); }, /** @deprecated Use `cacheMode` instead — kept only so internal call sites reading the old property name don't need to change too. */ get localCacheMode() { From d50ad50366cd7f5f58a974d78bd43d029e79e519 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 03:44:40 +0100 Subject: [PATCH 3/9] fix(orchestrator): reject unknown cache modes --- .../services/cache/cache-mode.test.ts | 5 +++++ .../orchestrator/services/cache/cache-mode.ts | 22 +++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts index 3b961bf2..cf15314a 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts @@ -12,6 +12,11 @@ describe('resolveCacheMode', () => { expect(resolveCacheMode('', 'copy-directory')).toBe('copy-directory'); }); + it('rejects an unknown mode instead of running it as a directory move', () => { + expect(() => resolveCacheMode('move-dir')).toThrow("unknown mode 'move-dir'"); + expect(() => resolveCacheMode('', 'Tar')).toThrow("unknown mode 'Tar'"); + }); + it('rejects the removed canonical-overlay mode, also through the deprecated alias', () => { expect(() => resolveCacheMode('canonical-overlay')).toThrow( 'canonical-overlay has been removed', diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts index 65211d39..242592b7 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts @@ -1,10 +1,18 @@ +import type { LocalCacheMode } from './local-cache-service'; + +const supportedCacheModes: readonly LocalCacheMode[] = ['move-directory', 'copy-directory', 'tar']; + /** * Resolves the cache mode both entry points (the GitHub Action and * `game-ci orchestrate`) run with: `cacheMode` first, then its deprecated alias - * `localCacheMode`, then `move-directory`. The removed `canonical-overlay` mode - * fails loudly rather than silently running as a directory move. + * `localCacheMode`, then `move-directory`. Anything else, including the removed + * `canonical-overlay` mode or a typo, fails loudly rather than silently running + * as a directory move. */ -export function resolveCacheMode(cacheMode?: string, legacyLocalCacheMode?: string): string { +export function resolveCacheMode( + cacheMode?: string, + legacyLocalCacheMode?: string, +): LocalCacheMode { const mode = cacheMode || legacyLocalCacheMode || 'move-directory'; if (mode === 'canonical-overlay') { @@ -16,5 +24,11 @@ export function resolveCacheMode(cacheMode?: string, legacyLocalCacheMode?: stri ); } - return mode; + if (!supportedCacheModes.includes(mode as LocalCacheMode)) { + throw new Error( + `cacheMode: unknown mode '${mode}'. Use one of: ${supportedCacheModes.join(', ')}.`, + ); + } + + return mode as LocalCacheMode; } From 74f78fd950b3a6ba3d37d21cb394e408d54a1929 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 10:17:27 +0100 Subject: [PATCH 4/9] build(orchestrator): @types/node 20 to match engines --- plugins/orchestrator/package.json | 2 +- plugins/orchestrator/yarn.lock | 11 ++--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/plugins/orchestrator/package.json b/plugins/orchestrator/package.json index 25676d93..e8f47917 100644 --- a/plugins/orchestrator/package.json +++ b/plugins/orchestrator/package.json @@ -88,7 +88,7 @@ }, "devDependencies": { "@types/base-64": "^1.0.2", - "@types/node": "^17.0.23", + "@types/node": "^20", "@types/semver": "^7.3.9", "@types/uuid": "^9.0.0", "@types/yargs": "^17.0.35", diff --git a/plugins/orchestrator/yarn.lock b/plugins/orchestrator/yarn.lock index fee4068d..9b0de02a 100644 --- a/plugins/orchestrator/yarn.lock +++ b/plugins/orchestrator/yarn.lock @@ -1386,7 +1386,7 @@ __metadata: "@kubernetes/client-node": "npm:^0.19.0" "@octokit/core": "npm:^5.1.0" "@types/base-64": "npm:^1.0.2" - "@types/node": "npm:^17.0.23" + "@types/node": "npm:^20" "@types/semver": "npm:^7.3.9" "@types/uuid": "npm:^9.0.0" "@types/yargs": "npm:^17.0.35" @@ -2992,14 +2992,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^17.0.23": - version: 17.0.45 - resolution: "@types/node@npm:17.0.45" - checksum: 10/b45fff7270b5e81be19ef91a66b764a8b21473a97a8d211218a52e3426b79ad48f371819ab9153370756b33ba284e5c875463de4d2cf48a472e9098d7f09e8a2 - languageName: node - linkType: hard - -"@types/node@npm:^20.1.1": +"@types/node@npm:^20, @types/node@npm:^20.1.1": version: 20.19.43 resolution: "@types/node@npm:20.19.43" dependencies: From ac496fa5aaa5574dab5c9b773d8cec78ad5caf31 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 12:00:37 +0100 Subject: [PATCH 5/9] build: refresh bun.lock for @types/node 20 --- bun.lock | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bun.lock b/bun.lock index 2bef2337..031ae357 100644 --- a/bun.lock +++ b/bun.lock @@ -130,7 +130,7 @@ }, "devDependencies": { "@types/base-64": "^1.0.2", - "@types/node": "^17.0.23", + "@types/node": "^20", "@types/semver": "^7.3.9", "@types/uuid": "^9.0.0", "@types/yargs": "^17.0.35", @@ -1673,6 +1673,8 @@ "@game-ci/itch-deploy/typescript": ["typescript@4.7.4", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ=="], + "@game-ci/orchestrator/@types/node": ["@types/node@20.19.43", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA=="], + "@game-ci/orchestrator/typescript": ["typescript@4.7.4", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ=="], "@game-ci/pseudo-localization/typescript": ["typescript@4.7.4", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ=="], @@ -1759,6 +1761,10 @@ "uri-js/punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], + "vite/@types/node": ["@types/node@20.19.43", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA=="], + + "vitest/@types/node": ["@types/node@20.19.43", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA=="], + "xml-js/sax": ["sax@1.6.1", "", {}, "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q=="], "@actions/glob/minimatch/brace-expansion": ["brace-expansion@1.1.18", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-Edep/X9fGqVNmzKBVsDYIOtD+z1tuezV70LBjdCst9Tqu76lsnvRiZ6oTic1n+/BIwX6QDGAO94PN4N2SADvtw=="], From b6b953aa52275442ae0a00a9acba33b97638d25a Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 16:31:11 +0100 Subject: [PATCH 6/9] fix(cache): tombstone a source before removing it A failed rmSync after a cross-volume copy left a half-deleted source, which a restore then accepted as a complete cache entry. The source is now renamed to a sibling tombstone first and only the tombstone is deleted, so it is either whole or gone. --- .../services/cache/move-directory.test.ts | 186 +++++++++++++++++- .../services/cache/move-directory.ts | 127 ++++++++++-- 2 files changed, 292 insertions(+), 21 deletions(-) diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts index af7628ef..ab83bd2c 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts @@ -2,12 +2,25 @@ import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { moveDirectory } from './move-directory'; +import { + copyIntoPlace, + moveDirectory, + removeDirectory, + sweepAbandonedLeftovers, +} from './move-directory'; function renameError(code: string): NodeJS.ErrnoException { return Object.assign(new Error(`${code}: rename refused`), { code }); } +/** Restores write access everywhere below a directory, so a test can clean it up. */ +function makeWritable(directory: string): void { + fs.chmodSync(directory, 0o755); + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory()) makeWritable(path.join(directory, entry.name)); + } +} + describe('moveDirectory', () => { let root: string; let source: string; @@ -84,12 +97,55 @@ describe('moveDirectory', () => { throw new Error('EBUSY: resource busy'); }); - expect(moveDirectory(source, destination)).toEqual({ + const move = moveDirectory(source, destination); + + expect(move).toEqual({ method: 'copy', renameError: 'EXDEV', - sourceCleanupError: 'EBUSY: resource busy', + sourceCleanupError: expect.stringMatching( + /^EBUSY: resource busy; left .*\.Library\.removing-/, + ), }); expect(fs.readFileSync(path.join(destination, 'a.asset'), 'utf8')).toBe('a'); + expect(fs.existsSync(source)).toBe(false); + }); + + // Root ignores directory permissions, and Windows does not use them to guard deletion. + it.skipIf(process.platform === 'win32' || process.getuid?.() === 0)( + 'never leaves a half-deleted source when a child cannot be removed after the copy', + () => { + fs.mkdirSync(path.join(source, 'locked')); + fs.writeFileSync(path.join(source, 'locked', 'c.asset'), 'c'); + fs.chmodSync(path.join(source, 'locked'), 0o555); + refuseFirstRename('EXDEV'); + + try { + const move = moveDirectory(source, destination); + + expect(move).toMatchObject({ method: 'copy', sourceCleanupError: expect.any(String) }); + expect(fs.existsSync(source)).toBe(false); + expect(fs.readFileSync(path.join(destination, 'locked', 'c.asset'), 'utf8')).toBe('c'); + } finally { + makeWritable(root); + } + }, + ); + + it('keeps the source whole when it cannot be set aside after a complete copy', () => { + const realRename = fs.renameSync; + vi.spyOn(fs, 'renameSync').mockImplementation((from, to) => { + if (from === source) throw renameError(to === destination ? 'EXDEV' : 'EBUSY'); + realRename(from, to); + }); + + const move = moveDirectory(source, destination); + + expect(move).toMatchObject({ + method: 'copy', + sourceCleanupError: expect.stringContaining(`kept ${source} whole`), + }); + expect(fs.readFileSync(path.join(source, 'nested', 'b.asset'), 'utf8')).toBe('b'); + expect(fs.readFileSync(path.join(destination, 'nested', 'b.asset'), 'utf8')).toBe('b'); }); it('rethrows rename failures a copy cannot fix', () => { @@ -109,3 +165,127 @@ describe('moveDirectory', () => { expect(fs.readFileSync(path.join(destination, 'link.asset'), 'utf8')).toBe('a'); }); }); + +describe('copyIntoPlace', () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'copy-into-place-')); + fs.mkdirSync(path.join(root, 'from', 'nested'), { recursive: true }); + fs.writeFileSync(path.join(root, 'from', 'nested', 'b.asset'), 'b'); + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('copies a directory and keeps the source', () => { + copyIntoPlace(path.join(root, 'from'), path.join(root, 'to')); + + expect(fs.readFileSync(path.join(root, 'to', 'nested', 'b.asset'), 'utf8')).toBe('b'); + expect(fs.existsSync(path.join(root, 'from', 'nested', 'b.asset'))).toBe(true); + expect(fs.readdirSync(root).sort()).toEqual(['from', 'to']); + }); + + it('leaves no destination and no staging directory when the copy fails', () => { + vi.spyOn(fs, 'cpSync').mockImplementation((_from, to) => { + fs.mkdirSync(String(to)); + throw new Error('ENOSPC: no space left on device'); + }); + + expect(() => copyIntoPlace(path.join(root, 'from'), path.join(root, 'to'))).toThrow('ENOSPC'); + expect(fs.readdirSync(root)).toEqual(['from']); + }); +}); + +describe('removeDirectory', () => { + let root: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'remove-directory-')); + fs.mkdirSync(path.join(root, 'Library', 'nested'), { recursive: true }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('removes the directory and its tombstone', () => { + expect(removeDirectory(path.join(root, 'Library'))).toEqual({ outcome: 'removed' }); + expect(fs.readdirSync(root)).toEqual([]); + }); + + it('keeps the directory whole when it cannot be set aside', () => { + vi.spyOn(fs, 'renameSync').mockImplementation(() => { + throw renameError('EBUSY'); + }); + + expect(removeDirectory(path.join(root, 'Library'))).toEqual({ + outcome: 'kept-whole', + error: 'EBUSY: rename refused', + }); + expect(fs.existsSync(path.join(root, 'Library', 'nested'))).toBe(true); + }); + + it('reports the tombstone it could not delete, with the directory already gone', () => { + vi.spyOn(fs, 'rmSync').mockImplementationOnce(() => { + throw new Error('EACCES: permission denied'); + }); + + const removal = removeDirectory(path.join(root, 'Library')); + + expect(removal).toEqual({ + outcome: 'tombstone-left', + tombstone: expect.stringContaining(path.join(root, `.Library.removing-${process.pid}-`)), + error: 'EACCES: permission denied', + }); + expect(fs.readdirSync(root)).toEqual([ + path.basename((removal as { tombstone: string }).tombstone), + ]); + }); +}); + +describe('sweepAbandonedLeftovers', () => { + let root: string; + let deadPid: number; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'sweep-leftovers-')); + deadPid = 999_999_999; + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('removes staging and tombstone directories of processes that are gone', () => { + fs.mkdirSync(path.join(root, `.Library.partial-${deadPid}-abc`, 'nested'), { recursive: true }); + fs.mkdirSync(path.join(root, `.Library.removing-${deadPid}-abc`)); + + const removed = sweepAbandonedLeftovers(root); + + expect(removed.map((leftover) => path.basename(leftover)).sort()).toEqual([ + `.Library.partial-${deadPid}-abc`, + `.Library.removing-${deadPid}-abc`, + ]); + expect(fs.readdirSync(root)).toEqual([]); + }); + + it('keeps leftovers of a live process, cache entries and look-alike files', () => { + const kept = [ + `.Library.partial-${process.pid}-abc`, + `.Library.removing-${process.ppid}-abc`, + 'Library', + '.Library.partial-notapid-abc', + ]; + for (const name of kept) fs.mkdirSync(path.join(root, name)); + fs.writeFileSync(path.join(root, `.Library.partial-${deadPid}-file`), ''); + + expect(sweepAbandonedLeftovers(root)).toEqual([]); + expect(fs.readdirSync(root).sort()).toEqual( + [...kept, `.Library.partial-${deadPid}-file`].sort(), + ); + }); +}); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts index 7925d707..26d7444d 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts @@ -4,6 +4,9 @@ import path from 'node:path'; /** Rename failures that a copy can work around: another volume, or a rename the OS refuses. */ const copyableRenameErrors = new Set(['EXDEV', 'EPERM']); +/** Sibling directories this module leaves behind only when a process dies mid-operation. */ +const leftoverPattern = /^\..+\.(partial|removing)-(\d+)-[\da-z]+$/; + export type DirectoryMove = | { method: 'rename' } | { @@ -14,6 +17,13 @@ export type DirectoryMove = sourceCleanupError?: string; }; +export type DirectoryRemoval = + | { outcome: 'removed' } + /** The directory could not be set aside, so it was left in place, whole. */ + | { outcome: 'kept-whole'; error: string } + /** The directory is gone, but its tombstone could not be deleted fully. */ + | { outcome: 'tombstone-left'; tombstone: string; error: string }; + /** * Moves a directory with an atomic rename, falling back to a copy when the * rename crosses filesystems (EXDEV) or is refused (EPERM), for instance a @@ -21,36 +31,47 @@ export type DirectoryMove = * * The copy is staged in a sibling directory and renamed into place only once * complete, so a failed copy never leaves a partial destination behind for a - * later restore to mistake for a valid cache. The source is removed only after - * the destination is in place. + * later restore to mistake for a valid cache. The source is then removed with + * removeDirectory, so a failed removal never leaves it half-deleted either. */ export function moveDirectory(source: string, destination: string): DirectoryMove { try { fs.renameSync(source, destination); return { method: 'rename' }; - } catch (error: any) { - if (!copyableRenameErrors.has(error?.code)) throw error; + } catch (error: unknown) { + const renameError = errorCode(error); + if (!renameError || !copyableRenameErrors.has(renameError)) throw error; copyIntoPlace(source, destination); - try { - fs.rmSync(source, { recursive: true, force: true }); - return { method: 'copy', renameError: error.code }; - } catch (cleanupError: any) { - return { - method: 'copy', - renameError: error.code, - sourceCleanupError: cleanupError?.message ?? String(cleanupError), - }; + const removal = removeDirectory(source); + switch (removal.outcome) { + case 'removed': + return { method: 'copy', renameError }; + case 'kept-whole': + return { + method: 'copy', + renameError, + sourceCleanupError: `${removal.error}; kept ${source} whole`, + }; + case 'tombstone-left': + return { + method: 'copy', + renameError, + sourceCleanupError: `${removal.error}; left ${removal.tombstone} for a later sweep`, + }; } } } -function copyIntoPlace(source: string, destination: string): void { - const staging = path.join( - path.dirname(destination), - `.${path.basename(destination)}.partial-${process.pid}-${Date.now().toString(36)}`, - ); +/** + * Copies a directory to a destination that does not exist yet. The copy is + * staged in a sibling `..partial--` directory and renamed into + * place only once complete; on failure the staging directory is removed and + * the error rethrown, so the destination is either whole or absent. + */ +export function copyIntoPlace(source: string, destination: string): void { + const staging = siblingPath(destination, 'partial'); try { // verbatimSymlinks keeps relative links relative; resolved, they would point into the removed source. @@ -61,3 +82,73 @@ function copyIntoPlace(source: string, destination: string): void { throw error; } } + +/** + * Removes a directory so that it is either whole or gone, never half-deleted: + * it is first renamed to a sibling `..removing--` tombstone + * (same volume, atomic), and only the tombstone is deleted. + */ +export function removeDirectory(directory: string): DirectoryRemoval { + const tombstone = siblingPath(directory, 'removing'); + + try { + fs.renameSync(directory, tombstone); + } catch (error: unknown) { + return { outcome: 'kept-whole', error: describe(error) }; + } + + try { + fs.rmSync(tombstone, { recursive: true, force: true }); + return { outcome: 'removed' }; + } catch (error: unknown) { + return { outcome: 'tombstone-left', tombstone, error: describe(error) }; + } +} + +/** + * Deletes the staging and tombstone directories under `parent` whose owning + * process is gone, e.g. after a runner was killed mid-copy. Directories of a + * process that is still alive, or that cannot be checked, are kept. Returns + * the paths removed; throws only if `parent` cannot be read. + */ +export function sweepAbandonedLeftovers(parent: string): string[] { + const removed: string[] = []; + + for (const entry of fs.readdirSync(parent, { withFileTypes: true })) { + const match = entry.isDirectory() && leftoverPattern.exec(entry.name); + if (!match || isProcessAlive(Number(match[2]))) continue; + + const leftover = path.join(parent, entry.name); + fs.rmSync(leftover, { recursive: true, force: true }); + removed.push(leftover); + } + + return removed; +} + +function siblingPath(directory: string, purpose: 'partial' | 'removing'): string { + return path.join( + path.dirname(directory), + `.${path.basename(directory)}.${purpose}-${process.pid}-${Date.now().toString(36)}`, + ); +} + +function isProcessAlive(pid: number): boolean { + if (pid === process.pid) return true; + + try { + process.kill(pid, 0); + return true; + } catch (error: unknown) { + // EPERM means the process exists under another user; only ESRCH proves it is gone. + return errorCode(error) !== 'ESRCH'; + } +} + +function errorCode(error: unknown): string | undefined { + return (error as NodeJS.ErrnoException)?.code; +} + +function describe(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} From 093469c8236dec40c3477a1d5254995721c41978 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 16:31:11 +0100 Subject: [PATCH 7/9] fix(cache): stage every directory copy Explicit copy-directory saves and restores, background saves and their synchronous fallback now copy through a staging directory, so a failed copy leaves no partial entry. Stale staging and tombstone directories are swept with the stale locks. --- .../cache/local-cache-service.fs.test.ts | 115 ++++++++++++++++ .../cache/local-cache-service.test.ts | 124 ++++++++++++++++-- .../services/cache/local-cache-service.ts | 120 +++++++++++++---- 3 files changed, 323 insertions(+), 36 deletions(-) create mode 100644 plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts new file mode 100644 index 00000000..15003675 --- /dev/null +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts @@ -0,0 +1,115 @@ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { LocalCacheService } from './local-cache-service'; + +const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() })); + +vi.mock('node:child_process', async (importOriginal) => ({ + ...(await importOriginal()), + spawn: spawnMock, +})); +vi.mock('../core/orchestrator-logger', () => ({ + __esModule: true, + default: { log: vi.fn(), logWarning: vi.fn(), error: vi.fn() }, +})); + +const deadPid = 999_999_999; + +/** LocalCacheService against a real filesystem, for what mocked fs calls cannot show. */ +describe('LocalCacheService on disk', () => { + let root: string; + let projectPath: string; + let cacheRoot: string; + let cachePath: string; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'local-cache-fs-')); + projectPath = path.join(root, 'project'); + cacheRoot = path.join(root, 'cache'); + cachePath = path.join(cacheRoot, 'key1', 'Library'); + fs.mkdirSync(path.join(projectPath, 'Library', 'nested'), { recursive: true }); + fs.writeFileSync(path.join(projectPath, 'Library', 'ArtifactDB'), 'new'); + fs.writeFileSync(path.join(projectPath, 'Library', 'nested', 'b.asset'), 'b'); + spawnMock.mockReset(); + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + describe('backgroundSaveScript', () => { + function runScript(): void { + const lockPath = path.join(cacheRoot, 'key1', '.game-ci-cache-save.lock'); + fs.mkdirSync(path.dirname(lockPath), { recursive: true }); + fs.writeFileSync(lockPath, 'pending'); + const script = LocalCacheService.backgroundSaveScript( + path.join(projectPath, 'Library'), + cachePath, + lockPath, + ); + + try { + execFileSync(process.execPath, ['-e', script], { stdio: 'pipe' }); + } finally { + expect(fs.existsSync(lockPath)).toBe(false); + } + } + + it('replaces the previous entry with a complete copy and leaves nothing beside it', () => { + fs.mkdirSync(cachePath, { recursive: true }); + fs.writeFileSync(path.join(cachePath, 'ArtifactDB'), 'old'); + + runScript(); + + expect(fs.readFileSync(path.join(cachePath, 'ArtifactDB'), 'utf8')).toBe('new'); + expect(fs.readFileSync(path.join(cachePath, 'nested', 'b.asset'), 'utf8')).toBe('b'); + expect(fs.readdirSync(path.dirname(cachePath))).toEqual(['Library']); + expect(fs.existsSync(path.join(projectPath, 'Library', 'ArtifactDB'))).toBe(true); + }); + + it('keeps the previous entry and removes its staging copy when the copy fails', () => { + fs.mkdirSync(cachePath, { recursive: true }); + fs.writeFileSync(path.join(cachePath, 'ArtifactDB'), 'old'); + fs.rmSync(path.join(projectPath, 'Library'), { recursive: true }); + + expect(() => runScript()).toThrow(); + + expect(fs.readFileSync(path.join(cachePath, 'ArtifactDB'), 'utf8')).toBe('old'); + expect(fs.readdirSync(path.dirname(cachePath))).toEqual(['Library']); + }); + }); + + it('falls back to a staged synchronous copy when the background save cannot spawn', async () => { + spawnMock.mockImplementation(() => { + throw new Error('EAGAIN: spawn failed'); + }); + + await LocalCacheService.saveEngineCache(projectPath, cacheRoot, 'key1', { + saveMode: 'copy-directory', + backgroundSave: true, + }); + + expect(spawnMock).toHaveBeenCalled(); + expect(fs.readFileSync(path.join(cachePath, 'nested', 'b.asset'), 'utf8')).toBe('b'); + expect(fs.readdirSync(path.dirname(cachePath))).toEqual(['Library']); + }); + + it('sweeps staging and tombstone directories abandoned by a dead process', () => { + const keyPath = path.join(cacheRoot, 'key1'); + const abandoned = [`.Library.partial-${deadPid}-abc`, `.Library.removing-${deadPid}-abc`]; + for (const name of abandoned) + fs.mkdirSync(path.join(keyPath, name, 'nested'), { recursive: true }); + fs.mkdirSync(path.join(keyPath, `.Library.partial-${process.pid}-abc`)); + fs.mkdirSync(cachePath); + + LocalCacheService.sweepStaleLocks(cacheRoot); + + expect(fs.readdirSync(keyPath).sort()).toEqual([ + `.Library.partial-${process.pid}-abc`, + 'Library', + ]); + }); +}); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts index 8224c433..b33801e1 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.test.ts @@ -192,10 +192,13 @@ describe('LocalCacheService', () => { }); expect(result).toBe(true); - expect(mockFs.cpSync).toHaveBeenCalledWith(fallbackPath, path.join('/project', 'Library'), { + const staging = expect.stringContaining(path.join('/project', '.Library.partial-')); + expect(mockFs.cpSync).toHaveBeenCalledWith(fallbackPath, staging, { recursive: true, + verbatimSymlinks: true, }); - expect(mockFs.renameSync).not.toHaveBeenCalled(); + expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, path.join('/project', 'Library')); + expect(mockFs.renameSync).not.toHaveBeenCalledWith(fallbackPath, expect.anything()); }); it('should copy, not move, a fallback directory cache when restoreMode is omitted', async () => { @@ -216,13 +219,40 @@ describe('LocalCacheService', () => { }); expect(result).toBe(true); - expect(mockFs.cpSync).toHaveBeenCalledWith(fallbackPath, path.join('/project', 'Library'), { + const staging = expect.stringContaining(path.join('/project', '.Library.partial-')); + expect(mockFs.cpSync).toHaveBeenCalledWith(fallbackPath, staging, { recursive: true, + verbatimSymlinks: true, }); - expect(mockFs.renameSync).not.toHaveBeenCalled(); + expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, path.join('/project', 'Library')); + expect(mockFs.renameSync).not.toHaveBeenCalledWith(fallbackPath, expect.anything()); expect(mockFs.rmSync).not.toHaveBeenCalledWith(fallbackPath, expect.anything()); }); + it('should leave no partial Library when a copy-directory restore fails mid-copy', async () => { + const cachePath = path.join('/cache', 'key1', 'Library'); + const destPath = path.join('/project', 'Library'); + + (mockFs.existsSync as vi.Mock).mockImplementation( + (filePath: string) => filePath === cachePath, + ); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.cpSync as vi.Mock).mockImplementation(() => { + throw new Error('ENOSPC: no space left on device'); + }); + + const result = await LocalCacheService.restoreEngineCache('/project', '/cache', 'key1', { + restoreMode: 'copy-directory', + }); + + expect(result).toBe(false); + const staging = expect.stringContaining(path.join('/project', '.Library.partial-')); + expect(mockFs.cpSync).toHaveBeenCalledWith(cachePath, staging, expect.anything()); + expect(mockFs.rmSync).toHaveBeenCalledWith(staging, { recursive: true, force: true }); + expect(mockFs.renameSync).not.toHaveBeenCalledWith(staging, destPath); + }); + it('should default to move-directory (rename) when no restoreMode is specified', async () => { const cachePath = path.join('/cache', 'key1', 'Library'); const destPath = path.join('/project', 'Library'); @@ -248,10 +278,10 @@ describe('LocalCacheService', () => { (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); (mockFs.mkdirSync as vi.Mock).mockReturnValue(undefined); - (mockFs.renameSync as vi.Mock).mockImplementationOnce(() => { - const error: any = new Error('cross-device link'); - error.code = 'EXDEV'; - throw error; + (mockFs.renameSync as vi.Mock).mockImplementation((from: string, to: string) => { + if (from === cachePath && to === destPath) { + throw Object.assign(new Error('cross-device link'), { code: 'EXDEV' }); + } }); (mockFs.cpSync as vi.Mock).mockReturnValue(undefined); (mockFs.rmSync as vi.Mock).mockReturnValue(undefined); @@ -265,8 +295,12 @@ describe('LocalCacheService', () => { recursive: true, verbatimSymlinks: true, }); - expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, destPath); - expect(mockFs.rmSync).toHaveBeenCalledWith(cachePath, { recursive: true, force: true }); + expect(mockFs.renameSync).toHaveBeenCalledWith(staging, destPath); + // The cache entry is set aside as a tombstone before deletion, never deleted in place. + const tombstone = expect.stringContaining(path.join('/cache', 'key1', '.Library.removing-')); + expect(mockFs.renameSync).toHaveBeenLastCalledWith(cachePath, tombstone); + expect(mockFs.rmSync).toHaveBeenCalledWith(tombstone, { recursive: true, force: true }); + expect(mockFs.rmSync).not.toHaveBeenCalledWith(cachePath, expect.anything()); }); }); @@ -350,8 +384,76 @@ describe('LocalCacheService', () => { recursive: true, verbatimSymlinks: true, }); + expect(mockFs.renameSync).toHaveBeenCalledWith(staging, cachePath); + const tombstone = expect.stringContaining(path.join('/project', '.Library.removing-')); + expect(mockFs.renameSync).toHaveBeenLastCalledWith(folderPath, tombstone); + expect(mockFs.rmSync).toHaveBeenCalledWith(tombstone, { recursive: true, force: true }); + expect(mockFs.rmSync).not.toHaveBeenCalledWith(folderPath, expect.anything()); + }); + + it('should stage an explicit copy-directory save and rename it into place', async () => { + const folderPath = path.join('/project', 'Library'); + const cachePath = path.join('/cache', 'key1', 'Library'); + + (mockFs.existsSync as vi.Mock).mockImplementation( + (filePath: string) => filePath === folderPath, + ); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + + await LocalCacheService.saveEngineCache('/project', '/cache', 'key1', { + saveMode: 'copy-directory', + }); + + const staging = expect.stringContaining(path.join('/cache', 'key1', '.Library.partial-')); + expect(mockFs.cpSync).toHaveBeenCalledWith(folderPath, staging, { + recursive: true, + verbatimSymlinks: true, + }); + expect(mockFs.cpSync).not.toHaveBeenCalledWith(folderPath, cachePath, expect.anything()); expect(mockFs.renameSync).toHaveBeenLastCalledWith(staging, cachePath); - expect(mockFs.rmSync).toHaveBeenCalledWith(folderPath, { recursive: true, force: true }); + }); + + it('should leave no partial cache entry when a copy-directory save fails mid-copy', async () => { + const folderPath = path.join('/project', 'Library'); + const cachePath = path.join('/cache', 'key1', 'Library'); + + (mockFs.existsSync as vi.Mock).mockImplementation( + (filePath: string) => filePath === folderPath, + ); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.cpSync as vi.Mock).mockImplementation(() => { + throw new Error('ENOSPC: no space left on device'); + }); + + await LocalCacheService.saveEngineCache('/project', '/cache', 'key1', { + saveMode: 'copy-directory', + }); + + const staging = expect.stringContaining(path.join('/cache', 'key1', '.Library.partial-')); + expect(mockFs.rmSync).toHaveBeenCalledWith(staging, { recursive: true, force: true }); + expect(mockFs.renameSync).not.toHaveBeenCalledWith(staging, cachePath); + }); + + it('should keep the previous cache entry whole when it cannot be set aside', async () => { + const folderPath = path.join('/project', 'Library'); + const cachePath = path.join('/cache', 'key1', 'Library'); + + (mockFs.existsSync as vi.Mock).mockReturnValue(true); + (mockFs.readdirSync as vi.Mock).mockReturnValue(['ArtifactDB']); + (mockFs.statSync as vi.Mock).mockReturnValue({ size: 1 }); + (mockFs.renameSync as vi.Mock).mockImplementation(() => { + throw Object.assign(new Error('EBUSY: resource busy'), { code: 'EBUSY' }); + }); + + await LocalCacheService.saveEngineCache('/project', '/cache', 'key1', { + saveMode: 'copy-directory', + }); + + expect(mockFs.rmSync).not.toHaveBeenCalledWith(cachePath, expect.anything()); + expect(mockFs.cpSync).not.toHaveBeenCalled(); + expect(mockFs.renameSync).not.toHaveBeenCalledWith(folderPath, expect.anything()); }); }); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts index c7d9f0c4..5f3fc3c7 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts @@ -3,7 +3,12 @@ import path from 'node:path'; import { execSync, spawn } from 'node:child_process'; import { OrchestratorSystem } from '../core/orchestrator-system'; import OrchestratorLogger from '../core/orchestrator-logger'; -import { moveDirectory } from './move-directory'; +import { + copyIntoPlace, + moveDirectory, + removeDirectory, + sweepAbandonedLeftovers, +} from './move-directory'; import { getEngine } from '../../../engine'; export type LocalCacheMode = 'tar' | 'move-directory' | 'copy-directory'; @@ -433,13 +438,11 @@ export class LocalCacheService { return false; } - if (fs.existsSync(dest)) { - fs.rmSync(dest, { recursive: true, force: true }); - } + if (!LocalCacheService.clearDirectory(dest)) return false; fs.mkdirSync(path.dirname(dest), { recursive: true }); if (options.restoreMode === 'copy-directory') { - fs.cpSync(cachePath, dest, { recursive: true }); + copyIntoPlace(cachePath, dest); } else { LocalCacheService.renameOrCopy(cachePath, dest); } @@ -879,13 +882,13 @@ export class LocalCacheService { return; } - if (fs.existsSync(cachePath)) { - fs.rmSync(cachePath, { recursive: true, force: true }); + if (!LocalCacheService.clearDirectory(cachePath)) { + throw new Error(`could not replace the existing cache entry at ${cachePath}`); } fs.mkdirSync(path.dirname(cachePath), { recursive: true }); if (saveMode === 'copy-directory') { - fs.cpSync(folderPath, cachePath, { recursive: true }); + copyIntoPlace(folderPath, cachePath); } else { LocalCacheService.renameOrCopy(folderPath, cachePath); } @@ -917,24 +920,70 @@ export class LocalCacheService { } /** - * Spawn a detached background process to copy the cache directory. - * Writes a lock file with PID before starting; removes it on completion. + * Removes a directory so it is whole or gone, never half-deleted (see + * removeDirectory). Returns false, with a warning, when it is still in place. */ - private static spawnBackgroundSave(folderPath: string, cachePath: string): void { - const lockPath = path.join(path.dirname(cachePath), BACKGROUND_LOCK_FILE); + private static clearDirectory(directory: string): boolean { + if (!fs.existsSync(directory)) return true; + + const removal = removeDirectory(directory); + if (removal.outcome === 'kept-whole') { + OrchestratorLogger.logWarning( + `[LocalCache] could not set ${directory} aside for removal, kept it whole: ${removal.error}`, + ); + + return false; + } + if (removal.outcome === 'tombstone-left') { + OrchestratorLogger.logWarning( + `[LocalCache] removed ${directory} but left ${removal.tombstone} for a later sweep: ${removal.error}`, + ); + } - // Build a Node.js one-liner that performs the copy - const script = [ + return true; + } + + /** + * The Node.js script a detached background save runs. It mirrors + * copyIntoPlace and removeDirectory, which it cannot import: the copy is + * staged in a sibling `.partial-` directory, the previous entry is set aside + * as a `.removing-` tombstone, and only then is the copy renamed into place, + * so the entry is never partial. The lock is always released. + */ + static backgroundSaveScript(folderPath: string, cachePath: string, lockPath: string): string { + return [ `const fs = require('node:fs');`, + `const path = require('node:path');`, + `const folderPath = ${JSON.stringify(folderPath)};`, + `const cachePath = ${JSON.stringify(cachePath)};`, `const lockPath = ${JSON.stringify(lockPath)};`, + `const sibling = (purpose) => path.join(path.dirname(cachePath), '.' + path.basename(cachePath) + '.' + purpose + '-' + process.pid + '-' + Date.now().toString(36));`, + `const staging = sibling('partial');`, `try {`, - ` if (fs.existsSync(${JSON.stringify(cachePath)})) fs.rmSync(${JSON.stringify(cachePath)}, { recursive: true, force: true });`, - ` fs.mkdirSync(require('node:path').dirname(${JSON.stringify(cachePath)}), { recursive: true });`, - ` fs.cpSync(${JSON.stringify(folderPath)}, ${JSON.stringify(cachePath)}, { recursive: true });`, + ` fs.mkdirSync(path.dirname(cachePath), { recursive: true });`, + ` fs.cpSync(folderPath, staging, { recursive: true, verbatimSymlinks: true });`, + ` if (fs.existsSync(cachePath)) {`, + ` const tombstone = sibling('removing');`, + ` fs.renameSync(cachePath, tombstone);`, + ` try { fs.rmSync(tombstone, { recursive: true, force: true }); } catch {}`, + ` }`, + ` fs.renameSync(staging, cachePath);`, + `} catch {`, + ` try { fs.rmSync(staging, { recursive: true, force: true }); } catch {}`, + ` process.exitCode = 1;`, `} finally {`, ` try { fs.unlinkSync(lockPath); } catch {}`, `}`, - ].join(' '); + ].join('\n'); + } + + /** + * Spawn a detached background process to copy the cache directory. + * Writes a lock file with PID before starting; removes it on completion. + */ + private static spawnBackgroundSave(folderPath: string, cachePath: string): void { + const lockPath = path.join(path.dirname(cachePath), BACKGROUND_LOCK_FILE); + const script = LocalCacheService.backgroundSaveScript(folderPath, cachePath, lockPath); try { // Write lock file before spawning to close the race window @@ -962,11 +1011,13 @@ export class LocalCacheService { fs.unlinkSync(lockPath); } catch {} // Fall back to synchronous save - if (fs.existsSync(cachePath)) { - fs.rmSync(cachePath, { recursive: true, force: true }); + if (!LocalCacheService.clearDirectory(cachePath)) { + throw new Error(`could not replace the existing cache entry at ${cachePath}`, { + cause: error, + }); } fs.mkdirSync(path.dirname(cachePath), { recursive: true }); - fs.cpSync(folderPath, cachePath, { recursive: true }); + copyIntoPlace(folderPath, cachePath); } } @@ -1005,6 +1056,8 @@ export class LocalCacheService { let swept = 0; for (const cacheKeyDir of cacheKeyDirs) { + LocalCacheService.sweepAbandonedCopies(path.join(cacheRoot, cacheKeyDir)); + const lockPath = path.join(cacheRoot, cacheKeyDir, BACKGROUND_LOCK_FILE); if (!fs.existsSync(lockPath)) continue; @@ -1016,8 +1069,8 @@ export class LocalCacheService { process.kill(pid, 0); // Signal 0 = existence check // Owning process is still alive -- a save is genuinely in progress. continue; - } catch (error: any) { - if (error?.code !== 'ESRCH') { + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { // EPERM means the process exists but is owned by another user; // unknown errors are likewise not proof that the lock is stale. continue; @@ -1044,6 +1097,23 @@ export class LocalCacheService { return swept; } + /** + * Removes the `.partial-` staging and `.removing-` tombstone directories a + * killed copy or removal left in a cache-key directory. They are never read + * as cache entries, but they hold disk space until swept. + */ + private static sweepAbandonedCopies(cacheKeyPath: string): void { + try { + for (const leftover of sweepAbandonedLeftovers(cacheKeyPath)) { + OrchestratorLogger.log(`[LocalCache] Swept abandoned copy: ${leftover}`); + } + } catch (error: unknown) { + OrchestratorLogger.logWarning( + `[LocalCache] Failed to sweep abandoned copies in ${cacheKeyPath}: ${(error as Error).message}`, + ); + } + } + /** * Wait for a background cache save lock to be released. * Polls the lock file for up to 5 minutes. @@ -1065,8 +1135,8 @@ export class LocalCacheService { if (pid > 0) { try { process.kill(pid, 0); // Signal 0 = existence check - } catch (error: any) { - if (error?.code !== 'ESRCH') { + } catch (error: unknown) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') { // Lack of permission is evidence that the process exists, not // that the lock is stale. Keep waiting in that case. continue; From 1841827fc5db4f9dcaae51441f66fecb7015a498 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 16:31:11 +0100 Subject: [PATCH 8/9] fix(orchestrator): warn on localCacheMode in CLI game-ci orchestrate now warns once when only the deprecated localCacheMode is set, as the Action does, from one shared message. Removes the unused localCacheMode config getter. --- .../cli-plugin/build-parameters-adapter.ts | 16 ++++++++++++- .../src/cli-plugin/cli-plugin.test.ts | 24 +++++++++++++++++++ .../services/cache/cache-mode.test.ts | 11 ++++++++- .../orchestrator/services/cache/cache-mode.ts | 17 +++++++++++++ plugins/orchestrator/src/plugin-lifecycle.ts | 19 ++++++--------- 5 files changed, 73 insertions(+), 14 deletions(-) diff --git a/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts b/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts index 441dc4b3..c34c136b 100644 --- a/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts +++ b/plugins/orchestrator/src/cli-plugin/build-parameters-adapter.ts @@ -2,7 +2,13 @@ import BuildParameters from '../model/build-parameters'; import { customAlphabet } from 'nanoid'; import OrchestratorConstants from '../model/orchestrator/options/orchestrator-constants'; import { initEngine } from '../model/engine'; -import { resolveCacheMode } from '../model/orchestrator/services/cache/cache-mode'; +import { + deprecatedCacheModeWarning, + resolveCacheMode, +} from '../model/orchestrator/services/cache/cache-mode'; +import OrchestratorLogger from '../model/orchestrator/services/core/orchestrator-logger'; + +let hasWarnedDeprecatedCacheMode = false; /** * Maps CLI yargs options (flat key-value pairs) into a BuildParameters object. @@ -123,6 +129,14 @@ export function createBuildParametersFromCliOptions(options: Record // cacheMode replaces localCacheMode, which stays as a deprecated alias. move-directory is an O(1) // rename (falling back to a staged copy across volumes); tar stays opt-in, e.g. for rclone/S3 hooks. bp.localCacheMode = resolveCacheMode(options.cacheMode, options.localCacheMode); + const cacheModeDeprecation = deprecatedCacheModeWarning( + options.cacheMode, + options.localCacheMode, + ); + if (cacheModeDeprecation && !hasWarnedDeprecatedCacheMode) { + hasWarnedDeprecatedCacheMode = true; + OrchestratorLogger.logWarning(`[orchestrate] Option ${cacheModeDeprecation}`); + } bp.localCacheSaveOnFailure = options.localCacheSaveOnFailure === true || options.localCacheSaveOnFailure === 'true'; bp.localCacheFloorCorruptionCategories = options.localCacheFloorCorruptionCategories || ''; diff --git a/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts b/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts index f3805caf..1cf10468 100644 --- a/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts +++ b/plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts @@ -118,6 +118,30 @@ describe('CLI Plugin Adapter', () => { ).toBe('tar'); }); + it('warns once when only the deprecated localCacheMode is set', async () => { + vi.resetModules(); + const { default: logger } = + await import('../model/orchestrator/services/core/orchestrator-logger'); + const { createBuildParametersFromCliOptions: createFresh } = + await import('./build-parameters-adapter'); + const warn = vi.spyOn(logger, 'logWarning').mockImplementation(() => {}); + + try { + createFresh({ cacheMode: 'tar' }); + createFresh({ cacheMode: 'tar', localCacheMode: 'copy-directory' }); + expect(warn).not.toHaveBeenCalled(); + + createFresh({ localCacheMode: 'copy-directory' }); + createFresh({ localCacheMode: 'tar' }); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("'localCacheMode' is deprecated: use 'cacheMode' instead"), + ); + } finally { + warn.mockRestore(); + } + }); + it('applies defaults for missing options', () => { const bp = createBuildParametersFromCliOptions({}); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts index cf15314a..c0049c48 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { resolveCacheMode } from './cache-mode'; +import { deprecatedCacheModeWarning, resolveCacheMode } from './cache-mode'; describe('resolveCacheMode', () => { it('defaults to move-directory', () => { @@ -26,3 +26,12 @@ describe('resolveCacheMode', () => { ); }); }); + +describe('deprecatedCacheModeWarning', () => { + it('warns only when the deprecated localCacheMode is used without cacheMode', () => { + expect(deprecatedCacheModeWarning('', 'tar')).toContain("use 'cacheMode' instead"); + expect(deprecatedCacheModeWarning('tar', 'copy-directory')).toBeUndefined(); + expect(deprecatedCacheModeWarning('tar', '')).toBeUndefined(); + expect(deprecatedCacheModeWarning()).toBeUndefined(); + }); +}); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts index 242592b7..f97db66f 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/cache-mode.ts @@ -32,3 +32,20 @@ export function resolveCacheMode( return mode as LocalCacheMode; } + +/** + * The warning to show when only the deprecated `localCacheMode` alias is set, + * or undefined when `cacheMode` is used (or neither is). + */ +export function deprecatedCacheModeWarning( + cacheMode?: string, + legacyLocalCacheMode?: string, +): string | undefined { + if (cacheMode || !legacyLocalCacheMode) return undefined; + + return ( + "'localCacheMode' is deprecated: use 'cacheMode' instead. 'local' no longer describes " + + 'this setting now that distributed rclone/S3 caching is a supported mode too. ' + + 'localCacheMode keeps working for now; see game-ci/roadmap#11.' + ); +} diff --git a/plugins/orchestrator/src/plugin-lifecycle.ts b/plugins/orchestrator/src/plugin-lifecycle.ts index 261e64fe..dca370e0 100644 --- a/plugins/orchestrator/src/plugin-lifecycle.ts +++ b/plugins/orchestrator/src/plugin-lifecycle.ts @@ -45,7 +45,10 @@ import { OutputService } from './model/orchestrator/services/output/output-servi import { OutputTypeRegistry } from './model/orchestrator/services/output/output-type-registry'; import { ArtifactUploadHandler } from './model/orchestrator/services/output/artifact-upload-handler'; import { IncrementalSyncService } from './model/orchestrator/services/sync'; -import { resolveCacheMode } from './model/orchestrator/services/cache/cache-mode'; +import { + deprecatedCacheModeWarning, + resolveCacheMode, +} from './model/orchestrator/services/cache/cache-mode'; // ── Input helpers ──────────────────────────────────────────────────── @@ -277,21 +280,13 @@ const config = { get cacheMode() { const explicitCacheMode = getInput('cacheMode'); const legacyLocalCacheMode = getInput('localCacheMode'); - if (!explicitCacheMode && legacyLocalCacheMode && !hasWarnedDeprecatedCacheMode) { + const deprecation = deprecatedCacheModeWarning(explicitCacheMode, legacyLocalCacheMode); + if (deprecation && !hasWarnedDeprecatedCacheMode) { hasWarnedDeprecatedCacheMode = true; - core.warning( - "[plugin-lifecycle] Input 'localCacheMode' is deprecated — use 'cacheMode' instead. " + - "'local' no longer accurately describes this input now that distributed rclone/S3 " + - 'caching is a supported mode too. localCacheMode will keep working for now, but ' + - 'prefer cacheMode going forward — see game-ci/roadmap#11.', - ); + core.warning(`[plugin-lifecycle] Input ${deprecation}`); } return resolveCacheMode(explicitCacheMode, legacyLocalCacheMode); }, - /** @deprecated Use `cacheMode` instead — kept only so internal call sites reading the old property name don't need to change too. */ - get localCacheMode() { - return config.cacheMode; - }, get maxCacheEntries() { return Number(getInput('maxCacheEntries')) || 2; }, From b592b1291d0ad738e3f0195331ae6e1464ef54e0 Mon Sep 17 00:00:00 2001 From: Webber Date: Sun, 15 Nov 2026 20:27:22 +0100 Subject: [PATCH 9/9] fix(cache): keep sweeping past a stuck leftover One leftover that cannot be removed no longer stops the sweep; it is reported with its path and error, and the rest are still removed. --- .../cache/local-cache-service.fs.test.ts | 18 ++++++++++++++ .../services/cache/local-cache-service.ts | 8 ++++++- .../services/cache/move-directory.test.ts | 24 +++++++++++++++++-- .../services/cache/move-directory.ts | 24 +++++++++++++------ 4 files changed, 64 insertions(+), 10 deletions(-) diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts index 15003675..b111b10b 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.fs.test.ts @@ -4,6 +4,7 @@ import os from 'node:os'; import path from 'node:path'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { LocalCacheService } from './local-cache-service'; +import OrchestratorLogger from '../core/orchestrator-logger'; const { spawnMock } = vi.hoisted(() => ({ spawnMock: vi.fn() })); @@ -37,6 +38,7 @@ describe('LocalCacheService on disk', () => { }); afterEach(() => { + vi.restoreAllMocks(); fs.rmSync(root, { recursive: true, force: true }); }); @@ -112,4 +114,20 @@ describe('LocalCacheService on disk', () => { 'Library', ]); }); + + it('warns about an abandoned copy it cannot remove', () => { + const stuck = path.join(cacheRoot, 'key1', `.Library.removing-${deadPid}-abc`); + fs.mkdirSync(stuck, { recursive: true }); + const realRm = fs.rmSync; + vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => { + if (target === stuck) throw new Error('EACCES: permission denied'); + realRm(target, options); + }); + + LocalCacheService.sweepStaleLocks(cacheRoot); + + expect(OrchestratorLogger.logWarning).toHaveBeenCalledWith( + `[LocalCache] Failed to sweep abandoned copy ${stuck}: EACCES: permission denied`, + ); + }); }); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts index 5f3fc3c7..5686e274 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/local-cache-service.ts @@ -1104,9 +1104,15 @@ export class LocalCacheService { */ private static sweepAbandonedCopies(cacheKeyPath: string): void { try { - for (const leftover of sweepAbandonedLeftovers(cacheKeyPath)) { + const { removed, failed } = sweepAbandonedLeftovers(cacheKeyPath); + for (const leftover of removed) { OrchestratorLogger.log(`[LocalCache] Swept abandoned copy: ${leftover}`); } + for (const { path: leftover, error } of failed) { + OrchestratorLogger.logWarning( + `[LocalCache] Failed to sweep abandoned copy ${leftover}: ${error}`, + ); + } } catch (error: unknown) { OrchestratorLogger.logWarning( `[LocalCache] Failed to sweep abandoned copies in ${cacheKeyPath}: ${(error as Error).message}`, diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts index ab83bd2c..132cf0ce 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.test.ts @@ -257,15 +257,35 @@ describe('sweepAbandonedLeftovers', () => { }); afterEach(() => { + vi.restoreAllMocks(); fs.rmSync(root, { recursive: true, force: true }); }); + it('reports a leftover it cannot remove and keeps sweeping the others', () => { + const stuck = path.join(root, `.Library.removing-${deadPid}-a`); + const removable = path.join(root, `.Library.partial-${deadPid}-b`); + fs.mkdirSync(stuck); + fs.mkdirSync(removable); + const realRm = fs.rmSync; + vi.spyOn(fs, 'rmSync').mockImplementation((target, options) => { + if (target === stuck) throw new Error('EACCES: permission denied'); + realRm(target, options); + }); + + expect(sweepAbandonedLeftovers(root)).toEqual({ + removed: [removable], + failed: [{ path: stuck, error: 'EACCES: permission denied' }], + }); + expect(fs.readdirSync(root)).toEqual([path.basename(stuck)]); + }); + it('removes staging and tombstone directories of processes that are gone', () => { fs.mkdirSync(path.join(root, `.Library.partial-${deadPid}-abc`, 'nested'), { recursive: true }); fs.mkdirSync(path.join(root, `.Library.removing-${deadPid}-abc`)); - const removed = sweepAbandonedLeftovers(root); + const { removed, failed } = sweepAbandonedLeftovers(root); + expect(failed).toEqual([]); expect(removed.map((leftover) => path.basename(leftover)).sort()).toEqual([ `.Library.partial-${deadPid}-abc`, `.Library.removing-${deadPid}-abc`, @@ -283,7 +303,7 @@ describe('sweepAbandonedLeftovers', () => { for (const name of kept) fs.mkdirSync(path.join(root, name)); fs.writeFileSync(path.join(root, `.Library.partial-${deadPid}-file`), ''); - expect(sweepAbandonedLeftovers(root)).toEqual([]); + expect(sweepAbandonedLeftovers(root)).toEqual({ removed: [], failed: [] }); expect(fs.readdirSync(root).sort()).toEqual( [...kept, `.Library.partial-${deadPid}-file`].sort(), ); diff --git a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts index 26d7444d..37452479 100644 --- a/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts +++ b/plugins/orchestrator/src/model/orchestrator/services/cache/move-directory.ts @@ -105,25 +105,35 @@ export function removeDirectory(directory: string): DirectoryRemoval { } } +export type LeftoverSweep = { + removed: string[]; + failed: { path: string; error: string }[]; +}; + /** * Deletes the staging and tombstone directories under `parent` whose owning * process is gone, e.g. after a runner was killed mid-copy. Directories of a - * process that is still alive, or that cannot be checked, are kept. Returns - * the paths removed; throws only if `parent` cannot be read. + * process that is still alive, or that cannot be checked, are kept. A leftover + * that cannot be removed is reported and the sweep moves on to the next one; + * this throws only if `parent` cannot be read. */ -export function sweepAbandonedLeftovers(parent: string): string[] { - const removed: string[] = []; +export function sweepAbandonedLeftovers(parent: string): LeftoverSweep { + const sweep: LeftoverSweep = { removed: [], failed: [] }; for (const entry of fs.readdirSync(parent, { withFileTypes: true })) { const match = entry.isDirectory() && leftoverPattern.exec(entry.name); if (!match || isProcessAlive(Number(match[2]))) continue; const leftover = path.join(parent, entry.name); - fs.rmSync(leftover, { recursive: true, force: true }); - removed.push(leftover); + try { + fs.rmSync(leftover, { recursive: true, force: true }); + sweep.removed.push(leftover); + } catch (error: unknown) { + sweep.failed.push({ path: leftover, error: describe(error) }); + } } - return removed; + return sweep; } function siblingPath(directory: string, purpose: 'partial' | 'removing'): string {