Skip to content
Merged
8 changes: 7 additions & 1 deletion bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 7 additions & 2 deletions plugins/orchestrator/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -485,10 +485,14 @@ inputs:
required: false
default: ''
description: 'Comma-separated explicit local cache fallback keys to try after the exact key.'
cacheMode:
required: false
default: ''
description: 'Library cache mode: move-directory (default, O(1) rename, falls back to copy across filesystem volumes), copy-directory, or tar (opt-in — needed when distributing cache entries via the rclone/S3 built-in container hooks). Renamed from localCacheMode, which still works as a deprecated alias — see game-ci/roadmap#11. NOTE: canonical-overlay has been removed — setting it now throws a clear error instead of running.'
localCacheMode:
required: false
default: 'tar'
description: 'Local Library cache mode: tar, copy-directory, or move-directory. move-directory is intended for same-volume Windows self-hosted runners. NOTE: canonical-overlay has been removed (see game-ci/roadmap#11) — setting it now throws a clear error instead of running.'
default: ''
description: 'Deprecated alias for cacheMode — "local" no longer accurately describes this input now that distributed rclone/S3 caching is a supported mode too. Still works, but prefer cacheMode. Will be removed in a future major version.'
upmOfflineEnabled:
required: false
default: 'false'
Expand Down Expand Up @@ -894,6 +898,7 @@ runs:
INPUT_LOCALCACHELFS: ${{ inputs.localCacheLfs }}
INPUT_LOCALCACHEFALLBACK: ${{ inputs.localCacheFallback }}
INPUT_LOCALCACHEFALLBACKKEYS: ${{ inputs.localCacheFallbackKeys }}
INPUT_CACHEMODE: ${{ inputs.cacheMode }}
INPUT_LOCALCACHEMODE: ${{ inputs.localCacheMode }}
INPUT_UPMOFFLINEENABLED: ${{ inputs.upmOfflineEnabled }}
INPUT_BACKGROUNDCACHESAVE: ${{ inputs.backgroundCacheSave }}
Expand Down
2 changes: 1 addition & 1 deletion plugins/orchestrator/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@
},
"devDependencies": {
"@types/base-64": "^1.0.2",
"@types/node": "^17.0.23",
"@types/node": "^20",
"@types/semver": "^7.3.9",
"@types/uuid": "^9.0.0",
"@types/yargs": "^17.0.35",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@ import BuildParameters from '../model/build-parameters';
import { customAlphabet } from 'nanoid';
import OrchestratorConstants from '../model/orchestrator/options/orchestrator-constants';
import { initEngine } from '../model/engine';
import {
deprecatedCacheModeWarning,
resolveCacheMode,
} from '../model/orchestrator/services/cache/cache-mode';
import OrchestratorLogger from '../model/orchestrator/services/core/orchestrator-logger';

let hasWarnedDeprecatedCacheMode = false;

/**
* Maps CLI yargs options (flat key-value pairs) into a BuildParameters object.
Expand Down Expand Up @@ -119,7 +126,17 @@ export function createBuildParametersFromCliOptions(options: Record<string, any>
bp.localCacheFallback =
options.localCacheFallback === true || options.localCacheFallback === 'true';
bp.localCacheFallbackKeys = options.localCacheFallbackKeys || '';
bp.localCacheMode = options.localCacheMode || 'tar';
// cacheMode replaces localCacheMode, which stays as a deprecated alias. move-directory is an O(1)
// rename (falling back to a staged copy across volumes); tar stays opt-in, e.g. for rclone/S3 hooks.
bp.localCacheMode = resolveCacheMode(options.cacheMode, options.localCacheMode);
const cacheModeDeprecation = deprecatedCacheModeWarning(
options.cacheMode,
options.localCacheMode,
);
if (cacheModeDeprecation && !hasWarnedDeprecatedCacheMode) {
hasWarnedDeprecatedCacheMode = true;
OrchestratorLogger.logWarning(`[orchestrate] Option ${cacheModeDeprecation}`);
}
bp.localCacheSaveOnFailure =
options.localCacheSaveOnFailure === true || options.localCacheSaveOnFailure === 'true';
bp.localCacheFloorCorruptionCategories = options.localCacheFloorCorruptionCategories || '';
Expand Down
39 changes: 39 additions & 0 deletions plugins/orchestrator/src/cli-plugin/cli-plugin.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,45 @@ describe('CLI Plugin Adapter', () => {
expect(bp.engine).toBe('unity');
});

it('defaults the cache mode to move-directory', () => {
expect(createBuildParametersFromCliOptions({}).localCacheMode).toBe('move-directory');
});

it('prefers cacheMode, keeping localCacheMode as a deprecated alias', () => {
expect(createBuildParametersFromCliOptions({ cacheMode: 'tar' }).localCacheMode).toBe('tar');
expect(
createBuildParametersFromCliOptions({ localCacheMode: 'copy-directory' }).localCacheMode,
).toBe('copy-directory');
expect(
createBuildParametersFromCliOptions({ cacheMode: 'tar', localCacheMode: 'copy-directory' })
.localCacheMode,
).toBe('tar');
});

it('warns once when only the deprecated localCacheMode is set', async () => {
vi.resetModules();
const { default: logger } =
await import('../model/orchestrator/services/core/orchestrator-logger');
const { createBuildParametersFromCliOptions: createFresh } =
await import('./build-parameters-adapter');
const warn = vi.spyOn(logger, 'logWarning').mockImplementation(() => {});

try {
createFresh({ cacheMode: 'tar' });
createFresh({ cacheMode: 'tar', localCacheMode: 'copy-directory' });
expect(warn).not.toHaveBeenCalled();

createFresh({ localCacheMode: 'copy-directory' });
createFresh({ localCacheMode: 'tar' });
expect(warn).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
expect.stringContaining("'localCacheMode' is deprecated: use 'cacheMode' instead"),
);
} finally {
warn.mockRestore();
}
});

it('applies defaults for missing options', () => {
const bp = createBuildParametersFromCliOptions({});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -513,10 +513,18 @@ export function configureOrchestratorOptions(yargs: any): void {
default: '',
});

yargs.option('cacheMode', {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
description:
'Cache save/restore mode: move-directory (default; falls back to a copy across volumes), copy-directory, or tar',
type: 'string',
default: '',
});

yargs.option('localCacheMode', {
description: 'Local cache save/restore mode: tar, move-directory, or copy-directory',
description: 'Deprecated alias for --cacheMode',
type: 'string',
default: 'tar',
default: '',
deprecated: 'use --cacheMode',
});

yargs.option('gcTimeoutMinutes', {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import { describe, expect, it } from 'vitest';
import { deprecatedCacheModeWarning, resolveCacheMode } from './cache-mode';

describe('resolveCacheMode', () => {
it('defaults to move-directory', () => {
expect(resolveCacheMode()).toBe('move-directory');
expect(resolveCacheMode('', '')).toBe('move-directory');
});

it('prefers cacheMode over the deprecated localCacheMode', () => {
expect(resolveCacheMode('tar', 'copy-directory')).toBe('tar');
expect(resolveCacheMode('', 'copy-directory')).toBe('copy-directory');
});

it('rejects an unknown mode instead of running it as a directory move', () => {
expect(() => resolveCacheMode('move-dir')).toThrow("unknown mode 'move-dir'");
expect(() => resolveCacheMode('', 'Tar')).toThrow("unknown mode 'Tar'");
});

it('rejects the removed canonical-overlay mode, also through the deprecated alias', () => {
expect(() => resolveCacheMode('canonical-overlay')).toThrow(
'canonical-overlay has been removed',
);
expect(() => resolveCacheMode('', 'canonical-overlay')).toThrow(
'canonical-overlay has been removed',
);
});
});

describe('deprecatedCacheModeWarning', () => {
it('warns only when the deprecated localCacheMode is used without cacheMode', () => {
expect(deprecatedCacheModeWarning('', 'tar')).toContain("use 'cacheMode' instead");
expect(deprecatedCacheModeWarning('tar', 'copy-directory')).toBeUndefined();
expect(deprecatedCacheModeWarning('tar', '')).toBeUndefined();
expect(deprecatedCacheModeWarning()).toBeUndefined();
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import type { LocalCacheMode } from './local-cache-service';

const supportedCacheModes: readonly LocalCacheMode[] = ['move-directory', 'copy-directory', 'tar'];

/**
* Resolves the cache mode both entry points (the GitHub Action and
* `game-ci orchestrate`) run with: `cacheMode` first, then its deprecated alias
* `localCacheMode`, then `move-directory`. Anything else, including the removed
* `canonical-overlay` mode or a typo, fails loudly rather than silently running
* as a directory move.
*/
export function resolveCacheMode(
cacheMode?: string,
legacyLocalCacheMode?: string,
): LocalCacheMode {
const mode = cacheMode || legacyLocalCacheMode || 'move-directory';

if (mode === 'canonical-overlay') {
throw new Error(
'cacheMode: canonical-overlay has been removed. ' +
"Use 'move-directory' (recommended for retained runners/build farms) or 'tar' " +
'(recommended for remote/ephemeral caches, or combine with rclone/S3 built-in ' +
'container hooks - see the caching docs for details).',
);
}

if (!supportedCacheModes.includes(mode as LocalCacheMode)) {
throw new Error(
`cacheMode: unknown mode '${mode}'. Use one of: ${supportedCacheModes.join(', ')}.`,
);
}

return mode as LocalCacheMode;
}

/**
* The warning to show when only the deprecated `localCacheMode` alias is set,
* or undefined when `cacheMode` is used (or neither is).
*/
export function deprecatedCacheModeWarning(
cacheMode?: string,
legacyLocalCacheMode?: string,
): string | undefined {
if (cacheMode || !legacyLocalCacheMode) return undefined;

return (
"'localCacheMode' is deprecated: use 'cacheMode' instead. 'local' no longer describes " +
'this setting now that distributed rclone/S3 caching is a supported mode too. ' +
'localCacheMode keeps working for now; see game-ci/roadmap#11.'
);
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import fs from 'node:fs';
import path from 'node:path';
import OrchestratorLogger from '../core/orchestrator-logger';
import { moveDirectory } from './move-directory';
import { getEngine } from '../../../engine';

export interface ChildWorkspaceConfig {
Expand Down Expand Up @@ -83,7 +84,7 @@ export class ChildWorkspaceService {
OrchestratorLogger.log(
`[ChildWorkspace] Restoring workspace: ${cachedWorkspacePath} -> ${projectPath}`,
);
fs.renameSync(cachedWorkspacePath, projectPath);
ChildWorkspaceService.renameOrCopy(cachedWorkspacePath, projectPath);
OrchestratorLogger.log(`[ChildWorkspace] Workspace restored via atomic move`);

// Restore Library cache separately if configured
Expand Down Expand Up @@ -151,7 +152,7 @@ export class ChildWorkspaceService {
OrchestratorLogger.log(
`[ChildWorkspace] Saving workspace: ${projectPath} -> ${cachedWorkspacePath}`,
);
fs.renameSync(projectPath, cachedWorkspacePath);
ChildWorkspaceService.renameOrCopy(projectPath, cachedWorkspacePath);
OrchestratorLogger.log(`[ChildWorkspace] Workspace saved via atomic move`);
} catch (error: any) {
OrchestratorLogger.logWarning(`[ChildWorkspace] Workspace save failed: ${error.message}`);
Expand Down Expand Up @@ -194,7 +195,7 @@ export class ChildWorkspaceService {
OrchestratorLogger.log(
`[ChildWorkspace] Restoring ${folder} cache: ${backupPath} -> ${destination}`,
);
fs.renameSync(backupPath, destination);
ChildWorkspaceService.renameOrCopy(backupPath, destination);
OrchestratorLogger.log(`[ChildWorkspace] ${folder} cache restored`);
restored = true;
} catch (error: any) {
Expand Down Expand Up @@ -246,7 +247,7 @@ export class ChildWorkspaceService {
OrchestratorLogger.log(
`[ChildWorkspace] Caching ${folder}: ${folderPath} -> ${backupPath}`,
);
fs.renameSync(folderPath, backupPath);
ChildWorkspaceService.renameOrCopy(folderPath, backupPath);
OrchestratorLogger.log(`[ChildWorkspace] ${folder} cached separately`);
} catch (error: any) {
OrchestratorLogger.logWarning(
Expand Down Expand Up @@ -409,4 +410,25 @@ export class ChildWorkspaceService {

return `${value.toFixed(2)} ${units[index]}`;
}

/**
* Moves a directory, degrading to a staged copy when parentCacheRoot sits on
* another volume than the workspace: slower, but the cache survives instead
* of falling through to "start fresh".
*/
private static renameOrCopy(source: string, destination: string): void {
const move = moveDirectory(source, destination);
if (move.method === 'rename') return;

OrchestratorLogger.logWarning(
`[ChildWorkspace] rename failed (${move.renameError}) from ${source} to ${destination}; ` +
`copied instead. This is slower - consider configuring parentCacheRoot on ` +
`the same volume as the workspace.`,
);
if (move.sourceCleanupError) {
OrchestratorLogger.logWarning(
`[ChildWorkspace] copied ${source} to ${destination} but could not remove the source: ${move.sourceCleanupError}`,
);
}
}
}
Loading
Loading