diff --git a/CHANGELOG.md b/CHANGELOG.md index 1475002553..bcc00c3d92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ This is the log of notable changes to EAS CLI and related packages. ### 🐛 Bug fixes +- [build-tools] Soft-stop EAS Simulator sessions before the backing job times out so cleanup can finish. ([#4117](https://github.com/expo/eas-cli/pull/4117) by [@sjchmiela](https://github.com/sjchmiela)) - [eas-cli] clean up error handling in local builds. ([#4105](https://github.com/expo/eas-cli/pull/4105) by [@douglowder](https://github.com/douglowder)) - [build-tools] Install ffmpeg when it is missing so Argent screen recording works in EAS Simulator sessions. ([#4110](https://github.com/expo/eas-cli/pull/4110) by [@szdziedzic](https://github.com/szdziedzic)) - [eas-cli] Stop simulator job runs when `eas simulator:start` is canceled before the session is ready. ([#4113](https://github.com/expo/eas-cli/pull/4113) by [@sjchmiela](https://github.com/sjchmiela)) diff --git a/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts b/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts index e73990886f..ed3fdda1cd 100644 --- a/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startAgentDeviceRemoteSession.ts @@ -55,6 +55,11 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( required: false, allowedValueTypeName: BuildStepInputValueTypeName.STRING, }), + BuildStepInput.createProvider({ + id: 'max_duration_seconds', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.NUMBER, + }), ], fn: async ({ logger, global }, { inputs, env, signal }) => { // Fail fast before any expensive setup if the injected env @@ -66,6 +71,7 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( const ngrokAuthtoken = getNgrokAuthtokenOrThrow(env); const packageVersion = inputs.package_version.value as string | undefined; + const maxDurationSeconds = inputs.max_duration_seconds?.value as number | undefined; const { runtimePlatform } = global; logger.info( `Starting agent-device remote session (version: ${packageVersion ?? 'latest'}, runtime: ${runtimePlatform}).` @@ -139,6 +145,7 @@ export function createStartAgentDeviceRemoteSessionBuildFunction( ctx, deviceRunSessionId, logger, + maxDurationSeconds, signal, }); } finally { diff --git a/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts b/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts index a62c2dfe91..3cafd5823e 100644 --- a/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startArgentRemoteSession.ts @@ -68,6 +68,11 @@ export function createStartArgentRemoteSessionBuildFunction( required: false, allowedValueTypeName: BuildStepInputValueTypeName.STRING, }), + BuildStepInput.createProvider({ + id: 'max_duration_seconds', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.NUMBER, + }), ], fn: async ({ logger, global }, { inputs, env, signal }) => { // Fail fast before any expensive setup if the injected env @@ -79,6 +84,7 @@ export function createStartArgentRemoteSessionBuildFunction( const ngrokAuthtoken = getNgrokAuthtokenOrThrow(env); const packageVersion = inputs.package_version.value as string | undefined; + const maxDurationSeconds = inputs.max_duration_seconds?.value as number | undefined; warnIfArgentPackageVersionCannotBeVerified({ packageVersion, logger }); const versionSpec = packageVersion ?? 'latest'; const { runtimePlatform } = global; @@ -214,6 +220,7 @@ export function createStartArgentRemoteSessionBuildFunction( ctx, deviceRunSessionId, logger, + maxDurationSeconds, signal, }); } finally { diff --git a/packages/build-tools/src/steps/functions/startServeSimRemoteSession.ts b/packages/build-tools/src/steps/functions/startServeSimRemoteSession.ts index eafe44cee4..5492e76db3 100644 --- a/packages/build-tools/src/steps/functions/startServeSimRemoteSession.ts +++ b/packages/build-tools/src/steps/functions/startServeSimRemoteSession.ts @@ -1,4 +1,9 @@ -import { BuildFunction, BuildRuntimePlatform } from '@expo/steps'; +import { + BuildFunction, + BuildRuntimePlatform, + BuildStepInput, + BuildStepInputValueTypeName, +} from '@expo/steps'; import { CustomBuildContext } from '../../customBuildContext'; import { @@ -21,9 +26,17 @@ export function createStartServeSimRemoteSessionBuildFunction( name: 'Start serve-sim remote session', __metricsId: 'eas/start_serve_sim_remote_session', supportedRuntimePlatforms: [BuildRuntimePlatform.DARWIN], - fn: async ({ logger }, { env, signal }) => { + inputProviders: [ + BuildStepInput.createProvider({ + id: 'max_duration_seconds', + required: false, + allowedValueTypeName: BuildStepInputValueTypeName.NUMBER, + }), + ], + fn: async ({ logger }, { inputs, env, signal }) => { const deviceRunSessionId = getDeviceRunSessionIdOrThrow(env); const ngrokTunnelDomain = getNgrokTunnelDomainOrThrow(env); + const maxDurationSeconds = inputs.max_duration_seconds?.value as number | undefined; logger.info('Starting serve-sim remote session.'); @@ -49,6 +62,7 @@ export function createStartServeSimRemoteSessionBuildFunction( ctx, deviceRunSessionId, logger, + maxDurationSeconds, signal, }); } finally { diff --git a/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts b/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts index 84908b4cec..c687c81986 100644 --- a/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts +++ b/packages/build-tools/src/steps/utils/__tests__/remoteDeviceRunSession.test.ts @@ -2,6 +2,10 @@ import { bunyan } from '@expo/logger'; import { BuildRuntimePlatform, BuildStepEnv } from '@expo/steps'; import spawn from '@expo/turtle-spawn'; import * as ngrok from '@ngrok/ngrok'; +import { + clearTimeout as clearTimeoutCallback, + setTimeout as setTimeoutCallback, +} from 'node:timers'; import { setTimeout as setTimeoutAsync } from 'node:timers/promises'; import { CustomBuildContext } from '../../../customBuildContext'; @@ -19,6 +23,7 @@ import { } from '../remoteDeviceRunSession'; jest.mock('@ngrok/ngrok'); +jest.mock('node:timers'); jest.mock('node:timers/promises'); jest.mock('../../../utils/turtleFetch'); jest.mock('../../../utils/retry', () => ({ sleepAsync: jest.fn() })); @@ -301,8 +306,13 @@ describe(fetchServeSimTurnArgsAsync, () => { }); describe(waitForDeviceRunSessionStoppedAsync, () => { + const durationTimeout = {} as NodeJS.Timeout; + beforeEach(() => { jest.mocked(Sentry).capture.mockReset(); + jest.mocked(setTimeoutCallback).mockReset(); + jest.mocked(setTimeoutCallback).mockReturnValue(durationTimeout); + jest.mocked(clearTimeoutCallback).mockReset(); jest.mocked(setTimeoutAsync).mockReset(); jest.mocked(setTimeoutAsync).mockResolvedValue(undefined); }); @@ -319,6 +329,57 @@ describe(waitForDeviceRunSessionStoppedAsync, () => { expect(ctx.graphqlClient.query).toHaveBeenCalledTimes(2); expect(logger.info).toHaveBeenCalledWith('Device run session drs-id was stopped.'); + expect(setTimeoutCallback).not.toHaveBeenCalled(); + }); + + it('returns normally when the maximum duration elapses', async () => { + const ctx = createStatusCtxMock([{ status: 'IN_PROGRESS' }]); + const logger = createLoggerMock(); + const waitPromise = waitForDeviceRunSessionStoppedAsync({ + ctx, + deviceRunSessionId: 'drs-id', + logger, + maxDurationSeconds: 1, + }); + + expect(setTimeoutCallback).toHaveBeenCalledWith(expect.any(Function), 1_000); + const durationTimeoutCallback = jest.mocked(setTimeoutCallback).mock.calls[0][0]; + durationTimeoutCallback(); + await waitPromise; + + expect(ctx.graphqlClient.query).toHaveBeenCalledTimes(1); + expect(logger.info).toHaveBeenCalledWith( + 'Device run session drs-id reached its maximum duration.' + ); + expect(clearTimeoutCallback).toHaveBeenCalledWith(durationTimeout); + }); + + it('clears the duration timeout when the session stops first', async () => { + await waitForDeviceRunSessionStoppedAsync({ + ctx: createStatusCtxMock([{ status: 'STOPPED' }]), + deviceRunSessionId: 'drs-id', + logger: createLoggerMock(), + maxDurationSeconds: 30, + }); + + expect(clearTimeoutCallback).toHaveBeenCalledWith(durationTimeout); + }); + + it('does not poll when the build step is already aborted', async () => { + const ctx = createStatusCtxMock([]); + const abortController = new AbortController(); + abortController.abort(); + + await waitForDeviceRunSessionStoppedAsync({ + ctx, + deviceRunSessionId: 'drs-id', + logger: createLoggerMock(), + maxDurationSeconds: 30, + signal: abortController.signal, + }); + + expect(ctx.graphqlClient.query).not.toHaveBeenCalled(); + expect(setTimeoutCallback).not.toHaveBeenCalled(); }); it('throws when the device run session errors', async () => { @@ -329,8 +390,28 @@ describe(waitForDeviceRunSessionStoppedAsync, () => { ctx, deviceRunSessionId: 'drs-id', logger: createLoggerMock(), + maxDurationSeconds: 30, }) ).rejects.toThrow('Device run session drs-id errored.'); + expect(clearTimeoutCallback).toHaveBeenCalledWith(durationTimeout); + }); + + it('clears the duration timeout when the build step is aborted', async () => { + const ctx = createStatusCtxMock([{ status: 'IN_PROGRESS' }]); + const abortController = new AbortController(); + const waitPromise = waitForDeviceRunSessionStoppedAsync({ + ctx, + deviceRunSessionId: 'drs-id', + logger: createLoggerMock(), + maxDurationSeconds: 30, + signal: abortController.signal, + }); + + abortController.abort(); + await waitPromise; + + expect(ctx.graphqlClient.query).toHaveBeenCalledTimes(1); + expect(clearTimeoutCallback).toHaveBeenCalledWith(durationTimeout); }); it('logs and retries transient polling errors', async () => { diff --git a/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts b/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts index 25c10084b1..c4cf8ccbce 100644 --- a/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts +++ b/packages/build-tools/src/steps/utils/remoteDeviceRunSession.ts @@ -10,6 +10,7 @@ import { z } from 'zod'; import { randomBytes } from 'node:crypto'; import fs from 'node:fs'; import { createServer } from 'node:net'; +import { clearTimeout, setTimeout } from 'node:timers'; import { setTimeout as setTimeoutAsync } from 'node:timers/promises'; import { CustomBuildContext } from '../../customBuildContext'; @@ -122,55 +123,80 @@ export async function waitForDeviceRunSessionStoppedAsync({ ctx, deviceRunSessionId, logger, - signal, + maxDurationSeconds, + signal: cancelSignal, }: { ctx: CustomBuildContext; deviceRunSessionId: string; logger: bunyan; + maxDurationSeconds?: number; signal?: AbortSignal; }): Promise { - logger.info( - `Remote session is live. Polling device run session ${deviceRunSessionId} until it is stopped.` - ); - let pollErrorCount = 0; + const durationAbortController = new AbortController(); + const signal = cancelSignal + ? AbortSignal.any([cancelSignal, durationAbortController.signal]) + : durationAbortController.signal; + const durationTimeout = + maxDurationSeconds === undefined || signal.aborted + ? undefined + : setTimeout(() => { + logger.info(`Device run session ${deviceRunSessionId} reached its maximum duration.`); + durationAbortController.abort(); + }, maxDurationSeconds * 1_000); - while (!signal?.aborted) { - try { - const result = await ctx.graphqlClient - .query(DEVICE_RUN_SESSION_STATUS_QUERY, { deviceRunSessionId }) - .toPromise(); - if (result.error) { - throw result.error; - } + try { + logger.info( + `Remote session is live. Polling device run session ${deviceRunSessionId} until it is stopped.` + ); + if (durationTimeout !== undefined) { + logger.info( + `The device run session will stop automatically after ${maxDurationSeconds} seconds.` + ); + } + let pollErrorCount = 0; - const status = result.data?.deviceRunSessions?.byId?.status; - if (!status) { - throw new Error(`Device run session ${deviceRunSessionId} status response was missing.`); - } - pollErrorCount = 0; - if (status === 'STOPPED') { - logger.info(`Device run session ${deviceRunSessionId} was stopped.`); - return; - } - if (status === 'ERRORED') { - throw new SystemError(`Device run session ${deviceRunSessionId} errored.`); - } - } catch (err) { - if (err instanceof SystemError) { - throw err; - } + while (!signal.aborted) { + try { + const result = await ctx.graphqlClient + .query(DEVICE_RUN_SESSION_STATUS_QUERY, { deviceRunSessionId }) + .toPromise(); + if (result.error) { + throw result.error; + } + + const status = result.data?.deviceRunSessions?.byId?.status; + if (!status) { + throw new Error(`Device run session ${deviceRunSessionId} status response was missing.`); + } + pollErrorCount = 0; + if (status === 'STOPPED') { + logger.info(`Device run session ${deviceRunSessionId} was stopped.`); + return; + } + if (status === 'ERRORED') { + throw new SystemError(`Device run session ${deviceRunSessionId} errored.`); + } + } catch (err) { + if (err instanceof SystemError) { + throw err; + } - const error = err instanceof Error ? err : new Error(String(err)); - pollErrorCount += 1; - if (pollErrorCount === 1 || pollErrorCount % 5 === 0) { - Sentry.capture('Could not poll device run session status', error, { level: 'warning' }); - logger.warn( - { err: error, failedStatusPollCount: pollErrorCount }, - 'Could not poll device run session status; will retry.' - ); + const error = err instanceof Error ? err : new Error(String(err)); + pollErrorCount += 1; + if (pollErrorCount === 1 || pollErrorCount % 5 === 0) { + Sentry.capture('Could not poll device run session status', error, { level: 'warning' }); + logger.warn( + { err: error, failedStatusPollCount: pollErrorCount }, + 'Could not poll device run session status; will retry.' + ); + } } + await sleepUntilAbortedAsync(DEVICE_RUN_SESSION_STATUS_POLL_INTERVAL_MS, signal); + } + } finally { + if (durationTimeout !== undefined) { + clearTimeout(durationTimeout); } - await sleepUntilAbortedAsync(DEVICE_RUN_SESSION_STATUS_POLL_INTERVAL_MS, signal); } }