From 49b8aee47b69c0c831e9a7040aa72c0abf9f4494 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 13:07:53 +0100 Subject: [PATCH 01/49] refactor(upload): add the upload engine as a queue of files with per-item events --- src/app/network/networkConstants.ts | 2 + src/app/network/types/index.ts | 13 +- src/app/network/types/upload.ts | 27 ++ src/app/network/uploadFailure.ts | 46 ++++ src/app/upload/UploadManager.test.ts | 286 ++++++++++++++++++++++ src/app/upload/UploadManager.ts | 214 ++++++++++++++++ src/testUtils/fixtures/upload.fixtures.ts | 18 ++ 7 files changed, 605 insertions(+), 1 deletion(-) create mode 100644 src/app/network/types/upload.ts create mode 100644 src/app/network/uploadFailure.ts create mode 100644 src/app/upload/UploadManager.test.ts create mode 100644 src/app/upload/UploadManager.ts create mode 100644 src/testUtils/fixtures/upload.fixtures.ts diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index 1e857231ac..e52923163a 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -10,3 +10,5 @@ export const USE_MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024; export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; +export const SINGLE_TRANSFER_CONCURRENCY = 1; +export const HIGH_BROWSER_MEMORY_USAGE_RATIO = 0.8; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index edb1f68756..3d87c2391c 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -35,7 +35,18 @@ export interface DownloadChunkPayload { options?: DownloadOptions; } -export type UploadErrorReason = 'connection-lost' | 'upload-failed'; +export enum UploadItemKind { + File = 'file', +} + +export enum UploadFailureReason { + ConnectionLost = 'connection-lost', + NoSpaceLeft = 'no-space', + FileTooLarge = 'file-too-large', + UploadFailed = 'upload-failed', +} + +export type UploadErrorReason = `${UploadFailureReason}`; export interface OwnerUserAuthenticationData { token: string; diff --git a/src/app/network/types/upload.ts b/src/app/network/types/upload.ts new file mode 100644 index 0000000000..6937394c5f --- /dev/null +++ b/src/app/network/types/upload.ts @@ -0,0 +1,27 @@ +import { DriveFileData } from 'app/drive/types'; +import { FileToUpload } from 'app/drive/services/file.service/types'; +import { UploadFailure } from '../uploadFailure'; +import { UploadItemKind } from '.'; + +export type UploadFileItem = { kind: UploadItemKind.File; file: FileToUpload }; + +export type UploadItem = UploadFileItem; + +export type UploadedItem = DriveFileData; + +/** + * Upload progress notifications emitted by the manager. The manager knows nothing + * about how uploads are presented to the user; observers subscribe to these events. + */ +export interface UploadManagerEvents { + onItemStarted?: (item: UploadItem) => void; + onItemProgress?: (item: UploadItem, progress: number) => void; + onItemSucceeded?: (item: UploadItem, uploadedItem: UploadedItem) => void | Promise; + onItemFailed?: (item: UploadItem, failure: UploadFailure) => void; + onItemCancelled?: (item: UploadItem) => void; +} + +export interface UploadResult { + succeeded: { item: UploadItem; uploadedItem: UploadedItem }[]; + failed: { item: UploadItem; failure: UploadFailure }[]; +} diff --git a/src/app/network/uploadFailure.ts b/src/app/network/uploadFailure.ts new file mode 100644 index 0000000000..7ac796a44e --- /dev/null +++ b/src/app/network/uploadFailure.ts @@ -0,0 +1,46 @@ +import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; +import errorService from 'services/error.service'; +import { ConnectionLostError } from './requests'; +import { UploadFailureReason } from './types'; + +export interface UploadFailure { + error: Error; + reason: UploadFailureReason; + isRetryable: boolean; +} + +const getStatusCode = (error: unknown) => (error as { status?: number } | undefined)?.status; + +const isNetworkErrorWhileOffline = (error: unknown) => + (error as Error | undefined)?.message === ErrorMessages.NetworkError && !navigator.onLine; + +const isConnectionLost = (error: unknown) => error instanceof ConnectionLostError || isNetworkErrorWhileOffline(error); + +const isStorageFull = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.MAX_SPACE_USED; + +const isFileTooLarge = (error: unknown) => + errorService.castError(error).code === HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED; + +const isPlanNotAllowingIt = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.PAYMENT_REQUIRED; + +const getFailureReason = (error: unknown): UploadFailureReason => { + if (isConnectionLost(error)) return UploadFailureReason.ConnectionLost; + if (isStorageFull(error)) return UploadFailureReason.NoSpaceLeft; + if (isFileTooLarge(error)) return UploadFailureReason.FileTooLarge; + return UploadFailureReason.UploadFailed; +}; + +export const toUploadFailure = (error: unknown): UploadFailure => { + const reason = getFailureReason(error); + const canSucceedLater = reason !== UploadFailureReason.FileTooLarge && !isPlanNotAllowingIt(error); + const genericUploadError = new Error('Upload failed'); + + return { + error: errorService.castError(error ?? genericUploadError), + reason, + isRetryable: canSucceedLater, + }; +}; + +export const canSucceedOnImmediateRetry = (error: unknown): boolean => + getFailureReason(error) === UploadFailureReason.UploadFailed && !isPlanNotAllowingIt(error); diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts new file mode 100644 index 0000000000..5078935370 --- /dev/null +++ b/src/app/upload/UploadManager.test.ts @@ -0,0 +1,286 @@ +import { afterEach, beforeEach, describe, expect, Mock, MockInstance, test, vi } from 'vitest'; +import { AppError } from '@internxt/sdk'; +import uploadFile from 'app/drive/services/file.service/uploadFile'; +import { DriveFileData } from 'app/drive/types'; +import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; +import errorService from 'services/error.service'; +import { ConnectionLostError } from 'app/network/requests'; +import { USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; +import { UploadManager } from './UploadManager'; +import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; +import { UploadFailureReason } from 'app/network/types'; + +vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); + +type UploadFileOptions = { abortCallback?: (abort?: () => void) => void }; + +const HANG_TIMEOUT_MS = 2000; +const networkError = new AppError(ErrorMessages.NetworkError); +const noSpaceLeftError = new AppError('No space left', HTTP_STATUS_CODES.MAX_SPACE_USED); +const paymentRequiredError = new AppError('Payment required', HTTP_STATUS_CODES.PAYMENT_REQUIRED); +const fileTooLargeError = new AppError('File too large', 400, HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED); + +const nameOf = (item: UploadItem) => item.file.name; + +const failIfHangs = (promise: Promise): Promise => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error('upload never settled')), HANG_TIMEOUT_MS)), + ]); + +const createManager = (props: Partial[0]> = {}) => + new UploadManager({ uploadRepository: getUploadRepository(), getPauseKey: nameOf, ...props }); + +const uploadedNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); + +const fakeSlowUploads = () => + (uploadFile as Mock).mockImplementation((file: { name: string }, _progress, options: UploadFileOptions) => { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => resolve({ uuid: `${file.name}-uuid` }), 500); + options.abortCallback?.(() => { + clearTimeout(timeout); + reject(new Error('aborted')); + }); + }); + }); + +const fakeUploadFailureOf = (failingName: string, error: unknown) => + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + if (file.name === failingName) throw error; + await new Promise((resolve) => setTimeout(resolve, 10)); + return { uuid: `${file.name}-uuid` } as DriveFileData; + }); + +let onlineSpy: MockInstance | undefined; +const goOffline = () => { + onlineSpy = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); +}; + +describe('Uploading files', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(errorService, 'reportError').mockReturnValue(); + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); + }); + + afterEach(() => onlineSpy?.mockRestore()); + + test('When files are uploaded, then each one succeeds with its uploaded data and its plain name', async () => { + const items = [getUploadFileItem('report'), getUploadFileItem('invoice')]; + const events: UploadManagerEvents = { onItemSucceeded: vi.fn() }; + + const { succeeded, failed } = await createManager({ events }).run(items); + + expect(failed).toEqual([]); + expect(succeeded).toEqual([ + { item: items[0], uploadedItem: { uuid: 'report-uuid', name: 'report' } }, + { item: items[1], uploadedItem: { uuid: 'invoice-uuid', name: 'invoice' } }, + ]); + expect(events.onItemSucceeded).toHaveBeenCalledWith(items[0], { uuid: 'report-uuid', name: 'report' }); + }); + + test('When an upload advances, then its progress is notified', async () => { + const item = getUploadFileItem('report'); + (uploadFile as Mock).mockImplementation(async (_file, onProgress: (progress: number) => void) => { + onProgress(0.5); + return { uuid: 'report-uuid' }; + }); + const events: UploadManagerEvents = { onItemStarted: vi.fn(), onItemProgress: vi.fn() }; + + await createManager({ events }).run([item]); + + expect(events.onItemStarted).toHaveBeenCalledWith(item); + expect(events.onItemProgress).toHaveBeenCalledWith(item, 0.5); + }); + + test('When an upload fails unexpectedly once, then it is retried and succeeds', async () => { + (uploadFile as Mock).mockRejectedValueOnce(new Error('Server error')); + + const { succeeded } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledTimes(2); + expect(succeeded).toHaveLength(1); + }); + + test('When an upload keeps failing, then it fails as retryable and the other files still upload', async () => { + const serverError = new Error('Server error'); + fakeUploadFailureOf('broken', serverError); + const events: UploadManagerEvents = { onItemFailed: vi.fn() }; + const brokenItem = getUploadFileItem('broken'); + + const { succeeded, failed } = await failIfHangs( + createManager({ events }).run([brokenItem, getUploadFileItem('a'), getUploadFileItem('b')]), + ); + + expect(uploadedNames().filter((name) => name === 'broken')).toHaveLength(2); + expect(succeeded).toHaveLength(2); + expect(failed).toEqual([ + { + item: brokenItem, + failure: { error: serverError, reason: UploadFailureReason.UploadFailed, isRetryable: true }, + }, + ]); + expect(events.onItemFailed).toHaveBeenCalledWith(brokenItem, failed[0].failure); + }); + + test.each([ + { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.UploadFailed }, + { problem: 'the file is too large', error: fileTooLargeError, reason: UploadFailureReason.FileTooLarge }, + ])('When $problem, then the file is not retried and cannot be retried later', async ({ error, reason }) => { + (uploadFile as Mock).mockRejectedValue(error); + + const { failed } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledOnce(); + expect(failed[0].failure).toEqual({ error, reason, isRetryable: false }); + }); + + test.each([ + { failure: 'the connection is lost', error: new ConnectionLostError(), arrange: () => undefined }, + { failure: 'a network error happens while offline', error: networkError, arrange: goOffline }, + ])('When $failure, then the file is not retried and fails as a retryable connection loss', async (scenario) => { + scenario.arrange(); + (uploadFile as Mock).mockRejectedValue(scenario.error); + + const { failed } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledOnce(); + expect(failed[0].failure).toEqual( + expect.objectContaining({ reason: UploadFailureReason.ConnectionLost, isRetryable: true }), + ); + }); + + test('When a network error happens while online, then the upload is retried and succeeds', async () => { + (uploadFile as Mock).mockRejectedValueOnce(networkError); + + const { succeeded } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledTimes(2); + expect(succeeded).toHaveLength(1); + }); + + test('When the upload rejects without an error, then the file fails instead of hanging', async () => { + (uploadFile as Mock).mockRejectedValue(undefined); + + const { failed } = await failIfHangs(createManager().run([getUploadFileItem('report')])); + + expect(failed).toHaveLength(1); + }); + + test('When the storage is full, then the queued files fail without being uploaded', async () => { + fakeUploadFailureOf('first', noSpaceLeftError); + const items = ['first', 'a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); + + const { failed } = await failIfHangs(createManager().run(items)); + + expect(uploadedNames()).toEqual(['first', 'a', 'b', 'c', 'd', 'e']); + expect(failed.map(({ item }) => nameOf(item))).toEqual(['first', 'f', 'g']); + expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); + }); + + test('When the upload is cancelled, then the queued files are never uploaded and are reported as cancelled', async () => { + fakeSlowUploads(); + const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => getUploadFileItem(name)); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const manager = createManager({ events }); + + const upload = manager.run(items); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(6)); + manager.cancelAll(); + const { succeeded, failed } = await failIfHangs(upload); + + expect(uploadFile).toHaveBeenCalledTimes(6); + expect(succeeded).toEqual([]); + expect(failed).toEqual([]); + expect(events.onItemCancelled).toHaveBeenCalledTimes(items.length); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + + test('When a single file is cancelled, then only that file stops', async () => { + fakeSlowUploads(); + const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; + const events: UploadManagerEvents = { onItemCancelled: vi.fn() }; + const manager = createManager({ events }); + + const upload = manager.run([cancelledItem, otherItem]); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(2)); + manager.cancelItem(cancelledItem); + const { succeeded } = await failIfHangs(upload); + + expect(succeeded.map(({ item }) => item)).toEqual([otherItem]); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(cancelledItem); + }); + + test('When small and large files are uploaded together, then the large ones go last and one at a time', async () => { + let activeLargeUploads = 0; + let maxActiveLargeUploads = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string; size: number }) => { + const isLarge = file.size >= USE_MULTIPART_THRESHOLD_BYTES; + if (isLarge) maxActiveLargeUploads = Math.max(maxActiveLargeUploads, ++activeLargeUploads); + await new Promise((resolve) => setTimeout(resolve, 10)); + if (isLarge) activeLargeUploads--; + return { uuid: `${file.name}-uuid` }; + }); + const items = [ + getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('small-1'), + getUploadFileItem('large-2', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('small-2'), + ]; + + await createManager().run(items); + + expect(uploadedNames()).toEqual(['small-1', 'small-2', 'large-1', 'large-2']); + expect(maxActiveLargeUploads).toBe(1); + }); + + test('When uploading to a workspace, then the workspace credentials are used', async () => { + const ownerUserAuthenticationData = { + bucketId: 'workspace-bucket', + workspaceId: 'workspace-id', + bridgeUser: 'bridge-user', + bridgePass: 'bridge-pass', + encryptionKey: 'encryption-key', + token: 'token', + workspacesToken: 'workspaces-token', + resourcesToken: 'resources-token', + }; + + await createManager({ ownerUserAuthenticationData }).run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Function), + expect.objectContaining({ isTeam: true, ownerUserAuthenticationData }), + expect.any(Object), + ); + }); + + test('When the upload of a file was paused, then it starts paused under its pause key', async () => { + const uploadRepository = getUploadRepository(['paused-key']); + + await createManager({ uploadRepository, getPauseKey: () => 'paused-key' }).run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Function), + expect.any(Object), + expect.objectContaining({ taskId: 'paused-key', isPaused: true }), + ); + }); + + test('When a success listener throws, then every file is uploaded once and the queue keeps going', async () => { + const items = [getUploadFileItem('a'), getUploadFileItem('b'), getUploadFileItem('c')]; + const events: UploadManagerEvents = { + onItemSucceeded: vi.fn(() => { + throw new Error('listener failure'); + }), + }; + + const { succeeded } = await failIfHangs(createManager({ events }).run(items)); + + expect(uploadFile).toHaveBeenCalledTimes(items.length); + expect(succeeded).toHaveLength(items.length); + }); +}); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts new file mode 100644 index 0000000000..00ee1b6e6b --- /dev/null +++ b/src/app/upload/UploadManager.ts @@ -0,0 +1,214 @@ +import { items as itemUtils } from '@internxt/lib'; +import { AsyncPriorityQueue, priorityQueue } from 'async'; +import errorService from 'services/error.service'; +import uploadFile from 'app/drive/services/file.service/uploadFile'; +import { DriveFileData } from 'app/drive/types'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { TaskStatus } from 'app/tasks/types'; +import { + HIGH_BROWSER_MEMORY_USAGE_RATIO, + MAX_CONCURRENT_UPLOADS, + MAX_UPLOAD_ATTEMPTS, + SINGLE_TRANSFER_CONCURRENCY, + USE_MULTIPART_THRESHOLD_BYTES, +} from 'app/network/networkConstants'; +import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; +import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; + +enum UploadPriority { + Normal = 1, + Last = 2, +} + +interface UploadManagerProps { + ownerUserAuthenticationData?: OwnerUserAuthenticationData; + uploadRepository?: PersistUploadRepository; + getPauseKey: (item: UploadItem) => string; + events?: UploadManagerEvents; +} + +export const createFileItem = (file: File, parentFolderUuid: string): UploadFileItem => { + const { filename, extension } = itemUtils.getFilenameAndExt(file.name); + + return { + kind: UploadItemKind.File, + file: { name: filename, type: extension, size: file.size, content: file, parentFolderId: parentFolderUuid }, + }; +}; + +const isMultipartUpload = (item: UploadItem) => item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; + +const getUploadPriority = (item: UploadItem): UploadPriority => { + if (isMultipartUpload(item)) return UploadPriority.Last; + return UploadPriority.Normal; +}; + +const isBrowserMemoryAlmostFull = (): boolean => { + const memory = window?.performance?.memory; + if (!memory?.jsHeapSizeLimit || !memory?.usedJSHeapSize) return false; + + return memory.usedJSHeapSize / memory.jsHeapSizeLimit >= HIGH_BROWSER_MEMORY_USAGE_RATIO; +}; + +/** + * Listeners are the caller's code. If one throws, the error is reported but never stops the queue nor turns an + * uploaded item into a failed one. + */ +const notifyIgnoringListenerErrors = async (notify: () => unknown): Promise => { + try { + await notify(); + } catch (error) { + errorService.reportError(error); + } +}; + +/** + * Uploads files through a single queue. Every item ends either succeeded, failed or cancelled: a failed + * item never stops the others, except when the storage is full, which fails the pending ones. An item that + * finishes uploading right after being cancelled is still reported as succeeded, since it does exist in + * the destination. + */ +export class UploadManager { + private readonly ownerUserAuthenticationData?: OwnerUserAuthenticationData; + private readonly uploadRepository?: PersistUploadRepository; + private readonly getPauseKey: (item: UploadItem) => string; + private readonly events?: UploadManagerEvents; + + private isEverythingCancelled = false; + private storageFullFailure?: UploadFailure; + private readonly cancelledItems = new WeakSet(); + private readonly abortActiveUploadOf = new Map void>(); + private result: UploadResult = { succeeded: [], failed: [] }; + + private readonly uploadQueue: AsyncPriorityQueue = priorityQueue((item, done) => { + this.processItem(item) + .catch((error) => errorService.reportError(error)) + .finally(() => done()); + }, MAX_CONCURRENT_UPLOADS); + + constructor(props: UploadManagerProps) { + this.ownerUserAuthenticationData = props.ownerUserAuthenticationData; + this.uploadRepository = props.uploadRepository; + this.getPauseKey = props.getPauseKey; + this.events = props.events; + } + + async run(items: UploadItem[]): Promise { + if (this.uploadQueue.idle()) { + this.result = { succeeded: [], failed: [] }; + this.storageFullFailure = undefined; + } + + const hasNothingToUpload = items.length === 0; + if (hasNothingToUpload) return this.result; + + this.queue(items); + await this.uploadQueue.drain(); + + return this.result; + } + + cancelAll(): void { + this.isEverythingCancelled = true; + this.abortActiveUploadOf.forEach((abort) => abort()); + } + + cancelItem(item: UploadItem): void { + this.cancelledItems.add(item); + this.abortActiveUploadOf.get(item)?.(); + } + + private queue(items: UploadItem[]): void { + for (const item of items) { + this.uploadQueue.push(item, getUploadPriority(item)); + } + } + + private async processItem(item: UploadItem): Promise { + if (this.isItemCancelled(item)) { + await notifyIgnoringListenerErrors(() => this.events?.onItemCancelled?.(item)); + return; + } + + if (this.storageFullFailure) { + await this.fail(item, this.storageFullFailure); + return; + } + + this.adjustConcurrencyFor(item); + await notifyIgnoringListenerErrors(() => this.events?.onItemStarted?.(item)); + + try { + const uploadedItem = await this.uploadWithRetries(item); + this.result.succeeded.push({ item, uploadedItem }); + + await notifyIgnoringListenerErrors(() => this.events?.onItemSucceeded?.(item, uploadedItem)); + } catch (error) { + await this.handleUploadError(item, error); + } + } + + private adjustConcurrencyFor(item: UploadItem): void { + const shouldUploadOneAtATime = isMultipartUpload(item) || isBrowserMemoryAlmostFull(); + this.uploadQueue.concurrency = shouldUploadOneAtATime ? SINGLE_TRANSFER_CONCURRENCY : MAX_CONCURRENT_UPLOADS; + } + + private async uploadWithRetries(item: UploadItem): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await this.uploadFile(item); + } catch (error) { + const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; + const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); + if (!shouldRetry) throw error; + } + } + } + + private async uploadFile(fileItem: UploadFileItem): Promise { + const pauseKey = this.getPauseKey(fileItem); + const uploadState = await this.uploadRepository?.getUploadState(pauseKey); + const isPaused = uploadState === TaskStatus.Paused; + + try { + const driveFile = await uploadFile( + fileItem.file, + (progress) => this.events?.onItemProgress?.(fileItem, progress), + { + isTeam: !!this.ownerUserAuthenticationData?.workspaceId, + ownerUserAuthenticationData: this.ownerUserAuthenticationData, + abortCallback: (abort) => this.abortActiveUploadOf.set(fileItem, () => abort?.()), + }, + { taskId: pauseKey, isPaused, isRetriedUpload: false }, + ); + + return { ...driveFile, name: fileItem.file.name }; + } finally { + this.abortActiveUploadOf.delete(fileItem); + } + } + + private isItemCancelled(item: UploadItem): boolean { + return this.isEverythingCancelled || this.cancelledItems.has(item); + } + + private async handleUploadError(item: UploadItem, error: unknown): Promise { + if (this.isItemCancelled(item)) { + await notifyIgnoringListenerErrors(() => this.events?.onItemCancelled?.(item)); + return; + } + + const failure = toUploadFailure(error); + const isStorageFull = failure.reason === UploadFailureReason.NoSpaceLeft; + if (isStorageFull) this.storageFullFailure = failure; + + errorService.reportError(failure.error); + await this.fail(item, failure); + } + + private async fail(item: UploadItem, failure: UploadFailure): Promise { + this.result.failed.push({ item, failure }); + await notifyIgnoringListenerErrors(() => this.events?.onItemFailed?.(item, failure)); + } +} diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts new file mode 100644 index 0000000000..51062cac1d --- /dev/null +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -0,0 +1,18 @@ +import { vi } from 'vitest'; +import { UploadItemKind } from 'app/network/types'; +import { UploadFileItem } from 'app/network/types/upload'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { TaskStatus } from 'app/tasks/types'; + +export const getFileToUpload = (name: string, size = 10): File => new File([new ArrayBuffer(size)], name); + +export const getUploadFileItem = (name: string, size = 1024): UploadFileItem => ({ + kind: UploadItemKind.File, + file: { name, size, type: 'txt', content: getFileToUpload(name), parentFolderId: 'folder-uuid' }, +}); + +export const getUploadRepository = (pausedKeys: string[] = []): PersistUploadRepository => ({ + setUploadState: vi.fn(async () => undefined), + removeUploadState: vi.fn(async () => undefined), + getUploadState: vi.fn(async (key: string) => (pausedKeys.includes(key) ? TaskStatus.Paused : undefined)), +}); From db2ee219c4c9256c178024c1273268d38c7c24ef Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 14:33:56 +0100 Subject: [PATCH 02/49] refactor(upload): upload small files before medium ones and adjust concurrency with the shared memory check --- src/app/network/networkConstants.ts | 1 - src/app/network/types/index.ts | 2 +- src/app/upload/UploadManager.test.ts | 36 +++++++++++++++++++++------- src/app/upload/UploadManager.ts | 30 ++++++++++++----------- src/utils/queueUtils.ts | 5 +++- 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index e52923163a..69f3b47c6a 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -11,4 +11,3 @@ export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; export const SINGLE_TRANSFER_CONCURRENCY = 1; -export const HIGH_BROWSER_MEMORY_USAGE_RATIO = 0.8; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index 3d87c2391c..f099c9a09b 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -46,7 +46,7 @@ export enum UploadFailureReason { UploadFailed = 'upload-failed', } -export type UploadErrorReason = `${UploadFailureReason}`; +export type UploadErrorReason = 'connection-lost' | 'upload-failed'; export interface OwnerUserAuthenticationData { token: string; diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 5078935370..c811b575f1 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -1,11 +1,12 @@ -import { afterEach, beforeEach, describe, expect, Mock, MockInstance, test, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, Mock, test, vi } from 'vitest'; import { AppError } from '@internxt/sdk'; import uploadFile from 'app/drive/services/file.service/uploadFile'; import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; @@ -52,10 +53,7 @@ const fakeUploadFailureOf = (failingName: string, error: unknown) => return { uuid: `${file.name}-uuid` } as DriveFileData; }); -let onlineSpy: MockInstance | undefined; -const goOffline = () => { - onlineSpy = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); -}; +const goOffline = () => vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); describe('Uploading files', () => { beforeEach(() => { @@ -64,7 +62,7 @@ describe('Uploading files', () => { (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); }); - afterEach(() => onlineSpy?.mockRestore()); + afterEach(() => vi.restoreAllMocks()); test('When files are uploaded, then each one succeeds with its uploaded data and its plain name', async () => { const items = [getUploadFileItem('report'), getUploadFileItem('invoice')]; @@ -212,7 +210,7 @@ describe('Uploading files', () => { expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(cancelledItem); }); - test('When small and large files are uploaded together, then the large ones go last and one at a time', async () => { + test('When small, medium and large files are uploaded together, then the small ones go first and the large ones last and one at a time', async () => { let activeLargeUploads = 0; let maxActiveLargeUploads = 0; (uploadFile as Mock).mockImplementation(async (file: { name: string; size: number }) => { @@ -224,17 +222,37 @@ describe('Uploading files', () => { }); const items = [ getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('medium-1', TWENTY_MEGABYTES), getUploadFileItem('small-1'), getUploadFileItem('large-2', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('medium-2', TWENTY_MEGABYTES), getUploadFileItem('small-2'), ]; await createManager().run(items); - expect(uploadedNames()).toEqual(['small-1', 'small-2', 'large-1', 'large-2']); + expect(uploadedNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); expect(maxActiveLargeUploads).toBe(1); }); + test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { + vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); + let activeUploads = 0; + let maxActiveUploads = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); + await new Promise((resolve) => setTimeout(resolve, 10)); + activeUploads--; + return { uuid: `${file.name}-uuid` }; + }); + const items = ['a', 'b', 'c', 'd'].map((name) => getUploadFileItem(name)); + + const { succeeded } = await createManager().run(items); + + expect(succeeded).toHaveLength(items.length); + expect(maxActiveUploads).toBe(1); + }); + test('When uploading to a workspace, then the workspace credentials are used', async () => { const ownerUserAuthenticationData = { bucketId: 'workspace-bucket', diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 00ee1b6e6b..4987c36993 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -1,15 +1,16 @@ import { items as itemUtils } from '@internxt/lib'; import { AsyncPriorityQueue, priorityQueue } from 'async'; import errorService from 'services/error.service'; +import { QueueUtilsService } from 'utils/queueUtils'; import uploadFile from 'app/drive/services/file.service/uploadFile'; import { DriveFileData } from 'app/drive/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { TaskStatus } from 'app/tasks/types'; import { - HIGH_BROWSER_MEMORY_USAGE_RATIO, MAX_CONCURRENT_UPLOADS, MAX_UPLOAD_ATTEMPTS, SINGLE_TRANSFER_CONCURRENCY, + TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES, } from 'app/network/networkConstants'; import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; @@ -17,8 +18,9 @@ import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadRe import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; enum UploadPriority { - Normal = 1, - Last = 2, + SmallFile = 1, + MediumFile = 2, + LargeFile = 3, } interface UploadManagerProps { @@ -39,16 +41,12 @@ export const createFileItem = (file: File, parentFolderUuid: string): UploadFile const isMultipartUpload = (item: UploadItem) => item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; -const getUploadPriority = (item: UploadItem): UploadPriority => { - if (isMultipartUpload(item)) return UploadPriority.Last; - return UploadPriority.Normal; -}; +const isMediumSizedFile = (item: UploadItem) => item.file.size >= TWENTY_MEGABYTES; -const isBrowserMemoryAlmostFull = (): boolean => { - const memory = window?.performance?.memory; - if (!memory?.jsHeapSizeLimit || !memory?.usedJSHeapSize) return false; - - return memory.usedJSHeapSize / memory.jsHeapSizeLimit >= HIGH_BROWSER_MEMORY_USAGE_RATIO; +const getUploadPriority = (item: UploadItem): UploadPriority => { + if (isMultipartUpload(item)) return UploadPriority.LargeFile; + if (isMediumSizedFile(item)) return UploadPriority.MediumFile; + return UploadPriority.SmallFile; }; /** @@ -150,8 +148,12 @@ export class UploadManager { } private adjustConcurrencyFor(item: UploadItem): void { - const shouldUploadOneAtATime = isMultipartUpload(item) || isBrowserMemoryAlmostFull(); - this.uploadQueue.concurrency = shouldUploadOneAtATime ? SINGLE_TRANSFER_CONCURRENCY : MAX_CONCURRENT_UPLOADS; + if (isMultipartUpload(item)) { + this.uploadQueue.concurrency = SINGLE_TRANSFER_CONCURRENCY; + return; + } + + QueueUtilsService.instance.adjustConcurrency(this.uploadQueue, MAX_CONCURRENT_UPLOADS); } private async uploadWithRetries(item: UploadItem): Promise { diff --git a/src/utils/queueUtils.ts b/src/utils/queueUtils.ts index 35bbaff53f..11b6bc4974 100644 --- a/src/utils/queueUtils.ts +++ b/src/utils/queueUtils.ts @@ -3,7 +3,10 @@ import { QueueObject } from 'async'; export class QueueUtilsService { public static readonly instance: QueueUtilsService = new QueueUtilsService(); - public readonly adjustConcurrency = (targetQueue: QueueObject, maxConcurrency: number): void => { + public readonly adjustConcurrency = ( + targetQueue: Pick, 'concurrency'>, + maxConcurrency: number, + ): void => { const newConcurrency = this.getConcurrencyUsingPerformance(targetQueue.concurrency, maxConcurrency); if (targetQueue.concurrency !== newConcurrency) { targetQueue.concurrency = newConcurrency; From 9e2d9da8791982961127aa5d39ee984b13129669 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 14:39:28 +0100 Subject: [PATCH 03/49] refactor(upload): run every file upload on the new engine --- src/app/network/UploadFolderManager.test.ts | 15 +- src/app/network/UploadFolderManager.ts | 12 +- src/app/repositories/uploadStateUpdates.ts | 11 + .../storage.thunks/uploadItemsThunk.ts | 1 - .../TaskToRetryItem/TaskToRetryItem.tsx | 2 +- .../tasks/upload/uploadFilesWithTasks.test.ts | 201 +++++++++----- src/app/tasks/upload/uploadFilesWithTasks.ts | 249 ++++++++++++------ .../upload/uploadFoldersWithTasks.test.ts | 9 +- src/app/tasks/upload/uploadTaskErrors.ts | 10 +- 9 files changed, 328 insertions(+), 182 deletions(-) create mode 100644 src/app/repositories/uploadStateUpdates.ts diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 60b096b09e..9d06a83351 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -13,13 +13,14 @@ import { import { FilesExceedsSizeLimitError } from 'app/drive/services/file.service/upload.errors'; import { uploadItemsParallelThunk } from 'app/store/slices/storage/storage.thunks/uploadItemsThunk'; import { deleteItemsThunk } from '../store/slices/storage/storage.thunks/deleteItemsThunk'; -import { uploadFileWithManager } from './UploadManager'; +import { uploadFilesWithTasks } from 'app/tasks/upload/uploadFilesWithTasks'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import uploadFile from 'app/drive/services/file.service/uploadFile'; import newStorageService from 'app/drive/services/new-storage.service'; import RetryManager from './RetryManager'; import { ConnectionLostError } from './requests'; import errorService from 'services/error.service'; +import { UploadFailureReason } from 'app/network/types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn(async (file: { name: string }) => ({ uuid: `${file.name}-uuid`, name: file.name })), @@ -51,6 +52,8 @@ vi.mock('app/store/slices/storage/folderUtils/getUniqueFolderName', () => ({ getUniqueFolderName: vi.fn(), })); +vi.mock('services/referral.service', () => ({ default: { trackFileUpload: vi.fn() } })); +vi.mock('app/network/networkInformation', () => ({ logNetworkInfoForUpload: vi.fn() })); vi.mock('services/error.service', () => ({ default: { castError: vi.fn().mockImplementation((e) => e), @@ -89,7 +92,7 @@ describe('uploadFoldersWithManager', () => { (uploadItemsParallelThunk as unknown as Mock).mockImplementation((thunkArgs) => thunkArgs); return vi.fn(({ files, parentFolderId, options, onFileUploadCallback }) => ({ unwrap: () => - uploadFileWithManager({ + uploadFilesWithTasks({ files: files.map((content: File, index: number) => ({ taskId: `file-task-${index}`, relatedTaskId: options.relatedTaskId, @@ -304,7 +307,7 @@ describe('uploadFoldersWithManager', () => { }); expect(events.onFolderUploadError).toHaveBeenCalledOnce(); - expect(events.onFolderUploadError).toHaveBeenCalledWith(taskId, 'upload-failed'); + expect(events.onFolderUploadError).toHaveBeenCalledWith(taskId, UploadFailureReason.UploadFailed); expect(events.onFolderUploadSuccess).not.toHaveBeenCalled(); }); @@ -420,7 +423,7 @@ describe('uploadFoldersWithManager', () => { await runFolderUpload(); - expect(events.onFolderUploadError).toHaveBeenLastCalledWith(taskId, 'connection-lost'); + expect(events.onFolderUploadError).toHaveBeenLastCalledWith(taskId, UploadFailureReason.ConnectionLost); expect(events.onFolderUploadSuccess).not.toHaveBeenCalled(); }); }); @@ -637,7 +640,7 @@ describe('uploadFoldersWithManager', () => { await expect(runFolderUpload()).resolves.toBeUndefined(); - expectEveryFolderFailedWith('connection-lost'); + expectEveryFolderFailedWith(UploadFailureReason.ConnectionLost); expect(errorService.reportError).toHaveBeenCalledWith(networkError); }); @@ -646,7 +649,7 @@ describe('uploadFoldersWithManager', () => { await runFolderUpload(); - expectEveryFolderFailedWith('upload-failed'); + expectEveryFolderFailedWith(UploadFailureReason.UploadFailed); }); }); diff --git a/src/app/network/UploadFolderManager.ts b/src/app/network/UploadFolderManager.ts index 33cac613c6..0822ba6acf 100644 --- a/src/app/network/UploadFolderManager.ts +++ b/src/app/network/UploadFolderManager.ts @@ -16,7 +16,7 @@ import { wait } from 'utils/timeUtils'; import { ConnectionLostError } from './requests'; import { FilesExceedsSizeLimitError } from 'app/drive/services/file.service/upload.errors'; import { filterFilesByMaxSize } from 'app/store/slices/storage/fileUtils/filterFilesByMaxSize'; -import { UploadErrorReason } from './types'; +import { UploadFailureReason } from './types'; import { MAX_CONCURRENT_UPLOADS } from './networkConstants'; export interface UploadFolderPayload { @@ -47,7 +47,7 @@ export interface UploadFolderManagerEvents { taskId: string, info: { folderName: string; rootFolderUUID?: string; hasFailedFiles: boolean }, ) => void; - onFolderUploadError?: (taskId: string, reason?: UploadErrorReason) => void; + onFolderUploadError?: (taskId: string, reason?: UploadFailureReason) => void; onFolderUploadCancelled?: (taskId: string) => void; stopRelatedUploads?: (taskId: string) => Promise[]; } @@ -335,7 +335,7 @@ export class UploadFoldersManager { private readonly killQueueAndNotifyError = (taskId: string) => { this.tasksInfo[taskId].hasErrored = true; this.uploadFoldersQueue.kill(); - this.events?.onFolderUploadError?.(taskId, 'upload-failed'); + this.events?.onFolderUploadError?.(taskId, UploadFailureReason.UploadFailed); }; /** @@ -370,7 +370,7 @@ export class UploadFoldersManager { }; private readonly notifyPreparationError = (error: unknown): void => { - const reason: UploadErrorReason = navigator.onLine ? 'upload-failed' : 'connection-lost'; + const reason = navigator.onLine ? UploadFailureReason.UploadFailed : UploadFailureReason.ConnectionLost; for (const { options } of this.payload) { this.events?.onFolderUploadError?.(options.taskId, reason); @@ -432,11 +432,11 @@ export class UploadFoldersManager { const castedError = errorService.castError(err); if (connectionLost) { - this.events?.onFolderUploadError?.(taskId, 'connection-lost'); + this.events?.onFolderUploadError?.(taskId, UploadFailureReason.ConnectionLost); errorService.reportError(castedError); break; } else if (!this.tasksInfo[taskId].isCancelled) { - this.events?.onFolderUploadError?.(taskId, 'upload-failed'); + this.events?.onFolderUploadError?.(taskId, UploadFailureReason.UploadFailed); // Log the error or report it but don't re-throw it to allow the next folder to be processed errorService.reportError(castedError); continue; diff --git a/src/app/repositories/uploadStateUpdates.ts b/src/app/repositories/uploadStateUpdates.ts new file mode 100644 index 0000000000..036452e990 --- /dev/null +++ b/src/app/repositories/uploadStateUpdates.ts @@ -0,0 +1,11 @@ +import errorService from 'services/error.service'; +import { TaskStatus } from '../tasks/types'; +import { PersistUploadRepository } from './DatabaseUploadRepository'; + +export const markUploadAsInProcess = (uploadRepository: PersistUploadRepository, taskId: string): void => { + uploadRepository.setUploadState(taskId, TaskStatus.InProcess).catch((error) => errorService.reportError(error)); +}; + +export const forgetUploadState = (uploadRepository: PersistUploadRepository, taskId: string): void => { + uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); +}; diff --git a/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts b/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts index e6b04e19a1..dcaa5e6a29 100644 --- a/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts +++ b/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts @@ -231,7 +231,6 @@ export const uploadItemsThunk = createAsyncThunk ({ - uploadFileWithManager: vi.fn(() => Promise.resolve({ uploadedFiles: [] })), +const { managerState } = vi.hoisted(() => ({ + managerState: { + result: { succeeded: [], failed: [] } as UploadResult, + instance: undefined as + | undefined + | { props: { events: UploadManagerEvents }; run: Mock; cancelAll: Mock; cancelItem: Mock; items: UploadItem[] }, + }, +})); + +vi.mock('app/upload/UploadManager', () => ({ + UploadManager: vi.fn(function (this: NonNullable, props) { + this.props = props; + this.cancelAll = vi.fn(); + this.cancelItem = vi.fn(); + this.run = vi.fn(async (items: UploadItem[]) => { + this.items = items; + return managerState.result; + }); + managerState.instance = this; + }), })); vi.mock('app/tasks/services/tasks.service', () => ({ @@ -16,8 +38,6 @@ vi.mock('app/tasks/services/tasks.service', () => ({ create: vi.fn(), updateTask: vi.fn(), findTask: vi.fn(), - addListener: vi.fn(), - removeListener: vi.fn(), }, })); @@ -48,8 +68,8 @@ const buildFile = (overrides: Record = {}) => ({ const uploadRepository = { getUploadState: vi.fn(), - setUploadState: vi.fn(), - removeUploadState: vi.fn(), + setUploadState: vi.fn(() => Promise.resolve()), + removeUploadState: vi.fn(() => Promise.resolve()), } as unknown as PersistUploadRepository; const runUpload = async (files: ReturnType[], props: Record = {}) => { @@ -59,19 +79,33 @@ const runUpload = async (files: ReturnType[], props: Record; + return { manager, events: manager.props.events, items: manager.items }; +}; + +const failWith = (failure: Partial) => { + vi.mocked(UploadManager).mockImplementationOnce(function (this: NonNullable, props) { + this.props = props; + this.run = vi.fn(async (items: UploadItem[]) => ({ + succeeded: [], + failed: [{ item: items[0], failure: { error: new Error('failed'), isRetryable: true, ...failure } }], + })); + managerState.instance = this; + } as never); }; -describe('uploadFilesWithTasks', () => { +describe('Uploading files with tasks', () => { beforeEach(() => { vi.clearAllMocks(); + RetryManager.clearTasks(); + managerState.result = { succeeded: [], failed: [] }; + (tasksService.findTask as Mock).mockReturnValue(undefined); }); test('When a new file upload begins, then a task is created to track it', async () => { (tasksService.create as Mock).mockReturnValue('new-task-id'); - const { managerProps } = await runUpload([buildFile()]); + const { items } = await runUpload([buildFile()]); expect(tasksService.create).toHaveBeenCalledWith( expect.objectContaining({ @@ -81,18 +115,17 @@ describe('uploadFilesWithTasks', () => { cancellable: true, }), ); - expect(managerProps.files).toEqual([expect.objectContaining({ taskId: 'new-task-id' })]); + expect(items).toEqual([{ kind: UploadItemKind.File, file: buildFile().filecontent }]); }); test('When an upload is retried, then its existing task resumes instead of creating a new one', async () => { - const { managerProps } = await runUpload([buildFile({ taskId: 'existing-task' })]); + await runUpload([buildFile({ taskId: 'existing-task' })]); expect(tasksService.create).not.toHaveBeenCalled(); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 'existing-task', - merge: { status: TaskStatus.Encrypting }, + merge: { status: TaskStatus.Encrypting, stop: expect.any(Function) }, }); - expect(managerProps.options.isRetriedUpload).toBe(true); }); test('When files from a folder upload begin, then the folder task shows as in progress', async () => { @@ -119,54 +152,47 @@ describe('uploadFilesWithTasks', () => { }); }); - test('When a file upload starts, then its task shows it is being prepared and becomes stoppable', async () => { - (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); - const stop = vi.fn(); + test('When a file upload starts, then only files uploaded outside a folder show as in progress', async () => { + (tasksService.create as Mock).mockReturnValueOnce('loose-task').mockReturnValueOnce('folder-file-task'); + const { events, items } = await runUpload([buildFile(), buildFile({ relatedTaskId: 'related' })]); - events.onUploadStart?.(managerProps.files[0], stop); + events.onItemStarted?.(items[0]); + events.onItemStarted?.(items[1]); expect(tasksService.updateTask).toHaveBeenCalledWith({ - taskId: 't1', - merge: { status: TaskStatus.Encrypting, stop }, + taskId: 'loose-task', + merge: { status: TaskStatus.InProcess }, + }); + expect(tasksService.updateTask).toHaveBeenCalledWith({ + taskId: 'folder-file-task', + merge: { status: TaskStatus.Encrypting }, }); - }); - - test('When an upload attempt begins, then only files uploaded outside a folder show as in progress', async () => { - (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); - - events.onUploadAttempt?.(managerProps.files[0]); - expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.InProcess } }); - - (tasksService.updateTask as Mock).mockClear(); - events.onUploadAttempt?.({ ...managerProps.files[0], relatedTaskId: 'related' }); - expect(tasksService.updateTask).not.toHaveBeenCalled(); }); test('When an upload advances, then the task progress updates unless the task is paused or cancelled', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const { events, items } = await runUpload([buildFile()]); (tasksService.findTask as Mock).mockReturnValue({ status: TaskStatus.InProcess }); - events.onUploadProgress?.(managerProps.files[0], 0.5); + events.onItemProgress?.(items[0], 0.5); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { progress: 0.5 } }); (tasksService.updateTask as Mock).mockClear(); (tasksService.findTask as Mock).mockReturnValue({ status: TaskStatus.Paused }); - events.onUploadProgress?.(managerProps.files[0], 0.6); - expect(tasksService.updateTask).not.toHaveBeenCalled(); - + events.onItemProgress?.(items[0], 0.6); (tasksService.findTask as Mock).mockReturnValue({ status: TaskStatus.Cancelled }); - events.onUploadProgress?.(managerProps.files[0], 0.7); + events.onItemProgress?.(items[0], 0.7); expect(tasksService.updateTask).not.toHaveBeenCalled(); }); - test('When a file finishes uploading, then its task succeeds and the upload is tracked', async () => { + test('When a file finishes uploading, then its task succeeds, the upload is tracked and the caller is told', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const onFinishUploadFile = vi.fn(); + const onFileUploadCallback = vi.fn(); + const { events, items } = await runUpload([buildFile({ onFinishUploadFile })], { onFileUploadCallback }); + const driveFile = { uuid: 'file-uuid' } as never; - await events.onUploadSuccess?.(managerProps.files[0], { uuid: 'file-uuid' } as never); + await events.onItemSucceeded?.(items[0], driveFile); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', @@ -174,16 +200,27 @@ describe('uploadFilesWithTasks', () => { }); expect(referralService.trackFileUpload).toHaveBeenCalledOnce(); expect(networkInformation.logNetworkInfoForUpload).toHaveBeenCalledWith({ fileName: 'file.txt', fileSize: 1024 }); + expect(onFinishUploadFile).toHaveBeenCalledWith(driveFile, 't1'); + expect(onFileUploadCallback).toHaveBeenCalledWith(driveFile); + }); + + test('When a retried file finishes uploading, then it leaves the retry list', async () => { + RetryManager.addTasks([{ taskId: 'existing-task', type: 'upload' as never, params: {} }]); + const { events, items } = await runUpload([buildFile({ taskId: 'existing-task' })]); + + await events.onItemSucceeded?.(items[0], { uuid: 'file-uuid' } as never); + + expect(RetryManager.getTasks()).toEqual([]); }); test('When a file inside a folder finishes uploading, then the folder progress advances', async () => { (tasksService.create as Mock).mockReturnValue('t1'); const relatedTaskProgress = { filesUploaded: 0, totalFilesToUpload: 4 }; - const { events, managerProps } = await runUpload([buildFile({ relatedTaskId: 'related-task' })], { + const { events, items } = await runUpload([buildFile({ relatedTaskId: 'related-task' })], { relatedTaskProgress, }); - await events.onUploadSuccess?.(managerProps.files[0], { uuid: 'file-uuid' } as never); + await events.onItemSucceeded?.(items[0], { uuid: 'file-uuid' } as never); expect(relatedTaskProgress.filesUploaded).toBe(1); expect(tasksService.updateTask).toHaveBeenCalledWith({ @@ -194,54 +231,74 @@ describe('uploadFilesWithTasks', () => { test('When an upload fails, then the task shows an error message matching the cause', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const { events, items } = await runUpload([buildFile()]); + const failure = { error: new Error('failed'), isRetryable: true }; - events.onUploadError?.(managerProps.files[0], 'connection-lost'); + events.onItemFailed?.(items[0], { ...failure, reason: UploadFailureReason.ConnectionLost }); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Error, subtitle: 'error.connectionLostError' }, }); - events.onUploadError?.(managerProps.files[0], 'upload-failed'); + events.onItemFailed?.(items[0], { ...failure, reason: UploadFailureReason.UploadFailed }); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Error, subtitle: 'tasks.subtitles.upload-failed' }, }); - - events.onUploadError?.(managerProps.files[0]); - expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Error } }); }); - test('When an upload is aborted, then its task shows as cancelled', async () => { + test('When an upload is cancelled, then its task shows as cancelled', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const { events, items } = await runUpload([buildFile()]); - events.onUploadAborted?.(managerProps.files[0]); + events.onItemCancelled?.(items[0]); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Cancelled } }); }); - test('When a task is cancelled by the user, then only that upload is stopped', async () => { + test('When a task is stopped by the user, then only that upload is cancelled', async () => { + const { manager, items } = await runUpload([buildFile(), buildFile()]); + const [{ stop }] = (tasksService.create as Mock).mock.calls[0] as [{ stop: () => Promise }]; + + await stop(); + + expect(manager.cancelItem).toHaveBeenCalledExactlyOnceWith(items[0]); + }); + + test('When the folder upload that owns the files is cancelled, then the whole upload is cancelled', async () => { + const abortController = new AbortController(); + const { manager } = await runUpload([buildFile()], { abortController }); + + abortController.abort(); + + expect(manager.cancelAll).toHaveBeenCalledOnce(); + }); + + test('When a file fails, then it is listed for retry and the upload rejects with its error', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - let registeredListener: ((task: { id: string }) => void) | undefined; - (tasksService.addListener as Mock).mockImplementation(({ listener }) => { - registeredListener = listener; - }); + const error = new Error('Server error'); + failWith({ error, reason: UploadFailureReason.UploadFailed, isRetryable: false }); + + await expect(runUpload([buildFile()])).rejects.toThrow(error); - const { events, managerProps } = await runUpload([buildFile()]); - const abort = vi.fn(); - const unsubscribe = events.registerUploadAbort?.(managerProps.files[0], abort); + expect(RetryManager.getTasks()).toEqual([ + expect.objectContaining({ taskId: 't1', retryable: false, params: expect.objectContaining({ taskId: 't1' }) }), + ]); + }); - registeredListener?.({ id: 'other-task' }); - expect(abort).not.toHaveBeenCalled(); + test.each([ + { problem: 'the storage is full', reason: UploadFailureReason.NoSpaceLeft, callback: 'maxSpaceOccupiedCallback' }, + { + problem: 'the file is too large', + reason: UploadFailureReason.FileTooLarge, + callback: 'fileSizeExceededCallback', + }, + ] as const)('When $problem, then the user is warned about it', async ({ reason, callback }) => { + const warn = vi.fn(); + failWith({ reason }); - registeredListener?.({ id: 't1' }); - expect(abort).toHaveBeenCalledOnce(); + await expect(runUpload([buildFile()], { [callback]: warn })).rejects.toThrow(); - unsubscribe?.(); - expect(tasksService.removeListener).toHaveBeenCalledWith({ - event: TaskEvent.TaskCancelled, - listener: registeredListener, - }); + expect(warn).toHaveBeenCalledOnce(); }); }); diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index c817b1b8b1..f505bec6ae 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -1,20 +1,45 @@ import tasksService from 'app/tasks/services/tasks.service'; -import { TaskData, TaskEvent, TaskStatus, TaskType, UploadFileTask } from 'app/tasks/types'; +import { TaskStatus, TaskType, UploadFileTask } from 'app/tasks/types'; import referralService from 'services/referral.service'; import { logNetworkInfoForUpload } from 'app/network/networkInformation'; import { DriveFileData } from 'app/drive/types'; -import { - uploadFileWithManager, - UploadFileParamsWithTaskId, - UploadManagerEvents, - UploadManagerFileParams, -} from 'app/network/UploadManager'; +import { FileToUpload } from 'app/drive/services/file.service/types'; +import { UploadManager } from 'app/upload/UploadManager'; +import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; +import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; +import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { forgetUploadState, markUploadAsInProcess } from 'app/repositories/uploadStateUpdates'; import { errorMerge } from './uploadTaskErrors'; -type UploadFileWithManagerProps = Parameters[0]; +export type UploadManagerFileParams = { + filecontent: FileToUpload; + taskId?: string; + relatedTaskId?: string; + fileType?: string; + userEmail: string; + parentFolderId: string; + onFinishUploadFile?: (driveItemData: DriveFileData, taskId: string) => void; + abortController?: AbortController; +}; + +type UploadFileParamsWithTaskId = UploadManagerFileParams & { taskId: string }; -interface UploadFilesWithTasksProps extends Omit { +interface UploadFilesWithTasksProps { files: UploadManagerFileParams[]; + maxSpaceOccupiedCallback: () => void; + fileSizeExceededCallback?: () => void; + onFileUploadCallback?: (driveFileData: DriveFileData) => void; + uploadRepository: PersistUploadRepository; + abortController?: AbortController; + options?: { + showNotifications?: boolean; + ownerUserAuthenticationData?: OwnerUserAuthenticationData; + sharedItemData?: { + isDeepFolder?: boolean; + currentFolderId?: string; + }; + }; relatedTaskProgress?: { filesUploaded: number; totalFilesToUpload: number }; } @@ -22,80 +47,47 @@ interface UploadFilesWithTasksProps extends Omit => { - const { files, options, abortController, uploadRepository, relatedTaskProgress, ...managerProps } = props; - - const isRetriedUpload = files.some((file) => !!file.taskId) || !!options?.isRetriedUpload; - - const filesWithTaskId: UploadFileParamsWithTaskId[] = files.map((file) => { - if (file.taskId) { - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.Encrypting } }); - return { ...file, taskId: file.taskId }; - } - - const sharedItemAuthenticationData = options?.ownerUserAuthenticationData - ? { - ownerUserAuthenticationData: { ...options.ownerUserAuthenticationData }, - isDeepFolder: !!options?.sharedItemData?.isDeepFolder, - currentFolderId: options?.sharedItemData?.currentFolderId as string, - } - : undefined; - - const taskId = tasksService.create({ - relatedTaskId: file.relatedTaskId, - action: TaskType.UploadFile, - item: { uploadFile: file.filecontent.content, parentFolderId: file.parentFolderId }, - fileName: file.filecontent.name, - fileType: file.filecontent.type, - isFileNameValidated: true, - showNotification: options?.showNotifications ?? true, - cancellable: true, - sharedItemAuthenticationData, - stop: async () => { - if (abortController) abortController.abort(); - else file.abortController?.abort(); - }, - }); - - return { ...file, taskId }; - }); - - const relatedTaskId = filesWithTaskId[0]?.relatedTaskId; - if (relatedTaskId) { - const isPaused = (await uploadRepository?.getUploadState(relatedTaskId)) === TaskStatus.Paused; - if (!isPaused) { - tasksService.updateTask({ taskId: relatedTaskId, merge: { status: TaskStatus.InProcess } }); - } - } +export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Promise => { + const { files, options, abortController, uploadRepository, relatedTaskProgress } = props; + const fileByItem = new Map(); + const getFile = (item: UploadItem) => fileByItem.get(item) as UploadFileParamsWithTaskId; + const getPauseKey = (item: UploadItem) => { + const file = getFile(item); + return file.relatedTaskId ?? file.taskId; + }; const events: UploadManagerEvents = { - onUploadStart: (file, stopUpload) => - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.Encrypting, stop: stopUpload } }), - - onUploadAttempt: (file) => { - if (!file.relatedTaskId) { - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.InProcess } }); - } + onItemStarted: (item) => { + const file = getFile(item); + tasksService.updateTask({ + taskId: file.taskId, + merge: { status: file.relatedTaskId ? TaskStatus.Encrypting : TaskStatus.InProcess }, + }); + if (!file.relatedTaskId) markUploadAsInProcess(uploadRepository, file.taskId); }, - onUploadProgress: (file, progress) => { - const taskStatus = tasksService.findTask(file.taskId)?.status; + onItemProgress: (item, progress) => { + const { taskId } = getFile(item); + const taskStatus = tasksService.findTask(taskId)?.status; const isTaskPaused = taskStatus === TaskStatus.Paused; const isTaskCancelled = taskStatus === TaskStatus.Cancelled; if (!isTaskCancelled && !isTaskPaused) { - tasksService.updateTask({ taskId: file.taskId, merge: { progress } }); + tasksService.updateTask({ taskId, merge: { progress } }); } }, - onUploadSuccess: async (file, driveFileData) => { + onItemSucceeded: async (item, uploadedItem) => { + const driveFileData = uploadedItem as DriveFileData; + const file = getFile(item); + if (relatedTaskProgress && file.relatedTaskId) { relatedTaskProgress.filesUploaded += 1; - const isPaused = (await uploadRepository?.getUploadState(file.relatedTaskId)) === TaskStatus.Paused; + const folderUploadState = await uploadRepository.getUploadState(file.relatedTaskId); + const isPaused = folderUploadState === TaskStatus.Paused; if (!isPaused) { tasksService.updateTask({ taskId: file.relatedTaskId, @@ -104,35 +96,118 @@ export const uploadFilesWithTasks = async ( } } - tasksService.updateTask({ - taskId: file.taskId, - merge: { status: TaskStatus.Success, itemUUID: { fileUUID: driveFileData.uuid } }, - }); + if (tasksService.findTask(file.taskId)?.status !== TaskStatus.Cancelled) { + tasksService.updateTask({ + taskId: file.taskId, + merge: { status: TaskStatus.Success, itemUUID: { fileUUID: driveFileData.uuid } }, + }); + } + RetryManager.removeTask(file.taskId); referralService.trackFileUpload(); logNetworkInfoForUpload({ fileName: file.filecontent.name, fileSize: file.filecontent.size }); + file.onFinishUploadFile?.(driveFileData, file.taskId); + props.onFileUploadCallback?.(driveFileData); }, - onUploadError: (file, reason) => tasksService.updateTask({ taskId: file.taskId, merge: errorMerge(reason) }), - - onUploadAborted: (file) => - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.Cancelled } }), + onItemFailed: (item, { reason }) => { + const { taskId } = getFile(item); + tasksService.updateTask({ taskId, merge: errorMerge(reason) }); + forgetUploadState(uploadRepository, taskId); + }, - registerUploadAbort: (file, abort) => { - const listener = (task: TaskData) => { - if (task.id === file.taskId) abort(); - }; - tasksService.addListener({ event: TaskEvent.TaskCancelled, listener }); - return () => tasksService.removeListener({ event: TaskEvent.TaskCancelled, listener }); + onItemCancelled: (item) => { + tasksService.updateTask({ taskId: getFile(item).taskId, merge: { status: TaskStatus.Cancelled } }); }, }; - return uploadFileWithManager({ - ...managerProps, - files: filesWithTaskId, + const manager = new UploadManager({ + ownerUserAuthenticationData: options?.ownerUserAuthenticationData, uploadRepository, - abortController, - options: { ...options, isRetriedUpload }, + getPauseKey, events, }); + + const items = files.map((file): UploadItem => { + const item: UploadItem = { + kind: UploadItemKind.File, + file: { ...file.filecontent, type: file.fileType ?? file.filecontent.type }, + }; + fileByItem.set(item, { ...file, taskId: file.taskId ?? createUploadTask(file, item, manager, options) }); + file.abortController?.signal.addEventListener('abort', () => manager.cancelItem(item), { once: true }); + + if (file.taskId) { + tasksService.updateTask({ + taskId: file.taskId, + merge: { status: TaskStatus.Encrypting, stop: async () => manager.cancelItem(item) }, + }); + } + + return item; + }); + + if (abortController?.signal.aborted) manager.cancelAll(); + abortController?.signal.addEventListener('abort', () => manager.cancelAll(), { once: true }); + + const relatedTaskId = files[0]?.relatedTaskId; + if (relatedTaskId) { + const folderUploadState = await uploadRepository.getUploadState(relatedTaskId); + const isPaused = folderUploadState === TaskStatus.Paused; + + if (!isPaused) { + tasksService.updateTask({ taskId: relatedTaskId, merge: { status: TaskStatus.InProcess } }); + } + } + + const { failed } = await manager.run(items); + + if (failed.length > 0) { + RetryManager.addTasks( + failed.map(({ item, failure }) => { + const file = getFile(item); + + return { + taskId: file.taskId, + type: RetryableTaskType.Upload, + params: file, + retryable: failure.isRetryable, + }; + }), + ); + + const failureReasons = new Set(failed.map(({ failure }) => failure.reason)); + + if (failureReasons.has(UploadFailureReason.NoSpaceLeft)) props.maxSpaceOccupiedCallback(); + if (failureReasons.has(UploadFailureReason.FileTooLarge)) props.fileSizeExceededCallback?.(); + + throw failed[0].failure.error; + } +}; + +const createUploadTask = ( + file: UploadManagerFileParams, + item: UploadItem, + manager: UploadManager, + options: UploadFilesWithTasksProps['options'], +): string => { + const sharedItemAuthenticationData = options?.ownerUserAuthenticationData + ? { + ownerUserAuthenticationData: { ...options.ownerUserAuthenticationData }, + isDeepFolder: !!options?.sharedItemData?.isDeepFolder, + currentFolderId: options?.sharedItemData?.currentFolderId as string, + } + : undefined; + + return tasksService.create({ + relatedTaskId: file.relatedTaskId, + action: TaskType.UploadFile, + item: { uploadFile: file.filecontent.content, parentFolderId: file.parentFolderId }, + fileName: file.filecontent.name, + fileType: file.filecontent.type, + isFileNameValidated: true, + showNotification: options?.showNotifications ?? true, + cancellable: true, + sharedItemAuthenticationData, + stop: async () => manager.cancelItem(item), + }); }; diff --git a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts index 3c3594c9c6..88efeef923 100644 --- a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts @@ -7,6 +7,7 @@ import { uploadFoldersWithManager, UploadFolderManagerEvents } from 'app/network import { IRoot } from 'app/store/slices/storage/types'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; import { uploadFoldersWithTasks } from './uploadFoldersWithTasks'; +import { UploadFailureReason } from 'app/network/types'; vi.mock('app/network/UploadFolderManager', () => ({ uploadFoldersWithManager: vi.fn(() => Promise.resolve()), @@ -179,7 +180,7 @@ describe('uploadFoldersWithTasks', () => { (tasksService.create as Mock).mockReturnValue('task-1'); const { events } = await runUpload([{ root, currentFolderId: 'parent' }]); - events.onFolderUploadError?.('task-1', 'connection-lost'); + events.onFolderUploadError?.('task-1', UploadFailureReason.ConnectionLost); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 'task-1', @@ -189,7 +190,7 @@ describe('uploadFoldersWithTasks', () => { test('When a retried folder upload fails before starting, then its resumed task ends in error instead of staying in progress', async () => { (uploadFoldersWithManager as Mock).mockImplementationOnce(async ({ events }) => - events.onFolderUploadError('existing-task', 'connection-lost'), + events.onFolderUploadError('existing-task', UploadFailureReason.ConnectionLost), ); await runUpload([{ root, currentFolderId: 'parent', options: { taskId: 'existing-task' } }]); @@ -226,8 +227,8 @@ describe('uploadFoldersWithTasks', () => { }); test.each([ - { outcome: 'the connection is lost', reason: 'connection-lost' as const }, - { outcome: 'a fatal error', reason: 'upload-failed' as const }, + { outcome: 'the connection is lost', reason: UploadFailureReason.ConnectionLost }, + { outcome: 'a fatal error', reason: UploadFailureReason.UploadFailed }, ])( 'when the folder upload ends because $outcome, then only its files are removed from the retry list', async ({ reason }) => { diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index 493f054115..f62e82df00 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -1,19 +1,19 @@ import { t } from 'i18next'; import { TaskStatus } from 'app/tasks/types'; -import { UploadErrorReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/network/types'; -const errorSubtitle = (reason?: UploadErrorReason): string | undefined => { +const errorSubtitle = (reason?: UploadFailureReason): string | undefined => { switch (reason) { - case 'connection-lost': + case UploadFailureReason.ConnectionLost: return t('error.connectionLostError') as string; - case 'upload-failed': + case UploadFailureReason.UploadFailed: return t('tasks.subtitles.upload-failed') as string; default: return undefined; } }; -export const errorMerge = (reason?: UploadErrorReason) => { +export const errorMerge = (reason?: UploadFailureReason) => { const subtitle = errorSubtitle(reason); return subtitle !== undefined ? { status: TaskStatus.Error, subtitle } : { status: TaskStatus.Error }; }; From 038286c69db6d4141950b3674badefb6cba1606b Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 15:10:52 +0100 Subject: [PATCH 04/49] refactor(upload): remove the legacy upload manager --- src/app/network/UploadManager.test.ts | 1089 ------------------------- src/app/network/UploadManager.ts | 555 ------------- src/app/network/types/index.ts | 2 - 3 files changed, 1646 deletions(-) delete mode 100644 src/app/network/UploadManager.test.ts delete mode 100644 src/app/network/UploadManager.ts diff --git a/src/app/network/UploadManager.test.ts b/src/app/network/UploadManager.test.ts deleted file mode 100644 index b36e397264..0000000000 --- a/src/app/network/UploadManager.test.ts +++ /dev/null @@ -1,1089 +0,0 @@ -import { afterEach, beforeEach, describe, expect, Mock, MockInstance, test, vi } from 'vitest'; -import { UploadFileParamsWithTaskId, uploadFileWithManager, UploadManagerEvents } from './UploadManager'; -import errorService from 'services/error.service'; -import { AppError } from '@internxt/sdk'; -import uploadFile from 'app/drive/services/file.service/uploadFile'; -import DatabaseUploadRepository, { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; -import { DriveFileData } from 'app/drive/types'; -import RetryManager, { RetryableTaskType } from './RetryManager'; -import { TaskStatus } from 'app/tasks/types'; -import { ErrorMessages } from 'app/core/constants'; -import { ConnectionLostError } from './requests'; - -vi.mock('app/drive/services/file.service/uploadFile', () => ({ - default: vi.fn(() => Promise.resolve({} as DriveFileData)), -})); - -vi.mock('services/error.service', () => ({ - default: { - castError: vi.fn().mockImplementation((e) => e), - reportError: vi.fn(), - }, -})); - -vi.mock('app/repositories/DatabaseUploadRepository', () => { - const uploadState: Record = {}; - - return { - default: { - getInstance: vi.fn(() => ({ - setUploadState: vi.fn(async (id: string, status: TaskStatus) => { - uploadState[id] = status; - }), - getUploadState: vi.fn(async (id: string) => uploadState[id]), - removeUploadState: vi.fn(async (id: string) => { - delete uploadState[id]; - }), - })), - }, - }; -}); - -const openMaxSpaceOccupiedDialogMock = vi.fn(); - -let offlineSpy: MockInstance | undefined; -const goOffline = () => { - offlineSpy = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); -}; -afterEach(() => offlineSpy?.mockRestore()); - -const mockFile1 = { - id: 1, - uuid: 'file-uuid1', - name: 'File1', - bucket: 'bucket', - folderId: 0, - userId: 0, - isFile: true, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), -} as unknown as DriveFileData; -const mockFile2 = { - id: 2, - uuid: 'file-uuid2', - name: 'File2', - bucket: 'bucket', - folderId: 1, - userId: 0, - isFile: true, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), -} as unknown as DriveFileData; - -describe('UploadManager memory usage conditions', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - test('When memory usage cannot be measured, then the upload still proceeds', async () => { - const originalPerformance = window.performance; - Object.defineProperty(window, 'performance', { - value: undefined, - writable: true, - configurable: true, - }); - - const uploadFileSpy = (uploadFile as Mock).mockResolvedValueOnce(mockFile1); - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(uploadFileSpy).toHaveBeenCalledOnce(); - - Object.defineProperty(window, 'performance', { - value: originalPerformance, - writable: true, - configurable: true, - }); - }); - - test('When memory usage readings are incomplete, then the upload still proceeds', async () => { - const originalPerformance = window.performance; - Object.defineProperty(window, 'performance', { - value: { - memory: { - jsHeapSizeLimit: null, - usedJSHeapSize: null, - }, - }, - writable: true, - configurable: true, - }); - - const uploadFileSpy = (uploadFile as Mock).mockResolvedValueOnce(mockFile1); - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(uploadFileSpy).toHaveBeenCalledOnce(); - - Object.defineProperty(window, 'performance', { - value: originalPerformance, - writable: true, - configurable: true, - }); - }); -}); - -describe('uploadFileWithManager', () => { - beforeEach(() => { - RetryManager.clearTasks(); - vi.clearAllMocks(); - }); - - test('When a file upload succeeds, then the success is notified with the uploaded file data', async () => { - const uploadFileSpy = (uploadFile as Mock).mockResolvedValueOnce(mockFile1); - const events: UploadManagerEvents = { onUploadSuccess: vi.fn() }; - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - events, - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(uploadFileSpy).toHaveBeenCalledOnce(); - expect(events.onUploadSuccess).toHaveBeenCalledWith( - expect.objectContaining({ taskId: 'taskId' }), - expect.objectContaining({ uuid: mockFile1.uuid, name: 'file.txt' }), - ); - }); - - test('When several files are uploaded, then every file is transferred', async () => { - const uploadFileSpy = (uploadFile as Mock).mockResolvedValueOnce(mockFile1).mockResolvedValueOnce(mockFile2); - - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId1', - filecontent: { - content: 'file-content1' as unknown as File, - type: 'text/plain1', - name: 'file1.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - { - taskId: 'taskId2', - filecontent: { - content: 'file-content2' as unknown as File, - type: 'text/plain2', - name: 'file2.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(uploadFileSpy).toHaveBeenCalledTimes(2); - }); - - test('When the upload is cancelled midway, then it is notified as cancelled and not as a failure', async () => { - const abortController = new AbortController(); - (uploadFile as Mock).mockImplementation(() => { - abortController.abort(); - return Promise.reject(new Error('abort')); - }); - const events: UploadManagerEvents = { onUploadAborted: vi.fn(), onUploadError: vi.fn() }; - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - abortController, - events, - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(events.onUploadAborted).toHaveBeenCalledWith(expect.objectContaining({ taskId: 'taskId' })); - expect(events.onUploadError).not.toHaveBeenCalled(); - }); - - test('When an upload succeeds, then it is no longer scheduled for retry', async () => { - (uploadFile as Mock).mockResolvedValueOnce(mockFile1); - vi.spyOn(Promise, 'all').mockResolvedValueOnce([mockFile1]); - const RetryAddFilesSpy = vi.spyOn(RetryManager, 'addTasks'); - const RetryRemoveFileSpy = vi.spyOn(RetryManager, 'removeTask'); - - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(RetryAddFilesSpy).not.toHaveBeenCalled(); - expect(RetryRemoveFileSpy).toHaveBeenCalledWith('taskId'); - }); - - test('When an upload keeps failing, then it is scheduled for retry', async () => { - //needs to fail twice because MAX_UPLOAD_ATTEMPTS = 2 - (uploadFile as Mock) - .mockResolvedValueOnce(mockFile1) - .mockRejectedValueOnce(new AppError('Retryable file')) - .mockRejectedValueOnce(new AppError('Retryable file')); - vi.spyOn(Promise, 'all').mockResolvedValueOnce([mockFile1, undefined]); - const RetryAddFilesSpy = vi.spyOn(RetryManager, 'addTasks'); - - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - { - taskId: 'taskId2', - filecontent: { - content: 'file-content2' as unknown as File, - type: 'text/plain2', - name: 'file2.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(RetryAddFilesSpy).toHaveBeenCalled(); - expect(RetryManager.getTasks()).toHaveLength(1); - }); - - test('When a retried upload fails again, then it is marked as failed', async () => { - //needs to fail twice because MAX_UPLOAD_ATTEMPTS = 2 - (uploadFile as Mock) - .mockRejectedValueOnce(new AppError('Retryable file')) - .mockRejectedValueOnce(new AppError('Retryable file')); - vi.spyOn(Promise, 'all').mockResolvedValueOnce([undefined]); - const RetryChangeStatusSpy = vi.spyOn(RetryManager, 'changeStatus'); - - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(RetryChangeStatusSpy).toHaveBeenCalledWith('taskId', 'failed'); - }); - - test('When the connection is lost during an upload, then the failure is notified as a connection loss', async () => { - goOffline(); - const lostConnectionError = new AppError(ErrorMessages.NetworkError); - (uploadFile as Mock).mockRejectedValueOnce(lostConnectionError); - - const events: UploadManagerEvents = { onUploadError: vi.fn() }; - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - vi.spyOn(errorService, 'reportError').mockReturnValue(); - - await expect( - uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - events, - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }), - ).rejects.toThrow(lostConnectionError); - - expect(errorService.reportError).toHaveBeenCalledWith(lostConnectionError); - expect(events.onUploadError).toHaveBeenCalledWith(expect.objectContaining({ taskId: 'taskId' }), 'connection-lost'); - }); - - test('When an upload fails unexpectedly, then the failure is notified and no success is reported', async () => { - const unexpectedError = new AppError(ErrorMessages.ServerUnavailable); - (uploadFile as Mock).mockRejectedValue(unexpectedError); - - const events: UploadManagerEvents = { onUploadError: vi.fn(), onUploadSuccess: vi.fn() }; - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - const errorServiceSpy = vi.spyOn(errorService, 'reportError').mockReturnValue(); - - await expect( - uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - events, - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }), - ).rejects.toThrow(unexpectedError); - - expect(errorServiceSpy).toHaveBeenCalledWith(unexpectedError); - expect(events.onUploadError).toHaveBeenCalledWith(expect.objectContaining({ taskId: 'taskId' }), 'upload-failed'); - expect(events.onUploadSuccess).not.toHaveBeenCalled(); - }); - - test('When uploading a file to a workspace, then the workspace credentials are used', async () => { - const workspaceBucket = 'workspace-bucket-123'; - const workspaceId = 'workspace-id-456'; - const uploadFileSpy = (uploadFile as Mock).mockResolvedValueOnce(mockFile1); - - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: 'user@test.com', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: { - bucketId: workspaceBucket, - workspaceId: workspaceId, - bridgeUser: 'bridge-user', - bridgePass: 'bridge-pass', - encryptionKey: 'encryption-key', - token: 'token', - workspacesToken: 'workspaces-token', - resourcesToken: 'resources-token', - }, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(uploadFileSpy).toHaveBeenCalledWith( - expect.any(Object), - expect.any(Function), - expect.objectContaining({ - isTeam: true, - ownerUserAuthenticationData: expect.objectContaining({ - bucketId: workspaceBucket, - workspaceId: workspaceId, - }), - }), - expect.any(Object), - ); - }); - - test('When uploading a personal file, then the personal credentials are used', async () => { - const uploadFileSpy = (uploadFile as Mock).mockResolvedValueOnce(mockFile1); - - vi.spyOn(errorService, 'castError').mockReturnValue(new AppError('error')); - - await uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: 'user@test.com', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }); - - expect(uploadFileSpy).toHaveBeenCalledWith( - expect.any(Object), - expect.any(Function), - expect.objectContaining({ - isTeam: false, - ownerUserAuthenticationData: undefined, - }), - expect.any(Object), - ); - }); - - test('When a file of a folder upload fails, then the remaining files still upload and the failure is reported at the end', async () => { - const err = new AppError(ErrorMessages.ServerUnavailable); - (uploadFile as Mock).mockImplementation((file: { name: string }) => - file.name === 'failing.txt' ? Promise.reject(err) : Promise.resolve(mockFile2), - ); - - const events: UploadManagerEvents = { onUploadSuccess: vi.fn(), onUploadError: vi.fn() }; - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - vi.spyOn(errorService, 'reportError').mockReturnValue(); - - await expect( - uploadFileWithManager({ - files: [ - { - taskId: 'failing-task', - relatedTaskId: 'folder-task', - filecontent: { - content: 'file-content1' as unknown as File, - type: 'text/plain', - name: 'failing.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - { - taskId: 'succeeding-task', - relatedTaskId: 'folder-task', - filecontent: { - content: 'file-content2' as unknown as File, - type: 'text/plain', - name: 'file2.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - events, - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }), - ).rejects.toThrow(err); - - expect(events.onUploadError).toHaveBeenCalledWith( - expect.objectContaining({ taskId: 'failing-task' }), - 'upload-failed', - ); - expect(events.onUploadSuccess).toHaveBeenCalledWith( - expect.objectContaining({ taskId: 'succeeding-task' }), - expect.objectContaining({ uuid: mockFile2.uuid }), - ); - - const retryTasks = RetryManager.getTasks(); - expect(retryTasks).toHaveLength(1); - expect(retryTasks[0]).toMatchObject({ taskId: 'failing-task', type: 'upload', retryable: true }); - }); - - test('When a retried upload fails again, then its retry entry is kept as-is instead of being duplicated', async () => { - const err = new AppError(ErrorMessages.ServerUnavailable); - (uploadFile as Mock).mockRejectedValue(err); - - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - vi.spyOn(errorService, 'reportError').mockReturnValue(); - - const fileParams = { - taskId: 'retried-task', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }; - RetryManager.addTasks([ - { taskId: 'retried-task', type: RetryableTaskType.Upload, params: fileParams, retryable: true }, - ]); - - await expect( - uploadFileWithManager({ - files: [fileParams], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - isRetriedUpload: true, - }, - }), - ).rejects.toThrow(err); - - const retryTasks = RetryManager.getTasks(); - expect(retryTasks).toHaveLength(1); - expect(retryTasks[0]).toMatchObject({ taskId: 'retried-task', retryable: true }); - }); - - test('When a file of a folder upload fails with a non-retryable error, then it is not retried and cannot be retried later', async () => { - const paymentRequiredError = new AppError('Payment required', 402); - (uploadFile as Mock).mockRejectedValue(paymentRequiredError); - - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - vi.spyOn(errorService, 'reportError').mockReturnValue(); - - await expect( - uploadFileWithManager({ - files: [ - { - taskId: 'not-allowed-task', - relatedTaskId: 'folder-task', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }), - ).rejects.toThrow(paymentRequiredError); - - expect(uploadFile).toHaveBeenCalledOnce(); - const retryTasks = RetryManager.getTasks(); - expect(retryTasks).toHaveLength(1); - expect(retryTasks[0]).toMatchObject({ taskId: 'not-allowed-task', retryable: false }); - }); - - test('When the user has no storage space left, then they are warned about it', async () => { - const err = new AppError('Max space used', 420); - // needs to fail twice because MAX_UPLOAD_ATTEMPTS = 2 - (uploadFile as Mock).mockRejectedValue(err); - - vi.spyOn(errorService, 'castError').mockImplementation((e) => e as AppError); - vi.spyOn(errorService, 'reportError').mockReturnValue(); - - await expect( - uploadFileWithManager({ - files: [ - { - taskId: 'taskId', - filecontent: { - content: 'file-content' as unknown as File, - type: 'text/plain', - name: 'file.txt', - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }, - ], - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: DatabaseUploadRepository.getInstance(), - options: { - ownerUserAuthenticationData: undefined, - sharedItemData: { - isDeepFolder: false, - currentFolderId: 'parentFolderId', - }, - isUploadedFromFolder: true, - }, - }), - ).rejects.toThrow(err); - - expect(openMaxSpaceOccupiedDialogMock).toHaveBeenCalledOnce(); - }); -}); - -describe('UploadManager settles every queued file', () => { - const HANG_TIMEOUT_MS = 2000; - const uploadStateError = new Error('IDB failure'); - - const failIfHangs = (promise: Promise): Promise => - Promise.race([ - promise, - new Promise((_, reject) => setTimeout(() => reject(new Error('upload never settled')), HANG_TIMEOUT_MS)), - ]); - - const buildFile = (name: string): UploadFileParamsWithTaskId => ({ - taskId: `${name}-task`, - filecontent: { - content: name as unknown as File, - type: 'text/plain', - name, - size: 1024, - parentFolderId: 'folder-1', - }, - userEmail: '', - parentFolderId: '', - }); - - const buildRepository = (failingUploadStateCalls: Record = {}): PersistUploadRepository => { - const callsByTaskId: Record = {}; - return { - setUploadState: vi.fn(async () => undefined), - removeUploadState: vi.fn(async () => undefined), - getUploadState: vi.fn(async (id: string) => { - callsByTaskId[id] = (callsByTaskId[id] ?? 0) + 1; - if (failingUploadStateCalls[id]?.includes(callsByTaskId[id])) throw uploadStateError; - return undefined; - }), - }; - }; - - const runUpload = ({ - files, - uploadRepository = buildRepository(), - events, - isUploadedFromFolder = false, - abortController, - }: { - files: UploadFileParamsWithTaskId[]; - uploadRepository?: PersistUploadRepository; - events?: UploadManagerEvents; - isUploadedFromFolder?: boolean; - abortController?: AbortController; - }) => - failIfHangs( - uploadFileWithManager({ - files, - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository, - events, - abortController, - options: { isUploadedFromFolder }, - }), - ); - - const uploadedNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); - - const failUploadOf = (failingFile: UploadFileParamsWithTaskId, error: Error) => - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - if (file.name === failingFile.filecontent.name) throw error; - await new Promise((resolve) => setTimeout(resolve, 10)); - return { ...mockFile1, name: file.name }; - }); - - beforeEach(() => { - RetryManager.clearTasks(); - vi.clearAllMocks(); - vi.spyOn(errorService, 'castError').mockImplementation((e) => (e ?? new AppError('Unknown error')) as AppError); - vi.spyOn(errorService, 'reportError').mockReturnValue(); - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ ...mockFile1, name: file.name })); - }); - - test('when reading the upload state keeps failing, then the upload settles and the file is reported as failed', async () => { - const file = buildFile('file.txt'); - const events: UploadManagerEvents = { onUploadError: vi.fn() }; - const uploadRepository = buildRepository({ [file.taskId]: [1, 2] }); - - await expect(runUpload({ files: [file], uploadRepository, events })).rejects.toThrow(uploadStateError); - - expect(uploadFile).not.toHaveBeenCalled(); - expect(events.onUploadError).toHaveBeenCalledWith( - expect.objectContaining({ taskId: file.taskId }), - 'upload-failed', - ); - }); - - test('when reading the upload state fails on the retry after a 502, then the folder upload settles with the file failed and the others uploaded', async () => { - const failingFile = buildFile('failing.txt'); - const otherFiles = [buildFile('a.txt'), buildFile('b.txt')]; - (uploadFile as Mock).mockImplementationOnce(() => Promise.reject({ status: 502 })); - const uploadRepository = buildRepository({ [failingFile.taskId]: [2] }); - const events: UploadManagerEvents = { onUploadSuccess: vi.fn(), onUploadError: vi.fn() }; - - await expect( - runUpload({ files: [failingFile, ...otherFiles], uploadRepository, events, isUploadedFromFolder: true }), - ).rejects.toThrow(uploadStateError); - - expect(events.onUploadSuccess).toHaveBeenCalledTimes(otherFiles.length); - expect(events.onUploadError).toHaveBeenCalledWith( - expect.objectContaining({ taskId: failingFile.taskId }), - 'upload-failed', - ); - expect(RetryManager.getTasks()).toEqual([expect.objectContaining({ taskId: failingFile.taskId, retryable: true })]); - }); - - test('when the upload rejects without an error value, then the upload settles and the file is reported as failed', async () => { - (uploadFile as Mock).mockRejectedValue(undefined); - const events: UploadManagerEvents = { onUploadError: vi.fn() }; - - await expect(runUpload({ files: [buildFile('file.txt')], events })).rejects.toThrow('Unknown error'); - - expect(events.onUploadError).toHaveBeenCalledWith(expect.anything(), 'upload-failed'); - }); - - test('when a success listener throws, then every file is uploaded once and the queue keeps processing', async () => { - const files = ['a.txt', 'b.txt', 'c.txt'].map(buildFile); - const onFileUploadCallback = vi.fn(); - const events: UploadManagerEvents = { - onUploadSuccess: vi.fn((file) => { - if (file.taskId === files[0].taskId) throw new Error('listener failure'); - }), - }; - - const { uploadedFiles } = await failIfHangs( - uploadFileWithManager({ - files, - maxSpaceOccupiedCallback: openMaxSpaceOccupiedDialogMock, - uploadRepository: buildRepository(), - events, - onFileUploadCallback, - }), - ); - - expect(uploadedNames()).toEqual(['a.txt', 'b.txt', 'c.txt']); - expect(uploadedFiles.map(({ name }) => name)).toEqual(['a.txt', 'b.txt', 'c.txt']); - expect(onFileUploadCallback).toHaveBeenCalledTimes(files.length); - }); - - test('when a listener throws while the next queued file starts, then finished files are not uploaded again and the folder upload settles', async () => { - const files = Array.from({ length: 7 }, (_, i) => buildFile(`f${i}.txt`)); - const startingFile = files[6]; - const listenerError = new Error('listener failure'); - const events: UploadManagerEvents = { - onUploadStart: vi.fn((file) => { - if (file.taskId === startingFile.taskId) throw listenerError; - }), - onUploadSuccess: vi.fn(), - }; - - await expect(runUpload({ files, events, isUploadedFromFolder: true })).rejects.toThrow(listenerError); - - expect(uploadedNames()).toEqual(files.slice(0, 6).map((file) => file.filecontent.name)); - expect(events.onUploadSuccess).toHaveBeenCalledTimes(6); - expect(RetryManager.getTasks()).toEqual([expect.objectContaining({ taskId: startingFile.taskId })]); - }); - - describe('when the failure listener throws', () => { - const listenerError = new Error('listener failure'); - const failingFile = buildFile('failing.txt'); - const events: UploadManagerEvents = { - onUploadSuccess: vi.fn(), - onUploadError: vi.fn(() => { - throw listenerError; - }), - }; - - beforeEach(() => { - failUploadOf(failingFile, new AppError('upload failure')); - }); - - test('when it throws for a file of a folder upload, then the other files are uploaded and the failed file is listed for retry', async () => { - const otherFiles = [buildFile('a.txt'), buildFile('b.txt')]; - - await expect( - runUpload({ files: [failingFile, ...otherFiles], events, isUploadedFromFolder: true }), - ).rejects.toThrow(listenerError); - - expect(events.onUploadSuccess).toHaveBeenCalledTimes(otherFiles.length); - expect(RetryManager.getTasks()).toEqual([expect.objectContaining({ taskId: failingFile.taskId })]); - }); - - test('when it throws outside a folder upload, then the upload rejects with that error', async () => { - await expect(runUpload({ files: [failingFile], events })).rejects.toThrow(listenerError); - }); - }); - - test('when an upload with more files than the concurrency is cancelled, then it settles and the queued files are never uploaded', async () => { - const files = Array.from({ length: 7 }, (_, i) => buildFile(`f${i}.txt`)); - const abortController = new AbortController(); - (uploadFile as Mock).mockImplementation(async () => { - abortController.abort(); - throw new Error('abort'); - }); - - await runUpload({ files, abortController }); - - expect(uploadedNames()).toEqual(files.slice(0, 6).map((file) => file.filecontent.name)); - }); - - describe('when the network fails', () => { - const networkError = new AppError(ErrorMessages.NetworkError); - const connectionLostError = new ConnectionLostError(); - - test.each([ - { failure: 'a network error', error: networkError }, - { failure: 'a lost connection', error: connectionLostError }, - ])( - 'when a file of a folder upload fails with $failure, then the other files finish first and the failed file is listed for retry', - async ({ error }) => { - const failingFile = buildFile('failing.txt'); - const otherFiles = [buildFile('a.txt'), buildFile('b.txt')]; - failUploadOf(failingFile, error); - const events: UploadManagerEvents = { onUploadSuccess: vi.fn() }; - - await expect( - runUpload({ files: [failingFile, ...otherFiles], events, isUploadedFromFolder: true }), - ).rejects.toThrow(error); - - expect(events.onUploadSuccess).toHaveBeenCalledTimes(otherFiles.length); - expect(RetryManager.getTasks()).toEqual([ - expect.objectContaining({ taskId: failingFile.taskId, retryable: true }), - ]); - }, - ); - - test('when a network error happens while the browser is online, then the upload is retried and succeeds', async () => { - const file = buildFile('file.txt'); - (uploadFile as Mock).mockRejectedValueOnce(networkError); - - const { uploadedFiles } = await runUpload({ files: [file] }); - - expect(uploadFile).toHaveBeenCalledTimes(2); - expect(uploadedFiles).toEqual([expect.objectContaining({ name: 'file.txt' })]); - }); - - test.each([ - { failure: 'the connection is lost', error: connectionLostError, arrange: () => undefined }, - { failure: 'a network error happens while the browser is offline', error: networkError, arrange: goOffline }, - ])( - 'when $failure, then the upload is not retried and is reported as a connection loss', - async ({ error, arrange }) => { - arrange(); - const file = buildFile('file.txt'); - (uploadFile as Mock).mockRejectedValue(error); - const events: UploadManagerEvents = { onUploadError: vi.fn() }; - - await expect(runUpload({ files: [file], events })).rejects.toThrow(error); - - expect(uploadFile).toHaveBeenCalledTimes(1); - expect(events.onUploadError).toHaveBeenCalledWith( - expect.objectContaining({ taskId: file.taskId }), - 'connection-lost', - ); - }, - ); - }); -}); diff --git a/src/app/network/UploadManager.ts b/src/app/network/UploadManager.ts deleted file mode 100644 index df782b74d0..0000000000 --- a/src/app/network/UploadManager.ts +++ /dev/null @@ -1,555 +0,0 @@ -import { queue, QueueObject } from 'async'; -import { randomBytes } from 'node:crypto'; -import errorService from 'services/error.service'; -import { HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from '../core/constants'; -import uploadFile from 'app/drive/services/file.service/uploadFile'; -import { DriveFileData } from 'app/drive/types'; -import { PersistUploadRepository } from '../repositories/DatabaseUploadRepository'; -import { TaskStatus } from '../tasks/types'; -import { ConnectionLostError } from './requests'; -import { FileToUpload } from 'app/drive/services/file.service/types'; -import RetryManager, { RetryableTask, RetryableTaskStatus, RetryableTaskType } from './RetryManager'; -import { ErrorMessages } from 'app/core/constants'; -import { MAX_UPLOAD_ATTEMPTS, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from './networkConstants'; -import { OwnerUserAuthenticationData, UploadErrorReason } from './types'; - -enum FileSizeType { - Big = 'big', - Medium = 'medium', - Small = 'small', -} - -type Options = { - isRetriedUpload?: boolean; - showNotifications?: boolean; - showErrors?: boolean; - ownerUserAuthenticationData?: OwnerUserAuthenticationData; - sharedItemData?: { - isDeepFolder?: boolean; - currentFolderId?: string; - }; - isUploadedFromFolder?: boolean; -}; - -export type UploadManagerFileParams = { - filecontent: FileToUpload; - taskId?: string; - relatedTaskId?: string; - fileType?: string; - userEmail: string; - parentFolderId: string; - onFinishUploadFile?: (driveItemData: DriveFileData, taskId: string) => void; - abortController?: AbortController; - isUploadedFromFolder?: boolean; -}; - -export type UploadFileParamsWithTaskId = UploadManagerFileParams & { taskId: string }; - -/** - * Upload progress notifications emitted by the manager. The manager knows nothing - * about how uploads are presented to the user; observers subscribe to these events. - */ -export interface UploadManagerEvents { - onUploadStart?: (file: UploadFileParamsWithTaskId, stopUpload: () => Promise) => void; - onUploadAttempt?: (file: UploadFileParamsWithTaskId) => void; - onUploadProgress?: (file: UploadFileParamsWithTaskId, progress: number) => void; - onUploadSuccess?: (file: UploadFileParamsWithTaskId, driveFileData: DriveFileData) => void | Promise; - onUploadError?: (file: UploadFileParamsWithTaskId, reason?: UploadErrorReason) => void; - onUploadAborted?: (file: UploadFileParamsWithTaskId) => void; - registerUploadAbort?: (file: UploadFileParamsWithTaskId, abort: () => void) => (() => void) | void; -} - -interface UploadFileWithManagerProps { - files: UploadFileParamsWithTaskId[]; - maxSpaceOccupiedCallback: () => void; - uploadRepository: PersistUploadRepository; - abortController?: AbortController; - options?: Options; - onFileUploadCallback?: (driveFileData: DriveFileData) => void; - fileSizeExceededCallback?: () => void; - events?: UploadManagerEvents; -} - -type UploadQueueCallback = (err: Error | null, res?: DriveFileData) => void; - -export const uploadFileWithManager = (props: UploadFileWithManagerProps): Promise<{ uploadedFiles: DriveFileData[] }> => - new UploadManager(props).run(); - -class UploadManager { - private currentGroupBeingUploaded: FileSizeType = FileSizeType.Small; - private hasErrored = false; - private hasSomeUploadFailed = false; - private firstUploadError?: Error; - private readonly nonRetryableTaskIds = new Set(); - private uploadsProgress: Record = {}; - private readonly abortController?: AbortController; - private readonly items: UploadFileParamsWithTaskId[]; - private readonly options?: Options; - private readonly maxSpaceOccupiedCallback: () => void; - private readonly fileSizeExceededCallback?: () => void; - private readonly onFileUploadCallback?: (driveFileData: DriveFileData) => void; - private readonly uploadRepository?: PersistUploadRepository; - private readonly events?: UploadManagerEvents; - private readonly filesUploadedList: (DriveFileData & { taskId: string })[] = []; - private readonly filesGroups: Record< - FileSizeType, - { - upperBound: number; - lowerBound: number; - concurrency: number; - } - > = { - [FileSizeType.Big]: { - upperBound: Infinity, - lowerBound: USE_MULTIPART_THRESHOLD_BYTES, - concurrency: 1, - }, - [FileSizeType.Medium]: { - upperBound: USE_MULTIPART_THRESHOLD_BYTES - 1, - lowerBound: TWENTY_MEGABYTES, - concurrency: 6, - }, - [FileSizeType.Small]: { - upperBound: TWENTY_MEGABYTES - 1, - lowerBound: 0, - concurrency: 6, - }, - }; - - private readonly uploadQueue: QueueObject = queue( - (fileData, next: UploadQueueCallback) => { - const isUploadCancelled = this.abortController?.signal.aborted ?? fileData.abortController?.signal.aborted; - if (isUploadCancelled) { - next(null); - return; - } - - const nextOnce = UploadManager.callOnce(next); - this.processFile(fileData, nextOnce).catch((error) => { - errorService.reportError(error); - const castedError = errorService.castError(error); - if (this.shouldContinueWithRemainingFiles(error)) this.continueWithRemainingFiles(castedError, nextOnce); - else nextOnce(castedError); - }); - }, - this.filesGroups.small.concurrency, - ); - - constructor(props: UploadFileWithManagerProps) { - this.items = props.files; - this.abortController = props.abortController; - this.options = props.options; - this.maxSpaceOccupiedCallback = props.maxSpaceOccupiedCallback; - this.fileSizeExceededCallback = props.fileSizeExceededCallback; - this.uploadRepository = props.uploadRepository; - this.onFileUploadCallback = props.onFileUploadCallback; - this.events = props.events; - } - - private static callOnce(next: UploadQueueCallback): UploadQueueCallback { - let hasBeenCalled = false; - return (err, res) => { - if (hasBeenCalled) return; - hasBeenCalled = true; - next(err, res); - }; - } - - private async processFile(fileData: UploadFileParamsWithTaskId, next: UploadQueueCallback): Promise { - this.manageMemoryUsage(); - - const uploadId = randomBytes(10).toString('hex'); - this.uploadsProgress[uploadId] = 0; - - let driveFileData: DriveFileData; - try { - this.events?.onUploadStart?.(fileData, async () => { - if (this.abortController) this.abortController?.abort(); - else fileData?.abortController?.abort(); - }); - - const existsRelatedTask = !!fileData.relatedTaskId; - if (!existsRelatedTask) { - this.uploadRepository - ?.setUploadState(fileData.taskId, TaskStatus.InProcess) - .catch((error) => errorService.reportError(error)); - } - - driveFileData = await this.uploadWithRetries(fileData, uploadId); - } catch (error) { - this.handleUploadErrors({ error, fileData, next }); - return; - } - - const driveFileDataWithNameParsed = { ...driveFileData, name: fileData.filecontent.name }; - await this.notifyUploadSuccess(fileData, driveFileDataWithNameParsed); - next(null, driveFileDataWithNameParsed); - } - - private async uploadWithRetries( - fileData: UploadFileParamsWithTaskId, - uploadId: string, - uploadAttempts = 1, - ): Promise { - try { - return await this.uploadAttempt(fileData, uploadId); - } catch (error) { - const { isUploadAborted, isLostConnectionError } = this.classifyUploadError(error, fileData); - const shouldRetry = - uploadAttempts < MAX_UPLOAD_ATTEMPTS && - !isUploadAborted && - !isLostConnectionError && - !UploadManager.isNonRetryableError(error); - - if (!shouldRetry) throw error; - return this.uploadWithRetries(fileData, uploadId, uploadAttempts + 1); - } - } - - private async uploadAttempt(fileData: UploadFileParamsWithTaskId, uploadId: string): Promise { - const file = fileData.filecontent; - const taskId = fileData.relatedTaskId ?? fileData.taskId; - const abortListener: { unsubscribe?: (() => void) | void } = {}; - - try { - this.events?.onUploadAttempt?.(fileData); - - const isPaused = (await this.uploadRepository?.getUploadState(taskId)) === TaskStatus.Paused; - const continueUploadOptions = { - taskId, - isPaused, - isRetriedUpload: !!this.options?.isRetriedUpload, - }; - - const driveFileData = await uploadFile( - { - name: file.name, - size: file.size, - type: fileData.fileType ?? file.type, - content: file.content, - parentFolderId: file.parentFolderId, - }, - (uploadProgress) => { - this.uploadsProgress[uploadId] = uploadProgress; - this.events?.onUploadProgress?.(fileData, uploadProgress); - }, - { - isTeam: !!this.options?.ownerUserAuthenticationData?.workspaceId, - abortController: this.abortController ?? fileData.abortController, - ownerUserAuthenticationData: this.options?.ownerUserAuthenticationData, - abortCallback: (abort?: () => void) => { - abortListener.unsubscribe = this.events?.registerUploadAbort?.(fileData, () => abort?.()); - }, - isUploadedFromFolder: fileData.isUploadedFromFolder, - }, - continueUploadOptions, - ); - - const isUploadAborted = this.abortController?.signal.aborted ?? fileData.abortController?.signal.aborted; - if (isUploadAborted) { - throw new Error('Upload task cancelled'); - } - - return driveFileData; - } finally { - abortListener.unsubscribe?.(); - } - } - - private async notifyUploadSuccess(fileData: UploadFileParamsWithTaskId, driveFileData: DriveFileData): Promise { - this.filesUploadedList.push({ ...driveFileData, taskId: fileData.taskId }); - - await UploadManager.runListener(() => this.events?.onUploadSuccess?.(fileData, driveFileData)); - await UploadManager.runListener(() => fileData.onFinishUploadFile?.(driveFileData, fileData.taskId)); - await UploadManager.runListener(() => this.onFileUploadCallback?.(driveFileData)); - } - - private static async runListener(listener: () => unknown): Promise { - try { - await listener(); - } catch (error) { - errorService.reportError(error); - } - } - - private classifyUploadError( - error: unknown, - fileData: UploadFileParamsWithTaskId, - ): { isUploadAborted: boolean; isLostConnectionError: boolean } { - const isUploadAborted = - !!this.abortController?.signal.aborted || !!fileData.abortController?.signal.aborted || error === 'abort'; - const isNetworkErrorWhileOffline = - (error as Error | undefined)?.message === ErrorMessages.NetworkError && !navigator.onLine; - const isLostConnectionError = error instanceof ConnectionLostError || isNetworkErrorWhileOffline; - return { isUploadAborted, isLostConnectionError }; - } - - private static isNonRetryableError(error: unknown): boolean { - return ( - (error as { status?: number })?.status === HTTP_STATUS_CODES.PAYMENT_REQUIRED || - errorService.castError(error).code === HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED - ); - } - - private applyUnexpectedUploadError({ - fileData, - castedError, - isMaxSpaceError, - shouldContinueWithRemainingFiles, - }: { - fileData: UploadFileParamsWithTaskId; - castedError: ReturnType; - isMaxSpaceError: boolean; - shouldContinueWithRemainingFiles: boolean; - }): void { - this.events?.onUploadError?.(fileData, 'upload-failed'); - errorService.reportError(castedError); - - if (castedError.code === HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED) { - this.fileSizeExceededCallback?.(); - } - - if (isMaxSpaceError) { - this.maxSpaceOccupiedCallback(); - } - - if (!shouldContinueWithRemainingFiles) { - if (!this.hasErrored) { - this.uploadQueue.kill(); - } - this.hasErrored = true; - } - } - - private handleLostConnectionError({ - fileData, - castedError, - shouldContinueWithRemainingFiles, - next, - }: { - fileData: UploadFileParamsWithTaskId; - castedError: ReturnType; - shouldContinueWithRemainingFiles: boolean; - next: UploadQueueCallback; - }): void { - errorService.reportError(castedError); - this.events?.onUploadError?.(fileData, 'connection-lost'); - - if (shouldContinueWithRemainingFiles) { - this.continueWithRemainingFiles(castedError, next); - return; - } - - this.hasErrored = true; - next(castedError); - } - - private shouldContinueWithRemainingFiles(error: unknown): boolean { - return ( - !!this.options?.isUploadedFromFolder && !this.options?.isRetriedUpload && !UploadManager.isMaxSpaceError(error) - ); - } - - private static isMaxSpaceError(error: unknown): boolean { - return (error as { status?: number })?.status === HTTP_STATUS_CODES.MAX_SPACE_USED; - } - - private continueWithRemainingFiles(castedError: Error, next: UploadQueueCallback): void { - this.hasSomeUploadFailed = true; - this.firstUploadError ??= castedError; - next(null); - } - - private handleUploadErrors({ - error, - fileData, - next, - }: { - error: unknown; - fileData: UploadFileParamsWithTaskId; - next: UploadQueueCallback; - }) { - const { isUploadAborted, isLostConnectionError } = this.classifyUploadError(error, fileData); - const castedError = errorService.castError(error); - // Handle retry error - if (castedError.message === 'Retryable file') { - next(null); - return; - } - - const isMaxSpaceError = UploadManager.isMaxSpaceError(error); - const shouldContinueWithRemainingFiles = this.shouldContinueWithRemainingFiles(error); - - if (isLostConnectionError) { - this.handleLostConnectionError({ fileData, castedError, shouldContinueWithRemainingFiles, next }); - return; - } - - if (UploadManager.isNonRetryableError(error)) { - this.nonRetryableTaskIds.add(fileData.taskId); - } - - // Handle unexpected errors - if (!isUploadAborted) { - this.applyUnexpectedUploadError({ fileData, castedError, isMaxSpaceError, shouldContinueWithRemainingFiles }); - } - - // Handle upload aborted - if (isUploadAborted) { - this.events?.onUploadAborted?.(fileData); - next(null); - return; - } - - if (shouldContinueWithRemainingFiles) { - this.continueWithRemainingFiles(castedError, next); - return; - } - - // If relatedTaskId is present, kill upload queue - if (fileData.relatedTaskId) { - this.uploadQueue.kill(); - } - - next(castedError); - } - - private getMemoryUsagePercentage(): number | null { - if (!window?.performance?.memory) { - return null; - } - - const memory = window.performance.memory; - if (!memory?.jsHeapSizeLimit || !memory?.usedJSHeapSize) { - return null; - } - - return memory.usedJSHeapSize / memory.jsHeapSizeLimit; - } - - private handleConcurrencyIncrease(memoryUsagePercentage: number): void { - const shouldIncrease = memoryUsagePercentage < 0.7 && this.currentGroupBeingUploaded !== FileSizeType.Big; - - if (!shouldIncrease) return; - - const newConcurrency = Math.min(this.uploadQueue.concurrency + 1, this.filesGroups[FileSizeType.Small].concurrency); - - if (newConcurrency !== this.uploadQueue.concurrency) { - console.warn(`Memory usage under 70%. Increasing upload concurrency to ${newConcurrency}`); - this.uploadQueue.concurrency = newConcurrency; - } - } - - private handleConcurrencyReduction(memoryUsagePercentage: number): void { - const shouldReduce = memoryUsagePercentage >= 0.8 && this.uploadQueue.concurrency > 1; - - if (shouldReduce) { - console.warn('Memory usage reached 80%. Reducing upload concurrency.'); - this.uploadQueue.concurrency = 1; - } - } - - private manageMemoryUsage() { - const memoryUsagePercentage = this.getMemoryUsagePercentage(); - console.log('Current memory usage percentage:', memoryUsagePercentage); - - if (memoryUsagePercentage === null) { - console.warn('Memory usage control is not available'); - return; - } - - this.handleConcurrencyIncrease(memoryUsagePercentage); - this.handleConcurrencyReduction(memoryUsagePercentage); - } - - private classifyFilesBySize( - files: UploadFileParamsWithTaskId[], - ): [UploadFileParamsWithTaskId[], UploadFileParamsWithTaskId[], UploadFileParamsWithTaskId[]] { - return [ - files.filter((f) => f.filecontent.size >= this.filesGroups.big.lowerBound), - files.filter( - (f) => - f.filecontent.size >= this.filesGroups.medium.lowerBound && - f.filecontent.size <= this.filesGroups.medium.upperBound, - ), - files.filter( - (f) => - f.filecontent.size >= this.filesGroups.small.lowerBound && - f.filecontent.size <= this.filesGroups.small.upperBound, - ), - ]; - } - - private handleFailedUploads(files: UploadFileParamsWithTaskId[]): void { - const filesUploadedTaskId = new Set(this.filesUploadedList.map((fileUploaded) => fileUploaded.taskId)); - const failedUploadFiles = files.filter((file) => !filesUploadedTaskId.has(file.taskId)); - - failedUploadFiles.forEach((fileErrored) => { - this.events?.onUploadError?.(fileErrored); - this.uploadRepository?.removeUploadState(fileErrored.taskId).catch((error) => errorService.reportError(error)); - }); - } - - async run(): Promise<{ uploadedFiles: DriveFileData[] }> { - try { - this.items.forEach((item) => (item.isUploadedFromFolder = this.options?.isUploadedFromFolder)); - - const [bigSizedFiles, mediumSizedFiles, smallSizedFiles] = this.classifyFilesBySize(this.items); - const uploadedFilesData: DriveFileData[] = []; - - const uploadFiles = async (files: UploadFileParamsWithTaskId[], concurrency: number) => { - if (this.abortController?.signal.aborted) return []; - - this.uploadQueue.concurrency = concurrency; - - const uploadPromises: Promise[] = await this.uploadQueue.pushAsync(files); - - let uploadedFiles: DriveFileData[] = []; - const filesToRetry: RetryableTask[] = []; - - uploadedFiles = await Promise.all(uploadPromises); - - for (let i = 0; i < uploadedFiles.length; i++) { - const uploadedFile = uploadedFiles[i]; - if (uploadedFile) uploadedFilesData.push(uploadedFile); - else - filesToRetry.push({ - taskId: files[i]?.taskId ?? files[i]?.relatedTaskId ?? '', - type: RetryableTaskType.Upload, - params: files[i], - retryable: !this.nonRetryableTaskIds.has(files[i]?.taskId ?? ''), - }); - } - - if (filesToRetry.length > 0) RetryManager.addTasks(filesToRetry); - const fileTaskId = files[0]?.taskId; - if (files.length === 1 && fileTaskId) { - const noFilesToRetry = filesToRetry.length === 0; - if (noFilesToRetry) RetryManager.removeTask(fileTaskId); - else RetryManager.changeStatus(fileTaskId, RetryableTaskStatus.Failed); - } - }; - - if (smallSizedFiles.length > 0) await uploadFiles(smallSizedFiles, this.filesGroups.small.concurrency); - - this.currentGroupBeingUploaded = FileSizeType.Medium; - - if (mediumSizedFiles.length > 0) await uploadFiles(mediumSizedFiles, this.filesGroups.medium.concurrency); - - this.currentGroupBeingUploaded = FileSizeType.Big; - - if (bigSizedFiles.length > 0) await uploadFiles(bigSizedFiles, this.filesGroups.big.concurrency); - - if (this.hasSomeUploadFailed) { - throw this.firstUploadError ?? new Error('Some files could not be uploaded'); - } - - return { uploadedFiles: uploadedFilesData }; - } catch (error) { - this.handleFailedUploads(this.items); - - errorService.reportError(error); - throw error; - } - } -} diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index f099c9a09b..57ba802da6 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -46,8 +46,6 @@ export enum UploadFailureReason { UploadFailed = 'upload-failed', } -export type UploadErrorReason = 'connection-lost' | 'upload-failed'; - export interface OwnerUserAuthenticationData { token: string; bridgeUser: string; From 946d37521c1757d0fb70aa0ac6c6a2727c35fbac Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 15:21:26 +0100 Subject: [PATCH 05/49] refactor(upload): upload folder trees through the engine, creating each folder before queueing its content --- .../folder.service/toDriveFolderData.ts | 19 +++ src/app/network/UploadFolderManager.test.ts | 2 + src/app/network/networkConstants.ts | 1 + src/app/network/types/index.ts | 1 + src/app/network/types/upload.ts | 16 +- .../storage/folderUtils/createFolder.ts | 18 +- src/app/upload/UploadManager.test.ts | 160 +++++++++++++++++- src/app/upload/UploadManager.ts | 140 +++++++++++++-- src/testUtils/fixtures/upload.fixtures.ts | 17 ++ 9 files changed, 337 insertions(+), 37 deletions(-) create mode 100644 src/app/drive/services/folder.service/toDriveFolderData.ts diff --git a/src/app/drive/services/folder.service/toDriveFolderData.ts b/src/app/drive/services/folder.service/toDriveFolderData.ts new file mode 100644 index 0000000000..47cce05e2b --- /dev/null +++ b/src/app/drive/services/folder.service/toDriveFolderData.ts @@ -0,0 +1,19 @@ +import { CreateFolderResponse } from '@internxt/sdk/dist/drive/storage/types'; +import { DriveFolderData } from 'app/drive/types'; + +export const toDriveFolderData = (createdFolder: CreateFolderResponse, folderName: string): DriveFolderData => ({ + ...createdFolder, + name: folderName, + parent_id: createdFolder.parentId, + user_id: createdFolder.userId, + icon: null, + iconId: null, + icon_id: null, + isFolder: true, + color: null, + encrypt_version: null, + plain_name: createdFolder.plainName, + deleted: false, + createdAt: new Date(createdFolder.createdAt || '').toISOString(), + updatedAt: new Date(createdFolder.updatedAt || '').toISOString(), +}); diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 9d06a83351..be9892dc6b 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -52,6 +52,8 @@ vi.mock('app/store/slices/storage/folderUtils/getUniqueFolderName', () => ({ getUniqueFolderName: vi.fn(), })); +vi.mock('app/drive/services/folder.service', () => ({ default: { createFolderByUuid: vi.fn() } })); +vi.mock('services/workspace.service', () => ({ default: { createFolder: vi.fn() } })); vi.mock('services/referral.service', () => ({ default: { trackFileUpload: vi.fn() } })); vi.mock('app/network/networkInformation', () => ({ logNetworkInfoForUpload: vi.fn() })); vi.mock('services/error.service', () => ({ diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index 69f3b47c6a..79bcad0b8c 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -11,3 +11,4 @@ export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; export const SINGLE_TRANSFER_CONCURRENCY = 1; +export const WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS = 600; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index 57ba802da6..852b5bdfaf 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -37,6 +37,7 @@ export interface DownloadChunkPayload { export enum UploadItemKind { File = 'file', + Folder = 'folder', } export enum UploadFailureReason { diff --git a/src/app/network/types/upload.ts b/src/app/network/types/upload.ts index 6937394c5f..4a3d721970 100644 --- a/src/app/network/types/upload.ts +++ b/src/app/network/types/upload.ts @@ -1,13 +1,21 @@ -import { DriveFileData } from 'app/drive/types'; +import { DriveFileData, DriveFolderData } from 'app/drive/types'; import { FileToUpload } from 'app/drive/services/file.service/types'; +import { IRoot } from 'app/store/slices/storage/types'; import { UploadFailure } from '../uploadFailure'; import { UploadItemKind } from '.'; -export type UploadFileItem = { kind: UploadItemKind.File; file: FileToUpload }; +export type UploadFileItem = { kind: UploadItemKind.File; file: FileToUpload; parent?: UploadFolderItem }; -export type UploadItem = UploadFileItem; +export type UploadFolderItem = { + kind: UploadItemKind.Folder; + folder: IRoot; + parentFolderUuid: string; + parent?: UploadFolderItem; +}; -export type UploadedItem = DriveFileData; +export type UploadItem = UploadFileItem | UploadFolderItem; + +export type UploadedItem = DriveFileData | DriveFolderData; /** * Upload progress notifications emitted by the manager. The manager knows nothing diff --git a/src/app/store/slices/storage/folderUtils/createFolder.ts b/src/app/store/slices/storage/folderUtils/createFolder.ts index dc029aca52..513ba1e548 100644 --- a/src/app/store/slices/storage/folderUtils/createFolder.ts +++ b/src/app/store/slices/storage/folderUtils/createFolder.ts @@ -10,6 +10,7 @@ import { RequestCanceler } from '@internxt/sdk/dist/shared/http/types'; import { AnyAction, ThunkDispatch } from '@reduxjs/toolkit'; import { RootState } from '../../../../store'; import { WorkspaceData } from '@internxt/sdk/dist/workspaces'; +import { toDriveFolderData } from 'app/drive/services/folder.service/toDriveFolderData'; interface CreateFolderOptions { relatedTaskId: string; @@ -57,22 +58,7 @@ export const createFolder = async ( const createdFolder = await createdFolderPromise; - const createdFolderNormalized: DriveFolderData = { - ...createdFolder, - name: folderName, - parent_id: createdFolder.parentId, - user_id: createdFolder.userId, - icon: null, - iconId: null, - icon_id: null, - isFolder: true, - color: null, - encrypt_version: null, - plain_name: createdFolder.plainName, - deleted: false, - createdAt: new Date(createdFolder.createdAt || '').toISOString(), - updatedAt: new Date(createdFolder.updatedAt || '').toISOString(), - }; + const createdFolderNormalized = toDriveFolderData(createdFolder, folderName); tasksService.updateTask({ taskId: taskId, diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index c811b575f1..15f5da5299 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -7,12 +7,22 @@ import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; import { QueueUtilsService } from 'utils/queueUtils'; -import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; -import { UploadManager } from './UploadManager'; +import workspacesService from 'services/workspace.service'; +import folderService from 'app/drive/services/folder.service'; +import { + getCreatedFolder, + getFileToUpload, + getFolderToUpload, + getUploadFileItem, + getUploadRepository, +} from 'testUtils/fixtures/upload.fixtures'; +import { createFolderItem, UploadManager } from './UploadManager'; import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason, UploadItemKind } from 'app/network/types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); +vi.mock('app/drive/services/folder.service', () => ({ default: { createFolderByUuid: vi.fn() } })); +vi.mock('services/workspace.service', () => ({ default: { createFolder: vi.fn() } })); type UploadFileOptions = { abortCallback?: (abort?: () => void) => void }; @@ -22,7 +32,7 @@ const noSpaceLeftError = new AppError('No space left', HTTP_STATUS_CODES.MAX_SPA const paymentRequiredError = new AppError('Payment required', HTTP_STATUS_CODES.PAYMENT_REQUIRED); const fileTooLargeError = new AppError('File too large', 400, HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED); -const nameOf = (item: UploadItem) => item.file.name; +const nameOf = (item: UploadItem) => (item.kind === UploadItemKind.File ? item.file.name : item.folder.name); const failIfHangs = (promise: Promise): Promise => Promise.race([ @@ -302,3 +312,145 @@ describe('Uploading files', () => { expect(succeeded).toHaveLength(items.length); }); }); + +const fakeFolderCreationTaking = (delayMs = 0) => + vi.mocked(folderService.createFolderByUuid).mockImplementation((_parentUuid, name) => { + const cancel = vi.fn(); + const promise = new Promise((resolve) => setTimeout(() => resolve(getCreatedFolder(name)), delayMs)); + return [promise, { cancel }] as never; + }); + +const uploadedFileParents = () => + Object.fromEntries((uploadFile as Mock).mock.calls.map(([file]) => [file.name, file.parentFolderId])); + +describe('Uploading folders', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(errorService, 'reportError').mockReturnValue(); + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); + fakeFolderCreationTaking(); + }); + + test('When a folder tree is uploaded, then each folder is created inside its parent and each file inside its folder', async () => { + const tree = getFolderToUpload('Work', { + childrenFiles: [getFileToUpload('report.pdf')], + childrenFolders: [getFolderToUpload('Invoices', { childrenFiles: [getFileToUpload('january.pdf')] })], + }); + + const { succeeded, failed } = await createManager().run([createFolderItem(tree, 'destination-uuid')]); + + expect(failed).toEqual([]); + expect(succeeded).toHaveLength(4); + expect(folderService.createFolderByUuid).toHaveBeenCalledWith('destination-uuid', 'Work'); + expect(folderService.createFolderByUuid).toHaveBeenCalledWith('Work-uuid', 'Invoices'); + expect(uploadedFileParents()).toEqual({ report: 'Work-uuid', january: 'Invoices-uuid' }); + }); + + test('When the content of a folder is uploaded, then every item knows the folder it belongs to', async () => { + const rootItem = createFolderItem( + getFolderToUpload('Work', { childrenFiles: [getFileToUpload('report.pdf')] }), + 'dest', + ); + + const { succeeded } = await createManager().run([rootItem]); + + const fileItem = succeeded.find(({ item }) => item.kind === UploadItemKind.File)?.item; + expect(fileItem?.parent).toBe(rootItem); + }); + + test('When a folder cannot be created, then its content is not uploaded', async () => { + vi.mocked(folderService.createFolderByUuid).mockImplementation( + () => [Promise.reject(new Error('Server error')), { cancel: vi.fn() }] as never, + ); + const rootItem = createFolderItem( + getFolderToUpload('Work', { childrenFiles: [getFileToUpload('report.pdf')] }), + 'dest', + ); + + const { failed } = await failIfHangs(createManager().run([rootItem])); + + expect(folderService.createFolderByUuid).toHaveBeenCalledTimes(2); + expect(failed).toEqual([ + { item: rootItem, failure: expect.objectContaining({ reason: UploadFailureReason.UploadFailed }) }, + ]); + expect(uploadFile).not.toHaveBeenCalled(); + }); + + test('When a folder is cancelled, then nothing else inside it is uploaded', async () => { + fakeSlowUploads(); + const files = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => getFileToUpload(`${name}.txt`)); + const rootItem = createFolderItem(getFolderToUpload('Work', { childrenFiles: files }), 'dest'); + const manager = createManager(); + + const upload = manager.run([rootItem]); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(6)); + manager.cancelItem(rootItem); + const { succeeded } = await failIfHangs(upload); + + expect(uploadFile).toHaveBeenCalledTimes(6); + expect(succeeded.map(({ item }) => item)).toEqual([rootItem]); + }); + + test('When folders appear while files are waiting, then the folders are created first', async () => { + fakeSlowUploads(); + const files = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); + const folderItem = createFolderItem(getFolderToUpload('Late'), 'dest'); + const manager = createManager(); + + const upload = manager.run([...files, folderItem]); + await vi.waitFor(() => expect(folderService.createFolderByUuid).toHaveBeenCalled()); + + expect(uploadedNames()).not.toContain('g'); + manager.cancelAll(); + await failIfHangs(upload); + }); + + test('When a folder is uploaded to a workspace, then it is created in the workspace and its content waits for it to be available', async () => { + vi.mocked(workspacesService.createFolder).mockImplementation( + ({ plainName }) => [Promise.resolve(getCreatedFolder(plainName)), { cancel: vi.fn() }] as never, + ); + let folderCreatedAt = 0; + vi.mocked(workspacesService.createFolder).mockImplementationOnce(({ plainName }) => { + folderCreatedAt = Date.now(); + return [Promise.resolve(getCreatedFolder(plainName)), { cancel: vi.fn() }] as never; + }); + let fileUploadedAt = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + fileUploadedAt = Date.now(); + return { uuid: `${file.name}-uuid` }; + }); + const ownerUserAuthenticationData = { workspaceId: 'workspace-id' } as never; + const rootItem = createFolderItem( + getFolderToUpload('Work', { childrenFiles: [getFileToUpload('report.pdf')] }), + 'dest', + ); + + await createManager({ ownerUserAuthenticationData }).run([rootItem]); + + expect(workspacesService.createFolder).toHaveBeenCalledWith({ + workspaceId: 'workspace-id', + parentFolderUuid: 'dest', + plainName: 'Work', + }); + expect(folderService.createFolderByUuid).not.toHaveBeenCalled(); + expect(fileUploadedAt - folderCreatedAt).toBeGreaterThanOrEqual(500); + }); + + test('When a large file is uploaded together with a folder, then it waits for the files inside the folder so they are not uploaded one at a time behind it', async () => { + fakeFolderCreationTaking(30); + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + await new Promise((resolve) => setTimeout(resolve, 10)); + return { uuid: `${file.name}-uuid` }; + }); + const largeFile = getUploadFileItem('large', USE_MULTIPART_THRESHOLD_BYTES); + const folderItem = createFolderItem( + getFolderToUpload('Photos', { childrenFiles: ['a', 'b', 'c'].map((name) => getFileToUpload(`${name}.png`)) }), + 'dest', + ); + + const { succeeded } = await failIfHangs(createManager().run([largeFile, folderItem])); + + expect(succeeded).toHaveLength(5); + expect(uploadedNames()).toEqual(['a', 'b', 'c', 'large']); + }); +}); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 4987c36993..fc8b93b4a6 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -1,10 +1,15 @@ import { items as itemUtils } from '@internxt/lib'; import { AsyncPriorityQueue, priorityQueue } from 'async'; import errorService from 'services/error.service'; +import workspacesService from 'services/workspace.service'; import { QueueUtilsService } from 'utils/queueUtils'; +import { wait } from 'utils/timeUtils'; import uploadFile from 'app/drive/services/file.service/uploadFile'; -import { DriveFileData } from 'app/drive/types'; +import folderService from 'app/drive/services/folder.service'; +import { toDriveFolderData } from 'app/drive/services/folder.service/toDriveFolderData'; +import { DriveFileData, DriveFolderData } from 'app/drive/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { IRoot } from 'app/store/slices/storage/types'; import { TaskStatus } from 'app/tasks/types'; import { MAX_CONCURRENT_UPLOADS, @@ -12,12 +17,21 @@ import { SINGLE_TRANSFER_CONCURRENCY, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES, + WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS, } from 'app/network/networkConstants'; import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; -import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; +import { + UploadedItem, + UploadFileItem, + UploadFolderItem, + UploadItem, + UploadManagerEvents, + UploadResult, +} from 'app/network/types/upload'; import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; enum UploadPriority { + Folder = 0, SmallFile = 1, MediumFile = 2, LargeFile = 3, @@ -30,20 +44,46 @@ interface UploadManagerProps { events?: UploadManagerEvents; } -export const createFileItem = (file: File, parentFolderUuid: string): UploadFileItem => { +export const createFileItem = (file: File, parentFolderUuid: string, parent?: UploadFolderItem): UploadFileItem => { const { filename, extension } = itemUtils.getFilenameAndExt(file.name); return { kind: UploadItemKind.File, file: { name: filename, type: extension, size: file.size, content: file, parentFolderId: parentFolderUuid }, + parent, }; }; -const isMultipartUpload = (item: UploadItem) => item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; +export const createFolderItem = ( + folder: IRoot, + parentFolderUuid: string, + parent?: UploadFolderItem, +): UploadFolderItem => ({ + kind: UploadItemKind.Folder, + folder, + parentFolderUuid, + parent, +}); + +const getItemAndItsFolders = (item: UploadItem): UploadItem[] => { + const itemAndItsFolders: UploadItem[] = []; + for (let current: UploadItem | undefined = item; current; current = current.parent) { + itemAndItsFolders.push(current); + } + return itemAndItsFolders; +}; + +const isPartOf = (item: UploadItem, otherItem: UploadItem): boolean => getItemAndItsFolders(item).includes(otherItem); -const isMediumSizedFile = (item: UploadItem) => item.file.size >= TWENTY_MEGABYTES; +const isFolderItem = (item: UploadItem): item is UploadFolderItem => item.kind === UploadItemKind.Folder; + +const isMultipartUpload = (item: UploadItem) => + item.kind === UploadItemKind.File && item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; + +const isMediumSizedFile = (item: UploadItem) => item.kind === UploadItemKind.File && item.file.size >= TWENTY_MEGABYTES; const getUploadPriority = (item: UploadItem): UploadPriority => { + if (isFolderItem(item)) return UploadPriority.Folder; if (isMultipartUpload(item)) return UploadPriority.LargeFile; if (isMediumSizedFile(item)) return UploadPriority.MediumFile; return UploadPriority.SmallFile; @@ -62,10 +102,12 @@ const notifyIgnoringListenerErrors = async (notify: () => unknown): Promise(); private readonly abortActiveUploadOf = new Map void>(); private result: UploadResult = { succeeded: [], failed: [] }; @@ -82,7 +126,10 @@ export class UploadManager { private readonly uploadQueue: AsyncPriorityQueue = priorityQueue((item, done) => { this.processItem(item) .catch((error) => errorService.reportError(error)) - .finally(() => done()); + .finally(() => { + if (isFolderItem(item)) this.finishFolder(); + done(); + }); }, MAX_CONCURRENT_UPLOADS); constructor(props: UploadManagerProps) { @@ -114,11 +161,12 @@ export class UploadManager { cancelItem(item: UploadItem): void { this.cancelledItems.add(item); - this.abortActiveUploadOf.get(item)?.(); + this.abortActiveUploadsPartOf(item); } private queue(items: UploadItem[]): void { for (const item of items) { + if (isFolderItem(item)) this.foldersStillQueueingContent++; this.uploadQueue.push(item, getUploadPriority(item)); } } @@ -134,6 +182,12 @@ export class UploadManager { return; } + const shouldWaitForFolders = isMultipartUpload(item) && this.foldersStillQueueingContent > 0; + if (shouldWaitForFolders) { + this.largeFilesWaitingForFolders.push(item); + return; + } + this.adjustConcurrencyFor(item); await notifyIgnoringListenerErrors(() => this.events?.onItemStarted?.(item)); @@ -141,12 +195,24 @@ export class UploadManager { const uploadedItem = await this.uploadWithRetries(item); this.result.succeeded.push({ item, uploadedItem }); + const shouldQueueFolderContent = isFolderItem(item) && !this.isItemCancelled(item); + if (shouldQueueFolderContent) { + await this.queueFolderContent(item, uploadedItem as DriveFolderData); + } + await notifyIgnoringListenerErrors(() => this.events?.onItemSucceeded?.(item, uploadedItem)); } catch (error) { await this.handleUploadError(item, error); } } + private finishFolder(): void { + this.foldersStillQueueingContent--; + + const haveAllFoldersQueuedTheirContent = this.foldersStillQueueingContent === 0; + if (haveAllFoldersQueuedTheirContent) this.queue(this.largeFilesWaitingForFolders.splice(0)); + } + private adjustConcurrencyFor(item: UploadItem): void { if (isMultipartUpload(item)) { this.uploadQueue.concurrency = SINGLE_TRANSFER_CONCURRENCY; @@ -156,10 +222,28 @@ export class UploadManager { QueueUtilsService.instance.adjustConcurrency(this.uploadQueue, MAX_CONCURRENT_UPLOADS); } + private async queueFolderContent(folderItem: UploadFolderItem, createdFolder: DriveFolderData): Promise { + const { childrenFolders, childrenFiles } = folderItem.folder; + const isFolderEmpty = childrenFolders.length === 0 && childrenFiles.length === 0; + if (isFolderEmpty) return; + + await this.waitUntilWorkspaceFolderIsAvailable(); + + this.queue([ + ...childrenFolders.map((childFolder) => createFolderItem(childFolder, createdFolder.uuid, folderItem)), + ...childrenFiles.map((childFile) => createFileItem(childFile, createdFolder.uuid, folderItem)), + ]); + } + + private async waitUntilWorkspaceFolderIsAvailable(): Promise { + const isWorkspaceUpload = !!this.ownerUserAuthenticationData?.workspaceId; + if (isWorkspaceUpload) await wait(WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS); + } + private async uploadWithRetries(item: UploadItem): Promise { for (let attempt = 1; ; attempt++) { try { - return await this.uploadFile(item); + return await this.uploadItem(item); } catch (error) { const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); @@ -168,6 +252,28 @@ export class UploadManager { } } + private uploadItem(item: UploadItem): Promise { + if (isFolderItem(item)) return this.createFolder(item); + return this.uploadFile(item); + } + + private async createFolder(folderItem: UploadFolderItem): Promise { + const workspaceId = this.ownerUserAuthenticationData?.workspaceId; + const { parentFolderUuid, folder } = folderItem; + const [createdFolderPromise, requestCanceler] = workspaceId + ? workspacesService.createFolder({ workspaceId, parentFolderUuid, plainName: folder.name }) + : folderService.createFolderByUuid(parentFolderUuid, folder.name); + + this.abortActiveUploadOf.set(folderItem, () => requestCanceler.cancel()); + + try { + const createdFolder = await createdFolderPromise; + return toDriveFolderData(createdFolder, folder.name); + } finally { + this.abortActiveUploadOf.delete(folderItem); + } + } + private async uploadFile(fileItem: UploadFileItem): Promise { const pauseKey = this.getPauseKey(fileItem); const uploadState = await this.uploadRepository?.getUploadState(pauseKey); @@ -192,7 +298,15 @@ export class UploadManager { } private isItemCancelled(item: UploadItem): boolean { - return this.isEverythingCancelled || this.cancelledItems.has(item); + if (this.isEverythingCancelled) return true; + + return getItemAndItsFolders(item).some((itemOrFolder) => this.cancelledItems.has(itemOrFolder)); + } + + private abortActiveUploadsPartOf(cancelledItem: UploadItem): void { + this.abortActiveUploadOf.forEach((abort, activeItem) => { + if (isPartOf(activeItem, cancelledItem)) abort(); + }); } private async handleUploadError(item: UploadItem, error: unknown): Promise { diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts index 51062cac1d..5ed38b7baf 100644 --- a/src/testUtils/fixtures/upload.fixtures.ts +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -2,15 +2,32 @@ import { vi } from 'vitest'; import { UploadItemKind } from 'app/network/types'; import { UploadFileItem } from 'app/network/types/upload'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { IRoot } from 'app/store/slices/storage/types'; import { TaskStatus } from 'app/tasks/types'; export const getFileToUpload = (name: string, size = 10): File => new File([new ArrayBuffer(size)], name); +export const getFolderToUpload = (name: string, content: Partial = {}): IRoot => ({ + name, + folderId: null, + isFolder: true, + childrenFiles: [], + childrenFolders: [], + ...content, +}); + export const getUploadFileItem = (name: string, size = 1024): UploadFileItem => ({ kind: UploadItemKind.File, file: { name, size, type: 'txt', content: getFileToUpload(name), parentFolderId: 'folder-uuid' }, }); +export const getCreatedFolder = (name: string) => ({ + uuid: `${name}-uuid`, + plainName: name, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), +}); + export const getUploadRepository = (pausedKeys: string[] = []): PersistUploadRepository => ({ setUploadState: vi.fn(async () => undefined), removeUploadState: vi.fn(async () => undefined), From 00a77faa0c42ccf440d3232aaf6e2853a5a89345 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:26:11 +0100 Subject: [PATCH 06/49] refactor(upload): mark the queue push as not awaited and simplify the upload fixtures --- src/app/network/types/upload.ts | 4 ---- src/app/upload/UploadManager.ts | 2 +- src/testUtils/fixtures/upload.fixtures.ts | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/app/network/types/upload.ts b/src/app/network/types/upload.ts index 6937394c5f..53cb34dae3 100644 --- a/src/app/network/types/upload.ts +++ b/src/app/network/types/upload.ts @@ -9,10 +9,6 @@ export type UploadItem = UploadFileItem; export type UploadedItem = DriveFileData; -/** - * Upload progress notifications emitted by the manager. The manager knows nothing - * about how uploads are presented to the user; observers subscribe to these events. - */ export interface UploadManagerEvents { onItemStarted?: (item: UploadItem) => void; onItemProgress?: (item: UploadItem, progress: number) => void; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 4987c36993..b327d479d4 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -119,7 +119,7 @@ export class UploadManager { private queue(items: UploadItem[]): void { for (const item of items) { - this.uploadQueue.push(item, getUploadPriority(item)); + void this.uploadQueue.push(item, getUploadPriority(item)); } } diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts index 51062cac1d..6e123b622e 100644 --- a/src/testUtils/fixtures/upload.fixtures.ts +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -12,7 +12,7 @@ export const getUploadFileItem = (name: string, size = 1024): UploadFileItem => }); export const getUploadRepository = (pausedKeys: string[] = []): PersistUploadRepository => ({ - setUploadState: vi.fn(async () => undefined), - removeUploadState: vi.fn(async () => undefined), - getUploadState: vi.fn(async (key: string) => (pausedKeys.includes(key) ? TaskStatus.Paused : undefined)), + setUploadState: vi.fn(() => Promise.resolve()), + removeUploadState: vi.fn(() => Promise.resolve()), + getUploadState: vi.fn((key: string) => Promise.resolve(pausedKeys.includes(key) ? TaskStatus.Paused : undefined)), }); From 28ac2168ecd85b389f16ba4391762b773bedcc67 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:35:36 +0100 Subject: [PATCH 07/49] refactor(upload): return the cancel promise without an empty async --- src/app/tasks/upload/uploadFilesWithTasks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index f505bec6ae..07fb0f0e6e 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -139,7 +139,7 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr if (file.taskId) { tasksService.updateTask({ taskId: file.taskId, - merge: { status: TaskStatus.Encrypting, stop: async () => manager.cancelItem(item) }, + merge: { status: TaskStatus.Encrypting, stop: () => Promise.resolve(manager.cancelItem(item)) }, }); } @@ -208,6 +208,6 @@ const createUploadTask = ( showNotification: options?.showNotifications ?? true, cancellable: true, sharedItemAuthenticationData, - stop: async () => manager.cancelItem(item), + stop: () => Promise.resolve(manager.cancelItem(item)), }); }; From 1a6ef191728672b9588d2390101c350df6603485 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:43:58 +0100 Subject: [PATCH 08/49] refactor(upload): retry an upload by calling it again instead of looping --- src/app/upload/UploadManager.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index b327d479d4..3c890d801c 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -156,15 +156,15 @@ export class UploadManager { QueueUtilsService.instance.adjustConcurrency(this.uploadQueue, MAX_CONCURRENT_UPLOADS); } - private async uploadWithRetries(item: UploadItem): Promise { - for (let attempt = 1; ; attempt++) { - try { - return await this.uploadFile(item); - } catch (error) { - const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; - const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); - if (!shouldRetry) throw error; - } + private async uploadWithRetries(item: UploadItem, attempt = 1): Promise { + try { + return await this.uploadFile(item); + } catch (error) { + const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; + const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); + if (!shouldRetry) throw error; + + return this.uploadWithRetries(item, attempt + 1); } } From 14c11daf2276a2840054a4a6febd9b2a4177268a Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:49:25 +0100 Subject: [PATCH 09/49] fix(upload): start every run with uploads not cancelled and the full concurrency --- src/app/upload/UploadManager.test.ts | 29 ++++++++++++++++++++++++++++ src/app/upload/UploadManager.ts | 2 ++ 2 files changed, 31 insertions(+) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index c811b575f1..845fec5ff7 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -235,6 +235,35 @@ describe('Uploading files', () => { expect(maxActiveLargeUploads).toBe(1); }); + test('When the same manager uploads again after a large file in a browser without memory information, then several files go at the same time again', async () => { + vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockImplementation((current) => current); + let activeUploads = 0; + let maxActiveUploads = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); + await new Promise((resolve) => setTimeout(resolve, 10)); + activeUploads--; + return { uuid: `${file.name}-uuid` }; + }); + const manager = createManager(); + await manager.run([getUploadFileItem('large', USE_MULTIPART_THRESHOLD_BYTES)]); + maxActiveUploads = 0; + + await manager.run(['a', 'b', 'c'].map((name) => getUploadFileItem(name))); + + expect(maxActiveUploads).toBe(3); + }); + + test('When the same manager uploads again after everything was cancelled, then the new files are uploaded', async () => { + const manager = createManager(); + manager.cancelAll(); + await manager.run([getUploadFileItem('cancelled')]); + + const { succeeded } = await manager.run([getUploadFileItem('report')]); + + expect(succeeded.map(({ item }) => item.file.name)).toEqual(['report']); + }); + test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); let activeUploads = 0; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 3c890d801c..9ff87bfb82 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -96,6 +96,8 @@ export class UploadManager { if (this.uploadQueue.idle()) { this.result = { succeeded: [], failed: [] }; this.storageFullFailure = undefined; + this.isEverythingCancelled = false; + this.uploadQueue.concurrency = MAX_CONCURRENT_UPLOADS; } const hasNothingToUpload = items.length === 0; From 53d1bbe6be60e156216aebbe2d538ada741b969f Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:51:14 +0100 Subject: [PATCH 10/49] test(upload): name uploaded items with the shared helper --- src/app/upload/UploadManager.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 845fec5ff7..38533c11a7 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -261,7 +261,7 @@ describe('Uploading files', () => { const { succeeded } = await manager.run([getUploadFileItem('report')]); - expect(succeeded.map(({ item }) => item.file.name)).toEqual(['report']); + expect(succeeded.map(({ item }) => nameOf(item))).toEqual(['report']); }); test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { From 99a8e276f62bb4020cbe350fcd0fdc3f9ed3c679 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:09:10 +0100 Subject: [PATCH 11/49] refactor(upload): remove cancelling everything from the engine, uploads are cancelled file by file --- src/app/upload/UploadManager.test.ts | 28 ---------------------------- src/app/upload/UploadManager.ts | 9 +-------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 38533c11a7..d4dc69b33f 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -177,24 +177,6 @@ describe('Uploading files', () => { expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); - test('When the upload is cancelled, then the queued files are never uploaded and are reported as cancelled', async () => { - fakeSlowUploads(); - const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => getUploadFileItem(name)); - const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; - const manager = createManager({ events }); - - const upload = manager.run(items); - await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(6)); - manager.cancelAll(); - const { succeeded, failed } = await failIfHangs(upload); - - expect(uploadFile).toHaveBeenCalledTimes(6); - expect(succeeded).toEqual([]); - expect(failed).toEqual([]); - expect(events.onItemCancelled).toHaveBeenCalledTimes(items.length); - expect(events.onItemFailed).not.toHaveBeenCalled(); - }); - test('When a single file is cancelled, then only that file stops', async () => { fakeSlowUploads(); const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; @@ -254,16 +236,6 @@ describe('Uploading files', () => { expect(maxActiveUploads).toBe(3); }); - test('When the same manager uploads again after everything was cancelled, then the new files are uploaded', async () => { - const manager = createManager(); - manager.cancelAll(); - await manager.run([getUploadFileItem('cancelled')]); - - const { succeeded } = await manager.run([getUploadFileItem('report')]); - - expect(succeeded.map(({ item }) => nameOf(item))).toEqual(['report']); - }); - test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); let activeUploads = 0; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 9ff87bfb82..0d7d602802 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -73,7 +73,6 @@ export class UploadManager { private readonly getPauseKey: (item: UploadItem) => string; private readonly events?: UploadManagerEvents; - private isEverythingCancelled = false; private storageFullFailure?: UploadFailure; private readonly cancelledItems = new WeakSet(); private readonly abortActiveUploadOf = new Map void>(); @@ -96,7 +95,6 @@ export class UploadManager { if (this.uploadQueue.idle()) { this.result = { succeeded: [], failed: [] }; this.storageFullFailure = undefined; - this.isEverythingCancelled = false; this.uploadQueue.concurrency = MAX_CONCURRENT_UPLOADS; } @@ -109,11 +107,6 @@ export class UploadManager { return this.result; } - cancelAll(): void { - this.isEverythingCancelled = true; - this.abortActiveUploadOf.forEach((abort) => abort()); - } - cancelItem(item: UploadItem): void { this.cancelledItems.add(item); this.abortActiveUploadOf.get(item)?.(); @@ -194,7 +187,7 @@ export class UploadManager { } private isItemCancelled(item: UploadItem): boolean { - return this.isEverythingCancelled || this.cancelledItems.has(item); + return this.cancelledItems.has(item); } private async handleUploadError(item: UploadItem, error: unknown): Promise { From 00797b0af13fbd2b9ef0dbf95f18d0f61233d015 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:10:18 +0100 Subject: [PATCH 12/49] fix(upload): cancel every file of a folder upload even when it is cancelled before the files start --- .../tasks/upload/uploadFilesWithTasks.test.ts | 18 +++++++++++++----- src/app/tasks/upload/uploadFilesWithTasks.ts | 5 +++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/app/tasks/upload/uploadFilesWithTasks.test.ts b/src/app/tasks/upload/uploadFilesWithTasks.test.ts index 9e7dd27e86..570556638f 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.test.ts @@ -16,14 +16,13 @@ const { managerState } = vi.hoisted(() => ({ result: { succeeded: [], failed: [] } as UploadResult, instance: undefined as | undefined - | { props: { events: UploadManagerEvents }; run: Mock; cancelAll: Mock; cancelItem: Mock; items: UploadItem[] }, + | { props: { events: UploadManagerEvents }; run: Mock; cancelItem: Mock; items: UploadItem[] }, }, })); vi.mock('app/upload/UploadManager', () => ({ UploadManager: vi.fn(function (this: NonNullable, props) { this.props = props; - this.cancelAll = vi.fn(); this.cancelItem = vi.fn(); this.run = vi.fn(async (items: UploadItem[]) => { this.items = items; @@ -265,13 +264,22 @@ describe('Uploading files with tasks', () => { expect(manager.cancelItem).toHaveBeenCalledExactlyOnceWith(items[0]); }); - test('When the folder upload that owns the files is cancelled, then the whole upload is cancelled', async () => { + test('When the folder upload that owns the files is cancelled, then every file is cancelled', async () => { const abortController = new AbortController(); - const { manager } = await runUpload([buildFile()], { abortController }); + const { manager, items } = await runUpload([buildFile(), buildFile()], { abortController }); abortController.abort(); - expect(manager.cancelAll).toHaveBeenCalledOnce(); + expect(manager.cancelItem.mock.calls).toEqual(items.map((item) => [item])); + }); + + test('When the folder upload that owns the files was cancelled before they start, then every file is cancelled', async () => { + const abortController = new AbortController(); + abortController.abort(); + + const { manager, items } = await runUpload([buildFile(), buildFile()], { abortController }); + + expect(manager.cancelItem.mock.calls).toEqual(items.map((item) => [item])); }); test('When a file fails, then it is listed for retry and the upload rejects with its error', async () => { diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index 07fb0f0e6e..98c3a4f5a2 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -146,8 +146,9 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr return item; }); - if (abortController?.signal.aborted) manager.cancelAll(); - abortController?.signal.addEventListener('abort', () => manager.cancelAll(), { once: true }); + const cancelEveryFile = () => items.forEach((item) => manager.cancelItem(item)); + if (abortController?.signal.aborted) cancelEveryFile(); + abortController?.signal.addEventListener('abort', cancelEveryFile, { once: true }); const relatedTaskId = files[0]?.relatedTaskId; if (relatedTaskId) { From 302d7c76e60217d2084af6c3fa32296af497f148 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:32:25 +0100 Subject: [PATCH 13/49] test(upload): show which files are uploading and which are waiting when the storage gets full --- src/app/upload/UploadManager.test.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index d4dc69b33f..c1f4d96116 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -5,7 +5,7 @@ import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { MAX_CONCURRENT_UPLOADS, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; @@ -33,7 +33,7 @@ const failIfHangs = (promise: Promise): Promise => const createManager = (props: Partial[0]> = {}) => new UploadManager({ uploadRepository: getUploadRepository(), getPauseKey: nameOf, ...props }); -const uploadedNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); +const startedUploadNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); const fakeSlowUploads = () => (uploadFile as Mock).mockImplementation((file: { name: string }, _progress, options: UploadFileOptions) => { @@ -111,7 +111,7 @@ describe('Uploading files', () => { createManager({ events }).run([brokenItem, getUploadFileItem('a'), getUploadFileItem('b')]), ); - expect(uploadedNames().filter((name) => name === 'broken')).toHaveLength(2); + expect(startedUploadNames().filter((name) => name === 'broken')).toHaveLength(2); expect(succeeded).toHaveLength(2); expect(failed).toEqual([ { @@ -166,14 +166,22 @@ describe('Uploading files', () => { expect(failed).toHaveLength(1); }); - test('When the storage is full, then the queued files fail without being uploaded', async () => { - fakeUploadFailureOf('first', noSpaceLeftError); - const items = ['first', 'a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); + test('When the storage gets full, then the files already uploading finish and the waiting ones fail without being uploaded', async () => { + fakeUploadFailureOf('fills-the-storage', noSpaceLeftError); + const uploadingAtTheSameTime = Array.from( + { length: MAX_CONCURRENT_UPLOADS - 1 }, + (_, index) => `uploading-${index}`, + ); + const stillWaiting = ['waiting-1', 'waiting-2']; + const items = ['fills-the-storage', ...uploadingAtTheSameTime, ...stillWaiting].map((name) => + getUploadFileItem(name), + ); - const { failed } = await failIfHangs(createManager().run(items)); + const { succeeded, failed } = await failIfHangs(createManager().run(items)); - expect(uploadedNames()).toEqual(['first', 'a', 'b', 'c', 'd', 'e']); - expect(failed.map(({ item }) => nameOf(item))).toEqual(['first', 'f', 'g']); + expect(startedUploadNames()).toEqual(['fills-the-storage', ...uploadingAtTheSameTime]); + expect(succeeded.map(({ item }) => nameOf(item))).toEqual(uploadingAtTheSameTime); + expect(failed.map(({ item }) => nameOf(item))).toEqual(['fills-the-storage', ...stillWaiting]); expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); @@ -213,7 +221,7 @@ describe('Uploading files', () => { await createManager().run(items); - expect(uploadedNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); + expect(startedUploadNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); expect(maxActiveLargeUploads).toBe(1); }); From 17874fd1f63c52cfdd1948f0102fb4ff53f98f71 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:42:36 +0100 Subject: [PATCH 14/49] test(upload): state each upload scenario plainly and count started uploads instead of tracking them in the test --- src/app/upload/UploadManager.test.ts | 96 +++++++++++----------------- 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index c1f4d96116..163bd89a57 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -5,7 +5,7 @@ import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { MAX_CONCURRENT_UPLOADS, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; @@ -53,6 +53,11 @@ const fakeUploadFailureOf = (failingName: string, error: unknown) => return { uuid: `${file.name}-uuid` } as DriveFileData; }); +const makeUploadsNeverFinish = () => (uploadFile as Mock).mockReturnValue(new Promise(() => undefined)); + +const uploadAtMostAtTheSameTime = (concurrency: number) => + vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(concurrency); + const goOffline = () => vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); describe('Uploading files', () => { @@ -125,7 +130,7 @@ describe('Uploading files', () => { test.each([ { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.UploadFailed }, { problem: 'the file is too large', error: fileTooLargeError, reason: UploadFailureReason.FileTooLarge }, - ])('When $problem, then the file is not retried and cannot be retried later', async ({ error, reason }) => { + ])('When $problem, then the file is never retried', async ({ error, reason }) => { (uploadFile as Mock).mockRejectedValue(error); const { failed } = await createManager().run([getUploadFileItem('report')]); @@ -149,7 +154,7 @@ describe('Uploading files', () => { ); }); - test('When a network error happens while online, then the upload is retried and succeeds', async () => { + test('When a network error happens once while online, then the upload is retried and succeeds', async () => { (uploadFile as Mock).mockRejectedValueOnce(networkError); const { succeeded } = await createManager().run([getUploadFileItem('report')]); @@ -158,30 +163,16 @@ describe('Uploading files', () => { expect(succeeded).toHaveLength(1); }); - test('When the upload rejects without an error, then the file fails instead of hanging', async () => { - (uploadFile as Mock).mockRejectedValue(undefined); - - const { failed } = await failIfHangs(createManager().run([getUploadFileItem('report')])); - - expect(failed).toHaveLength(1); - }); - - test('When the storage gets full, then the files already uploading finish and the waiting ones fail without being uploaded', async () => { + test('When the storage gets full, then the file uploading at the same time finishes and the waiting ones fail without being uploaded', async () => { + uploadAtMostAtTheSameTime(2); fakeUploadFailureOf('fills-the-storage', noSpaceLeftError); - const uploadingAtTheSameTime = Array.from( - { length: MAX_CONCURRENT_UPLOADS - 1 }, - (_, index) => `uploading-${index}`, - ); - const stillWaiting = ['waiting-1', 'waiting-2']; - const items = ['fills-the-storage', ...uploadingAtTheSameTime, ...stillWaiting].map((name) => - getUploadFileItem(name), - ); + const items = ['fills-the-storage', 'uploading', 'waiting-1', 'waiting-2'].map((name) => getUploadFileItem(name)); const { succeeded, failed } = await failIfHangs(createManager().run(items)); - expect(startedUploadNames()).toEqual(['fills-the-storage', ...uploadingAtTheSameTime]); - expect(succeeded.map(({ item }) => nameOf(item))).toEqual(uploadingAtTheSameTime); - expect(failed.map(({ item }) => nameOf(item))).toEqual(['fills-the-storage', ...stillWaiting]); + expect(startedUploadNames()).toEqual(['fills-the-storage', 'uploading']); + expect(succeeded.map(({ item }) => nameOf(item))).toEqual(['uploading']); + expect(failed.map(({ item }) => nameOf(item))).toEqual(['fills-the-storage', 'waiting-1', 'waiting-2']); expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); @@ -200,16 +191,7 @@ describe('Uploading files', () => { expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(cancelledItem); }); - test('When small, medium and large files are uploaded together, then the small ones go first and the large ones last and one at a time', async () => { - let activeLargeUploads = 0; - let maxActiveLargeUploads = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string; size: number }) => { - const isLarge = file.size >= USE_MULTIPART_THRESHOLD_BYTES; - if (isLarge) maxActiveLargeUploads = Math.max(maxActiveLargeUploads, ++activeLargeUploads); - await new Promise((resolve) => setTimeout(resolve, 10)); - if (isLarge) activeLargeUploads--; - return { uuid: `${file.name}-uuid` }; - }); + test('When small, medium and large files are uploaded together, then the small ones start first and the large ones last', async () => { const items = [ getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), getUploadFileItem('medium-1', TWENTY_MEGABYTES), @@ -222,44 +204,38 @@ describe('Uploading files', () => { await createManager().run(items); expect(startedUploadNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); - expect(maxActiveLargeUploads).toBe(1); }); - test('When the same manager uploads again after a large file in a browser without memory information, then several files go at the same time again', async () => { + test('When several large files are uploaded, then only one uploads at a time', async () => { + makeUploadsNeverFinish(); + const items = [ + getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('large-2', USE_MULTIPART_THRESHOLD_BYTES), + ]; + + void createManager().run(items); + + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['large-1'])); + }); + + test('When small files are uploaded after a large file in a browser without memory information, then they upload at the same time again', async () => { vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockImplementation((current) => current); - let activeUploads = 0; - let maxActiveUploads = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); - await new Promise((resolve) => setTimeout(resolve, 10)); - activeUploads--; - return { uuid: `${file.name}-uuid` }; - }); const manager = createManager(); await manager.run([getUploadFileItem('large', USE_MULTIPART_THRESHOLD_BYTES)]); - maxActiveUploads = 0; + makeUploadsNeverFinish(); - await manager.run(['a', 'b', 'c'].map((name) => getUploadFileItem(name))); + void manager.run(['report', 'invoice', 'photo'].map((name) => getUploadFileItem(name))); - expect(maxActiveUploads).toBe(3); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['large', 'report', 'invoice', 'photo'])); }); - test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { - vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); - let activeUploads = 0; - let maxActiveUploads = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); - await new Promise((resolve) => setTimeout(resolve, 10)); - activeUploads--; - return { uuid: `${file.name}-uuid` }; - }); - const items = ['a', 'b', 'c', 'd'].map((name) => getUploadFileItem(name)); + test('When the browser is running out of memory, then files are uploaded one at a time', async () => { + uploadAtMostAtTheSameTime(1); + makeUploadsNeverFinish(); - const { succeeded } = await createManager().run(items); + void createManager().run(['report', 'invoice', 'photo'].map((name) => getUploadFileItem(name))); - expect(succeeded).toHaveLength(items.length); - expect(maxActiveUploads).toBe(1); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['report'])); }); test('When uploading to a workspace, then the workspace credentials are used', async () => { From 614ec043c2da2f1473e19f2f2a391536d2754240 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:11:31 +0100 Subject: [PATCH 15/49] refactor(upload): save the upload state where it changes instead of through a shared helper --- src/app/repositories/uploadStateUpdates.ts | 11 ----------- src/app/tasks/upload/uploadFilesWithTasks.ts | 10 +++++++--- 2 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 src/app/repositories/uploadStateUpdates.ts diff --git a/src/app/repositories/uploadStateUpdates.ts b/src/app/repositories/uploadStateUpdates.ts deleted file mode 100644 index 036452e990..0000000000 --- a/src/app/repositories/uploadStateUpdates.ts +++ /dev/null @@ -1,11 +0,0 @@ -import errorService from 'services/error.service'; -import { TaskStatus } from '../tasks/types'; -import { PersistUploadRepository } from './DatabaseUploadRepository'; - -export const markUploadAsInProcess = (uploadRepository: PersistUploadRepository, taskId: string): void => { - uploadRepository.setUploadState(taskId, TaskStatus.InProcess).catch((error) => errorService.reportError(error)); -}; - -export const forgetUploadState = (uploadRepository: PersistUploadRepository, taskId: string): void => { - uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); -}; diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index 98c3a4f5a2..96a32f12bf 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -9,7 +9,7 @@ import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; -import { forgetUploadState, markUploadAsInProcess } from 'app/repositories/uploadStateUpdates'; +import errorService from 'services/error.service'; import { errorMerge } from './uploadTaskErrors'; export type UploadManagerFileParams = { @@ -65,7 +65,11 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr taskId: file.taskId, merge: { status: file.relatedTaskId ? TaskStatus.Encrypting : TaskStatus.InProcess }, }); - if (!file.relatedTaskId) markUploadAsInProcess(uploadRepository, file.taskId); + if (!file.relatedTaskId) { + uploadRepository + .setUploadState(file.taskId, TaskStatus.InProcess) + .catch((error) => errorService.reportError(error)); + } }, onItemProgress: (item, progress) => { @@ -113,7 +117,7 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr onItemFailed: (item, { reason }) => { const { taskId } = getFile(item); tasksService.updateTask({ taskId, merge: errorMerge(reason) }); - forgetUploadState(uploadRepository, taskId); + uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); }, onItemCancelled: (item) => { From d65eba25fec88787ae821a48df0bdd4d1580ec40 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:31:31 +0100 Subject: [PATCH 16/49] fix(upload): do not start a file upload that was cancelled while it was getting ready --- src/app/upload/UploadManager.test.ts | 17 +++++++++++++++++ src/app/upload/UploadManager.ts | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 163bd89a57..3d865a76e2 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -176,6 +176,23 @@ describe('Uploading files', () => { expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); + test('When a file is cancelled while its paused state is being read, then it is never uploaded and is reported as cancelled', async () => { + const report = getUploadFileItem('report'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const uploadRepository = getUploadRepository(); + const manager = createManager({ events, uploadRepository }); + vi.mocked(uploadRepository.getUploadState).mockImplementationOnce(async () => { + manager.cancelItem(report); + return undefined; + }); + + await manager.run([report]); + + expect(uploadFile).not.toHaveBeenCalled(); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(report); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + test('When a single file is cancelled, then only that file stops', async () => { fakeSlowUploads(); const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 0d7d602802..880e480ff4 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -30,6 +30,12 @@ interface UploadManagerProps { events?: UploadManagerEvents; } +class UploadCancelledError extends Error { + constructor() { + super('The upload was cancelled before it started'); + } +} + export const createFileItem = (file: File, parentFolderUuid: string): UploadFileItem => { const { filename, extension } = itemUtils.getFilenameAndExt(file.name); @@ -167,6 +173,7 @@ export class UploadManager { const pauseKey = this.getPauseKey(fileItem); const uploadState = await this.uploadRepository?.getUploadState(pauseKey); const isPaused = uploadState === TaskStatus.Paused; + this.throwIfCancelled(fileItem); try { const driveFile = await uploadFile( @@ -186,6 +193,10 @@ export class UploadManager { } } + private throwIfCancelled(item: UploadItem): void { + if (this.isItemCancelled(item)) throw new UploadCancelledError(); + } + private isItemCancelled(item: UploadItem): boolean { return this.cancelledItems.has(item); } From cac6c4973a89644a3eea36c0b88dc1d79af818f4 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:40:27 +0100 Subject: [PATCH 17/49] test(upload): a cancelled folder upload no longer starts the files that were still getting ready --- src/app/network/UploadFolderManager.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 9d06a83351..805b2a9b7b 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -502,12 +502,12 @@ describe('uploadFoldersWithManager', () => { RetryManager.clearTasks(); }); - test('when a folder upload with more files than the upload concurrency is cancelled right away, then the files already uploading/queued are aborted and the task ends as cancelled', async () => { + test('when a folder upload is cancelled while its first file is uploading, then no other file starts and the task ends as cancelled', async () => { const { events, stopRelatedUploads, run } = arrangeCancelledFolderUpload(10); await run(); - expect(uploadFile).toHaveBeenCalledTimes(6); + expect(uploadFile).toHaveBeenCalledOnce(); expect(stopRelatedUploads).toHaveBeenCalledWith(taskId); expect(events.onFolderUploadSuccess).not.toHaveBeenCalled(); expect(events.onFolderUploadError).not.toHaveBeenCalled(); From 407e5629855c5964a462b23ac2e1a9d341c447a9 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:41:11 +0100 Subject: [PATCH 18/49] fix(upload): do not create a folder that was cancelled while its upload was starting --- src/app/upload/UploadManager.test.ts | 13 +++++++++++++ src/app/upload/UploadManager.ts | 2 ++ 2 files changed, 15 insertions(+) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 71a0941a77..a4b440deb2 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -393,6 +393,19 @@ describe('Uploading folders', () => { expect(succeeded.map(({ item }) => item)).toEqual([rootItem]); }); + test('When a folder is cancelled while its upload is being announced, then it is never created and is reported as cancelled', async () => { + const rootItem = createFolderItem(getFolderToUpload('Work'), 'dest'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const manager = createManager({ events }); + events.onItemStarted = () => manager.cancelItem(rootItem); + + await manager.run([rootItem]); + + expect(folderService.createFolderByUuid).not.toHaveBeenCalled(); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(rootItem); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + test('When folders appear while files are waiting, then the folders are created first', async () => { fakeSlowUploads(); const files = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 9b8c9db928..253766e2c2 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -259,6 +259,8 @@ export class UploadManager { } private async createFolder(folderItem: UploadFolderItem): Promise { + this.throwIfCancelled(folderItem); + const workspaceId = this.ownerUserAuthenticationData?.workspaceId; const { parentFolderUuid, folder } = folderItem; const [createdFolderPromise, requestCanceler] = workspaceId From 7017fcf6797c768abd07ffa72a69664fa24a2d04 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:42:36 +0100 Subject: [PATCH 19/49] test(upload): state each folder upload scenario plainly instead of relying on timings and hidden limits --- src/app/upload/UploadManager.test.ts | 129 +++++++++++++++------------ 1 file changed, 72 insertions(+), 57 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index a4b440deb2..dfbeb7f722 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -5,7 +5,11 @@ import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { + TWENTY_MEGABYTES, + USE_MULTIPART_THRESHOLD_BYTES, + WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS, +} from 'app/network/networkConstants'; import { QueueUtilsService } from 'utils/queueUtils'; import workspacesService from 'services/workspace.service'; import folderService from 'app/drive/services/folder.service'; @@ -315,12 +319,13 @@ describe('Uploading files', () => { }); }); -const fakeFolderCreationTaking = (delayMs = 0) => - vi.mocked(folderService.createFolderByUuid).mockImplementation((_parentUuid, name) => { - const cancel = vi.fn(); - const promise = new Promise((resolve) => setTimeout(() => resolve(getCreatedFolder(name)), delayMs)); - return [promise, { cancel }] as never; - }); +const fakeFolderCreation = () => + vi + .mocked(folderService.createFolderByUuid) + .mockImplementation((_parentUuid, name) => [Promise.resolve(getCreatedFolder(name)), { cancel: vi.fn() }] as never); + +const makeFolderCreationNeverFinish = () => + vi.mocked(folderService.createFolderByUuid).mockReturnValue([new Promise(() => undefined), { cancel: vi.fn() }]); const uploadedFileParents = () => Object.fromEntries((uploadFile as Mock).mock.calls.map(([file]) => [file.name, file.parentFolderId])); @@ -330,7 +335,12 @@ describe('Uploading folders', () => { vi.clearAllMocks(); vi.spyOn(errorService, 'reportError').mockReturnValue(); (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); - fakeFolderCreationTaking(); + fakeFolderCreation(); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); }); test('When a folder tree is uploaded, then each folder is created inside its parent and each file inside its folder', async () => { @@ -356,14 +366,16 @@ describe('Uploading folders', () => { const { succeeded } = await createManager().run([rootItem]); - const fileItem = succeeded.find(({ item }) => item.kind === UploadItemKind.File)?.item; - expect(fileItem?.parent).toBe(rootItem); + const [uploadedFolder, uploadedFile] = succeeded; + expect(uploadedFolder.item).toBe(rootItem); + expect(uploadedFile.item.parent).toBe(rootItem); }); - test('When a folder cannot be created, then its content is not uploaded', async () => { - vi.mocked(folderService.createFolderByUuid).mockImplementation( - () => [Promise.reject(new Error('Server error')), { cancel: vi.fn() }] as never, - ); + test('When a folder cannot be created, then it fails and its content is not uploaded', async () => { + vi.mocked(folderService.createFolderByUuid).mockReturnValue([ + Promise.reject(new Error('Server error')), + { cancel: vi.fn() }, + ]); const rootItem = createFolderItem( getFolderToUpload('Work', { childrenFiles: [getFileToUpload('report.pdf')] }), 'dest', @@ -371,25 +383,29 @@ describe('Uploading folders', () => { const { failed } = await failIfHangs(createManager().run([rootItem])); - expect(folderService.createFolderByUuid).toHaveBeenCalledTimes(2); expect(failed).toEqual([ { item: rootItem, failure: expect.objectContaining({ reason: UploadFailureReason.UploadFailed }) }, ]); expect(uploadFile).not.toHaveBeenCalled(); }); - test('When a folder is cancelled, then nothing else inside it is uploaded', async () => { + test('When a folder is cancelled while the files inside are uploading, then the waiting files are never uploaded', async () => { + uploadAtMostAtTheSameTime(2); fakeSlowUploads(); - const files = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => getFileToUpload(`${name}.txt`)); - const rootItem = createFolderItem(getFolderToUpload('Work', { childrenFiles: files }), 'dest'); + const rootItem = createFolderItem( + getFolderToUpload('Work', { + childrenFiles: ['uploading-1.txt', 'uploading-2.txt', 'waiting.txt'].map((name) => getFileToUpload(name)), + }), + 'dest', + ); const manager = createManager(); const upload = manager.run([rootItem]); - await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(6)); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['uploading-1', 'uploading-2'])); manager.cancelItem(rootItem); const { succeeded } = await failIfHangs(upload); - expect(uploadFile).toHaveBeenCalledTimes(6); + expect(startedUploadNames()).toEqual(['uploading-1', 'uploading-2']); expect(succeeded.map(({ item }) => item)).toEqual([rootItem]); }); @@ -406,42 +422,24 @@ describe('Uploading folders', () => { expect(events.onItemFailed).not.toHaveBeenCalled(); }); - test('When folders appear while files are waiting, then the folders are created first', async () => { - fakeSlowUploads(); - const files = ['a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); - const folderItem = createFolderItem(getFolderToUpload('Late'), 'dest'); - const manager = createManager(); + test('When files and a folder are waiting to upload, then the folder is created first', async () => { + uploadAtMostAtTheSameTime(1); + makeFolderCreationNeverFinish(); + const items = [getUploadFileItem('report'), createFolderItem(getFolderToUpload('Work'), 'dest')]; - const items = [...files, folderItem]; - const upload = manager.run(items); - await vi.waitFor(() => expect(folderService.createFolderByUuid).toHaveBeenCalled()); + void createManager().run(items); - expect(startedUploadNames()).not.toContain('g'); - items.forEach((item) => manager.cancelItem(item)); - await failIfHangs(upload); + await vi.waitFor(() => expect(folderService.createFolderByUuid).toHaveBeenCalledWith('dest', 'Work')); + expect(uploadFile).not.toHaveBeenCalled(); }); - test('When a folder is uploaded to a workspace, then it is created in the workspace and its content waits for it to be available', async () => { + test('When a folder is uploaded to a workspace, then it is created in the workspace', async () => { vi.mocked(workspacesService.createFolder).mockImplementation( ({ plainName }) => [Promise.resolve(getCreatedFolder(plainName)), { cancel: vi.fn() }] as never, ); - let folderCreatedAt = 0; - vi.mocked(workspacesService.createFolder).mockImplementationOnce(({ plainName }) => { - folderCreatedAt = Date.now(); - return [Promise.resolve(getCreatedFolder(plainName)), { cancel: vi.fn() }] as never; - }); - let fileUploadedAt = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - fileUploadedAt = Date.now(); - return { uuid: `${file.name}-uuid` }; - }); const ownerUserAuthenticationData = { workspaceId: 'workspace-id' } as never; - const rootItem = createFolderItem( - getFolderToUpload('Work', { childrenFiles: [getFileToUpload('report.pdf')] }), - 'dest', - ); - await createManager({ ownerUserAuthenticationData }).run([rootItem]); + await createManager({ ownerUserAuthenticationData }).run([createFolderItem(getFolderToUpload('Work'), 'dest')]); expect(workspacesService.createFolder).toHaveBeenCalledWith({ workspaceId: 'workspace-id', @@ -449,24 +447,41 @@ describe('Uploading folders', () => { plainName: 'Work', }); expect(folderService.createFolderByUuid).not.toHaveBeenCalled(); - expect(fileUploadedAt - folderCreatedAt).toBeGreaterThanOrEqual(500); + }); + + test('When a folder is created in a workspace, then its content waits until the folder is available', async () => { + vi.useFakeTimers(); + vi.mocked(workspacesService.createFolder).mockImplementation( + ({ plainName }) => [Promise.resolve(getCreatedFolder(plainName)), { cancel: vi.fn() }] as never, + ); + const ownerUserAuthenticationData = { workspaceId: 'workspace-id' } as never; + const rootItem = createFolderItem( + getFolderToUpload('Work', { childrenFiles: [getFileToUpload('report.pdf')] }), + 'dest', + ); + + void createManager({ ownerUserAuthenticationData }).run([rootItem]); + await vi.advanceTimersByTimeAsync(WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS - 1); + + expect(workspacesService.createFolder).toHaveBeenCalled(); + expect(uploadFile).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(1); + + expect(startedUploadNames()).toEqual(['report']); }); test('When a large file is uploaded together with a folder, then it waits for the files inside the folder so they are not uploaded one at a time behind it', async () => { - fakeFolderCreationTaking(30); - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - await new Promise((resolve) => setTimeout(resolve, 10)); - return { uuid: `${file.name}-uuid` }; - }); const largeFile = getUploadFileItem('large', USE_MULTIPART_THRESHOLD_BYTES); const folderItem = createFolderItem( - getFolderToUpload('Photos', { childrenFiles: ['a', 'b', 'c'].map((name) => getFileToUpload(`${name}.png`)) }), + getFolderToUpload('Photos', { + childrenFiles: [getFileToUpload('beach.png'), getFileToUpload('city.png'), getFileToUpload('dog.png')], + }), 'dest', ); - const { succeeded } = await failIfHangs(createManager().run([largeFile, folderItem])); + await failIfHangs(createManager().run([largeFile, folderItem])); - expect(succeeded).toHaveLength(5); - expect(startedUploadNames()).toEqual(['a', 'b', 'c', 'large']); + expect(startedUploadNames()).toEqual(['beach', 'city', 'dog', 'large']); }); }); From 212bf47493282d949bee88421a1f349e564baf8d Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:12:08 +0100 Subject: [PATCH 20/49] refactor(upload): keep the upload engine types and failures next to the engine --- src/app/network/networkConstants.ts | 1 - src/app/network/types/index.ts | 11 ----------- src/app/upload/UploadManager.test.ts | 3 +-- src/app/upload/UploadManager.ts | 17 +++++++++++++---- .../types/upload.ts => upload/types.ts} | 14 ++++++++++++-- src/app/{network => upload}/uploadFailure.ts | 2 +- src/testUtils/fixtures/upload.fixtures.ts | 3 +-- 7 files changed, 28 insertions(+), 23 deletions(-) rename src/app/{network/types/upload.ts => upload/types.ts} (74%) rename src/app/{network => upload}/uploadFailure.ts (96%) diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index 69f3b47c6a..1e857231ac 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -10,4 +10,3 @@ export const USE_MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024; export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; -export const SINGLE_TRANSFER_CONCURRENCY = 1; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index f099c9a09b..edb1f68756 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -35,17 +35,6 @@ export interface DownloadChunkPayload { options?: DownloadOptions; } -export enum UploadItemKind { - File = 'file', -} - -export enum UploadFailureReason { - ConnectionLost = 'connection-lost', - NoSpaceLeft = 'no-space', - FileTooLarge = 'file-too-large', - UploadFailed = 'upload-failed', -} - export type UploadErrorReason = 'connection-lost' | 'upload-failed'; export interface OwnerUserAuthenticationData { diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 3d865a76e2..21fb5b7e46 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -9,8 +9,7 @@ import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/net import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; -import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason, UploadItem, UploadManagerEvents } from './types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 880e480ff4..ec187ddf5e 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -9,13 +9,22 @@ import { TaskStatus } from 'app/tasks/types'; import { MAX_CONCURRENT_UPLOADS, MAX_UPLOAD_ATTEMPTS, - SINGLE_TRANSFER_CONCURRENCY, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES, } from 'app/network/networkConstants'; -import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; -import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; -import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; +import { OwnerUserAuthenticationData } from 'app/network/types'; +import { + UploadedItem, + UploadFailureReason, + UploadFileItem, + UploadItem, + UploadItemKind, + UploadManagerEvents, + UploadResult, +} from './types'; +import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from './uploadFailure'; + +const SINGLE_TRANSFER_CONCURRENCY = 1; enum UploadPriority { SmallFile = 1, diff --git a/src/app/network/types/upload.ts b/src/app/upload/types.ts similarity index 74% rename from src/app/network/types/upload.ts rename to src/app/upload/types.ts index 53cb34dae3..bc4b98495a 100644 --- a/src/app/network/types/upload.ts +++ b/src/app/upload/types.ts @@ -1,7 +1,17 @@ import { DriveFileData } from 'app/drive/types'; import { FileToUpload } from 'app/drive/services/file.service/types'; -import { UploadFailure } from '../uploadFailure'; -import { UploadItemKind } from '.'; +import { UploadFailure } from './uploadFailure'; + +export enum UploadItemKind { + File = 'file', +} + +export enum UploadFailureReason { + ConnectionLost = 'connection-lost', + NoSpaceLeft = 'no-space', + FileTooLarge = 'file-too-large', + UploadFailed = 'upload-failed', +} export type UploadFileItem = { kind: UploadItemKind.File; file: FileToUpload }; diff --git a/src/app/network/uploadFailure.ts b/src/app/upload/uploadFailure.ts similarity index 96% rename from src/app/network/uploadFailure.ts rename to src/app/upload/uploadFailure.ts index 7ac796a44e..1edbebdd2b 100644 --- a/src/app/network/uploadFailure.ts +++ b/src/app/upload/uploadFailure.ts @@ -1,6 +1,6 @@ import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; -import { ConnectionLostError } from './requests'; +import { ConnectionLostError } from 'app/network/requests'; import { UploadFailureReason } from './types'; export interface UploadFailure { diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts index 6e123b622e..76b61e40a1 100644 --- a/src/testUtils/fixtures/upload.fixtures.ts +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -1,6 +1,5 @@ import { vi } from 'vitest'; -import { UploadItemKind } from 'app/network/types'; -import { UploadFileItem } from 'app/network/types/upload'; +import { UploadFileItem, UploadItemKind } from 'app/upload/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { TaskStatus } from 'app/tasks/types'; From 582e8225ee5026391c7aba5e02f6c1b485a48991 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:13:25 +0100 Subject: [PATCH 21/49] refactor(upload): import the upload types from their new place --- src/app/network/UploadFolderManager.test.ts | 2 +- src/app/network/UploadFolderManager.ts | 2 +- src/app/tasks/upload/uploadFilesWithTasks.test.ts | 5 ++--- src/app/tasks/upload/uploadFilesWithTasks.ts | 4 ++-- src/app/tasks/upload/uploadFoldersWithTasks.test.ts | 2 +- src/app/tasks/upload/uploadTaskErrors.ts | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 805b2a9b7b..e6249dd10e 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -20,7 +20,7 @@ import newStorageService from 'app/drive/services/new-storage.service'; import RetryManager from './RetryManager'; import { ConnectionLostError } from './requests'; import errorService from 'services/error.service'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/upload/types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn(async (file: { name: string }) => ({ uuid: `${file.name}-uuid`, name: file.name })), diff --git a/src/app/network/UploadFolderManager.ts b/src/app/network/UploadFolderManager.ts index 0822ba6acf..c60e567ccb 100644 --- a/src/app/network/UploadFolderManager.ts +++ b/src/app/network/UploadFolderManager.ts @@ -16,7 +16,7 @@ import { wait } from 'utils/timeUtils'; import { ConnectionLostError } from './requests'; import { FilesExceedsSizeLimitError } from 'app/drive/services/file.service/upload.errors'; import { filterFilesByMaxSize } from 'app/store/slices/storage/fileUtils/filterFilesByMaxSize'; -import { UploadFailureReason } from './types'; +import { UploadFailureReason } from 'app/upload/types'; import { MAX_CONCURRENT_UPLOADS } from './networkConstants'; export interface UploadFolderPayload { diff --git a/src/app/tasks/upload/uploadFilesWithTasks.test.ts b/src/app/tasks/upload/uploadFilesWithTasks.test.ts index 570556638f..09b0d84dfc 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.test.ts @@ -1,12 +1,11 @@ import { beforeEach, describe, expect, Mock, test, vi } from 'vitest'; -import { UploadFailure } from 'app/network/uploadFailure'; -import { UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { UploadFailure } from 'app/upload/uploadFailure'; +import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents, UploadResult } from 'app/upload/types'; import tasksService from 'app/tasks/services/tasks.service'; import { TaskStatus } from 'app/tasks/types'; import referralService from 'services/referral.service'; import * as networkInformation from 'app/network/networkInformation'; import { UploadManager } from 'app/upload/UploadManager'; -import { UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; import RetryManager from 'app/network/RetryManager'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { uploadFilesWithTasks } from './uploadFilesWithTasks'; diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index 96a32f12bf..b2c43faa2a 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -5,9 +5,9 @@ import { logNetworkInfoForUpload } from 'app/network/networkInformation'; import { DriveFileData } from 'app/drive/types'; import { FileToUpload } from 'app/drive/services/file.service/types'; import { UploadManager } from 'app/upload/UploadManager'; -import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; +import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents } from 'app/upload/types'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; -import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { OwnerUserAuthenticationData } from 'app/network/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import errorService from 'services/error.service'; import { errorMerge } from './uploadTaskErrors'; diff --git a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts index 88efeef923..9b4f3f041b 100644 --- a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts @@ -7,7 +7,7 @@ import { uploadFoldersWithManager, UploadFolderManagerEvents } from 'app/network import { IRoot } from 'app/store/slices/storage/types'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; import { uploadFoldersWithTasks } from './uploadFoldersWithTasks'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/upload/types'; vi.mock('app/network/UploadFolderManager', () => ({ uploadFoldersWithManager: vi.fn(() => Promise.resolve()), diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index f62e82df00..9e3c439001 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -1,6 +1,6 @@ import { t } from 'i18next'; import { TaskStatus } from 'app/tasks/types'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/upload/types'; const errorSubtitle = (reason?: UploadFailureReason): string | undefined => { switch (reason) { From 3e92a9fd50db5cb97e0a5d2ed25cb798c9cc1301 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:24:55 +0100 Subject: [PATCH 22/49] refactor(upload): classify plan limit failures once and retry automatically only generic upload failures --- src/app/upload/UploadManager.test.ts | 50 +++++++++++++++++++++++++++- src/app/upload/UploadManager.ts | 4 +-- src/app/upload/types.ts | 1 + src/app/upload/uploadFailure.ts | 10 +++--- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 21fb5b7e46..2488122320 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -127,7 +127,7 @@ describe('Uploading files', () => { }); test.each([ - { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.UploadFailed }, + { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.PlanLimit }, { problem: 'the file is too large', error: fileTooLargeError, reason: UploadFailureReason.FileTooLarge }, ])('When $problem, then the file is never retried', async ({ error, reason }) => { (uploadFile as Mock).mockRejectedValue(error); @@ -192,6 +192,54 @@ describe('Uploading files', () => { expect(events.onItemFailed).not.toHaveBeenCalled(); }); + test('When a file still waiting in the queue is cancelled, then it is never uploaded and the uploading one finishes', async () => { + uploadAtMostAtTheSameTime(1); + fakeSlowUploads(); + const [uploading, queued] = [getUploadFileItem('uploading'), getUploadFileItem('queued')]; + const events: UploadManagerEvents = { onItemCancelled: vi.fn() }; + const manager = createManager({ events }); + + const upload = manager.run([uploading, queued]); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['uploading'])); + manager.cancelItem(queued); + const { succeeded } = await failIfHangs(upload); + + expect(startedUploadNames()).toEqual(['uploading']); + expect(succeeded.map(({ item }) => item)).toEqual([uploading]); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(queued); + }); + + test('When a file is cancelled and its upload then fails, then it is not retried and is reported as cancelled', async () => { + const report = getUploadFileItem('report'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const manager = createManager({ events }); + (uploadFile as Mock).mockImplementationOnce(() => { + manager.cancelItem(report); + return Promise.reject(new Error('Server error')); + }); + + await manager.run([report]); + + expect(uploadFile).toHaveBeenCalledOnce(); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(report); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + + test('When a file finishes uploading right after being cancelled, then it is reported as uploaded', async () => { + const report = getUploadFileItem('report'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn() }; + const manager = createManager({ events }); + (uploadFile as Mock).mockImplementationOnce(() => { + manager.cancelItem(report); + return Promise.resolve({ uuid: 'report-uuid' }); + }); + + const { succeeded } = await manager.run([report]); + + expect(succeeded.map(({ item }) => item)).toEqual([report]); + expect(events.onItemCancelled).not.toHaveBeenCalled(); + }); + test('When a single file is cancelled, then only that file stops', async () => { fakeSlowUploads(); const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index ec187ddf5e..a9902a344f 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -22,7 +22,7 @@ import { UploadManagerEvents, UploadResult, } from './types'; -import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from './uploadFailure'; +import { shouldAutoRetry, toUploadFailure, UploadFailure } from './uploadFailure'; const SINGLE_TRANSFER_CONCURRENCY = 1; @@ -171,7 +171,7 @@ export class UploadManager { return await this.uploadFile(item); } catch (error) { const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; - const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); + const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && shouldAutoRetry(toUploadFailure(error)); if (!shouldRetry) throw error; return this.uploadWithRetries(item, attempt + 1); diff --git a/src/app/upload/types.ts b/src/app/upload/types.ts index bc4b98495a..f374d0504f 100644 --- a/src/app/upload/types.ts +++ b/src/app/upload/types.ts @@ -10,6 +10,7 @@ export enum UploadFailureReason { ConnectionLost = 'connection-lost', NoSpaceLeft = 'no-space', FileTooLarge = 'file-too-large', + PlanLimit = 'plan-limit', UploadFailed = 'upload-failed', } diff --git a/src/app/upload/uploadFailure.ts b/src/app/upload/uploadFailure.ts index 1edbebdd2b..78532669e3 100644 --- a/src/app/upload/uploadFailure.ts +++ b/src/app/upload/uploadFailure.ts @@ -21,18 +21,21 @@ const isStorageFull = (error: unknown) => getStatusCode(error) === HTTP_STATUS_C const isFileTooLarge = (error: unknown) => errorService.castError(error).code === HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED; -const isPlanNotAllowingIt = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.PAYMENT_REQUIRED; +const isPlanLimitReached = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.PAYMENT_REQUIRED; + +const REASONS_THAT_RETRYING_CANNOT_FIX = new Set([UploadFailureReason.FileTooLarge, UploadFailureReason.PlanLimit]); const getFailureReason = (error: unknown): UploadFailureReason => { if (isConnectionLost(error)) return UploadFailureReason.ConnectionLost; if (isStorageFull(error)) return UploadFailureReason.NoSpaceLeft; if (isFileTooLarge(error)) return UploadFailureReason.FileTooLarge; + if (isPlanLimitReached(error)) return UploadFailureReason.PlanLimit; return UploadFailureReason.UploadFailed; }; export const toUploadFailure = (error: unknown): UploadFailure => { const reason = getFailureReason(error); - const canSucceedLater = reason !== UploadFailureReason.FileTooLarge && !isPlanNotAllowingIt(error); + const canSucceedLater = !REASONS_THAT_RETRYING_CANNOT_FIX.has(reason); const genericUploadError = new Error('Upload failed'); return { @@ -42,5 +45,4 @@ export const toUploadFailure = (error: unknown): UploadFailure => { }; }; -export const canSucceedOnImmediateRetry = (error: unknown): boolean => - getFailureReason(error) === UploadFailureReason.UploadFailed && !isPlanNotAllowingIt(error); +export const shouldAutoRetry = ({ reason }: UploadFailure): boolean => reason === UploadFailureReason.UploadFailed; From 5448c6131f6a6a90d4eb4eddf573341f4ba3dcb4 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:25:40 +0100 Subject: [PATCH 23/49] fix(upload): keep showing the upload failed message when the plan limit is reached --- src/app/tasks/upload/uploadTaskErrors.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index 9e3c439001..ee64ed7d50 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -7,6 +7,7 @@ const errorSubtitle = (reason?: UploadFailureReason): string | undefined => { case UploadFailureReason.ConnectionLost: return t('error.connectionLostError') as string; case UploadFailureReason.UploadFailed: + case UploadFailureReason.PlanLimit: return t('tasks.subtitles.upload-failed') as string; default: return undefined; From f8bb4973a172b90128e8b963afa87778a176e638 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 16:22:00 +0100 Subject: [PATCH 24/49] fix(upload): show the storage full and file too large dialogs on the first failure, and keep finishing a file when the folder progress cannot be read --- .../tasks/upload/uploadFilesWithTasks.test.ts | 38 +++++++++++-- src/app/tasks/upload/uploadFilesWithTasks.ts | 57 ++++++++++++------- 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/src/app/tasks/upload/uploadFilesWithTasks.test.ts b/src/app/tasks/upload/uploadFilesWithTasks.test.ts index 09b0d84dfc..448ba8557d 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.test.ts @@ -4,6 +4,7 @@ import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents, U import tasksService from 'app/tasks/services/tasks.service'; import { TaskStatus } from 'app/tasks/types'; import referralService from 'services/referral.service'; +import errorService from 'services/error.service'; import * as networkInformation from 'app/network/networkInformation'; import { UploadManager } from 'app/upload/UploadManager'; import RetryManager from 'app/network/RetryManager'; @@ -227,6 +228,26 @@ describe('Uploading files with tasks', () => { }); }); + test('When the folder upload state cannot be read after a file inside it finishes, then the file is still shown and leaves the retry list', async () => { + vi.spyOn(errorService, 'reportError').mockReturnValue(); + (uploadRepository.getUploadState as Mock) + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('IndexedDB error')); + RetryManager.addTasks([{ taskId: 'existing-task', type: 'upload' as never, params: {} }]); + const onFinishUploadFile = vi.fn(); + const relatedTaskProgress = { filesUploaded: 0, totalFilesToUpload: 4 }; + const { events, items } = await runUpload( + [buildFile({ taskId: 'existing-task', relatedTaskId: 'related-task', onFinishUploadFile })], + { relatedTaskProgress }, + ); + + await events.onItemSucceeded?.(items[0], { uuid: 'file-uuid' } as never); + + expect(onFinishUploadFile).toHaveBeenCalledOnce(); + expect(RetryManager.getTasks()).toEqual([]); + expect(relatedTaskProgress.filesUploaded).toBe(1); + }); + test('When an upload fails, then the task shows an error message matching the cause', async () => { (tasksService.create as Mock).mockReturnValue('t1'); const { events, items } = await runUpload([buildFile()]); @@ -300,12 +321,17 @@ describe('Uploading files with tasks', () => { reason: UploadFailureReason.FileTooLarge, callback: 'fileSizeExceededCallback', }, - ] as const)('When $problem, then the user is warned about it', async ({ reason, callback }) => { - const warn = vi.fn(); - failWith({ reason }); + ] as const)( + 'When $problem, then the user is warned as soon as a file fails and only once', + async ({ reason, callback }) => { + const warn = vi.fn(); + const { events, items } = await runUpload([buildFile(), buildFile()], { [callback]: warn }); + const failure = { error: new Error('failed'), reason, isRetryable: false }; - await expect(runUpload([buildFile()], { [callback]: warn })).rejects.toThrow(); + events.onItemFailed?.(items[0], failure); + events.onItemFailed?.(items[1], failure); - expect(warn).toHaveBeenCalledOnce(); - }); + expect(warn).toHaveBeenCalledOnce(); + }, + ); }); diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index b2c43faa2a..168faca2b8 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -25,6 +25,8 @@ export type UploadManagerFileParams = { type UploadFileParamsWithTaskId = UploadManagerFileParams & { taskId: string }; +type FolderProgress = { filesUploaded: number; totalFilesToUpload: number }; + interface UploadFilesWithTasksProps { files: UploadManagerFileParams[]; maxSpaceOccupiedCallback: () => void; @@ -40,7 +42,7 @@ interface UploadFilesWithTasksProps { currentFolderId?: string; }; }; - relatedTaskProgress?: { filesUploaded: number; totalFilesToUpload: number }; + relatedTaskProgress?: FolderProgress; } /** @@ -58,6 +60,33 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr return file.relatedTaskId ?? file.taskId; }; + const failureDialogByReason: Partial void>> = { + [UploadFailureReason.NoSpaceLeft]: props.maxSpaceOccupiedCallback, + [UploadFailureReason.FileTooLarge]: props.fileSizeExceededCallback, + }; + const reasonsWithDialogShown = new Set(); + + const showFailureDialogOnce = (reason: UploadFailureReason) => { + const showDialog = failureDialogByReason[reason]; + const hasNoDialog = !showDialog; + const wasAlreadyShown = reasonsWithDialogShown.has(reason); + if (hasNoDialog || wasAlreadyShown) return; + + reasonsWithDialogShown.add(reason); + showDialog(); + }; + + const updateFolderProgress = async (folderTaskId: string, folderProgress: FolderProgress) => { + const folderUploadState = await uploadRepository.getUploadState(folderTaskId); + const isPaused = folderUploadState === TaskStatus.Paused; + if (isPaused) return; + + tasksService.updateTask({ + taskId: folderTaskId, + merge: { progress: folderProgress.filesUploaded / folderProgress.totalFilesToUpload }, + }); + }; + const events: UploadManagerEvents = { onItemStarted: (item) => { const file = getFile(item); @@ -87,19 +116,6 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr const driveFileData = uploadedItem as DriveFileData; const file = getFile(item); - if (relatedTaskProgress && file.relatedTaskId) { - relatedTaskProgress.filesUploaded += 1; - - const folderUploadState = await uploadRepository.getUploadState(file.relatedTaskId); - const isPaused = folderUploadState === TaskStatus.Paused; - if (!isPaused) { - tasksService.updateTask({ - taskId: file.relatedTaskId, - merge: { progress: relatedTaskProgress.filesUploaded / relatedTaskProgress.totalFilesToUpload }, - }); - } - } - if (tasksService.findTask(file.taskId)?.status !== TaskStatus.Cancelled) { tasksService.updateTask({ taskId: file.taskId, @@ -112,12 +128,20 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr logNetworkInfoForUpload({ fileName: file.filecontent.name, fileSize: file.filecontent.size }); file.onFinishUploadFile?.(driveFileData, file.taskId); props.onFileUploadCallback?.(driveFileData); + + if (relatedTaskProgress && file.relatedTaskId) { + relatedTaskProgress.filesUploaded += 1; + await updateFolderProgress(file.relatedTaskId, relatedTaskProgress).catch((error) => + errorService.reportError(error), + ); + } }, onItemFailed: (item, { reason }) => { const { taskId } = getFile(item); tasksService.updateTask({ taskId, merge: errorMerge(reason) }); uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); + showFailureDialogOnce(reason); }, onItemCancelled: (item) => { @@ -180,11 +204,6 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr }), ); - const failureReasons = new Set(failed.map(({ failure }) => failure.reason)); - - if (failureReasons.has(UploadFailureReason.NoSpaceLeft)) props.maxSpaceOccupiedCallback(); - if (failureReasons.has(UploadFailureReason.FileTooLarge)) props.fileSizeExceededCallback?.(); - throw failed[0].failure.error; } }; From 2b5ad5bea135cdb69eea920d7fdc545433794178 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 16:51:58 +0100 Subject: [PATCH 25/49] refactor(upload): add Upload to track each user upload with its tasks and cancel it with rollback --- src/app/upload/Upload.test.ts | 220 +++++++++++++++++++++ src/app/upload/Upload.ts | 276 +++++++++++++++++++++++++++ src/app/upload/UploadManager.test.ts | 22 +++ src/app/upload/UploadManager.ts | 6 +- src/app/upload/types.ts | 1 + 5 files changed, 524 insertions(+), 1 deletion(-) create mode 100644 src/app/upload/Upload.test.ts create mode 100644 src/app/upload/Upload.ts diff --git a/src/app/upload/Upload.test.ts b/src/app/upload/Upload.test.ts new file mode 100644 index 0000000000..b270b668c0 --- /dev/null +++ b/src/app/upload/Upload.test.ts @@ -0,0 +1,220 @@ +import { beforeEach, describe, expect, Mock, test, vi } from 'vitest'; +import uploadFile from 'app/drive/services/file.service/uploadFile'; +import folderService from 'app/drive/services/folder.service'; +import newStorageService from 'app/drive/services/new-storage.service'; +import { IRoot } from 'app/store/slices/storage/types'; +import tasksService from 'app/tasks/services/tasks.service'; +import { TaskStatus, TaskType } from 'app/tasks/types'; +import errorService from 'services/error.service'; +import { AppError } from '@internxt/sdk'; +import { HTTP_STATUS_CODES } from 'app/core/constants'; +import { + getCreatedFolder, + getFileToUpload, + getFolderToUpload, + getUploadRepository, +} from 'testUtils/fixtures/upload.fixtures'; +import { Upload } from './Upload'; + +vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); +vi.mock('app/drive/services/folder.service', () => ({ default: { createFolderByUuid: vi.fn() } })); +vi.mock('services/workspace.service', () => ({ default: { createFolder: vi.fn() } })); +vi.mock('app/drive/services/new-storage.service', () => ({ default: { deleteFolderByUuid: vi.fn() } })); +vi.mock('services/referral.service', () => ({ default: { trackFileUpload: vi.fn(), trackFolderUpload: vi.fn() } })); +vi.mock('app/network/networkInformation', () => ({ logNetworkInfoForUpload: vi.fn() })); +vi.mock('i18next', () => ({ t: (key: string) => key })); + +const DESTINATION_UUID = 'destination-uuid'; + +const uploadRepository = getUploadRepository(); + +const fakeFolderCreationTaking = (delayMs = 0) => + vi.mocked(folderService.createFolderByUuid).mockImplementation((_parentUuid, name) => { + const createdFolder = getCreatedFolder(name); + return [new Promise((resolve) => setTimeout(() => resolve(createdFolder), delayMs)), { cancel: vi.fn() }] as never; + }); + +const fakeUploadsTaking = (delayMs: number) => + (uploadFile as Mock).mockImplementation( + (file: { name: string }, _progress, options: { abortCallback?: (abort: () => void) => void }) => + new Promise((resolve, reject) => { + const timeout = setTimeout(() => resolve({ uuid: `${file.name}-uuid` }), delayMs); + options.abortCallback?.(() => { + clearTimeout(timeout); + reject(new Error('aborted')); + }); + }), + ); + +const createUpload = ( + roots: IRoot[], + files: File[], + callbacks: Partial[0]> = {}, +) => new Upload({ destinationFolderUuid: DESTINATION_UUID, roots, files, uploadRepository, ...callbacks }); + +const tasksOf = (action: TaskType) => tasksService.getTasks().filter((task) => task.action === action); + +const failUploadsOf = (failingNames: string[], error: unknown = new Error('Server error')) => + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + if (failingNames.includes(file.name)) throw error; + return { uuid: `${file.name}-uuid` }; + }); + +describe('Uploading what the user picked', () => { + beforeEach(() => { + vi.clearAllMocks(); + tasksService.clearTasks(); + vi.spyOn(errorService, 'reportError').mockReturnValue(); + vi.mocked(newStorageService.deleteFolderByUuid).mockResolvedValue(undefined as never); + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); + fakeFolderCreationTaking(); + }); + + test('When a folder and a file are uploaded, then each one is tracked by its own task until it succeeds', async () => { + const upload = createUpload( + [getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png'), getFileToUpload('city.png')] })], + [getFileToUpload('report.pdf')], + ); + + await upload.run(); + + const [folderTask] = tasksOf(TaskType.UploadFolder); + const [fileTask] = tasksOf(TaskType.UploadFile); + expect(tasksService.getTasks()).toHaveLength(2); + expect(folderTask).toEqual( + expect.objectContaining({ status: TaskStatus.Success, progress: 1, itemUUID: { rootFolderUUID: 'Photos-uuid' } }), + ); + expect(fileTask).toEqual( + expect.objectContaining({ status: TaskStatus.Success, itemUUID: { fileUUID: 'report-uuid' } }), + ); + }); + + test('When the items are uploaded, then each one is reported with the folder it was uploaded into', async () => { + const onItemUploaded = vi.fn(); + const upload = createUpload( + [getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], + [getFileToUpload('report.pdf')], + { + onItemUploaded, + }, + ); + + await upload.run(); + + expect(onItemUploaded).toHaveBeenCalledWith( + expect.objectContaining({ uuid: 'Photos-uuid' }), + DESTINATION_UUID, + true, + ); + expect(onItemUploaded).toHaveBeenCalledWith(expect.objectContaining({ name: 'beach' }), 'Photos-uuid', false); + expect(onItemUploaded).toHaveBeenCalledWith(expect.objectContaining({ name: 'report' }), DESTINATION_UUID, false); + }); + + test('When a file inside a folder fails, then the folder still succeeds with what could be uploaded', async () => { + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + if (file.name === 'broken') throw new Error('Server error'); + return { uuid: `${file.name}-uuid` }; + }); + const upload = createUpload( + [getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('broken.png'), getFileToUpload('beach.png')] })], + [], + ); + + const { failed } = await upload.run(); + + expect(failed).toHaveLength(1); + expect(tasksOf(TaskType.UploadFolder)[0]).toEqual( + expect.objectContaining({ status: TaskStatus.Success, progress: 2 / 3 }), + ); + }); + + test('When a folder cannot be created, then its task shows the error', async () => { + vi.mocked(folderService.createFolderByUuid).mockImplementation( + () => [Promise.reject(new Error('Server error')), { cancel: vi.fn() }] as never, + ); + + await createUpload([getFolderToUpload('Photos')], []).run(); + + expect(tasksOf(TaskType.UploadFolder)[0]).toEqual( + expect.objectContaining({ status: TaskStatus.Error, subtitle: 'tasks.subtitles.upload-failed' }), + ); + }); + + test('When a file fails, then its task shows the reason', async () => { + (uploadFile as Mock).mockRejectedValue(new Error('Server error')); + + await createUpload([], [getFileToUpload('report.pdf')]).run(); + + expect(tasksOf(TaskType.UploadFile)[0]).toEqual( + expect.objectContaining({ status: TaskStatus.Error, subtitle: 'tasks.subtitles.upload-failed' }), + ); + }); + + test('When the user cancels a file, then only that file stops', async () => { + fakeUploadsTaking(500); + const upload = createUpload([], [getFileToUpload('report.pdf'), getFileToUpload('invoice.pdf')]); + + const running = upload.run(); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(2)); + const [reportTask, invoiceTask] = tasksOf(TaskType.UploadFile); + await tasksService.cancelTask(reportTask.id); + await running; + + expect(tasksService.findTask(reportTask.id)?.status).toBe(TaskStatus.Cancelled); + expect(tasksService.findTask(invoiceTask.id)?.status).toBe(TaskStatus.Success); + }); + + test('When the user cancels a file, then its saved upload state is removed', async () => { + fakeUploadsTaking(500); + const upload = createUpload([], [getFileToUpload('report.pdf')]); + + const running = upload.run(); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalled()); + const [reportTask] = tasksOf(TaskType.UploadFile); + await tasksService.cancelTask(reportTask.id); + await running; + + expect(uploadRepository.removeUploadState).toHaveBeenCalledWith(reportTask.id); + }); + + test('When the user cancels a folder, then its upload stops and the folder already created is removed', async () => { + fakeUploadsTaking(500); + const onItemRemoved = vi.fn(); + const upload = createUpload([getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], [], { + onItemRemoved, + }); + + const running = upload.run(); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalled()); + await tasksService.cancelTask(tasksOf(TaskType.UploadFolder)[0].id); + await running; + + expect(newStorageService.deleteFolderByUuid).toHaveBeenCalledWith('Photos-uuid'); + await vi.waitFor(() => + expect(onItemRemoved).toHaveBeenCalledWith(expect.objectContaining({ uuid: 'Photos-uuid' }), DESTINATION_UUID), + ); + expect(tasksOf(TaskType.UploadFolder)[0].status).toBe(TaskStatus.Cancelled); + }); + + test('When the user cancels a folder while it is being created, then it is removed once it exists', async () => { + fakeFolderCreationTaking(300); + const upload = createUpload([getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], []); + + const running = upload.run(); + await vi.waitFor(() => expect(folderService.createFolderByUuid).toHaveBeenCalled()); + await tasksService.cancelTask(tasksOf(TaskType.UploadFolder)[0].id); + await running; + + expect(newStorageService.deleteFolderByUuid).toHaveBeenCalledWith('Photos-uuid'); + expect(uploadFile).not.toHaveBeenCalled(); + }); + + test('When the storage fills up while uploading, then the user is told once, however many items fail', async () => { + const onStorageFull = vi.fn(); + failUploadsOf(['report', 'invoice'], new AppError('No space left', HTTP_STATUS_CODES.MAX_SPACE_USED)); + + await createUpload([], [getFileToUpload('report.pdf'), getFileToUpload('invoice.pdf')], { onStorageFull }).run(); + + expect(onStorageFull).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/app/upload/Upload.ts b/src/app/upload/Upload.ts new file mode 100644 index 0000000000..7b7334e99d --- /dev/null +++ b/src/app/upload/Upload.ts @@ -0,0 +1,276 @@ +import newStorageService from 'app/drive/services/new-storage.service'; +import { DriveFolderData } from 'app/drive/types'; +import { logNetworkInfoForUpload } from 'app/network/networkInformation'; +import { OwnerUserAuthenticationData } from 'app/network/types'; +import { + UploadedItem, + UploadFileItem, + UploadFolderItem, + UploadItem, + UploadItemKind, + UploadManagerEvents, + UploadResult, +} from 'app/upload/types'; +import { UploadFailure } from 'app/upload/uploadFailure'; +import { createFileItem, createFolderItem, UploadManager } from './UploadManager'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { IRoot } from 'app/store/slices/storage/types'; +import tasksService from 'app/tasks/services/tasks.service'; +import { TaskStatus, TaskType, UploadFileTask, UploadFolderTask } from 'app/tasks/types'; +import { errorMerge } from 'app/tasks/upload/uploadTaskErrors'; +import errorService from 'services/error.service'; +import referralService from 'services/referral.service'; + +interface UploadProps { + destinationFolderUuid: string; + roots: IRoot[]; + files: File[]; + ownerUserAuthenticationData?: OwnerUserAuthenticationData; + uploadRepository: PersistUploadRepository; + onItemUploaded?: (uploadedItem: UploadedItem, parentFolderUuid: string, isFolder: boolean) => void; + onItemRemoved?: (removedItem: UploadedItem, parentFolderUuid: string) => void; + onStorageFull?: () => void; +} + +interface RootFolderUpload { + taskId: string; + totalItems: number; + uploadedItems: number; + isCancelled: boolean; + hasFailedItems: boolean; + createdFolder?: DriveFolderData; +} + +const countItemsIn = (folder: IRoot): number => + folder.childrenFolders.reduce((count, child) => count + countItemsIn(child), 1 + folder.childrenFiles.length); + +const getTopLevelItem = (item: UploadItem): UploadItem => { + let topLevelItem = item; + while (topLevelItem.parent) topLevelItem = topLevelItem.parent; + return topLevelItem; +}; + +const getParentFolderUuid = (item: UploadItem): string => + item.kind === UploadItemKind.File ? item.file.parentFolderId : item.parentFolderUuid; + +const createItemsToUpload = ({ roots, files, destinationFolderUuid }: UploadProps): UploadItem[] => [ + ...roots.map((root) => createFolderItem(root, destinationFolderUuid)), + ...files.map((file) => createFileItem(file, destinationFolderUuid)), +]; + +const isTaskStopped = (taskId: string): boolean => { + const status = tasksService.findTask(taskId)?.status; + return status === TaskStatus.Cancelled || status === TaskStatus.Paused; +}; + +export class Upload { + private readonly props: UploadProps; + private readonly manager: UploadManager; + private readonly items: UploadItem[]; + private readonly taskIdByItem = new Map(); + private readonly rootFolders = new Map(); + + constructor(props: UploadProps) { + this.props = props; + this.items = createItemsToUpload(props); + this.manager = new UploadManager({ + ownerUserAuthenticationData: props.ownerUserAuthenticationData, + uploadRepository: props.uploadRepository, + getPauseKey: (item) => this.getTaskIdOf(getTopLevelItem(item)), + events: this.createManagerEvents(), + }); + + for (const item of this.items) { + if (item.kind === UploadItemKind.Folder) this.trackRootFolder(item); + else this.trackFile(item); + } + } + + async run(): Promise { + const result = await this.manager.run(this.items); + this.rootFolders.forEach((rootFolder, item) => this.finishRootFolder(item as UploadFolderItem, rootFolder)); + return result; + } + + private createManagerEvents(): UploadManagerEvents { + return { + onItemStarted: (item) => this.onItemStarted(item), + onItemProgress: (item, progress) => this.onItemProgress(item, progress), + onItemSucceeded: (item, uploadedItem) => this.onItemSucceeded(item, uploadedItem), + onItemFailed: (item, failure) => this.onItemFailed(item, failure), + onItemCancelled: (item) => this.onItemCancelled(item), + onStorageFull: () => this.props.onStorageFull?.(), + }; + } + + private getTaskIdOf(item: UploadItem): string { + return this.taskIdByItem.get(item) as string; + } + + private trackRootFolder(item: UploadFolderItem): void { + const taskId = tasksService.create({ + action: TaskType.UploadFolder, + folderName: item.folder.name, + item: item.folder, + parentFolderId: item.parentFolderUuid, + showNotification: true, + cancellable: true, + stop: () => Promise.resolve(this.cancelRootFolder(item)), + }); + + this.taskIdByItem.set(item, taskId); + this.rootFolders.set(item, { + taskId, + totalItems: countItemsIn(item.folder), + uploadedItems: 0, + isCancelled: false, + hasFailedItems: false, + }); + } + + private trackFile(item: UploadFileItem): void { + const { ownerUserAuthenticationData, destinationFolderUuid } = this.props; + + const taskId = tasksService.create({ + action: TaskType.UploadFile, + item: { uploadFile: item.file.content, parentFolderId: item.file.parentFolderId }, + fileName: item.file.name, + fileType: item.file.type, + isFileNameValidated: true, + showNotification: true, + cancellable: true, + sharedItemAuthenticationData: ownerUserAuthenticationData && { + ownerUserAuthenticationData, + isDeepFolder: false, + currentFolderId: destinationFolderUuid, + }, + stop: () => Promise.resolve(this.manager.cancelItem(item)), + }); + + this.taskIdByItem.set(item, taskId); + } + + private onItemStarted(item: UploadItem): void { + const taskId = this.taskIdByItem.get(item); + if (!taskId) return; + + tasksService.updateTask({ taskId, merge: { status: TaskStatus.InProcess } }); + + const isFile = item.kind === UploadItemKind.File; + if (isFile) this.markUploadAsInProcess(taskId); + } + + private markUploadAsInProcess(taskId: string): void { + this.props.uploadRepository + .setUploadState(taskId, TaskStatus.InProcess) + .catch((error) => errorService.reportError(error)); + } + + private forgetUploadState(taskId: string): void { + this.props.uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); + } + + private onItemProgress(item: UploadItem, progress: number): void { + const taskId = this.taskIdByItem.get(item); + if (taskId && !isTaskStopped(taskId)) tasksService.updateTask({ taskId, merge: { progress } }); + } + + private onItemSucceeded(item: UploadItem, uploadedItem: UploadedItem): void { + const isFile = item.kind === UploadItemKind.File; + const isFolder = item.kind === UploadItemKind.Folder; + + this.props.onItemUploaded?.(uploadedItem, getParentFolderUuid(item), isFolder); + + if (isFile) { + referralService.trackFileUpload(); + logNetworkInfoForUpload({ fileName: item.file.name, fileSize: item.file.size }); + } + + const taskId = this.taskIdByItem.get(item); + const isFileWithItsOwnTask = !!taskId && isFile; + if (isFileWithItsOwnTask) { + this.finishFileTask(taskId, { fileUUID: uploadedItem.uuid }); + return; + } + + const topLevelItem = getTopLevelItem(item); + const rootFolder = this.rootFolders.get(topLevelItem) as RootFolderUpload; + + const isRootFolderItself = item === topLevelItem; + if (isRootFolderItself) { + rootFolder.createdFolder = uploadedItem as DriveFolderData; + if (rootFolder.isCancelled) void this.removeRootFolder(item as UploadFolderItem, rootFolder); + } + + rootFolder.uploadedItems += 1; + if (!isTaskStopped(rootFolder.taskId)) { + tasksService.updateTask({ + taskId: rootFolder.taskId, + merge: { progress: rootFolder.uploadedItems / rootFolder.totalItems }, + }); + } + } + + private finishFileTask(taskId: string, itemUUID: { fileUUID: string }): void { + const isTaskCancelled = tasksService.findTask(taskId)?.status === TaskStatus.Cancelled; + if (!isTaskCancelled) tasksService.updateTask({ taskId, merge: { status: TaskStatus.Success, itemUUID } }); + + this.forgetUploadState(taskId); + } + + private onItemFailed(item: UploadItem, failure: UploadFailure): void { + const rootFolder = this.rootFolders.get(getTopLevelItem(item)); + if (rootFolder) rootFolder.hasFailedItems = true; + + const taskId = this.taskIdByItem.get(item); + if (!taskId) return; + + tasksService.updateTask({ taskId, merge: errorMerge(failure.reason) }); + this.forgetUploadState(taskId); + } + + private onItemCancelled(item: UploadItem): void { + const taskId = this.taskIdByItem.get(item); + const isFileWithItsOwnTask = !!taskId && item.kind === UploadItemKind.File; + if (!isFileWithItsOwnTask) return; + + tasksService.updateTask({ taskId, merge: { status: TaskStatus.Cancelled } }); + this.forgetUploadState(taskId); + } + + private cancelRootFolder(item: UploadFolderItem): void { + const rootFolder = this.rootFolders.get(item) as RootFolderUpload; + rootFolder.isCancelled = true; + this.manager.cancelItem(item); + if (rootFolder.createdFolder) void this.removeRootFolder(item, rootFolder); + } + + private async removeRootFolder(item: UploadFolderItem, { createdFolder }: RootFolderUpload): Promise { + if (!createdFolder) return; + + try { + await newStorageService.deleteFolderByUuid(createdFolder.uuid); + this.props.onItemRemoved?.(createdFolder, item.parentFolderUuid); + } catch (error) { + errorService.reportError(error); + } + } + + private finishRootFolder(item: UploadFolderItem, rootFolder: RootFolderUpload): void { + this.forgetUploadState(rootFolder.taskId); + + const isRootFolderCreated = !!rootFolder.createdFolder; + if (rootFolder.isCancelled || !isRootFolderCreated) return; + + tasksService.updateTask({ + taskId: rootFolder.taskId, + merge: { status: TaskStatus.Success, itemUUID: { rootFolderUUID: rootFolder.createdFolder?.uuid } }, + }); + + const hasEverythingUploaded = !rootFolder.hasFailedItems; + if (hasEverythingUploaded) { + referralService.trackFolderUpload(); + logNetworkInfoForUpload({ folderName: item.folder.name }); + } + } +} diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 6593960eff..eb47cfd586 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -185,6 +185,28 @@ describe('Uploading files', () => { expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); + test('When several files fail because the storage is full, then it is announced only once', async () => { + uploadAtMostAtTheSameTime(2); + (uploadFile as Mock).mockRejectedValue(noSpaceLeftError); + const events: UploadManagerEvents = { onStorageFull: vi.fn() }; + const items = ['report', 'invoice', 'photo'].map((name) => getUploadFileItem(name)); + + await createManager({ events }).run(items); + + expect(events.onStorageFull).toHaveBeenCalledOnce(); + }); + + test('When the storage is still full on a new upload, then it is announced again', async () => { + (uploadFile as Mock).mockRejectedValue(noSpaceLeftError); + const events: UploadManagerEvents = { onStorageFull: vi.fn() }; + const manager = createManager({ events }); + + await manager.run([getUploadFileItem('report')]); + await manager.run([getUploadFileItem('invoice')]); + + expect(events.onStorageFull).toHaveBeenCalledTimes(2); + }); + test('When a file is cancelled while its paused state is being read, then it is never uploaded and is reported as cancelled', async () => { const report = getUploadFileItem('report'); const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 8d61c84a42..39fab71308 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -326,7 +326,11 @@ export class UploadManager { const failure = toUploadFailure(error); const isStorageFull = failure.reason === UploadFailureReason.NoSpaceLeft; - if (isStorageFull) this.storageFullFailure = failure; + const isFirstTimeStorageIsFull = isStorageFull && !this.storageFullFailure; + if (isFirstTimeStorageIsFull) { + this.storageFullFailure = failure; + await notifyIgnoringListenerErrors(() => this.events?.onStorageFull?.()); + } errorService.reportError(failure.error); await this.fail(item, failure); diff --git a/src/app/upload/types.ts b/src/app/upload/types.ts index 77837be645..bcc1f256b5 100644 --- a/src/app/upload/types.ts +++ b/src/app/upload/types.ts @@ -35,6 +35,7 @@ export interface UploadManagerEvents { onItemSucceeded?: (item: UploadItem, uploadedItem: UploadedItem) => void | Promise; onItemFailed?: (item: UploadItem, failure: UploadFailure) => void; onItemCancelled?: (item: UploadItem) => void; + onStorageFull?: () => void; } export interface UploadResult { From c1508539da5164065ccbb7353c245420719c441b Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 13:07:53 +0100 Subject: [PATCH 26/49] refactor(upload): add the upload engine as a queue of files with per-item events --- src/app/network/networkConstants.ts | 2 + src/app/network/types/index.ts | 13 +- src/app/network/types/upload.ts | 27 ++ src/app/network/uploadFailure.ts | 46 ++++ src/app/upload/UploadManager.test.ts | 286 ++++++++++++++++++++++ src/app/upload/UploadManager.ts | 214 ++++++++++++++++ src/testUtils/fixtures/upload.fixtures.ts | 18 ++ 7 files changed, 605 insertions(+), 1 deletion(-) create mode 100644 src/app/network/types/upload.ts create mode 100644 src/app/network/uploadFailure.ts create mode 100644 src/app/upload/UploadManager.test.ts create mode 100644 src/app/upload/UploadManager.ts create mode 100644 src/testUtils/fixtures/upload.fixtures.ts diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index 1e857231ac..e52923163a 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -10,3 +10,5 @@ export const USE_MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024; export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; +export const SINGLE_TRANSFER_CONCURRENCY = 1; +export const HIGH_BROWSER_MEMORY_USAGE_RATIO = 0.8; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index edb1f68756..3d87c2391c 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -35,7 +35,18 @@ export interface DownloadChunkPayload { options?: DownloadOptions; } -export type UploadErrorReason = 'connection-lost' | 'upload-failed'; +export enum UploadItemKind { + File = 'file', +} + +export enum UploadFailureReason { + ConnectionLost = 'connection-lost', + NoSpaceLeft = 'no-space', + FileTooLarge = 'file-too-large', + UploadFailed = 'upload-failed', +} + +export type UploadErrorReason = `${UploadFailureReason}`; export interface OwnerUserAuthenticationData { token: string; diff --git a/src/app/network/types/upload.ts b/src/app/network/types/upload.ts new file mode 100644 index 0000000000..6937394c5f --- /dev/null +++ b/src/app/network/types/upload.ts @@ -0,0 +1,27 @@ +import { DriveFileData } from 'app/drive/types'; +import { FileToUpload } from 'app/drive/services/file.service/types'; +import { UploadFailure } from '../uploadFailure'; +import { UploadItemKind } from '.'; + +export type UploadFileItem = { kind: UploadItemKind.File; file: FileToUpload }; + +export type UploadItem = UploadFileItem; + +export type UploadedItem = DriveFileData; + +/** + * Upload progress notifications emitted by the manager. The manager knows nothing + * about how uploads are presented to the user; observers subscribe to these events. + */ +export interface UploadManagerEvents { + onItemStarted?: (item: UploadItem) => void; + onItemProgress?: (item: UploadItem, progress: number) => void; + onItemSucceeded?: (item: UploadItem, uploadedItem: UploadedItem) => void | Promise; + onItemFailed?: (item: UploadItem, failure: UploadFailure) => void; + onItemCancelled?: (item: UploadItem) => void; +} + +export interface UploadResult { + succeeded: { item: UploadItem; uploadedItem: UploadedItem }[]; + failed: { item: UploadItem; failure: UploadFailure }[]; +} diff --git a/src/app/network/uploadFailure.ts b/src/app/network/uploadFailure.ts new file mode 100644 index 0000000000..7ac796a44e --- /dev/null +++ b/src/app/network/uploadFailure.ts @@ -0,0 +1,46 @@ +import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; +import errorService from 'services/error.service'; +import { ConnectionLostError } from './requests'; +import { UploadFailureReason } from './types'; + +export interface UploadFailure { + error: Error; + reason: UploadFailureReason; + isRetryable: boolean; +} + +const getStatusCode = (error: unknown) => (error as { status?: number } | undefined)?.status; + +const isNetworkErrorWhileOffline = (error: unknown) => + (error as Error | undefined)?.message === ErrorMessages.NetworkError && !navigator.onLine; + +const isConnectionLost = (error: unknown) => error instanceof ConnectionLostError || isNetworkErrorWhileOffline(error); + +const isStorageFull = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.MAX_SPACE_USED; + +const isFileTooLarge = (error: unknown) => + errorService.castError(error).code === HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED; + +const isPlanNotAllowingIt = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.PAYMENT_REQUIRED; + +const getFailureReason = (error: unknown): UploadFailureReason => { + if (isConnectionLost(error)) return UploadFailureReason.ConnectionLost; + if (isStorageFull(error)) return UploadFailureReason.NoSpaceLeft; + if (isFileTooLarge(error)) return UploadFailureReason.FileTooLarge; + return UploadFailureReason.UploadFailed; +}; + +export const toUploadFailure = (error: unknown): UploadFailure => { + const reason = getFailureReason(error); + const canSucceedLater = reason !== UploadFailureReason.FileTooLarge && !isPlanNotAllowingIt(error); + const genericUploadError = new Error('Upload failed'); + + return { + error: errorService.castError(error ?? genericUploadError), + reason, + isRetryable: canSucceedLater, + }; +}; + +export const canSucceedOnImmediateRetry = (error: unknown): boolean => + getFailureReason(error) === UploadFailureReason.UploadFailed && !isPlanNotAllowingIt(error); diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts new file mode 100644 index 0000000000..5078935370 --- /dev/null +++ b/src/app/upload/UploadManager.test.ts @@ -0,0 +1,286 @@ +import { afterEach, beforeEach, describe, expect, Mock, MockInstance, test, vi } from 'vitest'; +import { AppError } from '@internxt/sdk'; +import uploadFile from 'app/drive/services/file.service/uploadFile'; +import { DriveFileData } from 'app/drive/types'; +import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; +import errorService from 'services/error.service'; +import { ConnectionLostError } from 'app/network/requests'; +import { USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; +import { UploadManager } from './UploadManager'; +import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; +import { UploadFailureReason } from 'app/network/types'; + +vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); + +type UploadFileOptions = { abortCallback?: (abort?: () => void) => void }; + +const HANG_TIMEOUT_MS = 2000; +const networkError = new AppError(ErrorMessages.NetworkError); +const noSpaceLeftError = new AppError('No space left', HTTP_STATUS_CODES.MAX_SPACE_USED); +const paymentRequiredError = new AppError('Payment required', HTTP_STATUS_CODES.PAYMENT_REQUIRED); +const fileTooLargeError = new AppError('File too large', 400, HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED); + +const nameOf = (item: UploadItem) => item.file.name; + +const failIfHangs = (promise: Promise): Promise => + Promise.race([ + promise, + new Promise((_, reject) => setTimeout(() => reject(new Error('upload never settled')), HANG_TIMEOUT_MS)), + ]); + +const createManager = (props: Partial[0]> = {}) => + new UploadManager({ uploadRepository: getUploadRepository(), getPauseKey: nameOf, ...props }); + +const uploadedNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); + +const fakeSlowUploads = () => + (uploadFile as Mock).mockImplementation((file: { name: string }, _progress, options: UploadFileOptions) => { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => resolve({ uuid: `${file.name}-uuid` }), 500); + options.abortCallback?.(() => { + clearTimeout(timeout); + reject(new Error('aborted')); + }); + }); + }); + +const fakeUploadFailureOf = (failingName: string, error: unknown) => + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + if (file.name === failingName) throw error; + await new Promise((resolve) => setTimeout(resolve, 10)); + return { uuid: `${file.name}-uuid` } as DriveFileData; + }); + +let onlineSpy: MockInstance | undefined; +const goOffline = () => { + onlineSpy = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); +}; + +describe('Uploading files', () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(errorService, 'reportError').mockReturnValue(); + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); + }); + + afterEach(() => onlineSpy?.mockRestore()); + + test('When files are uploaded, then each one succeeds with its uploaded data and its plain name', async () => { + const items = [getUploadFileItem('report'), getUploadFileItem('invoice')]; + const events: UploadManagerEvents = { onItemSucceeded: vi.fn() }; + + const { succeeded, failed } = await createManager({ events }).run(items); + + expect(failed).toEqual([]); + expect(succeeded).toEqual([ + { item: items[0], uploadedItem: { uuid: 'report-uuid', name: 'report' } }, + { item: items[1], uploadedItem: { uuid: 'invoice-uuid', name: 'invoice' } }, + ]); + expect(events.onItemSucceeded).toHaveBeenCalledWith(items[0], { uuid: 'report-uuid', name: 'report' }); + }); + + test('When an upload advances, then its progress is notified', async () => { + const item = getUploadFileItem('report'); + (uploadFile as Mock).mockImplementation(async (_file, onProgress: (progress: number) => void) => { + onProgress(0.5); + return { uuid: 'report-uuid' }; + }); + const events: UploadManagerEvents = { onItemStarted: vi.fn(), onItemProgress: vi.fn() }; + + await createManager({ events }).run([item]); + + expect(events.onItemStarted).toHaveBeenCalledWith(item); + expect(events.onItemProgress).toHaveBeenCalledWith(item, 0.5); + }); + + test('When an upload fails unexpectedly once, then it is retried and succeeds', async () => { + (uploadFile as Mock).mockRejectedValueOnce(new Error('Server error')); + + const { succeeded } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledTimes(2); + expect(succeeded).toHaveLength(1); + }); + + test('When an upload keeps failing, then it fails as retryable and the other files still upload', async () => { + const serverError = new Error('Server error'); + fakeUploadFailureOf('broken', serverError); + const events: UploadManagerEvents = { onItemFailed: vi.fn() }; + const brokenItem = getUploadFileItem('broken'); + + const { succeeded, failed } = await failIfHangs( + createManager({ events }).run([brokenItem, getUploadFileItem('a'), getUploadFileItem('b')]), + ); + + expect(uploadedNames().filter((name) => name === 'broken')).toHaveLength(2); + expect(succeeded).toHaveLength(2); + expect(failed).toEqual([ + { + item: brokenItem, + failure: { error: serverError, reason: UploadFailureReason.UploadFailed, isRetryable: true }, + }, + ]); + expect(events.onItemFailed).toHaveBeenCalledWith(brokenItem, failed[0].failure); + }); + + test.each([ + { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.UploadFailed }, + { problem: 'the file is too large', error: fileTooLargeError, reason: UploadFailureReason.FileTooLarge }, + ])('When $problem, then the file is not retried and cannot be retried later', async ({ error, reason }) => { + (uploadFile as Mock).mockRejectedValue(error); + + const { failed } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledOnce(); + expect(failed[0].failure).toEqual({ error, reason, isRetryable: false }); + }); + + test.each([ + { failure: 'the connection is lost', error: new ConnectionLostError(), arrange: () => undefined }, + { failure: 'a network error happens while offline', error: networkError, arrange: goOffline }, + ])('When $failure, then the file is not retried and fails as a retryable connection loss', async (scenario) => { + scenario.arrange(); + (uploadFile as Mock).mockRejectedValue(scenario.error); + + const { failed } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledOnce(); + expect(failed[0].failure).toEqual( + expect.objectContaining({ reason: UploadFailureReason.ConnectionLost, isRetryable: true }), + ); + }); + + test('When a network error happens while online, then the upload is retried and succeeds', async () => { + (uploadFile as Mock).mockRejectedValueOnce(networkError); + + const { succeeded } = await createManager().run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledTimes(2); + expect(succeeded).toHaveLength(1); + }); + + test('When the upload rejects without an error, then the file fails instead of hanging', async () => { + (uploadFile as Mock).mockRejectedValue(undefined); + + const { failed } = await failIfHangs(createManager().run([getUploadFileItem('report')])); + + expect(failed).toHaveLength(1); + }); + + test('When the storage is full, then the queued files fail without being uploaded', async () => { + fakeUploadFailureOf('first', noSpaceLeftError); + const items = ['first', 'a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); + + const { failed } = await failIfHangs(createManager().run(items)); + + expect(uploadedNames()).toEqual(['first', 'a', 'b', 'c', 'd', 'e']); + expect(failed.map(({ item }) => nameOf(item))).toEqual(['first', 'f', 'g']); + expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); + }); + + test('When the upload is cancelled, then the queued files are never uploaded and are reported as cancelled', async () => { + fakeSlowUploads(); + const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => getUploadFileItem(name)); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const manager = createManager({ events }); + + const upload = manager.run(items); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(6)); + manager.cancelAll(); + const { succeeded, failed } = await failIfHangs(upload); + + expect(uploadFile).toHaveBeenCalledTimes(6); + expect(succeeded).toEqual([]); + expect(failed).toEqual([]); + expect(events.onItemCancelled).toHaveBeenCalledTimes(items.length); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + + test('When a single file is cancelled, then only that file stops', async () => { + fakeSlowUploads(); + const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; + const events: UploadManagerEvents = { onItemCancelled: vi.fn() }; + const manager = createManager({ events }); + + const upload = manager.run([cancelledItem, otherItem]); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(2)); + manager.cancelItem(cancelledItem); + const { succeeded } = await failIfHangs(upload); + + expect(succeeded.map(({ item }) => item)).toEqual([otherItem]); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(cancelledItem); + }); + + test('When small and large files are uploaded together, then the large ones go last and one at a time', async () => { + let activeLargeUploads = 0; + let maxActiveLargeUploads = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string; size: number }) => { + const isLarge = file.size >= USE_MULTIPART_THRESHOLD_BYTES; + if (isLarge) maxActiveLargeUploads = Math.max(maxActiveLargeUploads, ++activeLargeUploads); + await new Promise((resolve) => setTimeout(resolve, 10)); + if (isLarge) activeLargeUploads--; + return { uuid: `${file.name}-uuid` }; + }); + const items = [ + getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('small-1'), + getUploadFileItem('large-2', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('small-2'), + ]; + + await createManager().run(items); + + expect(uploadedNames()).toEqual(['small-1', 'small-2', 'large-1', 'large-2']); + expect(maxActiveLargeUploads).toBe(1); + }); + + test('When uploading to a workspace, then the workspace credentials are used', async () => { + const ownerUserAuthenticationData = { + bucketId: 'workspace-bucket', + workspaceId: 'workspace-id', + bridgeUser: 'bridge-user', + bridgePass: 'bridge-pass', + encryptionKey: 'encryption-key', + token: 'token', + workspacesToken: 'workspaces-token', + resourcesToken: 'resources-token', + }; + + await createManager({ ownerUserAuthenticationData }).run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Function), + expect.objectContaining({ isTeam: true, ownerUserAuthenticationData }), + expect.any(Object), + ); + }); + + test('When the upload of a file was paused, then it starts paused under its pause key', async () => { + const uploadRepository = getUploadRepository(['paused-key']); + + await createManager({ uploadRepository, getPauseKey: () => 'paused-key' }).run([getUploadFileItem('report')]); + + expect(uploadFile).toHaveBeenCalledWith( + expect.any(Object), + expect.any(Function), + expect.any(Object), + expect.objectContaining({ taskId: 'paused-key', isPaused: true }), + ); + }); + + test('When a success listener throws, then every file is uploaded once and the queue keeps going', async () => { + const items = [getUploadFileItem('a'), getUploadFileItem('b'), getUploadFileItem('c')]; + const events: UploadManagerEvents = { + onItemSucceeded: vi.fn(() => { + throw new Error('listener failure'); + }), + }; + + const { succeeded } = await failIfHangs(createManager({ events }).run(items)); + + expect(uploadFile).toHaveBeenCalledTimes(items.length); + expect(succeeded).toHaveLength(items.length); + }); +}); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts new file mode 100644 index 0000000000..00ee1b6e6b --- /dev/null +++ b/src/app/upload/UploadManager.ts @@ -0,0 +1,214 @@ +import { items as itemUtils } from '@internxt/lib'; +import { AsyncPriorityQueue, priorityQueue } from 'async'; +import errorService from 'services/error.service'; +import uploadFile from 'app/drive/services/file.service/uploadFile'; +import { DriveFileData } from 'app/drive/types'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { TaskStatus } from 'app/tasks/types'; +import { + HIGH_BROWSER_MEMORY_USAGE_RATIO, + MAX_CONCURRENT_UPLOADS, + MAX_UPLOAD_ATTEMPTS, + SINGLE_TRANSFER_CONCURRENCY, + USE_MULTIPART_THRESHOLD_BYTES, +} from 'app/network/networkConstants'; +import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; +import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; + +enum UploadPriority { + Normal = 1, + Last = 2, +} + +interface UploadManagerProps { + ownerUserAuthenticationData?: OwnerUserAuthenticationData; + uploadRepository?: PersistUploadRepository; + getPauseKey: (item: UploadItem) => string; + events?: UploadManagerEvents; +} + +export const createFileItem = (file: File, parentFolderUuid: string): UploadFileItem => { + const { filename, extension } = itemUtils.getFilenameAndExt(file.name); + + return { + kind: UploadItemKind.File, + file: { name: filename, type: extension, size: file.size, content: file, parentFolderId: parentFolderUuid }, + }; +}; + +const isMultipartUpload = (item: UploadItem) => item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; + +const getUploadPriority = (item: UploadItem): UploadPriority => { + if (isMultipartUpload(item)) return UploadPriority.Last; + return UploadPriority.Normal; +}; + +const isBrowserMemoryAlmostFull = (): boolean => { + const memory = window?.performance?.memory; + if (!memory?.jsHeapSizeLimit || !memory?.usedJSHeapSize) return false; + + return memory.usedJSHeapSize / memory.jsHeapSizeLimit >= HIGH_BROWSER_MEMORY_USAGE_RATIO; +}; + +/** + * Listeners are the caller's code. If one throws, the error is reported but never stops the queue nor turns an + * uploaded item into a failed one. + */ +const notifyIgnoringListenerErrors = async (notify: () => unknown): Promise => { + try { + await notify(); + } catch (error) { + errorService.reportError(error); + } +}; + +/** + * Uploads files through a single queue. Every item ends either succeeded, failed or cancelled: a failed + * item never stops the others, except when the storage is full, which fails the pending ones. An item that + * finishes uploading right after being cancelled is still reported as succeeded, since it does exist in + * the destination. + */ +export class UploadManager { + private readonly ownerUserAuthenticationData?: OwnerUserAuthenticationData; + private readonly uploadRepository?: PersistUploadRepository; + private readonly getPauseKey: (item: UploadItem) => string; + private readonly events?: UploadManagerEvents; + + private isEverythingCancelled = false; + private storageFullFailure?: UploadFailure; + private readonly cancelledItems = new WeakSet(); + private readonly abortActiveUploadOf = new Map void>(); + private result: UploadResult = { succeeded: [], failed: [] }; + + private readonly uploadQueue: AsyncPriorityQueue = priorityQueue((item, done) => { + this.processItem(item) + .catch((error) => errorService.reportError(error)) + .finally(() => done()); + }, MAX_CONCURRENT_UPLOADS); + + constructor(props: UploadManagerProps) { + this.ownerUserAuthenticationData = props.ownerUserAuthenticationData; + this.uploadRepository = props.uploadRepository; + this.getPauseKey = props.getPauseKey; + this.events = props.events; + } + + async run(items: UploadItem[]): Promise { + if (this.uploadQueue.idle()) { + this.result = { succeeded: [], failed: [] }; + this.storageFullFailure = undefined; + } + + const hasNothingToUpload = items.length === 0; + if (hasNothingToUpload) return this.result; + + this.queue(items); + await this.uploadQueue.drain(); + + return this.result; + } + + cancelAll(): void { + this.isEverythingCancelled = true; + this.abortActiveUploadOf.forEach((abort) => abort()); + } + + cancelItem(item: UploadItem): void { + this.cancelledItems.add(item); + this.abortActiveUploadOf.get(item)?.(); + } + + private queue(items: UploadItem[]): void { + for (const item of items) { + this.uploadQueue.push(item, getUploadPriority(item)); + } + } + + private async processItem(item: UploadItem): Promise { + if (this.isItemCancelled(item)) { + await notifyIgnoringListenerErrors(() => this.events?.onItemCancelled?.(item)); + return; + } + + if (this.storageFullFailure) { + await this.fail(item, this.storageFullFailure); + return; + } + + this.adjustConcurrencyFor(item); + await notifyIgnoringListenerErrors(() => this.events?.onItemStarted?.(item)); + + try { + const uploadedItem = await this.uploadWithRetries(item); + this.result.succeeded.push({ item, uploadedItem }); + + await notifyIgnoringListenerErrors(() => this.events?.onItemSucceeded?.(item, uploadedItem)); + } catch (error) { + await this.handleUploadError(item, error); + } + } + + private adjustConcurrencyFor(item: UploadItem): void { + const shouldUploadOneAtATime = isMultipartUpload(item) || isBrowserMemoryAlmostFull(); + this.uploadQueue.concurrency = shouldUploadOneAtATime ? SINGLE_TRANSFER_CONCURRENCY : MAX_CONCURRENT_UPLOADS; + } + + private async uploadWithRetries(item: UploadItem): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await this.uploadFile(item); + } catch (error) { + const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; + const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); + if (!shouldRetry) throw error; + } + } + } + + private async uploadFile(fileItem: UploadFileItem): Promise { + const pauseKey = this.getPauseKey(fileItem); + const uploadState = await this.uploadRepository?.getUploadState(pauseKey); + const isPaused = uploadState === TaskStatus.Paused; + + try { + const driveFile = await uploadFile( + fileItem.file, + (progress) => this.events?.onItemProgress?.(fileItem, progress), + { + isTeam: !!this.ownerUserAuthenticationData?.workspaceId, + ownerUserAuthenticationData: this.ownerUserAuthenticationData, + abortCallback: (abort) => this.abortActiveUploadOf.set(fileItem, () => abort?.()), + }, + { taskId: pauseKey, isPaused, isRetriedUpload: false }, + ); + + return { ...driveFile, name: fileItem.file.name }; + } finally { + this.abortActiveUploadOf.delete(fileItem); + } + } + + private isItemCancelled(item: UploadItem): boolean { + return this.isEverythingCancelled || this.cancelledItems.has(item); + } + + private async handleUploadError(item: UploadItem, error: unknown): Promise { + if (this.isItemCancelled(item)) { + await notifyIgnoringListenerErrors(() => this.events?.onItemCancelled?.(item)); + return; + } + + const failure = toUploadFailure(error); + const isStorageFull = failure.reason === UploadFailureReason.NoSpaceLeft; + if (isStorageFull) this.storageFullFailure = failure; + + errorService.reportError(failure.error); + await this.fail(item, failure); + } + + private async fail(item: UploadItem, failure: UploadFailure): Promise { + this.result.failed.push({ item, failure }); + await notifyIgnoringListenerErrors(() => this.events?.onItemFailed?.(item, failure)); + } +} diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts new file mode 100644 index 0000000000..51062cac1d --- /dev/null +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -0,0 +1,18 @@ +import { vi } from 'vitest'; +import { UploadItemKind } from 'app/network/types'; +import { UploadFileItem } from 'app/network/types/upload'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { TaskStatus } from 'app/tasks/types'; + +export const getFileToUpload = (name: string, size = 10): File => new File([new ArrayBuffer(size)], name); + +export const getUploadFileItem = (name: string, size = 1024): UploadFileItem => ({ + kind: UploadItemKind.File, + file: { name, size, type: 'txt', content: getFileToUpload(name), parentFolderId: 'folder-uuid' }, +}); + +export const getUploadRepository = (pausedKeys: string[] = []): PersistUploadRepository => ({ + setUploadState: vi.fn(async () => undefined), + removeUploadState: vi.fn(async () => undefined), + getUploadState: vi.fn(async (key: string) => (pausedKeys.includes(key) ? TaskStatus.Paused : undefined)), +}); From b6ec3d4b85ba330c3d01710ed72c28f0af6d6f5f Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 14:33:56 +0100 Subject: [PATCH 27/49] refactor(upload): upload small files before medium ones and adjust concurrency with the shared memory check --- src/app/network/networkConstants.ts | 1 - src/app/network/types/index.ts | 2 +- src/app/upload/UploadManager.test.ts | 36 +++++++++++++++++++++------- src/app/upload/UploadManager.ts | 30 ++++++++++++----------- src/utils/queueUtils.ts | 5 +++- 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index e52923163a..69f3b47c6a 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -11,4 +11,3 @@ export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; export const SINGLE_TRANSFER_CONCURRENCY = 1; -export const HIGH_BROWSER_MEMORY_USAGE_RATIO = 0.8; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index 3d87c2391c..f099c9a09b 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -46,7 +46,7 @@ export enum UploadFailureReason { UploadFailed = 'upload-failed', } -export type UploadErrorReason = `${UploadFailureReason}`; +export type UploadErrorReason = 'connection-lost' | 'upload-failed'; export interface OwnerUserAuthenticationData { token: string; diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 5078935370..c811b575f1 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -1,11 +1,12 @@ -import { afterEach, beforeEach, describe, expect, Mock, MockInstance, test, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, Mock, test, vi } from 'vitest'; import { AppError } from '@internxt/sdk'; import uploadFile from 'app/drive/services/file.service/uploadFile'; import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; @@ -52,10 +53,7 @@ const fakeUploadFailureOf = (failingName: string, error: unknown) => return { uuid: `${file.name}-uuid` } as DriveFileData; }); -let onlineSpy: MockInstance | undefined; -const goOffline = () => { - onlineSpy = vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); -}; +const goOffline = () => vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); describe('Uploading files', () => { beforeEach(() => { @@ -64,7 +62,7 @@ describe('Uploading files', () => { (uploadFile as Mock).mockImplementation(async (file: { name: string }) => ({ uuid: `${file.name}-uuid` })); }); - afterEach(() => onlineSpy?.mockRestore()); + afterEach(() => vi.restoreAllMocks()); test('When files are uploaded, then each one succeeds with its uploaded data and its plain name', async () => { const items = [getUploadFileItem('report'), getUploadFileItem('invoice')]; @@ -212,7 +210,7 @@ describe('Uploading files', () => { expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(cancelledItem); }); - test('When small and large files are uploaded together, then the large ones go last and one at a time', async () => { + test('When small, medium and large files are uploaded together, then the small ones go first and the large ones last and one at a time', async () => { let activeLargeUploads = 0; let maxActiveLargeUploads = 0; (uploadFile as Mock).mockImplementation(async (file: { name: string; size: number }) => { @@ -224,17 +222,37 @@ describe('Uploading files', () => { }); const items = [ getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('medium-1', TWENTY_MEGABYTES), getUploadFileItem('small-1'), getUploadFileItem('large-2', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('medium-2', TWENTY_MEGABYTES), getUploadFileItem('small-2'), ]; await createManager().run(items); - expect(uploadedNames()).toEqual(['small-1', 'small-2', 'large-1', 'large-2']); + expect(uploadedNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); expect(maxActiveLargeUploads).toBe(1); }); + test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { + vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); + let activeUploads = 0; + let maxActiveUploads = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); + await new Promise((resolve) => setTimeout(resolve, 10)); + activeUploads--; + return { uuid: `${file.name}-uuid` }; + }); + const items = ['a', 'b', 'c', 'd'].map((name) => getUploadFileItem(name)); + + const { succeeded } = await createManager().run(items); + + expect(succeeded).toHaveLength(items.length); + expect(maxActiveUploads).toBe(1); + }); + test('When uploading to a workspace, then the workspace credentials are used', async () => { const ownerUserAuthenticationData = { bucketId: 'workspace-bucket', diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 00ee1b6e6b..4987c36993 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -1,15 +1,16 @@ import { items as itemUtils } from '@internxt/lib'; import { AsyncPriorityQueue, priorityQueue } from 'async'; import errorService from 'services/error.service'; +import { QueueUtilsService } from 'utils/queueUtils'; import uploadFile from 'app/drive/services/file.service/uploadFile'; import { DriveFileData } from 'app/drive/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { TaskStatus } from 'app/tasks/types'; import { - HIGH_BROWSER_MEMORY_USAGE_RATIO, MAX_CONCURRENT_UPLOADS, MAX_UPLOAD_ATTEMPTS, SINGLE_TRANSFER_CONCURRENCY, + TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES, } from 'app/network/networkConstants'; import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; @@ -17,8 +18,9 @@ import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadRe import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; enum UploadPriority { - Normal = 1, - Last = 2, + SmallFile = 1, + MediumFile = 2, + LargeFile = 3, } interface UploadManagerProps { @@ -39,16 +41,12 @@ export const createFileItem = (file: File, parentFolderUuid: string): UploadFile const isMultipartUpload = (item: UploadItem) => item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; -const getUploadPriority = (item: UploadItem): UploadPriority => { - if (isMultipartUpload(item)) return UploadPriority.Last; - return UploadPriority.Normal; -}; +const isMediumSizedFile = (item: UploadItem) => item.file.size >= TWENTY_MEGABYTES; -const isBrowserMemoryAlmostFull = (): boolean => { - const memory = window?.performance?.memory; - if (!memory?.jsHeapSizeLimit || !memory?.usedJSHeapSize) return false; - - return memory.usedJSHeapSize / memory.jsHeapSizeLimit >= HIGH_BROWSER_MEMORY_USAGE_RATIO; +const getUploadPriority = (item: UploadItem): UploadPriority => { + if (isMultipartUpload(item)) return UploadPriority.LargeFile; + if (isMediumSizedFile(item)) return UploadPriority.MediumFile; + return UploadPriority.SmallFile; }; /** @@ -150,8 +148,12 @@ export class UploadManager { } private adjustConcurrencyFor(item: UploadItem): void { - const shouldUploadOneAtATime = isMultipartUpload(item) || isBrowserMemoryAlmostFull(); - this.uploadQueue.concurrency = shouldUploadOneAtATime ? SINGLE_TRANSFER_CONCURRENCY : MAX_CONCURRENT_UPLOADS; + if (isMultipartUpload(item)) { + this.uploadQueue.concurrency = SINGLE_TRANSFER_CONCURRENCY; + return; + } + + QueueUtilsService.instance.adjustConcurrency(this.uploadQueue, MAX_CONCURRENT_UPLOADS); } private async uploadWithRetries(item: UploadItem): Promise { diff --git a/src/utils/queueUtils.ts b/src/utils/queueUtils.ts index 35bbaff53f..11b6bc4974 100644 --- a/src/utils/queueUtils.ts +++ b/src/utils/queueUtils.ts @@ -3,7 +3,10 @@ import { QueueObject } from 'async'; export class QueueUtilsService { public static readonly instance: QueueUtilsService = new QueueUtilsService(); - public readonly adjustConcurrency = (targetQueue: QueueObject, maxConcurrency: number): void => { + public readonly adjustConcurrency = ( + targetQueue: Pick, 'concurrency'>, + maxConcurrency: number, + ): void => { const newConcurrency = this.getConcurrencyUsingPerformance(targetQueue.concurrency, maxConcurrency); if (targetQueue.concurrency !== newConcurrency) { targetQueue.concurrency = newConcurrency; From fb0f9bdaba0dadd84c1d173cfad6b8a836bb64d5 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:26:11 +0100 Subject: [PATCH 28/49] refactor(upload): mark the queue push as not awaited and simplify the upload fixtures --- src/app/network/types/upload.ts | 4 ---- src/app/upload/UploadManager.ts | 2 +- src/testUtils/fixtures/upload.fixtures.ts | 6 +++--- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/app/network/types/upload.ts b/src/app/network/types/upload.ts index 6937394c5f..53cb34dae3 100644 --- a/src/app/network/types/upload.ts +++ b/src/app/network/types/upload.ts @@ -9,10 +9,6 @@ export type UploadItem = UploadFileItem; export type UploadedItem = DriveFileData; -/** - * Upload progress notifications emitted by the manager. The manager knows nothing - * about how uploads are presented to the user; observers subscribe to these events. - */ export interface UploadManagerEvents { onItemStarted?: (item: UploadItem) => void; onItemProgress?: (item: UploadItem, progress: number) => void; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 4987c36993..b327d479d4 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -119,7 +119,7 @@ export class UploadManager { private queue(items: UploadItem[]): void { for (const item of items) { - this.uploadQueue.push(item, getUploadPriority(item)); + void this.uploadQueue.push(item, getUploadPriority(item)); } } diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts index 51062cac1d..6e123b622e 100644 --- a/src/testUtils/fixtures/upload.fixtures.ts +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -12,7 +12,7 @@ export const getUploadFileItem = (name: string, size = 1024): UploadFileItem => }); export const getUploadRepository = (pausedKeys: string[] = []): PersistUploadRepository => ({ - setUploadState: vi.fn(async () => undefined), - removeUploadState: vi.fn(async () => undefined), - getUploadState: vi.fn(async (key: string) => (pausedKeys.includes(key) ? TaskStatus.Paused : undefined)), + setUploadState: vi.fn(() => Promise.resolve()), + removeUploadState: vi.fn(() => Promise.resolve()), + getUploadState: vi.fn((key: string) => Promise.resolve(pausedKeys.includes(key) ? TaskStatus.Paused : undefined)), }); From 9bb6b2e112286a03f3a13932332f44e268a4eec5 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:43:58 +0100 Subject: [PATCH 29/49] refactor(upload): retry an upload by calling it again instead of looping --- src/app/upload/UploadManager.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index b327d479d4..3c890d801c 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -156,15 +156,15 @@ export class UploadManager { QueueUtilsService.instance.adjustConcurrency(this.uploadQueue, MAX_CONCURRENT_UPLOADS); } - private async uploadWithRetries(item: UploadItem): Promise { - for (let attempt = 1; ; attempt++) { - try { - return await this.uploadFile(item); - } catch (error) { - const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; - const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); - if (!shouldRetry) throw error; - } + private async uploadWithRetries(item: UploadItem, attempt = 1): Promise { + try { + return await this.uploadFile(item); + } catch (error) { + const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; + const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); + if (!shouldRetry) throw error; + + return this.uploadWithRetries(item, attempt + 1); } } From 50eb199a605eec26894ce9a450f2ff030da7556c Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:49:25 +0100 Subject: [PATCH 30/49] fix(upload): start every run with uploads not cancelled and the full concurrency --- src/app/upload/UploadManager.test.ts | 29 ++++++++++++++++++++++++++++ src/app/upload/UploadManager.ts | 2 ++ 2 files changed, 31 insertions(+) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index c811b575f1..845fec5ff7 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -235,6 +235,35 @@ describe('Uploading files', () => { expect(maxActiveLargeUploads).toBe(1); }); + test('When the same manager uploads again after a large file in a browser without memory information, then several files go at the same time again', async () => { + vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockImplementation((current) => current); + let activeUploads = 0; + let maxActiveUploads = 0; + (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { + maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); + await new Promise((resolve) => setTimeout(resolve, 10)); + activeUploads--; + return { uuid: `${file.name}-uuid` }; + }); + const manager = createManager(); + await manager.run([getUploadFileItem('large', USE_MULTIPART_THRESHOLD_BYTES)]); + maxActiveUploads = 0; + + await manager.run(['a', 'b', 'c'].map((name) => getUploadFileItem(name))); + + expect(maxActiveUploads).toBe(3); + }); + + test('When the same manager uploads again after everything was cancelled, then the new files are uploaded', async () => { + const manager = createManager(); + manager.cancelAll(); + await manager.run([getUploadFileItem('cancelled')]); + + const { succeeded } = await manager.run([getUploadFileItem('report')]); + + expect(succeeded.map(({ item }) => item.file.name)).toEqual(['report']); + }); + test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); let activeUploads = 0; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 3c890d801c..9ff87bfb82 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -96,6 +96,8 @@ export class UploadManager { if (this.uploadQueue.idle()) { this.result = { succeeded: [], failed: [] }; this.storageFullFailure = undefined; + this.isEverythingCancelled = false; + this.uploadQueue.concurrency = MAX_CONCURRENT_UPLOADS; } const hasNothingToUpload = items.length === 0; From 24944d2d46254f0f464721344fb4ad3f773b0d66 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:51:14 +0100 Subject: [PATCH 31/49] test(upload): name uploaded items with the shared helper --- src/app/upload/UploadManager.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 845fec5ff7..38533c11a7 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -261,7 +261,7 @@ describe('Uploading files', () => { const { succeeded } = await manager.run([getUploadFileItem('report')]); - expect(succeeded.map(({ item }) => item.file.name)).toEqual(['report']); + expect(succeeded.map(({ item }) => nameOf(item))).toEqual(['report']); }); test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { From e6cb85f83a285c4fc51499d5435d469e52a5fae0 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:09:10 +0100 Subject: [PATCH 32/49] refactor(upload): remove cancelling everything from the engine, uploads are cancelled file by file --- src/app/upload/UploadManager.test.ts | 28 ---------------------------- src/app/upload/UploadManager.ts | 9 +-------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 38533c11a7..d4dc69b33f 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -177,24 +177,6 @@ describe('Uploading files', () => { expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); - test('When the upload is cancelled, then the queued files are never uploaded and are reported as cancelled', async () => { - fakeSlowUploads(); - const items = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((name) => getUploadFileItem(name)); - const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; - const manager = createManager({ events }); - - const upload = manager.run(items); - await vi.waitFor(() => expect(uploadFile).toHaveBeenCalledTimes(6)); - manager.cancelAll(); - const { succeeded, failed } = await failIfHangs(upload); - - expect(uploadFile).toHaveBeenCalledTimes(6); - expect(succeeded).toEqual([]); - expect(failed).toEqual([]); - expect(events.onItemCancelled).toHaveBeenCalledTimes(items.length); - expect(events.onItemFailed).not.toHaveBeenCalled(); - }); - test('When a single file is cancelled, then only that file stops', async () => { fakeSlowUploads(); const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; @@ -254,16 +236,6 @@ describe('Uploading files', () => { expect(maxActiveUploads).toBe(3); }); - test('When the same manager uploads again after everything was cancelled, then the new files are uploaded', async () => { - const manager = createManager(); - manager.cancelAll(); - await manager.run([getUploadFileItem('cancelled')]); - - const { succeeded } = await manager.run([getUploadFileItem('report')]); - - expect(succeeded.map(({ item }) => nameOf(item))).toEqual(['report']); - }); - test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); let activeUploads = 0; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 9ff87bfb82..0d7d602802 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -73,7 +73,6 @@ export class UploadManager { private readonly getPauseKey: (item: UploadItem) => string; private readonly events?: UploadManagerEvents; - private isEverythingCancelled = false; private storageFullFailure?: UploadFailure; private readonly cancelledItems = new WeakSet(); private readonly abortActiveUploadOf = new Map void>(); @@ -96,7 +95,6 @@ export class UploadManager { if (this.uploadQueue.idle()) { this.result = { succeeded: [], failed: [] }; this.storageFullFailure = undefined; - this.isEverythingCancelled = false; this.uploadQueue.concurrency = MAX_CONCURRENT_UPLOADS; } @@ -109,11 +107,6 @@ export class UploadManager { return this.result; } - cancelAll(): void { - this.isEverythingCancelled = true; - this.abortActiveUploadOf.forEach((abort) => abort()); - } - cancelItem(item: UploadItem): void { this.cancelledItems.add(item); this.abortActiveUploadOf.get(item)?.(); @@ -194,7 +187,7 @@ export class UploadManager { } private isItemCancelled(item: UploadItem): boolean { - return this.isEverythingCancelled || this.cancelledItems.has(item); + return this.cancelledItems.has(item); } private async handleUploadError(item: UploadItem, error: unknown): Promise { From 7bfb76bfa67cb6881a9e88d47a6a625c07247b7a Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:32:25 +0100 Subject: [PATCH 33/49] test(upload): show which files are uploading and which are waiting when the storage gets full --- src/app/upload/UploadManager.test.ts | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index d4dc69b33f..c1f4d96116 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -5,7 +5,7 @@ import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { MAX_CONCURRENT_UPLOADS, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; @@ -33,7 +33,7 @@ const failIfHangs = (promise: Promise): Promise => const createManager = (props: Partial[0]> = {}) => new UploadManager({ uploadRepository: getUploadRepository(), getPauseKey: nameOf, ...props }); -const uploadedNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); +const startedUploadNames = () => (uploadFile as Mock).mock.calls.map(([file]) => file.name); const fakeSlowUploads = () => (uploadFile as Mock).mockImplementation((file: { name: string }, _progress, options: UploadFileOptions) => { @@ -111,7 +111,7 @@ describe('Uploading files', () => { createManager({ events }).run([brokenItem, getUploadFileItem('a'), getUploadFileItem('b')]), ); - expect(uploadedNames().filter((name) => name === 'broken')).toHaveLength(2); + expect(startedUploadNames().filter((name) => name === 'broken')).toHaveLength(2); expect(succeeded).toHaveLength(2); expect(failed).toEqual([ { @@ -166,14 +166,22 @@ describe('Uploading files', () => { expect(failed).toHaveLength(1); }); - test('When the storage is full, then the queued files fail without being uploaded', async () => { - fakeUploadFailureOf('first', noSpaceLeftError); - const items = ['first', 'a', 'b', 'c', 'd', 'e', 'f', 'g'].map((name) => getUploadFileItem(name)); + test('When the storage gets full, then the files already uploading finish and the waiting ones fail without being uploaded', async () => { + fakeUploadFailureOf('fills-the-storage', noSpaceLeftError); + const uploadingAtTheSameTime = Array.from( + { length: MAX_CONCURRENT_UPLOADS - 1 }, + (_, index) => `uploading-${index}`, + ); + const stillWaiting = ['waiting-1', 'waiting-2']; + const items = ['fills-the-storage', ...uploadingAtTheSameTime, ...stillWaiting].map((name) => + getUploadFileItem(name), + ); - const { failed } = await failIfHangs(createManager().run(items)); + const { succeeded, failed } = await failIfHangs(createManager().run(items)); - expect(uploadedNames()).toEqual(['first', 'a', 'b', 'c', 'd', 'e']); - expect(failed.map(({ item }) => nameOf(item))).toEqual(['first', 'f', 'g']); + expect(startedUploadNames()).toEqual(['fills-the-storage', ...uploadingAtTheSameTime]); + expect(succeeded.map(({ item }) => nameOf(item))).toEqual(uploadingAtTheSameTime); + expect(failed.map(({ item }) => nameOf(item))).toEqual(['fills-the-storage', ...stillWaiting]); expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); @@ -213,7 +221,7 @@ describe('Uploading files', () => { await createManager().run(items); - expect(uploadedNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); + expect(startedUploadNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); expect(maxActiveLargeUploads).toBe(1); }); From b872483c3a67e7263cf8f32fd93d08fe60128b59 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:42:36 +0100 Subject: [PATCH 34/49] test(upload): state each upload scenario plainly and count started uploads instead of tracking them in the test --- src/app/upload/UploadManager.test.ts | 96 +++++++++++----------------- 1 file changed, 36 insertions(+), 60 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index c1f4d96116..163bd89a57 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -5,7 +5,7 @@ import { DriveFileData } from 'app/drive/types'; import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; import { ConnectionLostError } from 'app/network/requests'; -import { MAX_CONCURRENT_UPLOADS, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; +import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/networkConstants'; import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; @@ -53,6 +53,11 @@ const fakeUploadFailureOf = (failingName: string, error: unknown) => return { uuid: `${file.name}-uuid` } as DriveFileData; }); +const makeUploadsNeverFinish = () => (uploadFile as Mock).mockReturnValue(new Promise(() => undefined)); + +const uploadAtMostAtTheSameTime = (concurrency: number) => + vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(concurrency); + const goOffline = () => vi.spyOn(navigator, 'onLine', 'get').mockReturnValue(false); describe('Uploading files', () => { @@ -125,7 +130,7 @@ describe('Uploading files', () => { test.each([ { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.UploadFailed }, { problem: 'the file is too large', error: fileTooLargeError, reason: UploadFailureReason.FileTooLarge }, - ])('When $problem, then the file is not retried and cannot be retried later', async ({ error, reason }) => { + ])('When $problem, then the file is never retried', async ({ error, reason }) => { (uploadFile as Mock).mockRejectedValue(error); const { failed } = await createManager().run([getUploadFileItem('report')]); @@ -149,7 +154,7 @@ describe('Uploading files', () => { ); }); - test('When a network error happens while online, then the upload is retried and succeeds', async () => { + test('When a network error happens once while online, then the upload is retried and succeeds', async () => { (uploadFile as Mock).mockRejectedValueOnce(networkError); const { succeeded } = await createManager().run([getUploadFileItem('report')]); @@ -158,30 +163,16 @@ describe('Uploading files', () => { expect(succeeded).toHaveLength(1); }); - test('When the upload rejects without an error, then the file fails instead of hanging', async () => { - (uploadFile as Mock).mockRejectedValue(undefined); - - const { failed } = await failIfHangs(createManager().run([getUploadFileItem('report')])); - - expect(failed).toHaveLength(1); - }); - - test('When the storage gets full, then the files already uploading finish and the waiting ones fail without being uploaded', async () => { + test('When the storage gets full, then the file uploading at the same time finishes and the waiting ones fail without being uploaded', async () => { + uploadAtMostAtTheSameTime(2); fakeUploadFailureOf('fills-the-storage', noSpaceLeftError); - const uploadingAtTheSameTime = Array.from( - { length: MAX_CONCURRENT_UPLOADS - 1 }, - (_, index) => `uploading-${index}`, - ); - const stillWaiting = ['waiting-1', 'waiting-2']; - const items = ['fills-the-storage', ...uploadingAtTheSameTime, ...stillWaiting].map((name) => - getUploadFileItem(name), - ); + const items = ['fills-the-storage', 'uploading', 'waiting-1', 'waiting-2'].map((name) => getUploadFileItem(name)); const { succeeded, failed } = await failIfHangs(createManager().run(items)); - expect(startedUploadNames()).toEqual(['fills-the-storage', ...uploadingAtTheSameTime]); - expect(succeeded.map(({ item }) => nameOf(item))).toEqual(uploadingAtTheSameTime); - expect(failed.map(({ item }) => nameOf(item))).toEqual(['fills-the-storage', ...stillWaiting]); + expect(startedUploadNames()).toEqual(['fills-the-storage', 'uploading']); + expect(succeeded.map(({ item }) => nameOf(item))).toEqual(['uploading']); + expect(failed.map(({ item }) => nameOf(item))).toEqual(['fills-the-storage', 'waiting-1', 'waiting-2']); expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); @@ -200,16 +191,7 @@ describe('Uploading files', () => { expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(cancelledItem); }); - test('When small, medium and large files are uploaded together, then the small ones go first and the large ones last and one at a time', async () => { - let activeLargeUploads = 0; - let maxActiveLargeUploads = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string; size: number }) => { - const isLarge = file.size >= USE_MULTIPART_THRESHOLD_BYTES; - if (isLarge) maxActiveLargeUploads = Math.max(maxActiveLargeUploads, ++activeLargeUploads); - await new Promise((resolve) => setTimeout(resolve, 10)); - if (isLarge) activeLargeUploads--; - return { uuid: `${file.name}-uuid` }; - }); + test('When small, medium and large files are uploaded together, then the small ones start first and the large ones last', async () => { const items = [ getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), getUploadFileItem('medium-1', TWENTY_MEGABYTES), @@ -222,44 +204,38 @@ describe('Uploading files', () => { await createManager().run(items); expect(startedUploadNames()).toEqual(['small-1', 'small-2', 'medium-1', 'medium-2', 'large-1', 'large-2']); - expect(maxActiveLargeUploads).toBe(1); }); - test('When the same manager uploads again after a large file in a browser without memory information, then several files go at the same time again', async () => { + test('When several large files are uploaded, then only one uploads at a time', async () => { + makeUploadsNeverFinish(); + const items = [ + getUploadFileItem('large-1', USE_MULTIPART_THRESHOLD_BYTES), + getUploadFileItem('large-2', USE_MULTIPART_THRESHOLD_BYTES), + ]; + + void createManager().run(items); + + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['large-1'])); + }); + + test('When small files are uploaded after a large file in a browser without memory information, then they upload at the same time again', async () => { vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockImplementation((current) => current); - let activeUploads = 0; - let maxActiveUploads = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); - await new Promise((resolve) => setTimeout(resolve, 10)); - activeUploads--; - return { uuid: `${file.name}-uuid` }; - }); const manager = createManager(); await manager.run([getUploadFileItem('large', USE_MULTIPART_THRESHOLD_BYTES)]); - maxActiveUploads = 0; + makeUploadsNeverFinish(); - await manager.run(['a', 'b', 'c'].map((name) => getUploadFileItem(name))); + void manager.run(['report', 'invoice', 'photo'].map((name) => getUploadFileItem(name))); - expect(maxActiveUploads).toBe(3); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['large', 'report', 'invoice', 'photo'])); }); - test('When the browser is running out of memory, then fewer files are uploaded at the same time', async () => { - vi.spyOn(QueueUtilsService.instance, 'getConcurrencyUsingPerformance').mockReturnValue(1); - let activeUploads = 0; - let maxActiveUploads = 0; - (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { - maxActiveUploads = Math.max(maxActiveUploads, ++activeUploads); - await new Promise((resolve) => setTimeout(resolve, 10)); - activeUploads--; - return { uuid: `${file.name}-uuid` }; - }); - const items = ['a', 'b', 'c', 'd'].map((name) => getUploadFileItem(name)); + test('When the browser is running out of memory, then files are uploaded one at a time', async () => { + uploadAtMostAtTheSameTime(1); + makeUploadsNeverFinish(); - const { succeeded } = await createManager().run(items); + void createManager().run(['report', 'invoice', 'photo'].map((name) => getUploadFileItem(name))); - expect(succeeded).toHaveLength(items.length); - expect(maxActiveUploads).toBe(1); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['report'])); }); test('When uploading to a workspace, then the workspace credentials are used', async () => { From 60fd915e40ffb489a10f18d9b3e3bfa93cb5f6d2 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:31:31 +0100 Subject: [PATCH 35/49] fix(upload): do not start a file upload that was cancelled while it was getting ready --- src/app/upload/UploadManager.test.ts | 17 +++++++++++++++++ src/app/upload/UploadManager.ts | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 163bd89a57..3d865a76e2 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -176,6 +176,23 @@ describe('Uploading files', () => { expect(failed.every(({ failure }) => failure.reason === UploadFailureReason.NoSpaceLeft)).toBe(true); }); + test('When a file is cancelled while its paused state is being read, then it is never uploaded and is reported as cancelled', async () => { + const report = getUploadFileItem('report'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const uploadRepository = getUploadRepository(); + const manager = createManager({ events, uploadRepository }); + vi.mocked(uploadRepository.getUploadState).mockImplementationOnce(async () => { + manager.cancelItem(report); + return undefined; + }); + + await manager.run([report]); + + expect(uploadFile).not.toHaveBeenCalled(); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(report); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + test('When a single file is cancelled, then only that file stops', async () => { fakeSlowUploads(); const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 0d7d602802..880e480ff4 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -30,6 +30,12 @@ interface UploadManagerProps { events?: UploadManagerEvents; } +class UploadCancelledError extends Error { + constructor() { + super('The upload was cancelled before it started'); + } +} + export const createFileItem = (file: File, parentFolderUuid: string): UploadFileItem => { const { filename, extension } = itemUtils.getFilenameAndExt(file.name); @@ -167,6 +173,7 @@ export class UploadManager { const pauseKey = this.getPauseKey(fileItem); const uploadState = await this.uploadRepository?.getUploadState(pauseKey); const isPaused = uploadState === TaskStatus.Paused; + this.throwIfCancelled(fileItem); try { const driveFile = await uploadFile( @@ -186,6 +193,10 @@ export class UploadManager { } } + private throwIfCancelled(item: UploadItem): void { + if (this.isItemCancelled(item)) throw new UploadCancelledError(); + } + private isItemCancelled(item: UploadItem): boolean { return this.cancelledItems.has(item); } From 560de20fb2f83c23456d2b90acf95bbb428b5deb Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:12:08 +0100 Subject: [PATCH 36/49] refactor(upload): keep the upload engine types and failures next to the engine --- src/app/network/networkConstants.ts | 1 - src/app/network/types/index.ts | 11 ----------- src/app/upload/UploadManager.test.ts | 3 +-- src/app/upload/UploadManager.ts | 17 +++++++++++++---- .../types/upload.ts => upload/types.ts} | 14 ++++++++++++-- src/app/{network => upload}/uploadFailure.ts | 2 +- src/testUtils/fixtures/upload.fixtures.ts | 3 +-- 7 files changed, 28 insertions(+), 23 deletions(-) rename src/app/{network/types/upload.ts => upload/types.ts} (74%) rename src/app/{network => upload}/uploadFailure.ts (96%) diff --git a/src/app/network/networkConstants.ts b/src/app/network/networkConstants.ts index 69f3b47c6a..1e857231ac 100644 --- a/src/app/network/networkConstants.ts +++ b/src/app/network/networkConstants.ts @@ -10,4 +10,3 @@ export const USE_MULTIPART_THRESHOLD_BYTES = 50 * 1024 * 1024; export const MAX_UPLOAD_ATTEMPTS = 2; export const UPLOAD_IDLE_TIMEOUT_MS = 60 * 1000; export const MAX_CONCURRENT_UPLOADS = 6; -export const SINGLE_TRANSFER_CONCURRENCY = 1; diff --git a/src/app/network/types/index.ts b/src/app/network/types/index.ts index f099c9a09b..edb1f68756 100644 --- a/src/app/network/types/index.ts +++ b/src/app/network/types/index.ts @@ -35,17 +35,6 @@ export interface DownloadChunkPayload { options?: DownloadOptions; } -export enum UploadItemKind { - File = 'file', -} - -export enum UploadFailureReason { - ConnectionLost = 'connection-lost', - NoSpaceLeft = 'no-space', - FileTooLarge = 'file-too-large', - UploadFailed = 'upload-failed', -} - export type UploadErrorReason = 'connection-lost' | 'upload-failed'; export interface OwnerUserAuthenticationData { diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 3d865a76e2..21fb5b7e46 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -9,8 +9,7 @@ import { TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES } from 'app/network/net import { QueueUtilsService } from 'utils/queueUtils'; import { getUploadFileItem, getUploadRepository } from 'testUtils/fixtures/upload.fixtures'; import { UploadManager } from './UploadManager'; -import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason, UploadItem, UploadManagerEvents } from './types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 880e480ff4..ec187ddf5e 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -9,13 +9,22 @@ import { TaskStatus } from 'app/tasks/types'; import { MAX_CONCURRENT_UPLOADS, MAX_UPLOAD_ATTEMPTS, - SINGLE_TRANSFER_CONCURRENCY, TWENTY_MEGABYTES, USE_MULTIPART_THRESHOLD_BYTES, } from 'app/network/networkConstants'; -import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; -import { UploadedItem, UploadFileItem, UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; -import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from 'app/network/uploadFailure'; +import { OwnerUserAuthenticationData } from 'app/network/types'; +import { + UploadedItem, + UploadFailureReason, + UploadFileItem, + UploadItem, + UploadItemKind, + UploadManagerEvents, + UploadResult, +} from './types'; +import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from './uploadFailure'; + +const SINGLE_TRANSFER_CONCURRENCY = 1; enum UploadPriority { SmallFile = 1, diff --git a/src/app/network/types/upload.ts b/src/app/upload/types.ts similarity index 74% rename from src/app/network/types/upload.ts rename to src/app/upload/types.ts index 53cb34dae3..bc4b98495a 100644 --- a/src/app/network/types/upload.ts +++ b/src/app/upload/types.ts @@ -1,7 +1,17 @@ import { DriveFileData } from 'app/drive/types'; import { FileToUpload } from 'app/drive/services/file.service/types'; -import { UploadFailure } from '../uploadFailure'; -import { UploadItemKind } from '.'; +import { UploadFailure } from './uploadFailure'; + +export enum UploadItemKind { + File = 'file', +} + +export enum UploadFailureReason { + ConnectionLost = 'connection-lost', + NoSpaceLeft = 'no-space', + FileTooLarge = 'file-too-large', + UploadFailed = 'upload-failed', +} export type UploadFileItem = { kind: UploadItemKind.File; file: FileToUpload }; diff --git a/src/app/network/uploadFailure.ts b/src/app/upload/uploadFailure.ts similarity index 96% rename from src/app/network/uploadFailure.ts rename to src/app/upload/uploadFailure.ts index 7ac796a44e..1edbebdd2b 100644 --- a/src/app/network/uploadFailure.ts +++ b/src/app/upload/uploadFailure.ts @@ -1,6 +1,6 @@ import { ErrorMessages, HTTP_CODE_ERRORS, HTTP_STATUS_CODES } from 'app/core/constants'; import errorService from 'services/error.service'; -import { ConnectionLostError } from './requests'; +import { ConnectionLostError } from 'app/network/requests'; import { UploadFailureReason } from './types'; export interface UploadFailure { diff --git a/src/testUtils/fixtures/upload.fixtures.ts b/src/testUtils/fixtures/upload.fixtures.ts index 6e123b622e..76b61e40a1 100644 --- a/src/testUtils/fixtures/upload.fixtures.ts +++ b/src/testUtils/fixtures/upload.fixtures.ts @@ -1,6 +1,5 @@ import { vi } from 'vitest'; -import { UploadItemKind } from 'app/network/types'; -import { UploadFileItem } from 'app/network/types/upload'; +import { UploadFileItem, UploadItemKind } from 'app/upload/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { TaskStatus } from 'app/tasks/types'; From e2304731dc61d013d0e50f2b2cd0c300f624b1ff Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:24:55 +0100 Subject: [PATCH 37/49] refactor(upload): classify plan limit failures once and retry automatically only generic upload failures --- src/app/upload/UploadManager.test.ts | 50 +++++++++++++++++++++++++++- src/app/upload/UploadManager.ts | 4 +-- src/app/upload/types.ts | 1 + src/app/upload/uploadFailure.ts | 10 +++--- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index 21fb5b7e46..2488122320 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -127,7 +127,7 @@ describe('Uploading files', () => { }); test.each([ - { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.UploadFailed }, + { problem: 'the plan does not allow it', error: paymentRequiredError, reason: UploadFailureReason.PlanLimit }, { problem: 'the file is too large', error: fileTooLargeError, reason: UploadFailureReason.FileTooLarge }, ])('When $problem, then the file is never retried', async ({ error, reason }) => { (uploadFile as Mock).mockRejectedValue(error); @@ -192,6 +192,54 @@ describe('Uploading files', () => { expect(events.onItemFailed).not.toHaveBeenCalled(); }); + test('When a file still waiting in the queue is cancelled, then it is never uploaded and the uploading one finishes', async () => { + uploadAtMostAtTheSameTime(1); + fakeSlowUploads(); + const [uploading, queued] = [getUploadFileItem('uploading'), getUploadFileItem('queued')]; + const events: UploadManagerEvents = { onItemCancelled: vi.fn() }; + const manager = createManager({ events }); + + const upload = manager.run([uploading, queued]); + await vi.waitFor(() => expect(startedUploadNames()).toEqual(['uploading'])); + manager.cancelItem(queued); + const { succeeded } = await failIfHangs(upload); + + expect(startedUploadNames()).toEqual(['uploading']); + expect(succeeded.map(({ item }) => item)).toEqual([uploading]); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(queued); + }); + + test('When a file is cancelled and its upload then fails, then it is not retried and is reported as cancelled', async () => { + const report = getUploadFileItem('report'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn(), onItemFailed: vi.fn() }; + const manager = createManager({ events }); + (uploadFile as Mock).mockImplementationOnce(() => { + manager.cancelItem(report); + return Promise.reject(new Error('Server error')); + }); + + await manager.run([report]); + + expect(uploadFile).toHaveBeenCalledOnce(); + expect(events.onItemCancelled).toHaveBeenCalledExactlyOnceWith(report); + expect(events.onItemFailed).not.toHaveBeenCalled(); + }); + + test('When a file finishes uploading right after being cancelled, then it is reported as uploaded', async () => { + const report = getUploadFileItem('report'); + const events: UploadManagerEvents = { onItemCancelled: vi.fn() }; + const manager = createManager({ events }); + (uploadFile as Mock).mockImplementationOnce(() => { + manager.cancelItem(report); + return Promise.resolve({ uuid: 'report-uuid' }); + }); + + const { succeeded } = await manager.run([report]); + + expect(succeeded.map(({ item }) => item)).toEqual([report]); + expect(events.onItemCancelled).not.toHaveBeenCalled(); + }); + test('When a single file is cancelled, then only that file stops', async () => { fakeSlowUploads(); const [cancelledItem, otherItem] = [getUploadFileItem('cancelled'), getUploadFileItem('other')]; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index ec187ddf5e..a9902a344f 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -22,7 +22,7 @@ import { UploadManagerEvents, UploadResult, } from './types'; -import { canSucceedOnImmediateRetry, toUploadFailure, UploadFailure } from './uploadFailure'; +import { shouldAutoRetry, toUploadFailure, UploadFailure } from './uploadFailure'; const SINGLE_TRANSFER_CONCURRENCY = 1; @@ -171,7 +171,7 @@ export class UploadManager { return await this.uploadFile(item); } catch (error) { const hasAttemptsLeft = attempt < MAX_UPLOAD_ATTEMPTS; - const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && canSucceedOnImmediateRetry(error); + const shouldRetry = hasAttemptsLeft && !this.isItemCancelled(item) && shouldAutoRetry(toUploadFailure(error)); if (!shouldRetry) throw error; return this.uploadWithRetries(item, attempt + 1); diff --git a/src/app/upload/types.ts b/src/app/upload/types.ts index bc4b98495a..f374d0504f 100644 --- a/src/app/upload/types.ts +++ b/src/app/upload/types.ts @@ -10,6 +10,7 @@ export enum UploadFailureReason { ConnectionLost = 'connection-lost', NoSpaceLeft = 'no-space', FileTooLarge = 'file-too-large', + PlanLimit = 'plan-limit', UploadFailed = 'upload-failed', } diff --git a/src/app/upload/uploadFailure.ts b/src/app/upload/uploadFailure.ts index 1edbebdd2b..78532669e3 100644 --- a/src/app/upload/uploadFailure.ts +++ b/src/app/upload/uploadFailure.ts @@ -21,18 +21,21 @@ const isStorageFull = (error: unknown) => getStatusCode(error) === HTTP_STATUS_C const isFileTooLarge = (error: unknown) => errorService.castError(error).code === HTTP_CODE_ERRORS.FILE_UPLOAD_SIZE_EXCEEDED; -const isPlanNotAllowingIt = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.PAYMENT_REQUIRED; +const isPlanLimitReached = (error: unknown) => getStatusCode(error) === HTTP_STATUS_CODES.PAYMENT_REQUIRED; + +const REASONS_THAT_RETRYING_CANNOT_FIX = new Set([UploadFailureReason.FileTooLarge, UploadFailureReason.PlanLimit]); const getFailureReason = (error: unknown): UploadFailureReason => { if (isConnectionLost(error)) return UploadFailureReason.ConnectionLost; if (isStorageFull(error)) return UploadFailureReason.NoSpaceLeft; if (isFileTooLarge(error)) return UploadFailureReason.FileTooLarge; + if (isPlanLimitReached(error)) return UploadFailureReason.PlanLimit; return UploadFailureReason.UploadFailed; }; export const toUploadFailure = (error: unknown): UploadFailure => { const reason = getFailureReason(error); - const canSucceedLater = reason !== UploadFailureReason.FileTooLarge && !isPlanNotAllowingIt(error); + const canSucceedLater = !REASONS_THAT_RETRYING_CANNOT_FIX.has(reason); const genericUploadError = new Error('Upload failed'); return { @@ -42,5 +45,4 @@ export const toUploadFailure = (error: unknown): UploadFailure => { }; }; -export const canSucceedOnImmediateRetry = (error: unknown): boolean => - getFailureReason(error) === UploadFailureReason.UploadFailed && !isPlanNotAllowingIt(error); +export const shouldAutoRetry = ({ reason }: UploadFailure): boolean => reason === UploadFailureReason.UploadFailed; From b6e3256a235390f00b9393f51c1dd74e26e8beac Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Thu, 29 Oct 2026 14:39:28 +0100 Subject: [PATCH 38/49] refactor(upload): run every file upload on the new engine --- src/app/network/UploadFolderManager.test.ts | 15 +- src/app/network/UploadFolderManager.ts | 12 +- src/app/repositories/uploadStateUpdates.ts | 11 + .../storage.thunks/uploadItemsThunk.ts | 1 - .../TaskToRetryItem/TaskToRetryItem.tsx | 2 +- .../tasks/upload/uploadFilesWithTasks.test.ts | 201 +++++++++----- src/app/tasks/upload/uploadFilesWithTasks.ts | 249 ++++++++++++------ .../upload/uploadFoldersWithTasks.test.ts | 9 +- src/app/tasks/upload/uploadTaskErrors.ts | 10 +- 9 files changed, 328 insertions(+), 182 deletions(-) create mode 100644 src/app/repositories/uploadStateUpdates.ts diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 60b096b09e..9d06a83351 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -13,13 +13,14 @@ import { import { FilesExceedsSizeLimitError } from 'app/drive/services/file.service/upload.errors'; import { uploadItemsParallelThunk } from 'app/store/slices/storage/storage.thunks/uploadItemsThunk'; import { deleteItemsThunk } from '../store/slices/storage/storage.thunks/deleteItemsThunk'; -import { uploadFileWithManager } from './UploadManager'; +import { uploadFilesWithTasks } from 'app/tasks/upload/uploadFilesWithTasks'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import uploadFile from 'app/drive/services/file.service/uploadFile'; import newStorageService from 'app/drive/services/new-storage.service'; import RetryManager from './RetryManager'; import { ConnectionLostError } from './requests'; import errorService from 'services/error.service'; +import { UploadFailureReason } from 'app/network/types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn(async (file: { name: string }) => ({ uuid: `${file.name}-uuid`, name: file.name })), @@ -51,6 +52,8 @@ vi.mock('app/store/slices/storage/folderUtils/getUniqueFolderName', () => ({ getUniqueFolderName: vi.fn(), })); +vi.mock('services/referral.service', () => ({ default: { trackFileUpload: vi.fn() } })); +vi.mock('app/network/networkInformation', () => ({ logNetworkInfoForUpload: vi.fn() })); vi.mock('services/error.service', () => ({ default: { castError: vi.fn().mockImplementation((e) => e), @@ -89,7 +92,7 @@ describe('uploadFoldersWithManager', () => { (uploadItemsParallelThunk as unknown as Mock).mockImplementation((thunkArgs) => thunkArgs); return vi.fn(({ files, parentFolderId, options, onFileUploadCallback }) => ({ unwrap: () => - uploadFileWithManager({ + uploadFilesWithTasks({ files: files.map((content: File, index: number) => ({ taskId: `file-task-${index}`, relatedTaskId: options.relatedTaskId, @@ -304,7 +307,7 @@ describe('uploadFoldersWithManager', () => { }); expect(events.onFolderUploadError).toHaveBeenCalledOnce(); - expect(events.onFolderUploadError).toHaveBeenCalledWith(taskId, 'upload-failed'); + expect(events.onFolderUploadError).toHaveBeenCalledWith(taskId, UploadFailureReason.UploadFailed); expect(events.onFolderUploadSuccess).not.toHaveBeenCalled(); }); @@ -420,7 +423,7 @@ describe('uploadFoldersWithManager', () => { await runFolderUpload(); - expect(events.onFolderUploadError).toHaveBeenLastCalledWith(taskId, 'connection-lost'); + expect(events.onFolderUploadError).toHaveBeenLastCalledWith(taskId, UploadFailureReason.ConnectionLost); expect(events.onFolderUploadSuccess).not.toHaveBeenCalled(); }); }); @@ -637,7 +640,7 @@ describe('uploadFoldersWithManager', () => { await expect(runFolderUpload()).resolves.toBeUndefined(); - expectEveryFolderFailedWith('connection-lost'); + expectEveryFolderFailedWith(UploadFailureReason.ConnectionLost); expect(errorService.reportError).toHaveBeenCalledWith(networkError); }); @@ -646,7 +649,7 @@ describe('uploadFoldersWithManager', () => { await runFolderUpload(); - expectEveryFolderFailedWith('upload-failed'); + expectEveryFolderFailedWith(UploadFailureReason.UploadFailed); }); }); diff --git a/src/app/network/UploadFolderManager.ts b/src/app/network/UploadFolderManager.ts index 33cac613c6..0822ba6acf 100644 --- a/src/app/network/UploadFolderManager.ts +++ b/src/app/network/UploadFolderManager.ts @@ -16,7 +16,7 @@ import { wait } from 'utils/timeUtils'; import { ConnectionLostError } from './requests'; import { FilesExceedsSizeLimitError } from 'app/drive/services/file.service/upload.errors'; import { filterFilesByMaxSize } from 'app/store/slices/storage/fileUtils/filterFilesByMaxSize'; -import { UploadErrorReason } from './types'; +import { UploadFailureReason } from './types'; import { MAX_CONCURRENT_UPLOADS } from './networkConstants'; export interface UploadFolderPayload { @@ -47,7 +47,7 @@ export interface UploadFolderManagerEvents { taskId: string, info: { folderName: string; rootFolderUUID?: string; hasFailedFiles: boolean }, ) => void; - onFolderUploadError?: (taskId: string, reason?: UploadErrorReason) => void; + onFolderUploadError?: (taskId: string, reason?: UploadFailureReason) => void; onFolderUploadCancelled?: (taskId: string) => void; stopRelatedUploads?: (taskId: string) => Promise[]; } @@ -335,7 +335,7 @@ export class UploadFoldersManager { private readonly killQueueAndNotifyError = (taskId: string) => { this.tasksInfo[taskId].hasErrored = true; this.uploadFoldersQueue.kill(); - this.events?.onFolderUploadError?.(taskId, 'upload-failed'); + this.events?.onFolderUploadError?.(taskId, UploadFailureReason.UploadFailed); }; /** @@ -370,7 +370,7 @@ export class UploadFoldersManager { }; private readonly notifyPreparationError = (error: unknown): void => { - const reason: UploadErrorReason = navigator.onLine ? 'upload-failed' : 'connection-lost'; + const reason = navigator.onLine ? UploadFailureReason.UploadFailed : UploadFailureReason.ConnectionLost; for (const { options } of this.payload) { this.events?.onFolderUploadError?.(options.taskId, reason); @@ -432,11 +432,11 @@ export class UploadFoldersManager { const castedError = errorService.castError(err); if (connectionLost) { - this.events?.onFolderUploadError?.(taskId, 'connection-lost'); + this.events?.onFolderUploadError?.(taskId, UploadFailureReason.ConnectionLost); errorService.reportError(castedError); break; } else if (!this.tasksInfo[taskId].isCancelled) { - this.events?.onFolderUploadError?.(taskId, 'upload-failed'); + this.events?.onFolderUploadError?.(taskId, UploadFailureReason.UploadFailed); // Log the error or report it but don't re-throw it to allow the next folder to be processed errorService.reportError(castedError); continue; diff --git a/src/app/repositories/uploadStateUpdates.ts b/src/app/repositories/uploadStateUpdates.ts new file mode 100644 index 0000000000..036452e990 --- /dev/null +++ b/src/app/repositories/uploadStateUpdates.ts @@ -0,0 +1,11 @@ +import errorService from 'services/error.service'; +import { TaskStatus } from '../tasks/types'; +import { PersistUploadRepository } from './DatabaseUploadRepository'; + +export const markUploadAsInProcess = (uploadRepository: PersistUploadRepository, taskId: string): void => { + uploadRepository.setUploadState(taskId, TaskStatus.InProcess).catch((error) => errorService.reportError(error)); +}; + +export const forgetUploadState = (uploadRepository: PersistUploadRepository, taskId: string): void => { + uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); +}; diff --git a/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts b/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts index e6b04e19a1..dcaa5e6a29 100644 --- a/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts +++ b/src/app/store/slices/storage/storage.thunks/uploadItemsThunk.ts @@ -231,7 +231,6 @@ export const uploadItemsThunk = createAsyncThunk ({ - uploadFileWithManager: vi.fn(() => Promise.resolve({ uploadedFiles: [] })), +const { managerState } = vi.hoisted(() => ({ + managerState: { + result: { succeeded: [], failed: [] } as UploadResult, + instance: undefined as + | undefined + | { props: { events: UploadManagerEvents }; run: Mock; cancelAll: Mock; cancelItem: Mock; items: UploadItem[] }, + }, +})); + +vi.mock('app/upload/UploadManager', () => ({ + UploadManager: vi.fn(function (this: NonNullable, props) { + this.props = props; + this.cancelAll = vi.fn(); + this.cancelItem = vi.fn(); + this.run = vi.fn(async (items: UploadItem[]) => { + this.items = items; + return managerState.result; + }); + managerState.instance = this; + }), })); vi.mock('app/tasks/services/tasks.service', () => ({ @@ -16,8 +38,6 @@ vi.mock('app/tasks/services/tasks.service', () => ({ create: vi.fn(), updateTask: vi.fn(), findTask: vi.fn(), - addListener: vi.fn(), - removeListener: vi.fn(), }, })); @@ -48,8 +68,8 @@ const buildFile = (overrides: Record = {}) => ({ const uploadRepository = { getUploadState: vi.fn(), - setUploadState: vi.fn(), - removeUploadState: vi.fn(), + setUploadState: vi.fn(() => Promise.resolve()), + removeUploadState: vi.fn(() => Promise.resolve()), } as unknown as PersistUploadRepository; const runUpload = async (files: ReturnType[], props: Record = {}) => { @@ -59,19 +79,33 @@ const runUpload = async (files: ReturnType[], props: Record; + return { manager, events: manager.props.events, items: manager.items }; +}; + +const failWith = (failure: Partial) => { + vi.mocked(UploadManager).mockImplementationOnce(function (this: NonNullable, props) { + this.props = props; + this.run = vi.fn(async (items: UploadItem[]) => ({ + succeeded: [], + failed: [{ item: items[0], failure: { error: new Error('failed'), isRetryable: true, ...failure } }], + })); + managerState.instance = this; + } as never); }; -describe('uploadFilesWithTasks', () => { +describe('Uploading files with tasks', () => { beforeEach(() => { vi.clearAllMocks(); + RetryManager.clearTasks(); + managerState.result = { succeeded: [], failed: [] }; + (tasksService.findTask as Mock).mockReturnValue(undefined); }); test('When a new file upload begins, then a task is created to track it', async () => { (tasksService.create as Mock).mockReturnValue('new-task-id'); - const { managerProps } = await runUpload([buildFile()]); + const { items } = await runUpload([buildFile()]); expect(tasksService.create).toHaveBeenCalledWith( expect.objectContaining({ @@ -81,18 +115,17 @@ describe('uploadFilesWithTasks', () => { cancellable: true, }), ); - expect(managerProps.files).toEqual([expect.objectContaining({ taskId: 'new-task-id' })]); + expect(items).toEqual([{ kind: UploadItemKind.File, file: buildFile().filecontent }]); }); test('When an upload is retried, then its existing task resumes instead of creating a new one', async () => { - const { managerProps } = await runUpload([buildFile({ taskId: 'existing-task' })]); + await runUpload([buildFile({ taskId: 'existing-task' })]); expect(tasksService.create).not.toHaveBeenCalled(); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 'existing-task', - merge: { status: TaskStatus.Encrypting }, + merge: { status: TaskStatus.Encrypting, stop: expect.any(Function) }, }); - expect(managerProps.options.isRetriedUpload).toBe(true); }); test('When files from a folder upload begin, then the folder task shows as in progress', async () => { @@ -119,54 +152,47 @@ describe('uploadFilesWithTasks', () => { }); }); - test('When a file upload starts, then its task shows it is being prepared and becomes stoppable', async () => { - (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); - const stop = vi.fn(); + test('When a file upload starts, then only files uploaded outside a folder show as in progress', async () => { + (tasksService.create as Mock).mockReturnValueOnce('loose-task').mockReturnValueOnce('folder-file-task'); + const { events, items } = await runUpload([buildFile(), buildFile({ relatedTaskId: 'related' })]); - events.onUploadStart?.(managerProps.files[0], stop); + events.onItemStarted?.(items[0]); + events.onItemStarted?.(items[1]); expect(tasksService.updateTask).toHaveBeenCalledWith({ - taskId: 't1', - merge: { status: TaskStatus.Encrypting, stop }, + taskId: 'loose-task', + merge: { status: TaskStatus.InProcess }, + }); + expect(tasksService.updateTask).toHaveBeenCalledWith({ + taskId: 'folder-file-task', + merge: { status: TaskStatus.Encrypting }, }); - }); - - test('When an upload attempt begins, then only files uploaded outside a folder show as in progress', async () => { - (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); - - events.onUploadAttempt?.(managerProps.files[0]); - expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.InProcess } }); - - (tasksService.updateTask as Mock).mockClear(); - events.onUploadAttempt?.({ ...managerProps.files[0], relatedTaskId: 'related' }); - expect(tasksService.updateTask).not.toHaveBeenCalled(); }); test('When an upload advances, then the task progress updates unless the task is paused or cancelled', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const { events, items } = await runUpload([buildFile()]); (tasksService.findTask as Mock).mockReturnValue({ status: TaskStatus.InProcess }); - events.onUploadProgress?.(managerProps.files[0], 0.5); + events.onItemProgress?.(items[0], 0.5); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { progress: 0.5 } }); (tasksService.updateTask as Mock).mockClear(); (tasksService.findTask as Mock).mockReturnValue({ status: TaskStatus.Paused }); - events.onUploadProgress?.(managerProps.files[0], 0.6); - expect(tasksService.updateTask).not.toHaveBeenCalled(); - + events.onItemProgress?.(items[0], 0.6); (tasksService.findTask as Mock).mockReturnValue({ status: TaskStatus.Cancelled }); - events.onUploadProgress?.(managerProps.files[0], 0.7); + events.onItemProgress?.(items[0], 0.7); expect(tasksService.updateTask).not.toHaveBeenCalled(); }); - test('When a file finishes uploading, then its task succeeds and the upload is tracked', async () => { + test('When a file finishes uploading, then its task succeeds, the upload is tracked and the caller is told', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const onFinishUploadFile = vi.fn(); + const onFileUploadCallback = vi.fn(); + const { events, items } = await runUpload([buildFile({ onFinishUploadFile })], { onFileUploadCallback }); + const driveFile = { uuid: 'file-uuid' } as never; - await events.onUploadSuccess?.(managerProps.files[0], { uuid: 'file-uuid' } as never); + await events.onItemSucceeded?.(items[0], driveFile); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', @@ -174,16 +200,27 @@ describe('uploadFilesWithTasks', () => { }); expect(referralService.trackFileUpload).toHaveBeenCalledOnce(); expect(networkInformation.logNetworkInfoForUpload).toHaveBeenCalledWith({ fileName: 'file.txt', fileSize: 1024 }); + expect(onFinishUploadFile).toHaveBeenCalledWith(driveFile, 't1'); + expect(onFileUploadCallback).toHaveBeenCalledWith(driveFile); + }); + + test('When a retried file finishes uploading, then it leaves the retry list', async () => { + RetryManager.addTasks([{ taskId: 'existing-task', type: 'upload' as never, params: {} }]); + const { events, items } = await runUpload([buildFile({ taskId: 'existing-task' })]); + + await events.onItemSucceeded?.(items[0], { uuid: 'file-uuid' } as never); + + expect(RetryManager.getTasks()).toEqual([]); }); test('When a file inside a folder finishes uploading, then the folder progress advances', async () => { (tasksService.create as Mock).mockReturnValue('t1'); const relatedTaskProgress = { filesUploaded: 0, totalFilesToUpload: 4 }; - const { events, managerProps } = await runUpload([buildFile({ relatedTaskId: 'related-task' })], { + const { events, items } = await runUpload([buildFile({ relatedTaskId: 'related-task' })], { relatedTaskProgress, }); - await events.onUploadSuccess?.(managerProps.files[0], { uuid: 'file-uuid' } as never); + await events.onItemSucceeded?.(items[0], { uuid: 'file-uuid' } as never); expect(relatedTaskProgress.filesUploaded).toBe(1); expect(tasksService.updateTask).toHaveBeenCalledWith({ @@ -194,54 +231,74 @@ describe('uploadFilesWithTasks', () => { test('When an upload fails, then the task shows an error message matching the cause', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const { events, items } = await runUpload([buildFile()]); + const failure = { error: new Error('failed'), isRetryable: true }; - events.onUploadError?.(managerProps.files[0], 'connection-lost'); + events.onItemFailed?.(items[0], { ...failure, reason: UploadFailureReason.ConnectionLost }); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Error, subtitle: 'error.connectionLostError' }, }); - events.onUploadError?.(managerProps.files[0], 'upload-failed'); + events.onItemFailed?.(items[0], { ...failure, reason: UploadFailureReason.UploadFailed }); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Error, subtitle: 'tasks.subtitles.upload-failed' }, }); - - events.onUploadError?.(managerProps.files[0]); - expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Error } }); }); - test('When an upload is aborted, then its task shows as cancelled', async () => { + test('When an upload is cancelled, then its task shows as cancelled', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - const { events, managerProps } = await runUpload([buildFile()]); + const { events, items } = await runUpload([buildFile()]); - events.onUploadAborted?.(managerProps.files[0]); + events.onItemCancelled?.(items[0]); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 't1', merge: { status: TaskStatus.Cancelled } }); }); - test('When a task is cancelled by the user, then only that upload is stopped', async () => { + test('When a task is stopped by the user, then only that upload is cancelled', async () => { + const { manager, items } = await runUpload([buildFile(), buildFile()]); + const [{ stop }] = (tasksService.create as Mock).mock.calls[0] as [{ stop: () => Promise }]; + + await stop(); + + expect(manager.cancelItem).toHaveBeenCalledExactlyOnceWith(items[0]); + }); + + test('When the folder upload that owns the files is cancelled, then the whole upload is cancelled', async () => { + const abortController = new AbortController(); + const { manager } = await runUpload([buildFile()], { abortController }); + + abortController.abort(); + + expect(manager.cancelAll).toHaveBeenCalledOnce(); + }); + + test('When a file fails, then it is listed for retry and the upload rejects with its error', async () => { (tasksService.create as Mock).mockReturnValue('t1'); - let registeredListener: ((task: { id: string }) => void) | undefined; - (tasksService.addListener as Mock).mockImplementation(({ listener }) => { - registeredListener = listener; - }); + const error = new Error('Server error'); + failWith({ error, reason: UploadFailureReason.UploadFailed, isRetryable: false }); + + await expect(runUpload([buildFile()])).rejects.toThrow(error); - const { events, managerProps } = await runUpload([buildFile()]); - const abort = vi.fn(); - const unsubscribe = events.registerUploadAbort?.(managerProps.files[0], abort); + expect(RetryManager.getTasks()).toEqual([ + expect.objectContaining({ taskId: 't1', retryable: false, params: expect.objectContaining({ taskId: 't1' }) }), + ]); + }); - registeredListener?.({ id: 'other-task' }); - expect(abort).not.toHaveBeenCalled(); + test.each([ + { problem: 'the storage is full', reason: UploadFailureReason.NoSpaceLeft, callback: 'maxSpaceOccupiedCallback' }, + { + problem: 'the file is too large', + reason: UploadFailureReason.FileTooLarge, + callback: 'fileSizeExceededCallback', + }, + ] as const)('When $problem, then the user is warned about it', async ({ reason, callback }) => { + const warn = vi.fn(); + failWith({ reason }); - registeredListener?.({ id: 't1' }); - expect(abort).toHaveBeenCalledOnce(); + await expect(runUpload([buildFile()], { [callback]: warn })).rejects.toThrow(); - unsubscribe?.(); - expect(tasksService.removeListener).toHaveBeenCalledWith({ - event: TaskEvent.TaskCancelled, - listener: registeredListener, - }); + expect(warn).toHaveBeenCalledOnce(); }); }); diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index c817b1b8b1..f505bec6ae 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -1,20 +1,45 @@ import tasksService from 'app/tasks/services/tasks.service'; -import { TaskData, TaskEvent, TaskStatus, TaskType, UploadFileTask } from 'app/tasks/types'; +import { TaskStatus, TaskType, UploadFileTask } from 'app/tasks/types'; import referralService from 'services/referral.service'; import { logNetworkInfoForUpload } from 'app/network/networkInformation'; import { DriveFileData } from 'app/drive/types'; -import { - uploadFileWithManager, - UploadFileParamsWithTaskId, - UploadManagerEvents, - UploadManagerFileParams, -} from 'app/network/UploadManager'; +import { FileToUpload } from 'app/drive/services/file.service/types'; +import { UploadManager } from 'app/upload/UploadManager'; +import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; +import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; +import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; +import { forgetUploadState, markUploadAsInProcess } from 'app/repositories/uploadStateUpdates'; import { errorMerge } from './uploadTaskErrors'; -type UploadFileWithManagerProps = Parameters[0]; +export type UploadManagerFileParams = { + filecontent: FileToUpload; + taskId?: string; + relatedTaskId?: string; + fileType?: string; + userEmail: string; + parentFolderId: string; + onFinishUploadFile?: (driveItemData: DriveFileData, taskId: string) => void; + abortController?: AbortController; +}; + +type UploadFileParamsWithTaskId = UploadManagerFileParams & { taskId: string }; -interface UploadFilesWithTasksProps extends Omit { +interface UploadFilesWithTasksProps { files: UploadManagerFileParams[]; + maxSpaceOccupiedCallback: () => void; + fileSizeExceededCallback?: () => void; + onFileUploadCallback?: (driveFileData: DriveFileData) => void; + uploadRepository: PersistUploadRepository; + abortController?: AbortController; + options?: { + showNotifications?: boolean; + ownerUserAuthenticationData?: OwnerUserAuthenticationData; + sharedItemData?: { + isDeepFolder?: boolean; + currentFolderId?: string; + }; + }; relatedTaskProgress?: { filesUploaded: number; totalFilesToUpload: number }; } @@ -22,80 +47,47 @@ interface UploadFilesWithTasksProps extends Omit => { - const { files, options, abortController, uploadRepository, relatedTaskProgress, ...managerProps } = props; - - const isRetriedUpload = files.some((file) => !!file.taskId) || !!options?.isRetriedUpload; - - const filesWithTaskId: UploadFileParamsWithTaskId[] = files.map((file) => { - if (file.taskId) { - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.Encrypting } }); - return { ...file, taskId: file.taskId }; - } - - const sharedItemAuthenticationData = options?.ownerUserAuthenticationData - ? { - ownerUserAuthenticationData: { ...options.ownerUserAuthenticationData }, - isDeepFolder: !!options?.sharedItemData?.isDeepFolder, - currentFolderId: options?.sharedItemData?.currentFolderId as string, - } - : undefined; - - const taskId = tasksService.create({ - relatedTaskId: file.relatedTaskId, - action: TaskType.UploadFile, - item: { uploadFile: file.filecontent.content, parentFolderId: file.parentFolderId }, - fileName: file.filecontent.name, - fileType: file.filecontent.type, - isFileNameValidated: true, - showNotification: options?.showNotifications ?? true, - cancellable: true, - sharedItemAuthenticationData, - stop: async () => { - if (abortController) abortController.abort(); - else file.abortController?.abort(); - }, - }); - - return { ...file, taskId }; - }); - - const relatedTaskId = filesWithTaskId[0]?.relatedTaskId; - if (relatedTaskId) { - const isPaused = (await uploadRepository?.getUploadState(relatedTaskId)) === TaskStatus.Paused; - if (!isPaused) { - tasksService.updateTask({ taskId: relatedTaskId, merge: { status: TaskStatus.InProcess } }); - } - } +export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Promise => { + const { files, options, abortController, uploadRepository, relatedTaskProgress } = props; + const fileByItem = new Map(); + const getFile = (item: UploadItem) => fileByItem.get(item) as UploadFileParamsWithTaskId; + const getPauseKey = (item: UploadItem) => { + const file = getFile(item); + return file.relatedTaskId ?? file.taskId; + }; const events: UploadManagerEvents = { - onUploadStart: (file, stopUpload) => - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.Encrypting, stop: stopUpload } }), - - onUploadAttempt: (file) => { - if (!file.relatedTaskId) { - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.InProcess } }); - } + onItemStarted: (item) => { + const file = getFile(item); + tasksService.updateTask({ + taskId: file.taskId, + merge: { status: file.relatedTaskId ? TaskStatus.Encrypting : TaskStatus.InProcess }, + }); + if (!file.relatedTaskId) markUploadAsInProcess(uploadRepository, file.taskId); }, - onUploadProgress: (file, progress) => { - const taskStatus = tasksService.findTask(file.taskId)?.status; + onItemProgress: (item, progress) => { + const { taskId } = getFile(item); + const taskStatus = tasksService.findTask(taskId)?.status; const isTaskPaused = taskStatus === TaskStatus.Paused; const isTaskCancelled = taskStatus === TaskStatus.Cancelled; if (!isTaskCancelled && !isTaskPaused) { - tasksService.updateTask({ taskId: file.taskId, merge: { progress } }); + tasksService.updateTask({ taskId, merge: { progress } }); } }, - onUploadSuccess: async (file, driveFileData) => { + onItemSucceeded: async (item, uploadedItem) => { + const driveFileData = uploadedItem as DriveFileData; + const file = getFile(item); + if (relatedTaskProgress && file.relatedTaskId) { relatedTaskProgress.filesUploaded += 1; - const isPaused = (await uploadRepository?.getUploadState(file.relatedTaskId)) === TaskStatus.Paused; + const folderUploadState = await uploadRepository.getUploadState(file.relatedTaskId); + const isPaused = folderUploadState === TaskStatus.Paused; if (!isPaused) { tasksService.updateTask({ taskId: file.relatedTaskId, @@ -104,35 +96,118 @@ export const uploadFilesWithTasks = async ( } } - tasksService.updateTask({ - taskId: file.taskId, - merge: { status: TaskStatus.Success, itemUUID: { fileUUID: driveFileData.uuid } }, - }); + if (tasksService.findTask(file.taskId)?.status !== TaskStatus.Cancelled) { + tasksService.updateTask({ + taskId: file.taskId, + merge: { status: TaskStatus.Success, itemUUID: { fileUUID: driveFileData.uuid } }, + }); + } + RetryManager.removeTask(file.taskId); referralService.trackFileUpload(); logNetworkInfoForUpload({ fileName: file.filecontent.name, fileSize: file.filecontent.size }); + file.onFinishUploadFile?.(driveFileData, file.taskId); + props.onFileUploadCallback?.(driveFileData); }, - onUploadError: (file, reason) => tasksService.updateTask({ taskId: file.taskId, merge: errorMerge(reason) }), - - onUploadAborted: (file) => - tasksService.updateTask({ taskId: file.taskId, merge: { status: TaskStatus.Cancelled } }), + onItemFailed: (item, { reason }) => { + const { taskId } = getFile(item); + tasksService.updateTask({ taskId, merge: errorMerge(reason) }); + forgetUploadState(uploadRepository, taskId); + }, - registerUploadAbort: (file, abort) => { - const listener = (task: TaskData) => { - if (task.id === file.taskId) abort(); - }; - tasksService.addListener({ event: TaskEvent.TaskCancelled, listener }); - return () => tasksService.removeListener({ event: TaskEvent.TaskCancelled, listener }); + onItemCancelled: (item) => { + tasksService.updateTask({ taskId: getFile(item).taskId, merge: { status: TaskStatus.Cancelled } }); }, }; - return uploadFileWithManager({ - ...managerProps, - files: filesWithTaskId, + const manager = new UploadManager({ + ownerUserAuthenticationData: options?.ownerUserAuthenticationData, uploadRepository, - abortController, - options: { ...options, isRetriedUpload }, + getPauseKey, events, }); + + const items = files.map((file): UploadItem => { + const item: UploadItem = { + kind: UploadItemKind.File, + file: { ...file.filecontent, type: file.fileType ?? file.filecontent.type }, + }; + fileByItem.set(item, { ...file, taskId: file.taskId ?? createUploadTask(file, item, manager, options) }); + file.abortController?.signal.addEventListener('abort', () => manager.cancelItem(item), { once: true }); + + if (file.taskId) { + tasksService.updateTask({ + taskId: file.taskId, + merge: { status: TaskStatus.Encrypting, stop: async () => manager.cancelItem(item) }, + }); + } + + return item; + }); + + if (abortController?.signal.aborted) manager.cancelAll(); + abortController?.signal.addEventListener('abort', () => manager.cancelAll(), { once: true }); + + const relatedTaskId = files[0]?.relatedTaskId; + if (relatedTaskId) { + const folderUploadState = await uploadRepository.getUploadState(relatedTaskId); + const isPaused = folderUploadState === TaskStatus.Paused; + + if (!isPaused) { + tasksService.updateTask({ taskId: relatedTaskId, merge: { status: TaskStatus.InProcess } }); + } + } + + const { failed } = await manager.run(items); + + if (failed.length > 0) { + RetryManager.addTasks( + failed.map(({ item, failure }) => { + const file = getFile(item); + + return { + taskId: file.taskId, + type: RetryableTaskType.Upload, + params: file, + retryable: failure.isRetryable, + }; + }), + ); + + const failureReasons = new Set(failed.map(({ failure }) => failure.reason)); + + if (failureReasons.has(UploadFailureReason.NoSpaceLeft)) props.maxSpaceOccupiedCallback(); + if (failureReasons.has(UploadFailureReason.FileTooLarge)) props.fileSizeExceededCallback?.(); + + throw failed[0].failure.error; + } +}; + +const createUploadTask = ( + file: UploadManagerFileParams, + item: UploadItem, + manager: UploadManager, + options: UploadFilesWithTasksProps['options'], +): string => { + const sharedItemAuthenticationData = options?.ownerUserAuthenticationData + ? { + ownerUserAuthenticationData: { ...options.ownerUserAuthenticationData }, + isDeepFolder: !!options?.sharedItemData?.isDeepFolder, + currentFolderId: options?.sharedItemData?.currentFolderId as string, + } + : undefined; + + return tasksService.create({ + relatedTaskId: file.relatedTaskId, + action: TaskType.UploadFile, + item: { uploadFile: file.filecontent.content, parentFolderId: file.parentFolderId }, + fileName: file.filecontent.name, + fileType: file.filecontent.type, + isFileNameValidated: true, + showNotification: options?.showNotifications ?? true, + cancellable: true, + sharedItemAuthenticationData, + stop: async () => manager.cancelItem(item), + }); }; diff --git a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts index 3c3594c9c6..88efeef923 100644 --- a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts @@ -7,6 +7,7 @@ import { uploadFoldersWithManager, UploadFolderManagerEvents } from 'app/network import { IRoot } from 'app/store/slices/storage/types'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; import { uploadFoldersWithTasks } from './uploadFoldersWithTasks'; +import { UploadFailureReason } from 'app/network/types'; vi.mock('app/network/UploadFolderManager', () => ({ uploadFoldersWithManager: vi.fn(() => Promise.resolve()), @@ -179,7 +180,7 @@ describe('uploadFoldersWithTasks', () => { (tasksService.create as Mock).mockReturnValue('task-1'); const { events } = await runUpload([{ root, currentFolderId: 'parent' }]); - events.onFolderUploadError?.('task-1', 'connection-lost'); + events.onFolderUploadError?.('task-1', UploadFailureReason.ConnectionLost); expect(tasksService.updateTask).toHaveBeenCalledWith({ taskId: 'task-1', @@ -189,7 +190,7 @@ describe('uploadFoldersWithTasks', () => { test('When a retried folder upload fails before starting, then its resumed task ends in error instead of staying in progress', async () => { (uploadFoldersWithManager as Mock).mockImplementationOnce(async ({ events }) => - events.onFolderUploadError('existing-task', 'connection-lost'), + events.onFolderUploadError('existing-task', UploadFailureReason.ConnectionLost), ); await runUpload([{ root, currentFolderId: 'parent', options: { taskId: 'existing-task' } }]); @@ -226,8 +227,8 @@ describe('uploadFoldersWithTasks', () => { }); test.each([ - { outcome: 'the connection is lost', reason: 'connection-lost' as const }, - { outcome: 'a fatal error', reason: 'upload-failed' as const }, + { outcome: 'the connection is lost', reason: UploadFailureReason.ConnectionLost }, + { outcome: 'a fatal error', reason: UploadFailureReason.UploadFailed }, ])( 'when the folder upload ends because $outcome, then only its files are removed from the retry list', async ({ reason }) => { diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index 493f054115..f62e82df00 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -1,19 +1,19 @@ import { t } from 'i18next'; import { TaskStatus } from 'app/tasks/types'; -import { UploadErrorReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/network/types'; -const errorSubtitle = (reason?: UploadErrorReason): string | undefined => { +const errorSubtitle = (reason?: UploadFailureReason): string | undefined => { switch (reason) { - case 'connection-lost': + case UploadFailureReason.ConnectionLost: return t('error.connectionLostError') as string; - case 'upload-failed': + case UploadFailureReason.UploadFailed: return t('tasks.subtitles.upload-failed') as string; default: return undefined; } }; -export const errorMerge = (reason?: UploadErrorReason) => { +export const errorMerge = (reason?: UploadFailureReason) => { const subtitle = errorSubtitle(reason); return subtitle !== undefined ? { status: TaskStatus.Error, subtitle } : { status: TaskStatus.Error }; }; From bd9908e66a68c34b06bf3f124bbf5061fa55bbc3 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 13:35:36 +0100 Subject: [PATCH 39/49] refactor(upload): return the cancel promise without an empty async --- src/app/tasks/upload/uploadFilesWithTasks.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index f505bec6ae..07fb0f0e6e 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -139,7 +139,7 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr if (file.taskId) { tasksService.updateTask({ taskId: file.taskId, - merge: { status: TaskStatus.Encrypting, stop: async () => manager.cancelItem(item) }, + merge: { status: TaskStatus.Encrypting, stop: () => Promise.resolve(manager.cancelItem(item)) }, }); } @@ -208,6 +208,6 @@ const createUploadTask = ( showNotification: options?.showNotifications ?? true, cancellable: true, sharedItemAuthenticationData, - stop: async () => manager.cancelItem(item), + stop: () => Promise.resolve(manager.cancelItem(item)), }); }; From 9f571bae0f221c47722d53f0102f558922b81a4a Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 16:10:18 +0100 Subject: [PATCH 40/49] fix(upload): cancel every file of a folder upload even when it is cancelled before the files start --- .../tasks/upload/uploadFilesWithTasks.test.ts | 18 +++++++++++++----- src/app/tasks/upload/uploadFilesWithTasks.ts | 5 +++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/app/tasks/upload/uploadFilesWithTasks.test.ts b/src/app/tasks/upload/uploadFilesWithTasks.test.ts index 9e7dd27e86..570556638f 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.test.ts @@ -16,14 +16,13 @@ const { managerState } = vi.hoisted(() => ({ result: { succeeded: [], failed: [] } as UploadResult, instance: undefined as | undefined - | { props: { events: UploadManagerEvents }; run: Mock; cancelAll: Mock; cancelItem: Mock; items: UploadItem[] }, + | { props: { events: UploadManagerEvents }; run: Mock; cancelItem: Mock; items: UploadItem[] }, }, })); vi.mock('app/upload/UploadManager', () => ({ UploadManager: vi.fn(function (this: NonNullable, props) { this.props = props; - this.cancelAll = vi.fn(); this.cancelItem = vi.fn(); this.run = vi.fn(async (items: UploadItem[]) => { this.items = items; @@ -265,13 +264,22 @@ describe('Uploading files with tasks', () => { expect(manager.cancelItem).toHaveBeenCalledExactlyOnceWith(items[0]); }); - test('When the folder upload that owns the files is cancelled, then the whole upload is cancelled', async () => { + test('When the folder upload that owns the files is cancelled, then every file is cancelled', async () => { const abortController = new AbortController(); - const { manager } = await runUpload([buildFile()], { abortController }); + const { manager, items } = await runUpload([buildFile(), buildFile()], { abortController }); abortController.abort(); - expect(manager.cancelAll).toHaveBeenCalledOnce(); + expect(manager.cancelItem.mock.calls).toEqual(items.map((item) => [item])); + }); + + test('When the folder upload that owns the files was cancelled before they start, then every file is cancelled', async () => { + const abortController = new AbortController(); + abortController.abort(); + + const { manager, items } = await runUpload([buildFile(), buildFile()], { abortController }); + + expect(manager.cancelItem.mock.calls).toEqual(items.map((item) => [item])); }); test('When a file fails, then it is listed for retry and the upload rejects with its error', async () => { diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index 07fb0f0e6e..98c3a4f5a2 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -146,8 +146,9 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr return item; }); - if (abortController?.signal.aborted) manager.cancelAll(); - abortController?.signal.addEventListener('abort', () => manager.cancelAll(), { once: true }); + const cancelEveryFile = () => items.forEach((item) => manager.cancelItem(item)); + if (abortController?.signal.aborted) cancelEveryFile(); + abortController?.signal.addEventListener('abort', cancelEveryFile, { once: true }); const relatedTaskId = files[0]?.relatedTaskId; if (relatedTaskId) { From 9d55c503bd5757a00c549f4e1753dea57a93ee19 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:11:31 +0100 Subject: [PATCH 41/49] refactor(upload): save the upload state where it changes instead of through a shared helper --- src/app/repositories/uploadStateUpdates.ts | 11 ----------- src/app/tasks/upload/uploadFilesWithTasks.ts | 10 +++++++--- 2 files changed, 7 insertions(+), 14 deletions(-) delete mode 100644 src/app/repositories/uploadStateUpdates.ts diff --git a/src/app/repositories/uploadStateUpdates.ts b/src/app/repositories/uploadStateUpdates.ts deleted file mode 100644 index 036452e990..0000000000 --- a/src/app/repositories/uploadStateUpdates.ts +++ /dev/null @@ -1,11 +0,0 @@ -import errorService from 'services/error.service'; -import { TaskStatus } from '../tasks/types'; -import { PersistUploadRepository } from './DatabaseUploadRepository'; - -export const markUploadAsInProcess = (uploadRepository: PersistUploadRepository, taskId: string): void => { - uploadRepository.setUploadState(taskId, TaskStatus.InProcess).catch((error) => errorService.reportError(error)); -}; - -export const forgetUploadState = (uploadRepository: PersistUploadRepository, taskId: string): void => { - uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); -}; diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index 98c3a4f5a2..96a32f12bf 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -9,7 +9,7 @@ import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; -import { forgetUploadState, markUploadAsInProcess } from 'app/repositories/uploadStateUpdates'; +import errorService from 'services/error.service'; import { errorMerge } from './uploadTaskErrors'; export type UploadManagerFileParams = { @@ -65,7 +65,11 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr taskId: file.taskId, merge: { status: file.relatedTaskId ? TaskStatus.Encrypting : TaskStatus.InProcess }, }); - if (!file.relatedTaskId) markUploadAsInProcess(uploadRepository, file.taskId); + if (!file.relatedTaskId) { + uploadRepository + .setUploadState(file.taskId, TaskStatus.InProcess) + .catch((error) => errorService.reportError(error)); + } }, onItemProgress: (item, progress) => { @@ -113,7 +117,7 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr onItemFailed: (item, { reason }) => { const { taskId } = getFile(item); tasksService.updateTask({ taskId, merge: errorMerge(reason) }); - forgetUploadState(uploadRepository, taskId); + uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); }, onItemCancelled: (item) => { From 51110c5286895db513d88ba9247c8445f0b60f35 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 30 Oct 2026 17:40:27 +0100 Subject: [PATCH 42/49] test(upload): a cancelled folder upload no longer starts the files that were still getting ready --- src/app/network/UploadFolderManager.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 9d06a83351..805b2a9b7b 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -502,12 +502,12 @@ describe('uploadFoldersWithManager', () => { RetryManager.clearTasks(); }); - test('when a folder upload with more files than the upload concurrency is cancelled right away, then the files already uploading/queued are aborted and the task ends as cancelled', async () => { + test('when a folder upload is cancelled while its first file is uploading, then no other file starts and the task ends as cancelled', async () => { const { events, stopRelatedUploads, run } = arrangeCancelledFolderUpload(10); await run(); - expect(uploadFile).toHaveBeenCalledTimes(6); + expect(uploadFile).toHaveBeenCalledOnce(); expect(stopRelatedUploads).toHaveBeenCalledWith(taskId); expect(events.onFolderUploadSuccess).not.toHaveBeenCalled(); expect(events.onFolderUploadError).not.toHaveBeenCalled(); From 6e0ded3f1a039b633dfa7e5a76471b39ce1e6cef Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:13:25 +0100 Subject: [PATCH 43/49] refactor(upload): import the upload types from their new place --- src/app/network/UploadFolderManager.test.ts | 2 +- src/app/network/UploadFolderManager.ts | 2 +- src/app/tasks/upload/uploadFilesWithTasks.test.ts | 5 ++--- src/app/tasks/upload/uploadFilesWithTasks.ts | 4 ++-- src/app/tasks/upload/uploadFoldersWithTasks.test.ts | 2 +- src/app/tasks/upload/uploadTaskErrors.ts | 2 +- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/src/app/network/UploadFolderManager.test.ts b/src/app/network/UploadFolderManager.test.ts index 805b2a9b7b..e6249dd10e 100644 --- a/src/app/network/UploadFolderManager.test.ts +++ b/src/app/network/UploadFolderManager.test.ts @@ -20,7 +20,7 @@ import newStorageService from 'app/drive/services/new-storage.service'; import RetryManager from './RetryManager'; import { ConnectionLostError } from './requests'; import errorService from 'services/error.service'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/upload/types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn(async (file: { name: string }) => ({ uuid: `${file.name}-uuid`, name: file.name })), diff --git a/src/app/network/UploadFolderManager.ts b/src/app/network/UploadFolderManager.ts index 0822ba6acf..c60e567ccb 100644 --- a/src/app/network/UploadFolderManager.ts +++ b/src/app/network/UploadFolderManager.ts @@ -16,7 +16,7 @@ import { wait } from 'utils/timeUtils'; import { ConnectionLostError } from './requests'; import { FilesExceedsSizeLimitError } from 'app/drive/services/file.service/upload.errors'; import { filterFilesByMaxSize } from 'app/store/slices/storage/fileUtils/filterFilesByMaxSize'; -import { UploadFailureReason } from './types'; +import { UploadFailureReason } from 'app/upload/types'; import { MAX_CONCURRENT_UPLOADS } from './networkConstants'; export interface UploadFolderPayload { diff --git a/src/app/tasks/upload/uploadFilesWithTasks.test.ts b/src/app/tasks/upload/uploadFilesWithTasks.test.ts index 570556638f..09b0d84dfc 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.test.ts @@ -1,12 +1,11 @@ import { beforeEach, describe, expect, Mock, test, vi } from 'vitest'; -import { UploadFailure } from 'app/network/uploadFailure'; -import { UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { UploadFailure } from 'app/upload/uploadFailure'; +import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents, UploadResult } from 'app/upload/types'; import tasksService from 'app/tasks/services/tasks.service'; import { TaskStatus } from 'app/tasks/types'; import referralService from 'services/referral.service'; import * as networkInformation from 'app/network/networkInformation'; import { UploadManager } from 'app/upload/UploadManager'; -import { UploadItem, UploadManagerEvents, UploadResult } from 'app/network/types/upload'; import RetryManager from 'app/network/RetryManager'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { uploadFilesWithTasks } from './uploadFilesWithTasks'; diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index 96a32f12bf..b2c43faa2a 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -5,9 +5,9 @@ import { logNetworkInfoForUpload } from 'app/network/networkInformation'; import { DriveFileData } from 'app/drive/types'; import { FileToUpload } from 'app/drive/services/file.service/types'; import { UploadManager } from 'app/upload/UploadManager'; -import { UploadItem, UploadManagerEvents } from 'app/network/types/upload'; +import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents } from 'app/upload/types'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; -import { OwnerUserAuthenticationData, UploadFailureReason, UploadItemKind } from 'app/network/types'; +import { OwnerUserAuthenticationData } from 'app/network/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import errorService from 'services/error.service'; import { errorMerge } from './uploadTaskErrors'; diff --git a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts index 88efeef923..9b4f3f041b 100644 --- a/src/app/tasks/upload/uploadFoldersWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFoldersWithTasks.test.ts @@ -7,7 +7,7 @@ import { uploadFoldersWithManager, UploadFolderManagerEvents } from 'app/network import { IRoot } from 'app/store/slices/storage/types'; import RetryManager, { RetryableTaskType } from 'app/network/RetryManager'; import { uploadFoldersWithTasks } from './uploadFoldersWithTasks'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/upload/types'; vi.mock('app/network/UploadFolderManager', () => ({ uploadFoldersWithManager: vi.fn(() => Promise.resolve()), diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index f62e82df00..9e3c439001 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -1,6 +1,6 @@ import { t } from 'i18next'; import { TaskStatus } from 'app/tasks/types'; -import { UploadFailureReason } from 'app/network/types'; +import { UploadFailureReason } from 'app/upload/types'; const errorSubtitle = (reason?: UploadFailureReason): string | undefined => { switch (reason) { From 3a2e56ae9bc3396683cad863890cf5d9f76bf0ef Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 15:25:40 +0100 Subject: [PATCH 44/49] fix(upload): keep showing the upload failed message when the plan limit is reached --- src/app/tasks/upload/uploadTaskErrors.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index 9e3c439001..ee64ed7d50 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -7,6 +7,7 @@ const errorSubtitle = (reason?: UploadFailureReason): string | undefined => { case UploadFailureReason.ConnectionLost: return t('error.connectionLostError') as string; case UploadFailureReason.UploadFailed: + case UploadFailureReason.PlanLimit: return t('tasks.subtitles.upload-failed') as string; default: return undefined; From 245674d861c003e84e6b12a513dfd3f8e93ede3e Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Mon, 2 Nov 2026 16:22:00 +0100 Subject: [PATCH 45/49] fix(upload): show the storage full and file too large dialogs on the first failure, and keep finishing a file when the folder progress cannot be read --- .../tasks/upload/uploadFilesWithTasks.test.ts | 38 +++++++++++-- src/app/tasks/upload/uploadFilesWithTasks.ts | 57 ++++++++++++------- 2 files changed, 70 insertions(+), 25 deletions(-) diff --git a/src/app/tasks/upload/uploadFilesWithTasks.test.ts b/src/app/tasks/upload/uploadFilesWithTasks.test.ts index 09b0d84dfc..448ba8557d 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.test.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.test.ts @@ -4,6 +4,7 @@ import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents, U import tasksService from 'app/tasks/services/tasks.service'; import { TaskStatus } from 'app/tasks/types'; import referralService from 'services/referral.service'; +import errorService from 'services/error.service'; import * as networkInformation from 'app/network/networkInformation'; import { UploadManager } from 'app/upload/UploadManager'; import RetryManager from 'app/network/RetryManager'; @@ -227,6 +228,26 @@ describe('Uploading files with tasks', () => { }); }); + test('When the folder upload state cannot be read after a file inside it finishes, then the file is still shown and leaves the retry list', async () => { + vi.spyOn(errorService, 'reportError').mockReturnValue(); + (uploadRepository.getUploadState as Mock) + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('IndexedDB error')); + RetryManager.addTasks([{ taskId: 'existing-task', type: 'upload' as never, params: {} }]); + const onFinishUploadFile = vi.fn(); + const relatedTaskProgress = { filesUploaded: 0, totalFilesToUpload: 4 }; + const { events, items } = await runUpload( + [buildFile({ taskId: 'existing-task', relatedTaskId: 'related-task', onFinishUploadFile })], + { relatedTaskProgress }, + ); + + await events.onItemSucceeded?.(items[0], { uuid: 'file-uuid' } as never); + + expect(onFinishUploadFile).toHaveBeenCalledOnce(); + expect(RetryManager.getTasks()).toEqual([]); + expect(relatedTaskProgress.filesUploaded).toBe(1); + }); + test('When an upload fails, then the task shows an error message matching the cause', async () => { (tasksService.create as Mock).mockReturnValue('t1'); const { events, items } = await runUpload([buildFile()]); @@ -300,12 +321,17 @@ describe('Uploading files with tasks', () => { reason: UploadFailureReason.FileTooLarge, callback: 'fileSizeExceededCallback', }, - ] as const)('When $problem, then the user is warned about it', async ({ reason, callback }) => { - const warn = vi.fn(); - failWith({ reason }); + ] as const)( + 'When $problem, then the user is warned as soon as a file fails and only once', + async ({ reason, callback }) => { + const warn = vi.fn(); + const { events, items } = await runUpload([buildFile(), buildFile()], { [callback]: warn }); + const failure = { error: new Error('failed'), reason, isRetryable: false }; - await expect(runUpload([buildFile()], { [callback]: warn })).rejects.toThrow(); + events.onItemFailed?.(items[0], failure); + events.onItemFailed?.(items[1], failure); - expect(warn).toHaveBeenCalledOnce(); - }); + expect(warn).toHaveBeenCalledOnce(); + }, + ); }); diff --git a/src/app/tasks/upload/uploadFilesWithTasks.ts b/src/app/tasks/upload/uploadFilesWithTasks.ts index b2c43faa2a..168faca2b8 100644 --- a/src/app/tasks/upload/uploadFilesWithTasks.ts +++ b/src/app/tasks/upload/uploadFilesWithTasks.ts @@ -25,6 +25,8 @@ export type UploadManagerFileParams = { type UploadFileParamsWithTaskId = UploadManagerFileParams & { taskId: string }; +type FolderProgress = { filesUploaded: number; totalFilesToUpload: number }; + interface UploadFilesWithTasksProps { files: UploadManagerFileParams[]; maxSpaceOccupiedCallback: () => void; @@ -40,7 +42,7 @@ interface UploadFilesWithTasksProps { currentFolderId?: string; }; }; - relatedTaskProgress?: { filesUploaded: number; totalFilesToUpload: number }; + relatedTaskProgress?: FolderProgress; } /** @@ -58,6 +60,33 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr return file.relatedTaskId ?? file.taskId; }; + const failureDialogByReason: Partial void>> = { + [UploadFailureReason.NoSpaceLeft]: props.maxSpaceOccupiedCallback, + [UploadFailureReason.FileTooLarge]: props.fileSizeExceededCallback, + }; + const reasonsWithDialogShown = new Set(); + + const showFailureDialogOnce = (reason: UploadFailureReason) => { + const showDialog = failureDialogByReason[reason]; + const hasNoDialog = !showDialog; + const wasAlreadyShown = reasonsWithDialogShown.has(reason); + if (hasNoDialog || wasAlreadyShown) return; + + reasonsWithDialogShown.add(reason); + showDialog(); + }; + + const updateFolderProgress = async (folderTaskId: string, folderProgress: FolderProgress) => { + const folderUploadState = await uploadRepository.getUploadState(folderTaskId); + const isPaused = folderUploadState === TaskStatus.Paused; + if (isPaused) return; + + tasksService.updateTask({ + taskId: folderTaskId, + merge: { progress: folderProgress.filesUploaded / folderProgress.totalFilesToUpload }, + }); + }; + const events: UploadManagerEvents = { onItemStarted: (item) => { const file = getFile(item); @@ -87,19 +116,6 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr const driveFileData = uploadedItem as DriveFileData; const file = getFile(item); - if (relatedTaskProgress && file.relatedTaskId) { - relatedTaskProgress.filesUploaded += 1; - - const folderUploadState = await uploadRepository.getUploadState(file.relatedTaskId); - const isPaused = folderUploadState === TaskStatus.Paused; - if (!isPaused) { - tasksService.updateTask({ - taskId: file.relatedTaskId, - merge: { progress: relatedTaskProgress.filesUploaded / relatedTaskProgress.totalFilesToUpload }, - }); - } - } - if (tasksService.findTask(file.taskId)?.status !== TaskStatus.Cancelled) { tasksService.updateTask({ taskId: file.taskId, @@ -112,12 +128,20 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr logNetworkInfoForUpload({ fileName: file.filecontent.name, fileSize: file.filecontent.size }); file.onFinishUploadFile?.(driveFileData, file.taskId); props.onFileUploadCallback?.(driveFileData); + + if (relatedTaskProgress && file.relatedTaskId) { + relatedTaskProgress.filesUploaded += 1; + await updateFolderProgress(file.relatedTaskId, relatedTaskProgress).catch((error) => + errorService.reportError(error), + ); + } }, onItemFailed: (item, { reason }) => { const { taskId } = getFile(item); tasksService.updateTask({ taskId, merge: errorMerge(reason) }); uploadRepository.removeUploadState(taskId).catch((error) => errorService.reportError(error)); + showFailureDialogOnce(reason); }, onItemCancelled: (item) => { @@ -180,11 +204,6 @@ export const uploadFilesWithTasks = async (props: UploadFilesWithTasksProps): Pr }), ); - const failureReasons = new Set(failed.map(({ failure }) => failure.reason)); - - if (failureReasons.has(UploadFailureReason.NoSpaceLeft)) props.maxSpaceOccupiedCallback(); - if (failureReasons.has(UploadFailureReason.FileTooLarge)) props.fileSizeExceededCallback?.(); - throw failed[0].failure.error; } }; From 133e94e15f70fee26cc0eab646794485a7380439 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Wed, 4 Nov 2026 13:50:57 +0100 Subject: [PATCH 46/49] fix(upload): let a cancelled folder finish being created so it can be removed --- src/app/upload/Upload.test.ts | 8 ++++++-- src/app/upload/Upload.ts | 22 +++++++++++----------- src/app/upload/UploadManager.ts | 12 +++--------- 3 files changed, 20 insertions(+), 22 deletions(-) diff --git a/src/app/upload/Upload.test.ts b/src/app/upload/Upload.test.ts index b270b668c0..3deb83d56e 100644 --- a/src/app/upload/Upload.test.ts +++ b/src/app/upload/Upload.test.ts @@ -30,8 +30,12 @@ const uploadRepository = getUploadRepository(); const fakeFolderCreationTaking = (delayMs = 0) => vi.mocked(folderService.createFolderByUuid).mockImplementation((_parentUuid, name) => { - const createdFolder = getCreatedFolder(name); - return [new Promise((resolve) => setTimeout(() => resolve(createdFolder), delayMs)), { cancel: vi.fn() }] as never; + let abortCreation: () => void = () => undefined; + const creation = new Promise((resolve, reject) => { + setTimeout(() => resolve(getCreatedFolder(name)), delayMs); + abortCreation = () => reject(new Error('aborted')); + }); + return [creation, { cancel: () => abortCreation() }] as never; }); const fakeUploadsTaking = (delayMs: number) => diff --git a/src/app/upload/Upload.ts b/src/app/upload/Upload.ts index 7b7334e99d..e6cd31e589 100644 --- a/src/app/upload/Upload.ts +++ b/src/app/upload/Upload.ts @@ -76,7 +76,7 @@ export class Upload { this.manager = new UploadManager({ ownerUserAuthenticationData: props.ownerUserAuthenticationData, uploadRepository: props.uploadRepository, - getPauseKey: (item) => this.getTaskIdOf(getTopLevelItem(item)), + getPauseKey: (item) => this.getTaskIdOf(getTopLevelItem(item)) as string, events: this.createManagerEvents(), }); @@ -103,8 +103,8 @@ export class Upload { }; } - private getTaskIdOf(item: UploadItem): string { - return this.taskIdByItem.get(item) as string; + private getTaskIdOf(item: UploadItem): string | undefined { + return this.taskIdByItem.get(item); } private trackRootFolder(item: UploadFolderItem): void { @@ -151,7 +151,7 @@ export class Upload { } private onItemStarted(item: UploadItem): void { - const taskId = this.taskIdByItem.get(item); + const taskId = this.getTaskIdOf(item); if (!taskId) return; tasksService.updateTask({ taskId, merge: { status: TaskStatus.InProcess } }); @@ -171,7 +171,7 @@ export class Upload { } private onItemProgress(item: UploadItem, progress: number): void { - const taskId = this.taskIdByItem.get(item); + const taskId = this.getTaskIdOf(item); if (taskId && !isTaskStopped(taskId)) tasksService.updateTask({ taskId, merge: { progress } }); } @@ -186,7 +186,7 @@ export class Upload { logNetworkInfoForUpload({ fileName: item.file.name, fileSize: item.file.size }); } - const taskId = this.taskIdByItem.get(item); + const taskId = this.getTaskIdOf(item); const isFileWithItsOwnTask = !!taskId && isFile; if (isFileWithItsOwnTask) { this.finishFileTask(taskId, { fileUUID: uploadedItem.uuid }); @@ -222,7 +222,7 @@ export class Upload { const rootFolder = this.rootFolders.get(getTopLevelItem(item)); if (rootFolder) rootFolder.hasFailedItems = true; - const taskId = this.taskIdByItem.get(item); + const taskId = this.getTaskIdOf(item); if (!taskId) return; tasksService.updateTask({ taskId, merge: errorMerge(failure.reason) }); @@ -230,7 +230,7 @@ export class Upload { } private onItemCancelled(item: UploadItem): void { - const taskId = this.taskIdByItem.get(item); + const taskId = this.getTaskIdOf(item); const isFileWithItsOwnTask = !!taskId && item.kind === UploadItemKind.File; if (!isFileWithItsOwnTask) return; @@ -259,12 +259,12 @@ export class Upload { private finishRootFolder(item: UploadFolderItem, rootFolder: RootFolderUpload): void { this.forgetUploadState(rootFolder.taskId); - const isRootFolderCreated = !!rootFolder.createdFolder; - if (rootFolder.isCancelled || !isRootFolderCreated) return; + const { createdFolder } = rootFolder; + if (rootFolder.isCancelled || !createdFolder) return; tasksService.updateTask({ taskId: rootFolder.taskId, - merge: { status: TaskStatus.Success, itemUUID: { rootFolderUUID: rootFolder.createdFolder?.uuid } }, + merge: { status: TaskStatus.Success, itemUUID: { rootFolderUUID: createdFolder.uuid } }, }); const hasEverythingUploaded = !rootFolder.hasFailedItems; diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 39fab71308..2f5c3f697b 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -266,18 +266,12 @@ export class UploadManager { const workspaceId = this.ownerUserAuthenticationData?.workspaceId; const { parentFolderUuid, folder } = folderItem; - const [createdFolderPromise, requestCanceler] = workspaceId + const [createdFolderPromise] = workspaceId ? workspacesService.createFolder({ workspaceId, parentFolderUuid, plainName: folder.name }) : folderService.createFolderByUuid(parentFolderUuid, folder.name); - this.abortActiveUploadOf.set(folderItem, () => requestCanceler.cancel()); - - try { - const createdFolder = await createdFolderPromise; - return toDriveFolderData(createdFolder, folder.name); - } finally { - this.abortActiveUploadOf.delete(folderItem); - } + const createdFolder = await createdFolderPromise; + return toDriveFolderData(createdFolder, folder.name); } private async uploadFile(fileItem: UploadFileItem): Promise { From 31d41fd44f00aad0c7a4e4d059509cb2fc5bd5b4 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 6 Nov 2026 10:16:53 +0100 Subject: [PATCH 47/49] refactor(upload): move the upload item tree helpers to their own file --- src/app/upload/Upload.ts | 15 ++-------- src/app/upload/UploadManager.test.ts | 3 +- src/app/upload/UploadManager.ts | 36 +--------------------- src/app/upload/uploadItemTree.ts | 45 ++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 49 deletions(-) create mode 100644 src/app/upload/uploadItemTree.ts diff --git a/src/app/upload/Upload.ts b/src/app/upload/Upload.ts index e6cd31e589..514ca5d3a1 100644 --- a/src/app/upload/Upload.ts +++ b/src/app/upload/Upload.ts @@ -12,7 +12,8 @@ import { UploadResult, } from 'app/upload/types'; import { UploadFailure } from 'app/upload/uploadFailure'; -import { createFileItem, createFolderItem, UploadManager } from './UploadManager'; +import { UploadManager } from './UploadManager'; +import { countItemsIn, createFileItem, createFolderItem, getParentFolderUuid, getTopLevelItem } from './uploadItemTree'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { IRoot } from 'app/store/slices/storage/types'; import tasksService from 'app/tasks/services/tasks.service'; @@ -41,18 +42,6 @@ interface RootFolderUpload { createdFolder?: DriveFolderData; } -const countItemsIn = (folder: IRoot): number => - folder.childrenFolders.reduce((count, child) => count + countItemsIn(child), 1 + folder.childrenFiles.length); - -const getTopLevelItem = (item: UploadItem): UploadItem => { - let topLevelItem = item; - while (topLevelItem.parent) topLevelItem = topLevelItem.parent; - return topLevelItem; -}; - -const getParentFolderUuid = (item: UploadItem): string => - item.kind === UploadItemKind.File ? item.file.parentFolderId : item.parentFolderUuid; - const createItemsToUpload = ({ roots, files, destinationFolderUuid }: UploadProps): UploadItem[] => [ ...roots.map((root) => createFolderItem(root, destinationFolderUuid)), ...files.map((file) => createFileItem(file, destinationFolderUuid)), diff --git a/src/app/upload/UploadManager.test.ts b/src/app/upload/UploadManager.test.ts index eb47cfd586..2d7e363828 100644 --- a/src/app/upload/UploadManager.test.ts +++ b/src/app/upload/UploadManager.test.ts @@ -16,7 +16,8 @@ import { getUploadFileItem, getUploadRepository, } from 'testUtils/fixtures/upload.fixtures'; -import { createFolderItem, UploadManager, WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS } from './UploadManager'; +import { UploadManager, WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS } from './UploadManager'; +import { createFolderItem } from './uploadItemTree'; import { UploadFailureReason, UploadItem, UploadItemKind, UploadManagerEvents } from './types'; vi.mock('app/drive/services/file.service/uploadFile', () => ({ default: vi.fn() })); diff --git a/src/app/upload/UploadManager.ts b/src/app/upload/UploadManager.ts index 2f5c3f697b..78f745a032 100644 --- a/src/app/upload/UploadManager.ts +++ b/src/app/upload/UploadManager.ts @@ -1,4 +1,3 @@ -import { items as itemUtils } from '@internxt/lib'; import { AsyncPriorityQueue, priorityQueue } from 'async'; import errorService from 'services/error.service'; import workspacesService from 'services/workspace.service'; @@ -9,7 +8,6 @@ import folderService from 'app/drive/services/folder.service'; import { toDriveFolderData } from 'app/drive/services/folder.service/toDriveFolderData'; import { DriveFileData, DriveFolderData } from 'app/drive/types'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; -import { IRoot } from 'app/store/slices/storage/types'; import { TaskStatus } from 'app/tasks/types'; import { MAX_CONCURRENT_UPLOADS, @@ -29,6 +27,7 @@ import { UploadResult, } from './types'; import { shouldAutoRetry, toUploadFailure, UploadFailure } from './uploadFailure'; +import { createFileItem, createFolderItem, getItemAndItsFolders, isFolderItem, isPartOf } from './uploadItemTree'; const SINGLE_TRANSFER_CONCURRENCY = 1; export const WORKSPACE_FOLDER_AVAILABILITY_DELAY_MS = 600; @@ -53,39 +52,6 @@ class UploadCancelledError extends Error { } } -export const createFileItem = (file: File, parentFolderUuid: string, parent?: UploadFolderItem): UploadFileItem => { - const { filename, extension } = itemUtils.getFilenameAndExt(file.name); - - return { - kind: UploadItemKind.File, - file: { name: filename, type: extension, size: file.size, content: file, parentFolderId: parentFolderUuid }, - parent, - }; -}; - -export const createFolderItem = ( - folder: IRoot, - parentFolderUuid: string, - parent?: UploadFolderItem, -): UploadFolderItem => ({ - kind: UploadItemKind.Folder, - folder, - parentFolderUuid, - parent, -}); - -const getItemAndItsFolders = (item: UploadItem): UploadItem[] => { - const itemAndItsFolders: UploadItem[] = []; - for (let current: UploadItem | undefined = item; current; current = current.parent) { - itemAndItsFolders.push(current); - } - return itemAndItsFolders; -}; - -const isPartOf = (item: UploadItem, otherItem: UploadItem): boolean => getItemAndItsFolders(item).includes(otherItem); - -const isFolderItem = (item: UploadItem): item is UploadFolderItem => item.kind === UploadItemKind.Folder; - const isMultipartUpload = (item: UploadItem) => item.kind === UploadItemKind.File && item.file.size >= USE_MULTIPART_THRESHOLD_BYTES; diff --git a/src/app/upload/uploadItemTree.ts b/src/app/upload/uploadItemTree.ts new file mode 100644 index 0000000000..85aa44ce00 --- /dev/null +++ b/src/app/upload/uploadItemTree.ts @@ -0,0 +1,45 @@ +import { items as itemUtils } from '@internxt/lib'; +import { IRoot } from 'app/store/slices/storage/types'; +import { UploadFileItem, UploadFolderItem, UploadItem, UploadItemKind } from './types'; + +export const createFileItem = (file: File, parentFolderUuid: string, parent?: UploadFolderItem): UploadFileItem => { + const { filename, extension } = itemUtils.getFilenameAndExt(file.name); + + return { + kind: UploadItemKind.File, + file: { name: filename, type: extension, size: file.size, content: file, parentFolderId: parentFolderUuid }, + parent, + }; +}; + +export const createFolderItem = ( + folder: IRoot, + parentFolderUuid: string, + parent?: UploadFolderItem, +): UploadFolderItem => ({ + kind: UploadItemKind.Folder, + folder, + parentFolderUuid, + parent, +}); + +export const isFolderItem = (item: UploadItem): item is UploadFolderItem => item.kind === UploadItemKind.Folder; + +export const getItemAndItsFolders = (item: UploadItem): UploadItem[] => { + const itemAndItsFolders: UploadItem[] = []; + for (let current: UploadItem | undefined = item; current; current = current.parent) { + itemAndItsFolders.push(current); + } + return itemAndItsFolders; +}; + +export const getTopLevelItem = (item: UploadItem): UploadItem => getItemAndItsFolders(item).at(-1) as UploadItem; + +export const isPartOf = (item: UploadItem, otherItem: UploadItem): boolean => + getItemAndItsFolders(item).includes(otherItem); + +export const getParentFolderUuid = (item: UploadItem): string => + isFolderItem(item) ? item.parentFolderUuid : item.file.parentFolderId; + +export const countItemsIn = (folder: IRoot): number => + folder.childrenFolders.reduce((count, child) => count + countItemsIn(child), 1 + folder.childrenFiles.length); From cf235b702ee6393f7c7dfc1bd104ae856a9b6ff7 Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 6 Nov 2026 11:55:27 +0100 Subject: [PATCH 48/49] fix(upload): update task before notifying --- src/app/upload/Upload.test.ts | 24 ++++++++++++++++++++++++ src/app/upload/Upload.ts | 27 ++++++++++++++++----------- src/services/referral.service.test.ts | 8 ++++++++ src/services/referral.service.ts | 8 +++++++- 4 files changed, 55 insertions(+), 12 deletions(-) diff --git a/src/app/upload/Upload.test.ts b/src/app/upload/Upload.test.ts index 3deb83d56e..d0b458560d 100644 --- a/src/app/upload/Upload.test.ts +++ b/src/app/upload/Upload.test.ts @@ -6,6 +6,7 @@ import { IRoot } from 'app/store/slices/storage/types'; import tasksService from 'app/tasks/services/tasks.service'; import { TaskStatus, TaskType } from 'app/tasks/types'; import errorService from 'services/error.service'; +import referralService from 'services/referral.service'; import { AppError } from '@internxt/sdk'; import { HTTP_STATUS_CODES } from 'app/core/constants'; import { @@ -114,6 +115,29 @@ describe('Uploading what the user picked', () => { expect(onItemUploaded).toHaveBeenCalledWith(expect.objectContaining({ name: 'report' }), DESTINATION_UUID, false); }); + test('When reporting an uploaded item fails, then the upload still finishes and each task succeeds', async () => { + const onItemUploaded = vi.fn(() => { + throw new Error('Listener error'); + }); + vi.mocked(referralService.trackFileUpload).mockImplementation(() => { + throw new Error('Corrupt banner state'); + }); + const upload = createUpload( + [getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], + [getFileToUpload('report.pdf')], + { onItemUploaded }, + ); + + await upload.run(); + + expect(tasksOf(TaskType.UploadFolder)[0]).toEqual( + expect.objectContaining({ status: TaskStatus.Success, progress: 1, itemUUID: { rootFolderUUID: 'Photos-uuid' } }), + ); + expect(tasksOf(TaskType.UploadFile)[0]).toEqual( + expect.objectContaining({ status: TaskStatus.Success, itemUUID: { fileUUID: 'report-uuid' } }), + ); + }); + test('When a file inside a folder fails, then the folder still succeeds with what could be uploaded', async () => { (uploadFile as Mock).mockImplementation(async (file: { name: string }) => { if (file.name === 'broken') throw new Error('Server error'); diff --git a/src/app/upload/Upload.ts b/src/app/upload/Upload.ts index 514ca5d3a1..a028828192 100644 --- a/src/app/upload/Upload.ts +++ b/src/app/upload/Upload.ts @@ -13,7 +13,14 @@ import { } from 'app/upload/types'; import { UploadFailure } from 'app/upload/uploadFailure'; import { UploadManager } from './UploadManager'; -import { countItemsIn, createFileItem, createFolderItem, getParentFolderUuid, getTopLevelItem } from './uploadItemTree'; +import { + countItemsIn, + createFileItem, + createFolderItem, + getParentFolderUuid, + getTopLevelItem, + isFolderItem, +} from './uploadItemTree'; import { PersistUploadRepository } from 'app/repositories/DatabaseUploadRepository'; import { IRoot } from 'app/store/slices/storage/types'; import tasksService from 'app/tasks/services/tasks.service'; @@ -165,23 +172,21 @@ export class Upload { } private onItemSucceeded(item: UploadItem, uploadedItem: UploadedItem): void { - const isFile = item.kind === UploadItemKind.File; - const isFolder = item.kind === UploadItemKind.Folder; + const taskId = this.getTaskIdOf(item); + const isFileWithItsOwnTask = !!taskId && !isFolderItem(item); + if (isFileWithItsOwnTask) this.finishFileTask(taskId, { fileUUID: uploadedItem.uuid }); + else this.countUploadedInRootFolder(item, uploadedItem); - this.props.onItemUploaded?.(uploadedItem, getParentFolderUuid(item), isFolder); + const isFile = item.kind === UploadItemKind.File; + this.props.onItemUploaded?.(uploadedItem, getParentFolderUuid(item), !isFile); if (isFile) { referralService.trackFileUpload(); logNetworkInfoForUpload({ fileName: item.file.name, fileSize: item.file.size }); } + } - const taskId = this.getTaskIdOf(item); - const isFileWithItsOwnTask = !!taskId && isFile; - if (isFileWithItsOwnTask) { - this.finishFileTask(taskId, { fileUUID: uploadedItem.uuid }); - return; - } - + private countUploadedInRootFolder(item: UploadItem, uploadedItem: UploadedItem): void { const topLevelItem = getTopLevelItem(item); const rootFolder = this.rootFolders.get(topLevelItem) as RootFolderUpload; diff --git a/src/services/referral.service.test.ts b/src/services/referral.service.test.ts index af6ae51f4d..d005c9c593 100644 --- a/src/services/referral.service.test.ts +++ b/src/services/referral.service.test.ts @@ -232,6 +232,14 @@ describe('referralService', () => { expectBannerStateSaved(expectedFragment); }); + + it('when the saved banner state is corrupt, then tracking starts again from the default state', () => { + localStorageService.set(LocalStorageItem.BannerStateKey, '{not json'); + + referralService.trackFileUpload(); + + expectBannerStateSaved('"fileUploadCount":1'); + }); }); describe('trackAppOpenDay', () => { diff --git a/src/services/referral.service.ts b/src/services/referral.service.ts index c72b7b72cb..2d4cecbdd9 100644 --- a/src/services/referral.service.ts +++ b/src/services/referral.service.ts @@ -45,7 +45,13 @@ const DEFAULT_BANNER_STATE: BannerState = { const getBannerState = (): BannerState => { const stored = localStorageService.get(LocalStorageItem.BannerStateKey); - return stored ? { ...DEFAULT_BANNER_STATE, ...JSON.parse(stored) } : { ...DEFAULT_BANNER_STATE }; + if (!stored) return { ...DEFAULT_BANNER_STATE }; + + try { + return { ...DEFAULT_BANNER_STATE, ...JSON.parse(stored) }; + } catch { + return { ...DEFAULT_BANNER_STATE }; + } }; const updateBannerState = (update: Partial): void => { From 690c24fd15f6039d356410049845d9c7eb3630bb Mon Sep 17 00:00:00 2001 From: Xavier Abad Date: Fri, 6 Nov 2026 12:01:16 +0100 Subject: [PATCH 49/49] fix(upload): await cancelled folder deletion --- src/app/i18n/locales/de.json | 3 +- src/app/i18n/locales/en.json | 3 +- src/app/i18n/locales/es.json | 3 +- src/app/i18n/locales/fr.json | 3 +- src/app/i18n/locales/it.json | 3 +- src/app/i18n/locales/pt.json | 3 +- src/app/i18n/locales/ru.json | 3 +- src/app/i18n/locales/tw.json | 3 +- src/app/i18n/locales/zh.json | 3 +- src/app/tasks/upload/uploadTaskErrors.ts | 5 +++ src/app/upload/Upload.test.ts | 44 ++++++++++++++++++++++-- src/app/upload/Upload.ts | 18 +++++++--- 12 files changed, 78 insertions(+), 16 deletions(-) diff --git a/src/app/i18n/locales/de.json b/src/app/i18n/locales/de.json index 5e2284e37a..6505c5a325 100644 --- a/src/app/i18n/locales/de.json +++ b/src/app/i18n/locales/de.json @@ -1373,7 +1373,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Upload fehlgeschlagen" + "upload-failed": "Upload fehlgeschlagen", + "folder-removal-failed": "Der hochgeladene Ordner konnte nicht entfernt werden" }, "generalErrorMessages": { "retryUploadFailed": "Beim erneuten Hochladen ist etwas schiefgegangen. Versuche es erneut", diff --git a/src/app/i18n/locales/en.json b/src/app/i18n/locales/en.json index e2d137546d..3ba79f3bcc 100644 --- a/src/app/i18n/locales/en.json +++ b/src/app/i18n/locales/en.json @@ -1397,7 +1397,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Upload failed" + "upload-failed": "Upload failed", + "folder-removal-failed": "Couldn't remove the uploaded folder" }, "generalErrorMessages": { "retryUploadFailed": "Something went wrong when retrying the upload. Try uploading again", diff --git a/src/app/i18n/locales/es.json b/src/app/i18n/locales/es.json index d43941b69a..058ae21150 100644 --- a/src/app/i18n/locales/es.json +++ b/src/app/i18n/locales/es.json @@ -1377,7 +1377,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Error al subir" + "upload-failed": "Error al subir", + "folder-removal-failed": "No se pudo eliminar la carpeta subida" }, "generalErrorMessages": { "retryUploadFailed": "Algo salió mal al reintentar la subida. Intenta subirlo de nuevo", diff --git a/src/app/i18n/locales/fr.json b/src/app/i18n/locales/fr.json index a65aa45ce5..3c711b5829 100644 --- a/src/app/i18n/locales/fr.json +++ b/src/app/i18n/locales/fr.json @@ -1376,7 +1376,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Échec du téléchargement" + "upload-failed": "Échec du téléchargement", + "folder-removal-failed": "Impossible de supprimer le dossier importé" }, "generalErrorMessages": { "retryUploadFailed": "Quelque chose s'est mal passé en réessayant de télécharger. Essayez de télécharger à nouveau", diff --git a/src/app/i18n/locales/it.json b/src/app/i18n/locales/it.json index c740263d48..6e592f641c 100644 --- a/src/app/i18n/locales/it.json +++ b/src/app/i18n/locales/it.json @@ -1397,7 +1397,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Caricamento fallito" + "upload-failed": "Caricamento fallito", + "folder-removal-failed": "Impossibile rimuovere la cartella caricata" }, "generalErrorMessages": { "retryUploadFailed": "Qualcosa è andato storto nel riprovare a caricare. Prova a caricare di nuovo", diff --git a/src/app/i18n/locales/pt.json b/src/app/i18n/locales/pt.json index ceaf96953b..7e7808fd3c 100644 --- a/src/app/i18n/locales/pt.json +++ b/src/app/i18n/locales/pt.json @@ -1407,7 +1407,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Falha no envio" + "upload-failed": "Falha no envio", + "folder-removal-failed": "Não foi possível remover a pasta enviada" }, "generalErrorMessages": { "retryUploadFailed": "Algo deu errado ao tentar enviar novamente. Tente fazer o envio de novo", diff --git a/src/app/i18n/locales/ru.json b/src/app/i18n/locales/ru.json index e14531e32a..2d11a62a06 100644 --- a/src/app/i18n/locales/ru.json +++ b/src/app/i18n/locales/ru.json @@ -1376,7 +1376,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "Не удалось загрузить" + "upload-failed": "Не удалось загрузить", + "folder-removal-failed": "Не удалось удалить загруженную папку" }, "generalErrorMessages": { "retryUploadFailed": "Что-то пошло не так при повторной попытке загрузки. Попробуйте загрузить снова", diff --git a/src/app/i18n/locales/tw.json b/src/app/i18n/locales/tw.json index 5cbdc87a44..6765b9ea99 100644 --- a/src/app/i18n/locales/tw.json +++ b/src/app/i18n/locales/tw.json @@ -1397,7 +1397,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "上傳失敗" + "upload-failed": "上傳失敗", + "folder-removal-failed": "無法移除已上傳的資料夾" }, "generalErrorMessages": { "retryUploadFailed": "重試上傳時出現問題。請重新上傳", diff --git a/src/app/i18n/locales/zh.json b/src/app/i18n/locales/zh.json index a864fba01a..1e0c1ba20f 100644 --- a/src/app/i18n/locales/zh.json +++ b/src/app/i18n/locales/zh.json @@ -1397,7 +1397,8 @@ "errors": {} }, "subtitles": { - "upload-failed": "上传失败" + "upload-failed": "上传失败", + "folder-removal-failed": "无法删除已上传的文件夹" }, "generalErrorMessages": { "retryUploadFailed": "重试上传时出了问题。请再次上传", diff --git a/src/app/tasks/upload/uploadTaskErrors.ts b/src/app/tasks/upload/uploadTaskErrors.ts index ee64ed7d50..2a504063ad 100644 --- a/src/app/tasks/upload/uploadTaskErrors.ts +++ b/src/app/tasks/upload/uploadTaskErrors.ts @@ -18,3 +18,8 @@ export const errorMerge = (reason?: UploadFailureReason) => { const subtitle = errorSubtitle(reason); return subtitle !== undefined ? { status: TaskStatus.Error, subtitle } : { status: TaskStatus.Error }; }; + +export const folderRemovalFailedMerge = () => ({ + status: TaskStatus.Error, + subtitle: t('tasks.subtitles.folder-removal-failed') as string, +}); diff --git a/src/app/upload/Upload.test.ts b/src/app/upload/Upload.test.ts index d0b458560d..34d5904b47 100644 --- a/src/app/upload/Upload.test.ts +++ b/src/app/upload/Upload.test.ts @@ -218,12 +218,50 @@ describe('Uploading what the user picked', () => { await running; expect(newStorageService.deleteFolderByUuid).toHaveBeenCalledWith('Photos-uuid'); - await vi.waitFor(() => - expect(onItemRemoved).toHaveBeenCalledWith(expect.objectContaining({ uuid: 'Photos-uuid' }), DESTINATION_UUID), - ); + expect(onItemRemoved).toHaveBeenCalledWith(expect.objectContaining({ uuid: 'Photos-uuid' }), DESTINATION_UUID); expect(tasksOf(TaskType.UploadFolder)[0].status).toBe(TaskStatus.Cancelled); }); + test('When the user cancels a folder, then the upload finishes only once the folder is removed', async () => { + fakeUploadsTaking(500); + let finishRemoval: () => void = () => undefined; + vi.mocked(newStorageService.deleteFolderByUuid).mockReturnValue( + new Promise((resolve) => (finishRemoval = resolve)) as never, + ); + const upload = createUpload([getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], []); + const onFinished = vi.fn(); + + const running = upload.run().then(onFinished); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalled()); + await tasksService.cancelTask(tasksOf(TaskType.UploadFolder)[0].id); + await vi.waitFor(() => expect(newStorageService.deleteFolderByUuid).toHaveBeenCalled()); + await new Promise((resolve) => setTimeout(resolve, 600)); + + expect(onFinished).not.toHaveBeenCalled(); + finishRemoval(); + await running; + expect(onFinished).toHaveBeenCalled(); + }); + + test('When the cancelled folder cannot be removed, then its task shows the error', async () => { + fakeUploadsTaking(500); + vi.mocked(newStorageService.deleteFolderByUuid).mockRejectedValue(new Error('Server error')); + const onItemRemoved = vi.fn(); + const upload = createUpload([getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], [], { + onItemRemoved, + }); + + const running = upload.run(); + await vi.waitFor(() => expect(uploadFile).toHaveBeenCalled()); + await tasksService.cancelTask(tasksOf(TaskType.UploadFolder)[0].id); + await running; + + expect(tasksOf(TaskType.UploadFolder)[0]).toEqual( + expect.objectContaining({ status: TaskStatus.Error, subtitle: 'tasks.subtitles.folder-removal-failed' }), + ); + expect(onItemRemoved).not.toHaveBeenCalled(); + }); + test('When the user cancels a folder while it is being created, then it is removed once it exists', async () => { fakeFolderCreationTaking(300); const upload = createUpload([getFolderToUpload('Photos', { childrenFiles: [getFileToUpload('beach.png')] })], []); diff --git a/src/app/upload/Upload.ts b/src/app/upload/Upload.ts index a028828192..73a2675865 100644 --- a/src/app/upload/Upload.ts +++ b/src/app/upload/Upload.ts @@ -25,7 +25,7 @@ import { PersistUploadRepository } from 'app/repositories/DatabaseUploadReposito import { IRoot } from 'app/store/slices/storage/types'; import tasksService from 'app/tasks/services/tasks.service'; import { TaskStatus, TaskType, UploadFileTask, UploadFolderTask } from 'app/tasks/types'; -import { errorMerge } from 'app/tasks/upload/uploadTaskErrors'; +import { errorMerge, folderRemovalFailedMerge } from 'app/tasks/upload/uploadTaskErrors'; import errorService from 'services/error.service'; import referralService from 'services/referral.service'; @@ -65,6 +65,7 @@ export class Upload { private readonly items: UploadItem[]; private readonly taskIdByItem = new Map(); private readonly rootFolders = new Map(); + private readonly cancelledRootFolderDeletions: Promise[] = []; constructor(props: UploadProps) { this.props = props; @@ -84,6 +85,7 @@ export class Upload { async run(): Promise { const result = await this.manager.run(this.items); + await Promise.all(this.cancelledRootFolderDeletions); this.rootFolders.forEach((rootFolder, item) => this.finishRootFolder(item as UploadFolderItem, rootFolder)); return result; } @@ -193,7 +195,9 @@ export class Upload { const isRootFolderItself = item === topLevelItem; if (isRootFolderItself) { rootFolder.createdFolder = uploadedItem as DriveFolderData; - if (rootFolder.isCancelled) void this.removeRootFolder(item as UploadFolderItem, rootFolder); + if (rootFolder.isCancelled) { + this.cancelledRootFolderDeletions.push(this.deleteCancelledRootFolder(item as UploadFolderItem, rootFolder)); + } } rootFolder.uploadedItems += 1; @@ -236,10 +240,15 @@ export class Upload { const rootFolder = this.rootFolders.get(item) as RootFolderUpload; rootFolder.isCancelled = true; this.manager.cancelItem(item); - if (rootFolder.createdFolder) void this.removeRootFolder(item, rootFolder); + if (rootFolder.createdFolder) { + this.cancelledRootFolderDeletions.push(this.deleteCancelledRootFolder(item, rootFolder)); + } } - private async removeRootFolder(item: UploadFolderItem, { createdFolder }: RootFolderUpload): Promise { + private async deleteCancelledRootFolder( + item: UploadFolderItem, + { taskId, createdFolder }: RootFolderUpload, + ): Promise { if (!createdFolder) return; try { @@ -247,6 +256,7 @@ export class Upload { this.props.onItemRemoved?.(createdFolder, item.parentFolderUuid); } catch (error) { errorService.reportError(error); + tasksService.updateTask({ taskId, merge: folderRemovalFailedMerge() }); } }