diff --git a/README.md b/README.md index e293a7a..a488515 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,13 @@ ol document move --collection ol document archive ol document unarchive +# Attachments (alias: ol attach) +ol attachment upload ./plan.pdf --doc # prints the markdown link for the file +ol attachment upload ./plan.pdf --doc --append # also appends that link to the document +ol attachment list --doc # attachments and images linked in a document +ol attachment download # saves under the original name; --output , --force +ol attachment delete --confirm + # Collections (alias: ol col) ol collection list ol collection get diff --git a/skills/outline-cli/SKILL.md b/skills/outline-cli/SKILL.md index 0fb3409..e19f674 100644 --- a/skills/outline-cli/SKILL.md +++ b/skills/outline-cli/SKILL.md @@ -1,6 +1,6 @@ --- name: outline-cli -description: "Search and manage Outline wiki documents and collections via the ol CLI" +description: "Search and manage Outline wiki documents, collections and attachments via the ol CLI" --- # Outline CLI (ol) @@ -15,6 +15,8 @@ Use this skill when the user wants to interact with their Outline wiki/knowledge - `ol doc open ` - Open document in browser - `ol doc create --title "Title" --collection ` - Create document - `ol col list` - List collections +- `ol attachment upload --doc ` - Upload a file to a document +- `ol attachment download ` - Download an attachment - `ol account` - List stored accounts; `ol account use ` sets the default ## Output Formats @@ -83,6 +85,23 @@ ol col update --name "New Name" ol col delete --confirm ``` +### Attachments (alias: attach) +```bash +ol attachment upload ./plan.pdf --doc # Upload; prints the markdown link to paste into the doc +ol attachment upload ./plan.pdf --doc --append # Upload and append the link to the end of the doc +ol attachment upload ./photo.jpg --doc --name "Site photo.jpg" --content-type image/jpeg +ol attachment upload ./plan.pdf --doc --json # {id, name, contentType, size, documentId, url, markdown} +ol attachment list --doc # Attachments and images linked in the doc (id, name, size, type) +ol attachment list --doc --json +ol attachment download # Save under the original file name in the current directory +ol attachment download --output ./files/ # Into a folder (created when the path ends in /), or --output +ol attachment download --output - # Write the file to stdout +ol attachment download --force # Overwrite an existing file (refused otherwise) +ol attachment delete --confirm # Delete the file; links to it in documents stay +``` +An attachment is referenced by its ID or any link containing `/api/attachments.redirect?id=`. Images are linked as `![alt](/api/attachments.redirect?id=)`, other files as `[ ](/api/attachments.redirect?id=)`; keep that form when writing it into a document yourself. +`attachment list` reads those links from the document text, so a plain link whose text ends in a number (`[report 2024](…attachments.redirect?id=…)`) lists as name `report`, size 2024, and links to another Outline instance's attachments are listed but cannot be downloaded from this one. + ### Authentication ```bash ol auth login # OAuth login (opens browser); prompts for base URL + client ID if needed @@ -149,6 +168,14 @@ ol doc update --file doc.md --if-updated-at "$ts" ``` If the document was saved after `$ts`, the update fails with `DOCUMENT_CHANGED`, or with Outline's "Document has been modified since the provided revision" when the save lands during the update; re-read and retry. Edits not yet saved from an open editor are not detected. +### Attach a file and download every attachment of a document +```bash +ol attachment upload ./drawing.pdf --doc --append +ol attachment list --doc --json | jq -r '.[].id' | while read -r id; do + ol attachment download "$id" --output "./downloads/$id/" # one folder each: names often repeat +done +``` + ### List all collections and their documents ```bash ol col list --json diff --git a/src/commands/attachment.test.ts b/src/commands/attachment.test.ts new file mode 100644 index 0000000..0c2f3fb --- /dev/null +++ b/src/commands/attachment.test.ts @@ -0,0 +1,190 @@ +import { captureConsole, createTestProgram } from '@todoist/cli-core/testing' +import { type MockInstance, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { lines, mockOutlineAuthModule } from '../_fixtures/testing.js' + +vi.mock('../lib/auth.js', () => mockOutlineAuthModule()) + +vi.mock('../lib/api.js', () => ({ + apiRequest: vi.fn(), +})) + +vi.mock('../lib/refs.js', () => ({ + resolveDocumentRef: vi.fn(async () => ({ + id: 'doc-1', + title: 'Spec', + url: '/doc/spec-abc123', + urlId: 'abc123', + })), +})) + +vi.mock('../lib/attachments.js', async (importOriginal) => ({ + ...(await importOriginal()), + uploadAttachment: vi.fn(), + downloadAttachment: vi.fn(), +})) + +const ID = '3e7bbd8c-907b-4b10-9160-9b820d5ce255' + +const uploaded = { + id: ID, + name: 'plan.pdf', + contentType: 'application/pdf', + size: 2048, + documentId: 'doc-1', + url: `/api/files.get?key=uploads/${ID}/plan.pdf`, +} + +async function mocks() { + const { apiRequest } = await import('../lib/api.js') + const attachments = await import('../lib/attachments.js') + return { + apiRequest: apiRequest as ReturnType, + uploadAttachment: attachments.uploadAttachment as ReturnType, + downloadAttachment: attachments.downloadAttachment as ReturnType, + } +} + +async function run(...args: string[]) { + const { registerAttachmentCommand } = await import('./attachment.js') + const program = createTestProgram(registerAttachmentCommand) + await program.parseAsync(['node', 'ol', ...args]) +} + +describe('attachment commands', () => { + let log: MockInstance + + beforeEach(() => { + log = captureConsole() + }) + + afterEach(() => { + vi.clearAllMocks() + vi.restoreAllMocks() + }) + + it('upload passes the file and options and prints the markdown link', async () => { + const { uploadAttachment, apiRequest } = await mocks() + uploadAttachment.mockResolvedValue(uploaded) + + await run( + 'attachment', + 'upload', + './plan.pdf', + '--doc', + 'spec', + '--name', + 'plan.pdf', + '--content-type', + 'application/pdf', + ) + + expect(uploadAttachment).toHaveBeenCalledWith({ + filePath: './plan.pdf', + documentId: 'doc-1', + name: 'plan.pdf', + contentType: 'application/pdf', + }) + expect(apiRequest).not.toHaveBeenCalled() + const out = lines(log).join('\n') + expect(out).toContain('Uploaded: plan.pdf') + expect(out).toContain(`[plan.pdf 2048](/api/attachments.redirect?id=${ID})`) + }) + + it('upload --append appends the link with the version-safe append flag', async () => { + const { uploadAttachment, apiRequest } = await mocks() + uploadAttachment.mockResolvedValue({ ...uploaded, contentType: 'image/png', name: 'a.png' }) + apiRequest.mockResolvedValue({ data: {} }) + + await run('attach', 'upload', 'a.png', '--doc', 'spec', '--append') + + expect(apiRequest).toHaveBeenCalledWith('documents.update', { + id: 'doc-1', + text: `\n\n![a.png](/api/attachments.redirect?id=${ID})`, + append: true, + }) + expect(lines(log).join('\n')).toContain('Appended to: Spec') + }) + + it('upload --json prints the attachment with its redirect URL and markdown', async () => { + const { uploadAttachment } = await mocks() + uploadAttachment.mockResolvedValue(uploaded) + + await run('attachment', 'upload', 'plan.pdf', '--doc', 'spec', '--json') + + expect(JSON.parse(lines(log)[0])).toEqual({ + ...uploaded, + url: `/api/attachments.redirect?id=${ID}`, + markdown: `[plan.pdf 2048](/api/attachments.redirect?id=${ID})`, + }) + }) + + it('download passes output and force through', async () => { + const { downloadAttachment } = await mocks() + downloadAttachment.mockResolvedValue({ id: ID, name: 'a.pdf', path: 'out/a.pdf', size: 10 }) + + await run('attachment', 'download', ID, '--output', 'out', '--force') + + expect(downloadAttachment).toHaveBeenCalledWith(ID, { output: 'out', force: true }) + expect(lines(log)[0]).toContain('Downloaded: out/a.pdf') + }) + + it('download --output - prints nothing besides the file content', async () => { + const { downloadAttachment } = await mocks() + downloadAttachment.mockResolvedValue({ id: ID, name: 'a.pdf', path: '-', size: 10 }) + + await run('attachment', 'download', ID, '--output', '-') + + expect(downloadAttachment).toHaveBeenCalledWith(ID, { output: '-' }) + expect(log).not.toHaveBeenCalled() + }) + + it('list reads attachment links from the document text', async () => { + const { apiRequest } = await mocks() + apiRequest.mockResolvedValue({ + data: { + id: 'doc-1', + text: `See [plan.pdf 2048](/api/attachments.redirect?id=${ID})`, + }, + }) + + await run('attachment', 'list', '--doc', 'spec', '--json') + + expect(apiRequest).toHaveBeenCalledWith('documents.info', { id: 'doc-1' }) + expect(JSON.parse(lines(log)[0])).toEqual([ + { + id: ID, + name: 'plan.pdf', + size: 2048, + type: 'file', + url: `/api/attachments.redirect?id=${ID}`, + }, + ]) + }) + + it('list says so when a document has no attachments', async () => { + const { apiRequest } = await mocks() + apiRequest.mockResolvedValue({ data: { id: 'doc-1', text: 'Plain text' } }) + + await run('attachment', 'list', '--doc', 'spec') + + expect(lines(log).join('\n')).toContain('No attachments found.') + }) + + it('delete requires --confirm', async () => { + const { apiRequest } = await mocks() + await expect(run('attachment', 'delete', ID)).rejects.toThrow( + 'Delete operation requires confirmation.', + ) + expect(apiRequest).not.toHaveBeenCalled() + }) + + it('delete accepts a link and deletes by ID', async () => { + const { apiRequest } = await mocks() + apiRequest.mockResolvedValue({ data: undefined }) + + await run('attachment', 'delete', `/api/attachments.redirect?id=${ID}`, '--confirm') + + expect(apiRequest).toHaveBeenCalledWith('attachments.delete', { id: ID }) + expect(lines(log)[0]).toBe('Deleted.') + }) +}) diff --git a/src/commands/attachment.ts b/src/commands/attachment.ts new file mode 100644 index 0000000..6d03a5b --- /dev/null +++ b/src/commands/attachment.ts @@ -0,0 +1,175 @@ +import chalk from 'chalk' +import type { Command } from 'commander' +import { apiRequest } from '../lib/api.js' +import { + type Attachment, + type LinkedAttachment, + attachmentMarkdown, + attachmentPath, + downloadAttachment, + extractLinkedAttachments, + parseAttachmentId, + uploadAttachment, +} from '../lib/attachments.js' +import { CliError } from '../lib/errors.js' +import { getOutputOptions, outputItem, outputList } from '../lib/output.js' +import { resolveDocumentRef } from '../lib/refs.js' +import { withSpinner } from '../lib/spinner.js' + +type UploadedAttachment = Attachment & { markdown: string } + +const uploadKeys: (keyof UploadedAttachment)[] = [ + 'id', + 'name', + 'contentType', + 'size', + 'documentId', + 'url', + 'markdown', +] + +const linkedKeys: (keyof LinkedAttachment)[] = ['id', 'name', 'size', 'type', 'url'] + +function formatSize(bytes: number | null): string { + if (bytes === null) return '' + if (bytes < 1024) return `${bytes} B` + const units = ['KB', 'MB', 'GB'] + let value = bytes / 1024 + let unit = 0 + while (value >= 1024 && unit < units.length - 1) { + value /= 1024 + unit++ + } + return `${value.toFixed(1)} ${units[unit]}` +} + +function formatLinked(item: LinkedAttachment): string { + const name = chalk.bold(item.name || '(no name)') + const meta = chalk.dim([item.type, formatSize(item.size)].filter(Boolean).join(', ')) + return `${name} ${chalk.dim(item.id)} ${meta}` +} + +async function documentText(ref: string): Promise<{ id: string; text: string }> { + const resolved = await resolveDocumentRef(ref) + if (resolved.text !== undefined) return { id: resolved.id, text: resolved.text } + const { data } = await apiRequest<{ id: string; text: string }>('documents.info', { + id: resolved.id, + }) + return { id: data.id, text: data.text } +} + +export function registerAttachmentCommand(program: Command): void { + const att = program + .command('attachment') + .alias('attach') + .description('Upload, download and manage document attachments') + + att.command('upload ') + .description('Upload a file as an attachment of a document') + .requiredOption('--doc ', 'Document ID, URL, or name to attach the file to') + .option('--name ', 'File name shown in Outline (default: the file name)') + .option('--content-type ', 'MIME type (default: guessed from the file extension)') + .option('--append', 'Also append a link to the attachment at the end of the document') + .option('--json', 'Output JSON') + .action(async (file: string, opts) => { + const doc = await resolveDocumentRef(opts.doc) + const attachment = await uploadAttachment({ + filePath: file, + documentId: doc.id, + name: opts.name, + contentType: opts.contentType, + }) + const markdown = attachmentMarkdown(attachment) + + if (opts.append) { + // `append` (rather than `editMode`) is understood by every + // Outline version with text edits; older servers ignore an + // unknown `editMode` and would replace the whole document. + // The leading blank line keeps the link in its own paragraph. + await apiRequest('documents.update', { + id: doc.id, + text: `\n\n${markdown}`, + append: true, + }) + } + + const result: UploadedAttachment = { + ...attachment, + url: attachmentPath(attachment.id), + markdown, + } + if (opts.json) { + outputItem(result, () => '', uploadKeys, { json: true }) + return + } + console.log( + chalk.green(`Uploaded: ${attachment.name}`), + chalk.dim(`${attachment.id} ${formatSize(attachment.size)}`), + ) + if (opts.append) { + console.log(chalk.dim(`Appended to: ${doc.title}`)) + } else { + console.log(chalk.dim('Markdown:'), markdown) + } + }) + + att.command('download ') + .description('Download an attachment by ID or link') + .option( + '--output ', + 'File path, folder (ending in / is created if missing), or - for stdout (default: current directory)', + ) + .option('--force', 'Overwrite an existing file') + .option('--json', 'Output JSON') + .action(async (ref: string, opts) => { + const toStdout = opts.output === '-' + const result = toStdout + ? await downloadAttachment(ref, { output: '-' }) + : await withSpinner({ text: 'Downloading file...', color: 'blue' }, () => + downloadAttachment(ref, { output: opts.output, force: opts.force }), + ) + if (toStdout) return + if (opts.json) { + outputItem(result, () => '', undefined, { json: true }) + return + } + console.log( + chalk.green(`Downloaded: ${result.path}`), + chalk.dim(formatSize(result.size)), + ) + }) + + att.command('list') + .description('List the attachments and images linked in a document') + .requiredOption('--doc ', 'Document ID, URL, or name') + .option('--json', 'Output JSON') + .option('--ndjson', 'Output NDJSON') + .option('--full', 'Include all fields in JSON output') + .action(async (opts) => { + const { text } = await documentText(opts.doc) + outputList( + extractLinkedAttachments(text), + formatLinked, + linkedKeys, + getOutputOptions(opts), + undefined, + 'No attachments found.', + ) + }) + + att.command('delete ') + .description('Delete an attachment by ID or link (links to it in documents remain)') + .option('--confirm', 'Skip confirmation') + .action(async (ref: string, opts) => { + if (!opts.confirm) { + throw new CliError( + 'CONFIRMATION_REQUIRED', + 'Delete operation requires confirmation.', + ['Use --confirm flag to proceed with deletion'], + ) + } + const id = parseAttachmentId(ref) + await apiRequest('attachments.delete', { id }) + console.log('Deleted.') + }) +} diff --git a/src/index.ts b/src/index.ts index b9c5634..5a4d763 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,6 +2,7 @@ import { Command } from 'commander' import packageJson from '../package.json' with { type: 'json' } import { registerAccountCommand } from './commands/account.js' +import { registerAttachmentCommand } from './commands/attachment.js' import { registerAuthCommand } from './commands/auth.js' import { registerChangelogCommand } from './commands/changelog.js' import { registerCollectionCommand } from './commands/collection.js' @@ -38,6 +39,7 @@ registerAccountCommand(program) registerSearchCommand(program) registerDocumentCommand(program) registerCollectionCommand(program) +registerAttachmentCommand(program) registerSkillCommand(program) registerChangelogCommand(program) registerUpdateCommand(program) diff --git a/src/lib/api.test.ts b/src/lib/api.test.ts index 58ad268..a1cfa03 100644 --- a/src/lib/api.test.ts +++ b/src/lib/api.test.ts @@ -132,6 +132,85 @@ describe('apiRequest', () => { expect(headers.Authorization).toBe('Bearer rotated-proactive') }) + it('authorizedFetch sends the bearer token to paths on the instance', async () => { + const { fetchWithRetry } = await import('../transport/fetch-with-retry.js') + const f = fetchWithRetry as ReturnType + f.mockResolvedValue(new Response(null, { status: 302, headers: { location: '/x' } })) + + const { authorizedFetch } = await import('./api.js') + const result = await authorizedFetch('/api/attachments.redirect?id=1', { + method: 'GET', + redirect: 'manual', + }) + + expect(result.url).toBe('https://test.outline.com/api/attachments.redirect?id=1') + expect(result.onInstance).toBe(true) + expect(result.response.status).toBe(302) + expect(f).toHaveBeenCalledWith({ + url: 'https://test.outline.com/api/attachments.redirect?id=1', + options: { + method: 'GET', + headers: { Authorization: 'Bearer test-token' }, + redirect: 'manual', + }, + }) + }) + + it('authorizedFetch keeps a sub-path in the base URL, like apiRequest', async () => { + authMocks.getRequestContext.mockResolvedValue({ baseUrl: 'https://example.com/wiki' }) + const { fetchWithRetry } = await import('../transport/fetch-with-retry.js') + const f = fetchWithRetry as ReturnType + f.mockResolvedValue(new Response('ok')) + + const { authorizedFetch } = await import('./api.js') + const result = await authorizedFetch('/api/files.create', { method: 'POST' }) + + expect(f.mock.calls[0][0].url).toBe('https://example.com/wiki/api/files.create') + expect(result.onInstance).toBe(true) + }) + + it('authorizedFetch never sends the token to another origin', async () => { + const { fetchWithRetry } = await import('../transport/fetch-with-retry.js') + const f = fetchWithRetry as ReturnType + f.mockResolvedValue(new Response(null, { status: 401 })) + const body = new Blob(['x']) + + const { authorizedFetch } = await import('./api.js') + const result = await authorizedFetch('https://bucket.s3.amazonaws.com/', { + method: 'POST', + headers: { 'Content-Type': 'multipart/form-data; boundary=b' }, + body, + }) + + expect(f).toHaveBeenCalledTimes(1) + expect(f.mock.calls[0][0].options).toEqual({ + method: 'POST', + headers: { 'Content-Type': 'multipart/form-data; boundary=b' }, + body, + }) + expect(result.onInstance).toBe(false) + // A 401 from third-party storage is not a token problem: no refresh. + expect(authMocks.reactiveRefresh).not.toHaveBeenCalled() + }) + + it('authorizedFetch refreshes and retries once on a 401 from the instance', async () => { + const { fetchWithRetry } = await import('../transport/fetch-with-retry.js') + const f = fetchWithRetry as ReturnType + f.mockResolvedValueOnce(errResponse(401, 'Unauthorized')) + f.mockResolvedValueOnce(new Response('ok')) + authMocks.reactiveRefresh.mockResolvedValueOnce(true) + authMocks.getApiToken + .mockResolvedValueOnce('stale-token') + .mockResolvedValueOnce('rotated-token') + + const { authorizedFetch } = await import('./api.js') + const { response } = await authorizedFetch('/api/files.create', { method: 'POST' }) + + expect(response.ok).toBe(true) + const retryHeaders = f.mock.calls[1][0].options.headers as Record + expect(retryHeaders.Authorization).toBe('Bearer rotated-token') + }) + it('does not refresh when OUTLINE_API_TOKEN is set (unmanaged token)', async () => { vi.stubEnv('OUTLINE_API_TOKEN', 'env-tok') const { fetchWithRetry } = await import('../transport/fetch-with-retry.js') diff --git a/src/lib/api.ts b/src/lib/api.ts index c142ff2..f7a0dbe 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -25,6 +25,8 @@ const API_SPINNER_CONFIG: Record = { 'collections.create': { text: 'Creating collection...', color: 'green' }, 'collections.update': { text: 'Updating collection...', color: 'yellow' }, 'collections.delete': { text: 'Deleting collection...', color: 'yellow' }, + 'attachments.create': { text: 'Creating attachment...', color: 'green' }, + 'attachments.delete': { text: 'Deleting attachment...', color: 'yellow' }, } export interface Pagination { @@ -75,14 +77,18 @@ async function resolveRequestToken( return getApiToken() } +type AuthorizedTarget = { + baseUrl: string + token: string + managed: boolean + handshake?: RefreshHandshake +} + /** - * Core API request function without spinner wrapping. + * Resolve the instance base URL and bearer token for a request, the same way + * for JSON API calls and raw transfers (attachment uploads and downloads). */ -async function rawApiRequest( - path: string, - body: object = {}, - overrides: ApiRequestOverrides = {}, -): Promise> { +async function resolveTarget(overrides: ApiRequestOverrides): Promise { // Only stored credentials can be refreshed: a caller-supplied token // override or the `OUTLINE_API_TOKEN` env var is taken as-is. const managed = !overrides.token && !process.env[TOKEN_ENV_VAR]?.trim() @@ -101,11 +107,53 @@ async function rawApiRequest( handshake = ctx.handshake } - const resolvedToken = await resolveRequestToken(managed, overrides.token, handshake) + const token = await resolveRequestToken(managed, overrides.token, handshake) + return { baseUrl: resolvedBaseUrl, token, managed, handshake } +} - const performRequest = (token: string) => +/** + * Send a request with the target's token. Reactive path: a 401 on a managed + * token triggers a forced rotation and a single retry. `reactiveRefresh` + * throws `NoTokenError` when the refresh token is gone, so an unrecoverable + * 401 surfaces the re-login hint. + */ +async function sendWithAuth( + target: AuthorizedTarget, + send: (token: string) => Promise, +): Promise { + let res = await send(target.token) + if (res.status === 401 && target.managed && (await reactiveRefresh(target.handshake))) { + res = await send(await getApiToken()) + } + return res +} + +/** + * Build the error thrown for a failed Outline API response, preferring the + * message from Outline's JSON error body. + */ +export async function toApiError(res: Response): Promise { + let message = `API error: ${res.status} ${res.statusText}` + try { + const err = (await res.json()) as ApiError + if (err.message) message = `API error: ${err.message}` + } catch {} + return new Error(message) +} + +/** + * Core API request function without spinner wrapping. + */ +async function rawApiRequest( + path: string, + body: object = {}, + overrides: ApiRequestOverrides = {}, +): Promise> { + const target = await resolveTarget(overrides) + + const res = await sendWithAuth(target, (token) => fetchWithRetry({ - url: `${resolvedBaseUrl}/api/${path}`, + url: `${target.baseUrl}/api/${path}`, options: { method: 'POST', headers: { @@ -114,30 +162,67 @@ async function rawApiRequest( }, body: JSON.stringify(body), }, - }) - - let res = await performRequest(resolvedToken) - - // Reactive path: a 401 on a managed token triggers a forced rotation and - // a single retry. `reactiveRefresh` throws `NoTokenError` when the refresh - // token is gone, so an unrecoverable 401 surfaces the re-login hint. - if (res.status === 401 && managed && (await reactiveRefresh(handshake))) { - res = await performRequest(await getApiToken()) - } + }), + ) - if (!res.ok) { - let message = `API error: ${res.status} ${res.statusText}` - try { - const err = (await res.json()) as ApiError - if (err.message) message = `API error: ${err.message}` - } catch {} - throw new Error(message) - } + if (!res.ok) throw await toApiError(res) const json = (await res.json()) as ApiResponse return { data: json.data, pagination: json.pagination } } +export type AuthorizedFetchInit = { + method: string + headers?: Record + body?: BodyInit + redirect?: RequestRedirect +} + +export type AuthorizedFetchResult = { + response: Response + /** Absolute URL the request went to. */ + url: string + /** Whether that URL is on the Outline instance (and so carried the token). */ + onInstance: boolean +} + +/** + * Send a raw request with the account's bearer token, for transfers that are + * not JSON API calls (multipart uploads, file downloads). `url` is either a + * path on the Outline instance (`/api/...`, appended to the base URL the same + * way `apiRequest` builds its URLs, so a sub-path install keeps its prefix) or + * an absolute URL. The token is only attached when the URL is on the + * instance's own origin, so it never reaches third-party storage such as S3. + * The response is returned as-is: callers check `ok` and read the body. + */ +export async function authorizedFetch( + url: string, + init: AuthorizedFetchInit, + overrides: ApiRequestOverrides = {}, +): Promise { + const target = await resolveTarget(overrides) + const instancePath = url.startsWith('/') && !url.startsWith('//') + const resolved = new URL(instancePath ? `${target.baseUrl}${url}` : url) + const onInstance = resolved.origin === new URL(target.baseUrl).origin + + const send = (token?: string) => + fetchWithRetry({ + url: resolved.toString(), + options: { + method: init.method, + headers: { + ...init.headers, + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + ...(init.body === undefined ? {} : { body: init.body }), + ...(init.redirect ? { redirect: init.redirect } : {}), + }, + }) + + const response = onInstance ? await sendWithAuth(target, send) : await send() + return { response, url: resolved.toString(), onInstance } +} + /** * Public API request function that wraps rawApiRequest with automatic spinners. * Spinner messages are configured per API path in API_SPINNER_CONFIG. diff --git a/src/lib/attachments.test.ts b/src/lib/attachments.test.ts new file mode 100644 index 0000000..a62c5d5 --- /dev/null +++ b/src/lib/attachments.test.ts @@ -0,0 +1,594 @@ +import { mkdtemp, readFile, readlink, rm, symlink, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const apiMocks = vi.hoisted(() => ({ + apiRequest: vi.fn(), + authorizedFetch: vi.fn(), + toApiError: vi.fn(async (res: Response) => new Error(`API error: ${res.status}`)), +})) + +vi.mock('./api.js', () => apiMocks) + +const ID = '3e7bbd8c-907b-4b10-9160-9b820d5ce255' +const DOC_ID = 'dfb46f8e-4d2d-4c7e-b478-4f07d2c34f18' +const BASE = 'https://test.outline.com' + +function attachment(overrides: Record = {}) { + return { + id: ID, + name: 'note.txt', + contentType: 'text/plain', + size: 5, + documentId: DOC_ID, + url: `/api/files.get?key=uploads/${ID}/note.txt`, + ...overrides, + } +} + +describe('attachment helpers', () => { + it('parses IDs from UUIDs, redirect paths and full URLs', async () => { + const { parseAttachmentId } = await import('./attachments.js') + expect(parseAttachmentId(ID.toUpperCase())).toBe(ID) + expect(parseAttachmentId(`/api/attachments.redirect?id=${ID}`)).toBe(ID) + expect(parseAttachmentId(`${BASE}/api/attachments.redirect?foo=1&id=${ID}`)).toBe(ID) + expect(() => parseAttachmentId('nonsense')).toThrow('Not an attachment ID or link') + }) + + it('builds image markdown for images and sized links for files', async () => { + const { attachmentMarkdown } = await import('./attachments.js') + expect(attachmentMarkdown(attachment({ contentType: 'image/png', name: 'a.png' }))).toBe( + `![a.png](/api/attachments.redirect?id=${ID})`, + ) + expect(attachmentMarkdown(attachment({ name: 'a [draft].pdf', size: 1234 }))).toBe( + `[a draft.pdf 1234](/api/attachments.redirect?id=${ID})`, + ) + }) + + it('extracts linked attachments and images once each, in order', async () => { + const { extractLinkedAttachments } = await import('./attachments.js') + const other = '0d4fff26-6b65-4873-ae45-a79647413550' + const text = [ + 'Intro [a normal link](https://example.com)', + `[Door schedule rev B.pdf 3000](/api/attachments.redirect?id=${ID})`, + ` ![dot.png](${BASE}/api/attachments.redirect?id=${other} "title")`, + `[again 1](/api/attachments.redirect?id=${ID})`, + `[no size](/api/attachments.redirect?id=${other.replace('0d', '1d')})`, + ].join('\n\n') + + expect(extractLinkedAttachments(text)).toEqual([ + { + id: ID, + name: 'Door schedule rev B.pdf', + size: 3000, + type: 'file', + url: `/api/attachments.redirect?id=${ID}`, + }, + { + id: other, + name: 'dot.png', + size: null, + type: 'image', + url: `${BASE}/api/attachments.redirect?id=${other}`, + }, + { + id: other.replace('0d', '1d'), + name: 'no size', + size: null, + type: 'file', + url: `/api/attachments.redirect?id=${other.replace('0d', '1d')}`, + }, + ]) + }) + + it('reads escaped and nested brackets in labels', async () => { + const { extractLinkedAttachments } = await import('./attachments.js') + const other = ID.replace('3e', '4e') + const text = [ + `[plan \\[final\\].pdf 12](/api/attachments.redirect?id=${ID})`, + `![dot [x] y](/api/attachments.redirect?id=${other})`, + ].join('\n\n') + expect(extractLinkedAttachments(text).map((a) => [a.name, a.size, a.type])).toEqual([ + ['plan [final].pdf', 12, 'file'], + ['dot [x] y', null, 'image'], + ]) + }) + + it('ignores attachment links inside inline and fenced code', async () => { + const { extractLinkedAttachments } = await import('./attachments.js') + const link = `[example 1](/api/attachments.redirect?id=${ID})` + const text = ['`' + link + '`', '```', link, '```', ' ' + link].join('\n\n') + expect(extractLinkedAttachments(text)).toEqual([]) + }) + + it('reads file names from Content-Disposition, keeping only the base name', async () => { + const { fileNameFromDisposition } = await import('./attachments.js') + expect(fileNameFromDisposition('attachment; filename="a b.pdf"')).toBe('a b.pdf') + expect( + fileNameFromDisposition( + `inline; filename="x.pdf"; filename*=UTF-8''%E9%97%A8%E8%A1%A8.pdf`, + ), + ).toBe('门表.pdf') + expect(fileNameFromDisposition('attachment; filename="../../etc/passwd"')).toBe('passwd') + expect(fileNameFromDisposition('attachment; filename=".."')).toBeUndefined() + expect(fileNameFromDisposition(null)).toBeUndefined() + }) + + it('strips control characters from server-supplied names', async () => { + const { fileNameFromDisposition, sanitizeFileName } = await import('./attachments.js') + expect(fileNameFromDisposition('attachment; filename="\u001b[31mred.txt"')).toBe( + '[31mred.txt', + ) + expect(sanitizeFileName('a\u0000b\u007fc\n.txt')).toBe('abc.txt') + expect(sanitizeFileName('\u001b')).toBeUndefined() + }) + + it('falls back to the storage key in the served URL for the file name', async () => { + const { fileNameFromUrl } = await import('./attachments.js') + expect( + fileNameFromUrl( + 'https://bucket.s3.amazonaws.com/uploads/u1/a1/Site%20plan.pdf?X-Amz-Signature=s', + ), + ).toBe('Site plan.pdf') + expect(fileNameFromUrl(`${BASE}/api/files.get?key=uploads/u1/a1/a%25b.pdf`)).toBe('a%b.pdf') + expect(fileNameFromUrl(`${BASE}/api/files.get?sig=signed`)).toBeUndefined() + expect(fileNameFromUrl('not a url')).toBeUndefined() + }) + + it('encodes multipart bodies with escaped names and the file last', async () => { + const { buildMultipartBody } = await import('./attachments.js') + const { body, contentType } = buildMultipartBody( + { key: 'uploads/x', sig: 'abc' }, + { + field: 'file', + name: 'a"b\r\n.txt', + contentType: 'text/plain', + blob: new Blob(['hello']), + }, + ) + const boundary = contentType.split('boundary=')[1] + const text = await body.text() + expect(contentType).toMatch(/^multipart\/form-data; boundary=/) + expect(body.size).toBe(Buffer.byteLength(text)) + expect(text).toBe( + [ + `--${boundary}`, + 'Content-Disposition: form-data; name="key"', + '', + 'uploads/x', + `--${boundary}`, + 'Content-Disposition: form-data; name="sig"', + '', + 'abc', + `--${boundary}`, + 'Content-Disposition: form-data; name="file"; filename="a%22b%0D%0A.txt"', + 'Content-Type: text/plain', + '', + 'hello', + `--${boundary}--`, + '', + ].join('\r\n'), + ) + }) + + it('guesses content types from extensions', async () => { + const { guessContentType } = await import('./attachments.js') + expect(guessContentType('A.PDF')).toBe('application/pdf') + expect(guessContentType('photo.jpeg')).toBe('image/jpeg') + expect(guessContentType('drawing.dwg')).toBe('application/octet-stream') + }) +}) + +describe('uploadAttachment', () => { + let dir: string + let file: string + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), `ol-attach-${process.pid}-`)) + file = join(dir, 'note.txt') + await writeFile(file, 'hello') + apiMocks.apiRequest.mockReset() + apiMocks.authorizedFetch.mockReset() + apiMocks.toApiError.mockClear() + }) + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }) + }) + + it('registers the attachment then posts the form to the upload URL', async () => { + apiMocks.apiRequest.mockResolvedValueOnce({ + data: { + mode: 'post', + uploadUrl: '/api/files.create', + form: { key: 'uploads/x/note.txt', sig: 'signed' }, + attachment: attachment(), + }, + }) + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: Response.json({ success: true }), + url: `${BASE}/api/x`, + onInstance: true, + }) + + const { uploadAttachment } = await import('./attachments.js') + const result = await uploadAttachment({ filePath: file, documentId: DOC_ID }) + + expect(apiMocks.apiRequest).toHaveBeenCalledWith('attachments.create', { + name: 'note.txt', + documentId: DOC_ID, + contentType: 'text/plain', + size: 5, + preset: 'documentAttachment', + }) + const [url, init] = apiMocks.authorizedFetch.mock.calls[0] + expect(url).toBe('/api/files.create') + expect(init.method).toBe('POST') + expect(init.headers['Content-Type']).toMatch(/^multipart\/form-data; boundary=/) + const body = await (init.body as Blob).text() + expect(body).toContain('name="sig"\r\n\r\nsigned\r\n') + expect(body).toContain('filename="note.txt"\r\nContent-Type: text/plain\r\n\r\nhello\r\n') + expect(result).toMatchObject({ id: ID, name: 'note.txt', size: 5 }) + }) + + it('treats a response without a mode as a form upload (older servers)', async () => { + apiMocks.apiRequest.mockResolvedValueOnce({ + data: { + uploadUrl: 'https://bucket.s3.amazonaws.com', + form: { policy: 'p' }, + attachment: attachment(), + }, + }) + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response(null, { status: 204 }), + url: 'https://bucket.s3.amazonaws.com/', + onInstance: false, + }) + + const { uploadAttachment } = await import('./attachments.js') + await uploadAttachment({ filePath: file, documentId: DOC_ID }) + + expect(apiMocks.authorizedFetch.mock.calls[0][0]).toBe('https://bucket.s3.amazonaws.com') + expect(apiMocks.authorizedFetch.mock.calls[0][1].method).toBe('POST') + }) + + it('uses a presigned PUT with the returned headers in put mode', async () => { + apiMocks.apiRequest.mockResolvedValueOnce({ + data: { + mode: 'put', + url: 'https://bucket.s3.amazonaws.com/x?X-Amz-Signature=s', + headers: { 'x-amz-acl': 'private' }, + attachment: attachment(), + }, + }) + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response(null, { status: 200 }), + url: 'https://bucket.s3.amazonaws.com/x', + onInstance: false, + }) + + const { uploadAttachment } = await import('./attachments.js') + await uploadAttachment({ + filePath: file, + documentId: DOC_ID, + name: 'renamed.md', + contentType: 'text/markdown', + }) + + expect(apiMocks.apiRequest.mock.calls[0][1]).toMatchObject({ + name: 'renamed.md', + contentType: 'text/markdown', + }) + const [url, init] = apiMocks.authorizedFetch.mock.calls[0] + expect(url).toBe('https://bucket.s3.amazonaws.com/x?X-Amz-Signature=s') + expect(init.method).toBe('PUT') + expect(init.headers).toEqual({ 'Content-Type': 'text/markdown', 'x-amz-acl': 'private' }) + expect(await (init.body as Blob).text()).toBe('hello') + }) + + it('deletes the registered attachment when the transfer fails', async () => { + apiMocks.apiRequest + .mockResolvedValueOnce({ + data: { mode: 'post', uploadUrl: '/api/files.create', attachment: attachment() }, + }) + .mockResolvedValueOnce({ data: undefined }) + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response('nope', { status: 413 }), + url: `${BASE}/api/x`, + onInstance: true, + }) + + const { uploadAttachment } = await import('./attachments.js') + await expect(uploadAttachment({ filePath: file, documentId: DOC_ID })).rejects.toThrow( + 'Upload of note.txt failed: API error: 413', + ) + expect(apiMocks.apiRequest).toHaveBeenLastCalledWith('attachments.delete', { id: ID }) + }) + + it('reports S3 failures as storage errors with the XML message', async () => { + apiMocks.apiRequest + .mockResolvedValueOnce({ + data: { + mode: 'post', + uploadUrl: 'https://bucket.s3.amazonaws.com', + attachment: attachment(), + }, + }) + .mockResolvedValueOnce({ data: undefined }) + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response( + 'AccessDeniedAccess Denied', + { status: 403, statusText: 'Forbidden' }, + ), + url: 'https://bucket.s3.amazonaws.com/', + onInstance: false, + }) + + const { uploadAttachment } = await import('./attachments.js') + await expect(uploadAttachment({ filePath: file, documentId: DOC_ID })).rejects.toThrow( + 'Upload of note.txt failed: Storage error: 403 Forbidden (Access Denied)', + ) + expect(apiMocks.toApiError).not.toHaveBeenCalled() + expect(apiMocks.apiRequest).toHaveBeenLastCalledWith('attachments.delete', { id: ID }) + }) + + it('fails before any request when the file is missing', async () => { + const { uploadAttachment } = await import('./attachments.js') + await expect( + uploadAttachment({ filePath: join(dir, 'missing.txt'), documentId: DOC_ID }), + ).rejects.toThrow('File not found') + await expect(uploadAttachment({ filePath: dir, documentId: DOC_ID })).rejects.toThrow( + 'File not found', + ) + expect(apiMocks.apiRequest).not.toHaveBeenCalled() + }) +}) + +describe('downloadAttachment', () => { + let dir: string + const cwd = process.cwd() + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), `ol-download-${process.pid}-`)) + process.chdir(dir) + apiMocks.authorizedFetch.mockReset() + apiMocks.toApiError.mockClear() + }) + + afterEach(async () => { + process.chdir(cwd) + await rm(dir, { recursive: true, force: true }) + }) + + function redirectThenFile(content: string, disposition = 'attachment; filename="a.txt"') { + apiMocks.authorizedFetch + .mockResolvedValueOnce({ + response: new Response(null, { + status: 302, + headers: { location: '/api/files.get?sig=signed' }, + }), + url: `${BASE}/api/attachments.redirect?id=${ID}`, + onInstance: true, + }) + .mockResolvedValueOnce({ + response: new Response(content, { + headers: { 'content-disposition': disposition }, + }), + url: `${BASE}/api/x`, + onInstance: true, + }) + } + + it('follows the redirect by hand and saves under the server file name', async () => { + redirectThenFile('content') + + const { downloadAttachment } = await import('./attachments.js') + const result = await downloadAttachment(`/api/attachments.redirect?id=${ID}`) + + expect(apiMocks.authorizedFetch.mock.calls[0]).toEqual([ + `/api/attachments.redirect?id=${ID}`, + { method: 'GET', redirect: 'manual' }, + ]) + expect(apiMocks.authorizedFetch.mock.calls[1][0]).toBe(`${BASE}/api/files.get?sig=signed`) + expect(result).toEqual({ id: ID, name: 'a.txt', path: 'a.txt', size: 7 }) + expect(await readFile(join(dir, 'a.txt'), 'utf-8')).toBe('content') + }) + + it('writes into an existing directory passed as --output', async () => { + redirectThenFile('content') + const { downloadAttachment } = await import('./attachments.js') + const result = await downloadAttachment(ID, { output: dir }) + expect(result.path).toBe(join(dir, 'a.txt')) + }) + + it('refuses to overwrite unless forced, and keeps the existing file', async () => { + await writeFile(join(dir, 'a.txt'), 'original') + redirectThenFile('new') + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID)).rejects.toThrow('File already exists: a.txt') + expect(await readFile(join(dir, 'a.txt'), 'utf-8')).toBe('original') + + redirectThenFile('new') + await downloadAttachment(ID, { force: true }) + expect(await readFile(join(dir, 'a.txt'), 'utf-8')).toBe('new') + }) + + it('keeps an existing file when the response stream fails before the file opens', async () => { + await writeFile(join(dir, 'a.txt'), 'original') + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error('connection reset')) + }, + }), + { headers: { 'content-disposition': 'attachment; filename="a.txt"' } }, + ), + url: `${BASE}/api/x`, + onInstance: true, + }) + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID)).rejects.toThrow() + expect(await readFile(join(dir, 'a.txt'), 'utf-8')).toBe('original') + }) + + it('removes the partial new file when the response stream fails', async () => { + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('part')) + controller.error(new Error('connection reset')) + }, + }), + { headers: { 'content-disposition': 'attachment; filename="b.txt"' } }, + ), + url: `${BASE}/api/x`, + onInstance: true, + }) + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID)).rejects.toThrow('connection reset') + await expect(readFile(join(dir, 'b.txt'))).rejects.toThrow('ENOENT') + }) + + it('creates a missing folder when --output ends in a slash', async () => { + redirectThenFile('content') + const { downloadAttachment } = await import('./attachments.js') + const result = await downloadAttachment(ID, { output: './new/nested/' }) + expect(result.path).toBe(join('new', 'nested', 'a.txt')) + expect(await readFile(join(dir, 'new', 'nested', 'a.txt'), 'utf-8')).toBe('content') + }) + + it('says the folder is missing instead of a raw ENOENT', async () => { + redirectThenFile('content') + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID, { output: './missing/a.txt' })).rejects.toThrow( + 'Folder does not exist: ./missing', + ) + }) + + it('names the file from the served URL when there is no Content-Disposition', async () => { + apiMocks.authorizedFetch + .mockResolvedValueOnce({ + response: new Response(null, { + status: 302, + headers: { + location: 'https://bucket.s3.amazonaws.com/uploads/u/a/plan.pdf?s=1', + }, + }), + url: `${BASE}/api/attachments.redirect?id=${ID}`, + onInstance: true, + }) + .mockResolvedValueOnce({ + response: new Response('pdf'), + url: 'https://bucket.s3.amazonaws.com/uploads/u/a/plan.pdf?s=1', + onInstance: false, + }) + const { downloadAttachment } = await import('./attachments.js') + const result = await downloadAttachment(ID) + expect(apiMocks.authorizedFetch.mock.calls[1][0]).toBe( + 'https://bucket.s3.amazonaws.com/uploads/u/a/plan.pdf?s=1', + ) + expect(result.name).toBe('plan.pdf') + }) + + it.each([ + [ + 'a storage redirect', + 'https://bucket.s3.amazonaws.com/a?s=1', + '/signed/file?s=2', + 'https://bucket.s3.amazonaws.com/signed/file?s=2', + ], + [ + 'a sub-path install', + 'https://example.com/wiki/api/attachments.redirect?id=1', + '/wiki/api/files.get?sig=s', + 'https://example.com/wiki/api/files.get?sig=s', + ], + ])( + 'resolves a root-relative redirect against the requested URL (%s)', + async (_case, requested, location, expected) => { + apiMocks.authorizedFetch + .mockResolvedValueOnce({ + response: new Response(null, { status: 302, headers: { location } }), + url: requested, + onInstance: false, + }) + .mockResolvedValueOnce({ + response: new Response('x', { + headers: { 'content-disposition': 'attachment; filename="a.txt"' }, + }), + url: expected, + onInstance: false, + }) + const { downloadAttachment } = await import('./attachments.js') + await downloadAttachment(ID) + expect(apiMocks.authorizedFetch.mock.calls[1][0]).toBe(expected) + }, + ) + + it('cancels the redirect response body before following it', async () => { + const cancel = vi.fn() + apiMocks.authorizedFetch + .mockResolvedValueOnce({ + response: new Response(new ReadableStream({ cancel }), { + status: 302, + headers: { location: '/api/files.get?sig=signed' }, + }), + url: `${BASE}/api/x`, + onInstance: true, + }) + .mockResolvedValueOnce({ + response: new Response('content', { + headers: { 'content-disposition': 'attachment; filename="a.txt"' }, + }), + url: `${BASE}/api/x`, + onInstance: true, + }) + const { downloadAttachment } = await import('./attachments.js') + await downloadAttachment(ID) + expect(cancel).toHaveBeenCalled() + }) + + it('keeps a dangling symlink when the response stream fails before the file opens', async () => { + await symlink(join(dir, 'nowhere.txt'), join(dir, 'a.txt')) + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error('connection reset')) + }, + }), + { headers: { 'content-disposition': 'attachment; filename="a.txt"' } }, + ), + url: `${BASE}/api/x`, + onInstance: true, + }) + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID)).rejects.toThrow() + expect(await readlink(join(dir, 'a.txt'))).toBe(join(dir, 'nowhere.txt')) + }) + + it('reports Outline errors such as a missing attachment', async () => { + apiMocks.authorizedFetch.mockResolvedValueOnce({ + response: new Response('{}', { status: 404 }), + url: `${BASE}/api/x`, + onInstance: true, + }) + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID)).rejects.toThrow( + `Download of attachment ${ID} failed: API error: 404`, + ) + }) + + it('stops after too many redirects', async () => { + apiMocks.authorizedFetch.mockResolvedValue({ + response: new Response(null, { status: 302, headers: { location: '/loop' } }), + url: `${BASE}/api/x`, + onInstance: true, + }) + const { downloadAttachment } = await import('./attachments.js') + await expect(downloadAttachment(ID)).rejects.toThrow('Too many redirects') + }) +}) diff --git a/src/lib/attachments.ts b/src/lib/attachments.ts new file mode 100644 index 0000000..d8960e1 --- /dev/null +++ b/src/lib/attachments.ts @@ -0,0 +1,477 @@ +import { randomUUID } from 'node:crypto' +import { createWriteStream, openAsBlob } from 'node:fs' +import { lstat, mkdir, stat, unlink } from 'node:fs/promises' +import { basename, dirname, extname, join, sep } from 'node:path' +import { Readable } from 'node:stream' +import { pipeline } from 'node:stream/promises' +import type { ReadableStream as WebReadableStream } from 'node:stream/web' +import { type Tokens, marked } from 'marked' +import { type AuthorizedFetchResult, apiRequest, authorizedFetch, toApiError } from './api.js' +import { CliError } from './errors.js' +import { withSpinner } from './spinner.js' + +export type Attachment = { + id: string + name: string + contentType: string + size: number + documentId: string | null + url: string +} + +/** Response of `attachments.create`; servers before upload modes omit `mode`. */ +type CreateAttachmentResponse = { + mode?: 'post' | 'put' + uploadUrl?: string + form?: Record + url?: string + headers?: Record + attachment: Attachment +} + +/** An attachment or image linked from a document's markdown. */ +export type LinkedAttachment = { + id: string + name: string + /** Size in bytes from the link label (file attachments only). */ + size: number | null + type: 'image' | 'file' + url: string +} + +const UUID = '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}' +const UUID_RE = new RegExp(`^${UUID}$`) +const REDIRECT_ID_RE = new RegExp(`attachments\\.redirect\\?(?:[^#\\s)]*&)?id=(${UUID})`) +const MAX_REDIRECTS = 5 + +const CONTENT_TYPES: Record = { + '.avif': 'image/avif', + '.bmp': 'image/bmp', + '.csv': 'text/csv', + '.doc': 'application/msword', + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.gif': 'image/gif', + '.gz': 'application/gzip', + '.heic': 'image/heic', + '.htm': 'text/html', + '.html': 'text/html', + '.jpeg': 'image/jpeg', + '.jpg': 'image/jpeg', + '.json': 'application/json', + '.md': 'text/markdown', + '.mov': 'video/quicktime', + '.mp3': 'audio/mpeg', + '.mp4': 'video/mp4', + '.pdf': 'application/pdf', + '.png': 'image/png', + '.ppt': 'application/vnd.ms-powerpoint', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + '.svg': 'image/svg+xml', + '.tar': 'application/x-tar', + '.txt': 'text/plain', + '.wav': 'audio/wav', + '.webm': 'video/webm', + '.webp': 'image/webp', + '.xls': 'application/vnd.ms-excel', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.xml': 'application/xml', + '.zip': 'application/zip', +} + +/** Guess a content type from a file name; unknown extensions are binary. */ +export function guessContentType(name: string): string { + return CONTENT_TYPES[extname(name).toLowerCase()] ?? 'application/octet-stream' +} + +/** The path Outline uses in document markdown to link an attachment. */ +export function attachmentPath(id: string): string { + return `/api/attachments.redirect?id=${id}` +} + +/** + * Markdown that Outline renders as the attachment: an image for image types, + * otherwise a file attachment, whose link label is ` `. + * Brackets and backslashes are dropped from the label rather than escaped: + * Outline's attachment rule reads only the first text token of the label, so + * an escaped `\[` cuts the stored title short. The file keeps its real name. + */ +export function attachmentMarkdown(attachment: Attachment): string { + const path = attachmentPath(attachment.id) + const name = attachment.name.replace(/[[\]\\]/g, '') + if (attachment.contentType.startsWith('image/')) { + return `![${name}](${path})` + } + return `[${name} ${attachment.size}](${path})` +} + +/** + * Extract an attachment ID from a UUID, an `attachments.redirect` path, or a + * full URL containing one. + */ +export function parseAttachmentId(ref: string): string { + const trimmed = ref.trim() + if (UUID_RE.test(trimmed)) return trimmed.toLowerCase() + const match = trimmed.match(REDIRECT_ID_RE) + if (match) return match[1].toLowerCase() + throw new CliError('INVALID_ATTACHMENT', `Not an attachment ID or link: "${ref}".`, [ + 'Pass the attachment ID, or a link containing /api/attachments.redirect?id=', + 'Run ol attachment list --doc to see the attachments in a document', + ]) +} + +/** + * Find the attachments and images a document links to, in order of first + * appearance. Outline writes file attachments as `[ ]()` + * and images as `![]()`. The markdown is tokenized rather than + * scanned with a regex, so escaped or nested brackets in a label are read + * correctly and links inside inline or fenced code are not counted. + */ +export function extractLinkedAttachments(markdown: string): LinkedAttachment[] { + const seen = new Set() + const result: LinkedAttachment[] = [] + marked.walkTokens(marked.lexer(markdown), (token) => { + if (token.type !== 'link' && token.type !== 'image') return + const { href, text } = token as Tokens.Link | Tokens.Image + const match = href.match(REDIRECT_ID_RE) + if (!match) return + const id = match[1].toLowerCase() + if (seen.has(id)) return + seen.add(id) + if (token.type === 'image') { + result.push({ id, name: text, size: null, type: 'image', url: href }) + return + } + const label = text.trim() + const sizeMatch = label.match(/^(.+) (\d+)$/) + result.push({ + id, + name: sizeMatch ? sizeMatch[1] : label, + size: sizeMatch ? Number(sizeMatch[2]) : null, + type: 'file', + url: href, + }) + }) + return result +} + +/** Escape a multipart header value the way browsers do (WHATWG HTML form encoding). */ +function escapeMultipartValue(value: string): string { + return value.replace(/\r/g, '%0D').replace(/\n/g, '%0A').replace(/"/g, '%22') +} + +/** + * Encode form fields plus one file as a `multipart/form-data` body. + * + * The body is a `Blob` rather than a `FormData` because requests go through + * the bundled undici `fetch` (see `http-dispatcher.ts`), which does not accept + * Node's global `FormData` and would send it as the string + * `[object FormData]`. A `Blob` assembled from the file's lazy `Blob` is + * accepted by both `fetch` implementations, keeps a known `Content-Length` + * (presigned S3 POSTs reject chunked bodies), and is never read into memory. + */ +export function buildMultipartBody( + fields: Record, + file: { field: string; name: string; contentType: string; blob: Blob }, +): { body: Blob; contentType: string } { + const boundary = `----olFormBoundary${randomUUID().replace(/-/g, '')}` + const parts: (string | Blob)[] = [] + for (const [key, value] of Object.entries(fields)) { + parts.push( + `--${boundary}\r\nContent-Disposition: form-data; name="${escapeMultipartValue(key)}"\r\n\r\n${value}\r\n`, + ) + } + parts.push( + `--${boundary}\r\nContent-Disposition: form-data; name="${escapeMultipartValue(file.field)}"; filename="${escapeMultipartValue(file.name)}"\r\nContent-Type: ${file.contentType}\r\n\r\n`, + file.blob, + `\r\n--${boundary}--\r\n`, + ) + return { body: new Blob(parts), contentType: `multipart/form-data; boundary=${boundary}` } +} + +/** + * Error for a failed transfer to third-party storage (S3 and compatibles), + * which answers with an XML error body rather than Outline's JSON. + */ +async function toStorageError(res: Response): Promise { + let detail = '' + try { + detail = (await res.text()).match(/([^<]*)<\/Message>/)?.[1]?.trim() ?? '' + } catch {} + const status = `${res.status} ${res.statusText}`.trim() + return new Error(`Storage error: ${status}${detail ? ` (${detail})` : ''}`) +} + +export type UploadOptions = { + filePath: string + documentId: string + name?: string + contentType?: string +} + +/** + * Upload a local file as an attachment of a document: register it with + * `attachments.create`, then send the bytes to the upload target Outline + * returns (its own `files.create` endpoint for local storage, or a presigned + * S3 POST/PUT). A failed transfer deletes the registered attachment. + */ +export async function uploadAttachment(opts: UploadOptions): Promise { + let size: number + try { + const info = await stat(opts.filePath) + if (!info.isFile()) throw new Error('not a file') + size = info.size + } catch { + throw new CliError('FILE_NOT_FOUND', `File not found: ${opts.filePath}`, [ + 'Pass the path of an existing file', + ]) + } + + const name = opts.name ?? basename(opts.filePath) + const contentType = opts.contentType ?? guessContentType(name) + + const { data } = await apiRequest('attachments.create', { + name, + documentId: opts.documentId, + contentType, + size, + preset: 'documentAttachment', + }) + + try { + await withSpinner({ text: 'Uploading file...', color: 'green' }, async () => { + const blob = await openAsBlob(opts.filePath, { type: contentType }) + let result: AuthorizedFetchResult + if (data.mode === 'put') { + if (!data.url) throw new Error('Outline did not return an upload URL') + result = await authorizedFetch(data.url, { + method: 'PUT', + headers: { 'Content-Type': contentType, ...data.headers }, + body: blob, + }) + } else { + if (!data.uploadUrl) throw new Error('Outline did not return an upload URL') + const multipart = buildMultipartBody(data.form ?? {}, { + field: 'file', + name, + contentType, + blob, + }) + result = await authorizedFetch(data.uploadUrl, { + method: 'POST', + headers: { 'Content-Type': multipart.contentType }, + body: multipart.body, + }) + } + if (!result.response.ok) { + throw result.onInstance + ? await toApiError(result.response) + : await toStorageError(result.response) + } + }) + } catch (error) { + try { + await apiRequest('attachments.delete', { id: data.attachment.id }) + } catch {} + const message = error instanceof Error ? error.message : String(error) + throw new CliError('UPLOAD_FAILED', `Upload of ${name} failed: ${message}`) + } + + return { ...data.attachment, size } +} + +/** + * Reduce a server-supplied name to a safe base name: no directories (so it + * cannot point outside the chosen folder) and no control characters (so it + * cannot write escape sequences to the terminal when printed). + */ +export function sanitizeFileName(name: string): string | undefined { + let clean = '' + for (const char of basename(name.replace(/\\/g, '/'))) { + const code = char.codePointAt(0) ?? 0 + if (code >= 0x20 && code !== 0x7f) clean += char + } + clean = clean.trim() + return clean && clean !== '.' && clean !== '..' ? clean : undefined +} + +/** + * File name from a `Content-Disposition` header, preferring the RFC 5987 + * `filename*` form. + */ +export function fileNameFromDisposition(header: string | null): string | undefined { + if (!header) return undefined + let name: string | undefined + const extended = header.match(/filename\*\s*=\s*(?:UTF-8|utf-8)?''([^;]+)/) + if (extended) { + try { + name = decodeURIComponent(extended[1].trim()) + } catch {} + } + if (!name) { + const plain = header.match(/filename\s*=\s*"((?:[^"\\]|\\.)*)"|filename\s*=\s*([^;]+)/) + name = plain ? (plain[1] ?? plain[2])?.replace(/\\(.)/g, '$1').trim() : undefined + } + return name ? sanitizeFileName(name) : undefined +} + +/** + * File name from the URL the content was served from, for responses without + * a `Content-Disposition` name: the storage key's last segment, from the + * `key` parameter of Outline's `files.get` or from an S3 object path. + */ +export function fileNameFromUrl(url: string): string | undefined { + let parsed: URL + try { + parsed = new URL(url) + } catch { + return undefined + } + if (parsed.pathname.endsWith('/api/files.get')) { + const key = parsed.searchParams.get('key') + return key ? sanitizeFileName(key.split('/').pop() ?? '') : undefined + } + const last = parsed.pathname.split('/').filter(Boolean).pop() + if (!last) return undefined + try { + return sanitizeFileName(decodeURIComponent(last)) + } catch { + return sanitizeFileName(last) + } +} + +export type DownloadTarget = { + /** + * `-` for stdout, a directory (an existing one, or any path ending in `/`, + * which is created), or a file path; defaults to the current directory. + */ + output?: string + force?: boolean +} + +export type DownloadResult = { + id: string + name: string + /** Written path, or `-` for stdout. */ + path: string + size: number +} + +/** + * Open the attachment's content. `attachments.redirect` answers with a + * redirect to a signed URL; redirects are followed by hand so the bearer + * token is only sent to the Outline instance itself. + */ +async function openAttachment(id: string): Promise<{ response: Response; url: string }> { + let url = attachmentPath(id) + for (let hop = 0; hop <= MAX_REDIRECTS; hop++) { + const { response, url: requested } = await authorizedFetch(url, { + method: 'GET', + redirect: 'manual', + }) + const location = response.headers.get('location') + if (response.status >= 300 && response.status < 400 && location) { + // Release the connection before following the redirect. + await response.body?.cancel().catch(() => {}) + // Standard redirect semantics: resolve against the URL just + // requested. authorizedFetch scopes the token by origin, so a + // storage URL never receives it. + url = new URL(location, requested).toString() + continue + } + if (!response.ok) throw await toApiError(response) + return { response, url: requested } + } + throw new Error(`Too many redirects downloading attachment ${id}`) +} + +async function resolveOutputPath(output: string | undefined, name: string): Promise { + if (!output) return name + if (output.endsWith('/') || output.endsWith(sep)) { + try { + await mkdir(output, { recursive: true }) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new CliError('DOWNLOAD_FAILED', `Cannot create folder ${output}: ${message}`) + } + return join(output, name) + } + try { + if ((await stat(output)).isDirectory()) return join(output, name) + } catch {} + return output +} + +/** Download an attachment to a file (or stdout when `output` is `-`). */ +export async function downloadAttachment( + ref: string, + target: DownloadTarget = {}, +): Promise { + const id = parseAttachmentId(ref) + let response: Response + let servedFrom: string + try { + ;({ response, url: servedFrom } = await openAttachment(id)) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + throw new CliError('DOWNLOAD_FAILED', `Download of attachment ${id} failed: ${message}`) + } + const name = + fileNameFromDisposition(response.headers.get('content-disposition')) ?? + fileNameFromUrl(servedFrom) ?? + id + if (!response.body) { + throw new CliError('DOWNLOAD_FAILED', `Attachment ${id} returned no content.`) + } + const webBody = response.body as WebReadableStream + // Wrap the body only right before piping it: a Node stream that errors + // while nothing listens (e.g. during the awaits below) is an uncaught error. + const toNode = () => Readable.fromWeb(webBody) + let size = 0 + // Count bytes inside the pipeline: a 'data' listener would switch the + // stream to flowing mode and drop chunks emitted before the pipe is set up. + async function* countBytes(source: AsyncIterable) { + for await (const chunk of source) { + size += chunk.length + yield chunk + } + } + + if (target.output === '-') { + await pipeline(toNode(), countBytes, process.stdout, { end: false }) + return { id, name, path: '-', size } + } + + const path = await resolveOutputPath(target.output, name) + // Only a file this download created or opened may be removed on failure; + // a failure before the open (the response stream erroring first) must + // never touch a file that was already there. + // `lstat` so a dangling symlink counts as existing; only ENOENT is absence. + const existedBefore = await lstat(path).then( + () => true, + (error: NodeJS.ErrnoException) => error.code !== 'ENOENT', + ) + const destination = createWriteStream(path, { flags: target.force ? 'w' : 'wx' }) + let opened = false + destination.once('open', () => { + opened = true + }) + try { + await pipeline(toNode(), countBytes, destination) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new CliError('FILE_EXISTS', `File already exists: ${path}`, [ + 'Use --force to overwrite it, or --output to choose another path', + ]) + } + // Do not leave a partial file behind. + if (opened || !existedBefore) await unlink(path).catch(() => {}) + if (!opened && (error as NodeJS.ErrnoException).code === 'ENOENT') { + throw new CliError('DOWNLOAD_FAILED', `Folder does not exist: ${dirname(path)}`, [ + 'End --output with / to have the folder created', + ]) + } + const message = error instanceof Error ? error.message : String(error) + throw new CliError('DOWNLOAD_FAILED', `Download of attachment ${id} failed: ${message}`) + } + return { id, name, path, size } +} diff --git a/src/lib/errors.ts b/src/lib/errors.ts index 13a6c0c..f0d33e5 100644 --- a/src/lib/errors.ts +++ b/src/lib/errors.ts @@ -9,6 +9,10 @@ export type ErrorCode = | 'CONFIRMATION_REQUIRED' | 'CONFLICTING_OPTIONS' | 'DOCUMENT_CHANGED' + | 'DOWNLOAD_FAILED' + | 'FILE_EXISTS' + | 'FILE_NOT_FOUND' + | 'INVALID_ATTACHMENT' | 'INVALID_PARENT' | 'INVALID_TIMESTAMP' | 'INVALID_USER_FLAG' @@ -19,6 +23,7 @@ export type ErrorCode = | 'OAUTH_CLIENT_ID_REQUIRED' | 'OAUTH_LOGIN_FAILED' | 'UNKNOWN_AGENT' + | 'UPLOAD_FAILED' export class CliError extends CoreCliError { constructor( diff --git a/src/lib/skills/content.ts b/src/lib/skills/content.ts index 0387a83..d197ab5 100644 --- a/src/lib/skills/content.ts +++ b/src/lib/skills/content.ts @@ -1,6 +1,6 @@ export const SKILL_NAME = 'outline-cli' export const SKILL_DESCRIPTION = - 'Search and manage Outline wiki documents and collections via the ol CLI' + 'Search and manage Outline wiki documents, collections and attachments via the ol CLI' export const SKILL_CONTENT = `# Outline CLI (ol) @@ -14,6 +14,8 @@ Use this skill when the user wants to interact with their Outline wiki/knowledge - \`ol doc open \` - Open document in browser - \`ol doc create --title "Title" --collection \` - Create document - \`ol col list\` - List collections +- \`ol attachment upload --doc \` - Upload a file to a document +- \`ol attachment download \` - Download an attachment - \`ol account\` - List stored accounts; \`ol account use \` sets the default ## Output Formats @@ -82,6 +84,23 @@ ol col update --name "New Name" ol col delete --confirm \`\`\` +### Attachments (alias: attach) +\`\`\`bash +ol attachment upload ./plan.pdf --doc # Upload; prints the markdown link to paste into the doc +ol attachment upload ./plan.pdf --doc --append # Upload and append the link to the end of the doc +ol attachment upload ./photo.jpg --doc --name "Site photo.jpg" --content-type image/jpeg +ol attachment upload ./plan.pdf --doc --json # {id, name, contentType, size, documentId, url, markdown} +ol attachment list --doc # Attachments and images linked in the doc (id, name, size, type) +ol attachment list --doc --json +ol attachment download # Save under the original file name in the current directory +ol attachment download --output ./files/ # Into a folder (created when the path ends in /), or --output +ol attachment download --output - # Write the file to stdout +ol attachment download --force # Overwrite an existing file (refused otherwise) +ol attachment delete --confirm # Delete the file; links to it in documents stay +\`\`\` +An attachment is referenced by its ID or any link containing \`/api/attachments.redirect?id=\`. Images are linked as \`![alt](/api/attachments.redirect?id=)\`, other files as \`[ ](/api/attachments.redirect?id=)\`; keep that form when writing it into a document yourself. +\`attachment list\` reads those links from the document text, so a plain link whose text ends in a number (\`[report 2024](…attachments.redirect?id=…)\`) lists as name \`report\`, size 2024, and links to another Outline instance's attachments are listed but cannot be downloaded from this one. + ### Authentication \`\`\`bash ol auth login # OAuth login (opens browser); prompts for base URL + client ID if needed @@ -148,6 +167,14 @@ ol doc update --file doc.md --if-updated-at "$ts" \`\`\` If the document was saved after \`$ts\`, the update fails with \`DOCUMENT_CHANGED\`, or with Outline's "Document has been modified since the provided revision" when the save lands during the update; re-read and retry. Edits not yet saved from an open editor are not detected. +### Attach a file and download every attachment of a document +\`\`\`bash +ol attachment upload ./drawing.pdf --doc --append +ol attachment list --doc --json | jq -r '.[].id' | while read -r id; do + ol attachment download "$id" --output "./downloads/$id/" # one folder each: names often repeat +done +\`\`\` + ### List all collections and their documents \`\`\`bash ol col list --json