From 240132d49e5de975ca04102ead8d8ec9a64360d8 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:26:54 +0000 Subject: [PATCH 1/6] lift the oauth sign-in routes out of mcp-connect into connect-flow --- packages/api/src/connections/connect-flow.ts | 229 ++++++++++++++++ packages/api/src/connections/custom/route.ts | 31 ++- .../api/src/connections/mcp/mcp-connect.ts | 246 +++--------------- packages/api/src/connections/mcp/mcp-oauth.ts | 8 - 4 files changed, 278 insertions(+), 236 deletions(-) create mode 100644 packages/api/src/connections/connect-flow.ts diff --git a/packages/api/src/connections/connect-flow.ts b/packages/api/src/connections/connect-flow.ts new file mode 100644 index 00000000..9df6edd5 --- /dev/null +++ b/packages/api/src/connections/connect-flow.ts @@ -0,0 +1,229 @@ +import {createHash, randomBytes} from 'node:crypto'; + +import type {Response} from 'express'; +import {Err, type Result} from 'ts-results-es'; +import {z} from 'zod'; + +import {ApiErr, type ApiError, Do, parseCookieHeader, requirePresent} from '@proxy/utils'; +import {createConnection, listConnections, updateConnectionCredential, updateConnectionName, updateConnectionTools, type ConnectionRow} from '@proxy/db/connection'; +import type {IntegrationId, McpTool} from '@proxy/integrations'; + +import {queryParam, timingSafeEqualString, useSecureCookies} from '../auth/session'; +import {log, serializeError} from '../observability/log'; +import type {AuthenticatedRequest} from '../server/middleware/require-auth'; +import {env} from '../utils/env'; +import {encryptSecret} from '../utils/secret-crypto'; +import {readStoredJson} from '../utils/stored-json'; +import {requireUserOrgId} from '../utils/user-org'; + +/** + * Connecting an integration by signing in to it with OAuth and PKCE, as Granola, Notion and Linear + * are through their MCP servers, and a custom MCP server found from its address. A sign-in begins + * on the api, which keeps the handshake in a short-lived encrypted cookie, and the browser goes to + * the provider's sign-in page: from the start route, a top-level navigation under + * `/api/connections/`, for a provider known in advance (`signInRoutes`), or from the + * answer of the route that found it, as a custom server's connect route (`signInFlow`). The + * callback checks the provider's answer against the cookie, trades the code for tokens and makes + * the connection, or gives an existing one of the same account its new tokens, which is how + * signing in again works, and ends on the connection's page, or back on the catalog with + * `?error=`; a sign-in begun from the welcome flow (`returnTo: 'welcome'`) ends on its + * page for the connection, or its catalog, instead. What the provider's sign-in page is and how a + * code becomes tokens and a connection is the integration's. + */ + +const stateCookieMaxAgeMs = 15 * 60 * 1000; + +/** Where a sign-in ends when not on the connection's page: the welcome flow it began from. */ +export const returnToSchema = z.enum(['welcome']); + +type ReturnTo = z.infer; + +type Route = (request: AuthenticatedRequest, res: Response) => Promise; + +/** What the provider's sign-in is sent: where it comes back to, the state, and the PKCE challenge. */ +export type SignInRequest = {redirectUri: string; state: string; challenge: string}; + +/** The provider's sign-in page, and what to keep in the cookie for the callback. */ +type SignInStart = (request: SignInRequest) => Promise>; + +/** The provider's answer coming back, with the verifier and what the start kept for it. */ +export type SignInAnswer = {redirectUri: string; code: string; verifier: string; kept: E}; + +/** + * A connection to save: the account it shows, its credential, stored encrypted, the actions agents + * get with a new one, and a custom MCP server's tools and the name the person gave it. + */ +export type Connected = {account: string; credential: C; defaults: string[]; tools?: McpTool[]; name?: string}; + +/** A PKCE verifier and the challenge the provider is sent in its place. */ +export function makePkce(): {verifier: string; challenge: string} { + const verifier = randomBytes(32).toString('base64url'); + return {verifier, challenge: createHash('sha256').update(verifier).digest('base64url')}; +} + +/** + * Saves a connection: a new one, or an existing one of the same account given the new credential, + * and the tools and the name, where it has its own. + */ +export function saveConnection(orgId: string, integrationId: IntegrationId, connected: Connected): Promise> { + return Do(async ($) => { + const encrypted = $(encryptSecret(JSON.stringify(connected.credential))); + const existing = $(await listConnections(orgId)).find((connection) => connection.integrationId === integrationId && connection.account === connected.account); + const name = connected.name === undefined ? {} : {name: connected.name}; + if (!existing) { + const tools = connected.tools === undefined ? {} : {tools: JSON.stringify(connected.tools)}; + return $(await createConnection({orgId, integrationId, account: connected.account, credential: encrypted, defaults: connected.defaults, ...tools, ...name})); + } + $(await updateConnectionCredential({connectionId: existing.id, credential: encrypted})); + if (connected.name !== undefined) { + $(await updateConnectionName({connectionId: existing.id, name: connected.name})); + } + if (connected.tools === undefined) { + return {...existing, ...name}; + } + const tools = JSON.stringify(connected.tools); + $(await updateConnectionTools({connectionId: existing.id, tools, actions: connected.tools.map((tool) => tool.name)})); + return {...existing, tools, ...name}; + }); +} + +type SignInArgs = { + integrationId: IntegrationId; + // The provider's name in errors and logs: "Notion". + name: string; + // What the start keeps in the cookie for the callback beyond the handshake itself, such as the + // client an MCP server registered for this sign-in. + keptSchema: z.ZodType; + // The code traded for tokens, and the connection they make. + finish: (answer: SignInAnswer) => Promise, ApiError>>; +}; + +/** + * The sign-in to one integration: `beginSignIn` for the route that knows the provider, and the + * callback route. + */ +export function signInFlow(args: SignInArgs) { + const {integrationId, name} = args; + const stateCookieName = `proxy.${integrationId}-state`; + // Only sent to these routes. + const basePath = `/api/connections/${integrationId}`; + const statePayloadSchema = z.object({state: z.string(), verifier: z.string(), userId: z.string(), returnTo: returnToSchema.optional(), kept: args.keptSchema}); + + type StatePayload = {state: string; verifier: string; userId: string; returnTo?: ReturnTo; kept: E}; + + function redirectUri(appUrl: string): string { + return `${appUrl}${basePath}/callback`; + } + + function stateCookieOptions() { + return {httpOnly: true, sameSite: 'lax' as const, secure: useSecureCookies(), path: basePath}; + } + + function readStateCookie(request: AuthenticatedRequest): Result { + return Do(($) => { + const cookie = parseCookieHeader(request.headers.cookie ?? '').get(stateCookieName); + const stored = $(requirePresent(cookie, ApiErr.validationError(`No ${name} sign-in under way`))); + return $(readStoredJson(stored, statePayloadSchema)); + }); + } + + // Where the sign-in ends: the catalog, or the welcome flow's connections step it started from. + function catalogUrl(appUrl: string, returnTo: ReturnTo | undefined): string { + if (returnTo === 'welcome') { + return `${appUrl}/welcome/connections`; + } + return `${appUrl}/connections/new`; + } + + function redirectToConnectError(res: Response, appUrl: string, returnTo: ReturnTo | undefined, error: ApiError): void { + log.warn(`Connecting ${name} failed`, {kind: error.kind, ...('message' in error ? {message: error.message} : {}), ...('cause' in error ? serializeError(error.cause) : {})}); + res.redirect(`${catalogUrl(appUrl, returnTo)}?error=${integrationId}`); + } + + /** Keeps the handshake, and what `start` keeps beside it, in a cookie on `res`, and answers the provider's sign-in page, for the browser to go to. */ + function beginSignIn(res: Response, begin: {userId: string; returnTo: ReturnTo | undefined; start: SignInStart}): Promise> { + return Do(async ($) => { + const appUrl = $(requirePresent(env.APP_URL, ApiErr.internalError(new Error('APP_URL is not configured')))); + const state = randomBytes(16).toString('hex'); + const {verifier, challenge} = makePkce(); + const {url, kept} = $(await begin.start({redirectUri: redirectUri(appUrl), state, challenge})); + const payload: StatePayload = {state, verifier, userId: begin.userId, ...(begin.returnTo ? {returnTo: begin.returnTo} : {}), kept}; + const cookie = $(encryptSecret(JSON.stringify(payload))); + res.cookie(stateCookieName, cookie, {...stateCookieOptions(), maxAge: stateCookieMaxAgeMs}); + return url; + }); + } + + // The handshake the cookie holds, checked against the provider's answer, then the connection + // made or given its new tokens. + function finishConnect(request: AuthenticatedRequest, appUrl: string, payload: StatePayload): Promise> { + return Do(async ($) => { + const state = queryParam(request, 'state') ?? ''; + if (!timingSafeEqualString(state, payload.state) || payload.userId !== request.user.userId) { + return $(Err(ApiErr.validationError(`${name} sign-in state does not match`))); + } + + const code = queryParam(request, 'code'); + if (!code) { + return $(Err(ApiErr.validationError(`${name} sign-in ended without a code: ${queryParam(request, 'error') ?? 'unknown'}`))); + } + + const connected = $(await args.finish({redirectUri: redirectUri(appUrl), code, verifier: payload.verifier, kept: payload.kept})); + const orgId = $(await requireUserOrgId(request)); + return $(await saveConnection(orgId, integrationId, connected)).id; + }); + } + + /** + * `GET /api/connections//callback` — where the sign-in comes back to; lands on the + * connection, saying it was added, or on the welcome flow's page for it when it started there. + */ + async function handleCallback(request: AuthenticatedRequest, res: Response): Promise { + const appUrl = env.APP_URL; + if (!appUrl) { + res.status(500).json({error: 'app_url_not_configured'}); + return; + } + + // The handshake is single-use: cleared however the callback ends. + res.clearCookie(stateCookieName, stateCookieOptions()); + const payload = readStateCookie(request); + const returnTo = payload.isOk() ? payload.value.returnTo : undefined; + const connectionId = payload.isErr() ? Err(payload.error) : await finishConnect(request, appUrl, payload.value); + if (connectionId.isErr()) { + redirectToConnectError(res, appUrl, returnTo, connectionId.error); + return; + } + if (returnTo === 'welcome') { + res.redirect(`${appUrl}/welcome/connections/${connectionId.value}`); + return; + } + res.redirect(`${appUrl}/connections/${connectionId.value}?added`); + } + + return {beginSignIn, handleCallback, redirectToConnectError}; +} + +/** The routes for a provider known in advance: the start, a top-level navigation, and the callback. */ +export function signInRoutes(args: SignInArgs & {start: SignInStart}): {handleStart: Route; handleCallback: Route} { + const flow = signInFlow(args); + + /** `GET /api/connections//start` — sends the browser to the provider's sign-in. */ + async function handleStart(request: AuthenticatedRequest, res: Response): Promise { + const appUrl = env.APP_URL; + if (!appUrl) { + res.status(500).json({error: 'app_url_not_configured'}); + return; + } + + const returnTo = returnToSchema.safeParse(queryParam(request, 'returnTo')).data; + const begun = await flow.beginSignIn(res, {userId: request.user.userId, returnTo, start: args.start}); + if (begun.isErr()) { + flow.redirectToConnectError(res, appUrl, returnTo, begun.error); + return; + } + res.redirect(begun.value); + } + + return {handleStart, handleCallback: flow.handleCallback}; +} diff --git a/packages/api/src/connections/custom/route.ts b/packages/api/src/connections/custom/route.ts index a2009d7e..7f1ebcb4 100644 --- a/packages/api/src/connections/custom/route.ts +++ b/packages/api/src/connections/custom/route.ts @@ -9,9 +9,10 @@ import type {AuthenticatedRequest} from '../../server/middleware/require-auth'; import {sendResult} from '../../server/response'; import {requireUserOrgId} from '../../utils/user-org'; import {toConnectionResponse, type ConnectionResponse} from '../connection-response'; +import {returnToSchema, saveConnection, signInFlow, type SignInRequest} from '../connect-flow'; import {listMcpTools} from '../mcp/mcp-client'; -import {mcpSignInRoutes, returnToSchema, saveMcpConnection} from '../mcp/mcp-connect'; -import {mcpEndpoints} from '../mcp/mcp-oauth'; +import {startMcpSignIn} from '../mcp/mcp-connect'; +import {exchangeMcpCode, mcpEndpoints, mcpEndpointsSchema} from '../mcp/mcp-oauth'; import {listCustomTools, mcpAccount, readOnlyToolNames, type CustomCredential} from './custom-mcp'; import {discoverMcpServer, parseMcpServerUrl, probeMcpServer} from './mcp-discovery'; @@ -44,19 +45,22 @@ const connectBodySchema = z.object({ /** Connected, with the new connection, or the server asks for a sign-in, begun, at the page the browser goes to. */ export type ConnectCustomMcpResponse = {kind: 'connected'; connection: ConnectionResponse} | {kind: 'signIn'; url: string}; -const {beginSignIn, handleCallback: handleCustomMcpCallbackRoute} = mcpSignInRoutes({ +const {beginSignIn, handleCallback: handleCustomMcpCallbackRoute} = signInFlow({ integrationId: 'mcp', name: 'MCP server', - server: null, + // The server found from the address and the name the person gave the connection, beside the + // client registered for the sign-in. + keptSchema: z.object({clientId: z.string(), server: mcpEndpointsSchema, name: z.string()}), // Signed in, the server is asked for its tools with the new token, and the connection keeps its // endpoints beside the tokens, for refreshing them. - connection: async (signIn, server) => - (await listMcpTools(server, signIn.credential.accessToken)).map((tools) => ({ - account: mcpAccount(new URL(server.url)), - credential: {auth: 'oauth', url: server.url, server: mcpEndpoints(server), ...signIn.credential} satisfies CustomCredential, - defaults: readOnlyToolNames(tools), - tools, - })), + finish: ({redirectUri, code, verifier, kept}) => + Do(async ($) => { + const {server, name} = kept; + const signIn = $(await exchangeMcpCode(server, {clientId: kept.clientId, redirectUri, code, verifier})); + const tools = $(await listMcpTools(server, signIn.credential.accessToken)); + const credential: CustomCredential = {auth: 'oauth', url: server.url, server, ...signIn.credential}; + return {account: mcpAccount(new URL(server.url)), credential, defaults: readOnlyToolNames(tools), tools, name}; + }), }); export {handleCustomMcpCallbackRoute}; @@ -69,11 +73,12 @@ function connectCustomMcp(request: AuthenticatedRequest, res: Response): Promise const probe = $(await probeMcpServer(url, token)); if (probe.kind === 'signIn') { const server = $(await discoverMcpServer(url)); - return {kind: 'signIn' as const, url: $(await beginSignIn(res, {userId: request.user.userId, server, returnTo: body.returnTo, name: body.name}))}; + const start = (signIn: SignInRequest) => startMcpSignIn(server, signIn, {server: mcpEndpoints(server), name: body.name}); + return {kind: 'signIn' as const, url: $(await beginSignIn(res, {userId: request.user.userId, returnTo: body.returnTo, start}))}; } const credential: CustomCredential = token === null ? {auth: 'none', url: url.toString()} : {auth: 'token', url: url.toString(), token}; const orgId = $(await requireUserOrgId(request)); - const row = $(await saveMcpConnection(orgId, 'mcp', {account: mcpAccount(url), credential, defaults: readOnlyToolNames(probe.tools), tools: probe.tools, name: body.name})); + const row = $(await saveConnection(orgId, 'mcp', {account: mcpAccount(url), credential, defaults: readOnlyToolNames(probe.tools), tools: probe.tools, name: body.name})); return {kind: 'connected' as const, connection: toConnectionResponse(row)}; }); } diff --git a/packages/api/src/connections/mcp/mcp-connect.ts b/packages/api/src/connections/mcp/mcp-connect.ts index 4d0a3560..f853269a 100644 --- a/packages/api/src/connections/mcp/mcp-connect.ts +++ b/packages/api/src/connections/mcp/mcp-connect.ts @@ -1,231 +1,47 @@ -import {randomBytes} from 'node:crypto'; - -import type {Response} from 'express'; -import {Err, Ok, Result} from 'ts-results-es'; +import type {Result} from 'ts-results-es'; import {z} from 'zod'; -import {ApiErr, type ApiError, Do, parseCookieHeader, requirePresent} from '@proxy/utils'; -import {createConnection, listConnections, updateConnectionCredential, updateConnectionName, updateConnectionTools, type ConnectionRow} from '@proxy/db/connection'; -import type {IntegrationId, McpTool} from '@proxy/integrations'; +import {Do, type ApiError} from '@proxy/utils'; +import type {IntegrationId} from '@proxy/integrations'; -import {queryParam, timingSafeEqualString, useSecureCookies} from '../../auth/session'; -import {log, serializeError} from '../../observability/log'; -import type {AuthenticatedRequest} from '../../server/middleware/require-auth'; -import {env} from '../../utils/env'; -import {encryptSecret} from '../../utils/secret-crypto'; -import {readStoredJson} from '../../utils/stored-json'; -import {requireUserOrgId} from '../../utils/user-org'; -import {exchangeMcpCode, makePkce, mcpAuthorizeUrl, mcpEndpoints, mcpEndpointsSchema, registerMcpClient, type McpServer, type McpSignIn} from './mcp-oauth'; +import {signInRoutes, type Connected, type SignInRequest} from '../connect-flow'; +import {exchangeMcpCode, mcpAuthorizeUrl, registerMcpClient, type McpServer, type McpSignIn} from './mcp-oauth'; /** * Connecting an integration by signing in to its MCP server, as Granola, Notion and Linear are, and - * a custom server by its address. A sign-in begins on the api, which registers Personal Agent Proxy - * with the server and keeps the handshake in a short-lived encrypted cookie, and the browser goes - * to the server's sign-in page: from the start route, a top-level navigation, for a server known - * in advance, or from the custom connect route's answer for one found from its address. The - * callback, under `/api/connections//callback`, trades the code for tokens and makes - * the connection, or gives an existing one of the same account its new tokens, and ends on the - * connection's page, or back on the catalog with `?error=`; a sign-in begun from the - * welcome flow (`returnTo: 'welcome'`) ends on its page for the connection, or its catalog, instead. + * a custom server by its address, through the sign-in flow every OAuth integration shares + * (`connect-flow.ts`): the start registers Personal Agent Proxy with the server, keeping the client + * it registered for the callback, and sends the browser to the server's sign-in. */ -const stateCookieMaxAgeMs = 15 * 60 * 1000; - -/** Where a sign-in ends when not on the connection's page: the welcome flow it began from. */ -export const returnToSchema = z.enum(['welcome']); - -export type ReturnTo = z.infer; - -// The server is kept only when it was found from the request, as a custom one is. -const statePayloadSchema = z.object({ - state: z.string(), - verifier: z.string(), - clientId: z.string(), - userId: z.string(), - returnTo: returnToSchema.optional(), - server: mcpEndpointsSchema.optional(), - // What the person called the connection, typed in before the sign-in, for a custom server. - name: z.string().optional(), -}); - -type StatePayload = z.infer; - -type Route = (request: AuthenticatedRequest, res: Response) => Promise; - -/** The connection a finished sign-in makes: its account, what it stores (encrypted), what agents get by default, and a custom server's tools and the name the person gave it. */ -export type McpConnection = {account: string; credential: Record; defaults: string[]; tools?: McpTool[]; name?: string}; - -type MakeConnection = (signIn: McpSignIn, server: McpServer) => Promise>; - /** - * Saves a connection to an MCP server: a new one, or an existing one of the same account given - * the new credential, and the tools and the name, where the server has its own. + * Registers with the server for this sign-in, and its sign-in page, keeping the client for the + * callback beside `kept`. */ -export function saveMcpConnection(orgId: string, integrationId: IntegrationId, made: McpConnection): Promise> { +export function startMcpSignIn(server: McpServer, request: SignInRequest, kept: K): Promise> { return Do(async ($) => { - const encrypted = $(encryptSecret(JSON.stringify(made.credential))); - const existing = $(await listConnections(orgId)).find((connection) => connection.integrationId === integrationId && connection.account === made.account); - const name = made.name === undefined ? {} : {name: made.name}; - if (!existing) { - const tools = made.tools === undefined ? {} : {tools: JSON.stringify(made.tools)}; - return $(await createConnection({orgId, integrationId, account: made.account, credential: encrypted, defaults: made.defaults, ...tools, ...name})); - } - $(await updateConnectionCredential({connectionId: existing.id, credential: encrypted})); - if (made.name !== undefined) { - $(await updateConnectionName({connectionId: existing.id, name: made.name})); - } - if (made.tools === undefined) { - return {...existing, ...name}; - } - const tools = JSON.stringify(made.tools); - $(await updateConnectionTools({connectionId: existing.id, tools, actions: made.tools.map((tool) => tool.name)})); - return {...existing, tools, ...name}; + const clientId = $(await registerMcpClient(server, request.redirectUri)); + return {url: mcpAuthorizeUrl(server, {clientId, ...request}), kept: {...kept, clientId}}; }); } -/** - * The sign-in to one integration's server, known in advance as Granola's is, or found per request - * as a custom one is from its address (`server: null`) and kept in the handshake cookie for the - * callback: `beginSignIn` for the route that has the server, and the callback route. - */ -export function mcpSignInRoutes(flow: {integrationId: IntegrationId; name: string; server: McpServer | null; connection: MakeConnection}) { - const {integrationId, name} = flow; - const stateCookieName = `proxy.${integrationId}-state`; - // Only sent to these routes. - const basePath = `/api/connections/${integrationId}`; - - function redirectUri(appUrl: string): string { - return `${appUrl}${basePath}/callback`; - } - - function stateCookieOptions() { - return {httpOnly: true, sameSite: 'lax' as const, secure: useSecureCookies(), path: basePath}; - } - - function readStateCookie(request: AuthenticatedRequest): Result { - return Do(($) => { - const cookie = parseCookieHeader(request.headers.cookie ?? '').get(stateCookieName); - const stored = $(requirePresent(cookie, ApiErr.validationError(`No ${name} sign-in under way`))); - return $(readStoredJson(stored, statePayloadSchema)); - }); - } - - // The server the sign-in was to: the one known in advance, or the one the cookie kept. - function serverAtCallback(payload: StatePayload): Result { - if (flow.server) { - return Ok(flow.server); - } - return requirePresent(payload.server, ApiErr.validationError(`The ${name} sign-in under way names no server`)); - } - - // Where the sign-in ends: the catalog, or the welcome flow's connections step it started from. - function catalogUrl(appUrl: string, returnTo: ReturnTo | undefined): string { - if (returnTo === 'welcome') { - return `${appUrl}/welcome/connections`; - } - return `${appUrl}/connections/new`; - } - - function redirectToConnectError(res: Response, appUrl: string, returnTo: ReturnTo | undefined, error: ApiError): void { - log.warn(`Connecting ${name} failed`, {kind: error.kind, ...('message' in error ? {message: error.message} : {}), ...('cause' in error ? serializeError(error.cause) : {})}); - res.redirect(`${catalogUrl(appUrl, returnTo)}?error=${integrationId}`); - } - - /** Registers with the server, keeps the handshake in a cookie on `res`, with the name the person gave the connection, and answers the server's sign-in page, for the browser to go to. */ - function beginSignIn(res: Response, args: {userId: string; server: McpServer; returnTo: ReturnTo | undefined; name?: string}): Promise> { - return Do(async ($) => { - const appUrl = $(requirePresent(env.APP_URL, ApiErr.internalError(new Error('APP_URL is not configured')))); - const clientId = $(await registerMcpClient(args.server, redirectUri(appUrl))); - const state = randomBytes(16).toString('hex'); - const {verifier, challenge} = makePkce(); - const kept = flow.server ? {} : {server: mcpEndpoints(args.server)}; - const payload: StatePayload = { - state, - verifier, - clientId, - userId: args.userId, - ...(args.returnTo ? {returnTo: args.returnTo} : {}), - ...kept, - ...(args.name === undefined ? {} : {name: args.name}), - }; - const cookie = $(encryptSecret(JSON.stringify(payload))); - res.cookie(stateCookieName, cookie, {...stateCookieOptions(), maxAge: stateCookieMaxAgeMs}); - return mcpAuthorizeUrl(args.server, {clientId, redirectUri: redirectUri(appUrl), state, challenge}); - }); - } - - // The handshake the cookie holds, checked against the server's answer, then the connection made - // or given its new tokens. - function finishConnect(request: AuthenticatedRequest, appUrl: string, payload: StatePayload): Promise> { - return Do(async ($) => { - const state = queryParam(request, 'state') ?? ''; - if (!timingSafeEqualString(state, payload.state) || payload.userId !== request.user.userId) { - return $(Err(ApiErr.validationError(`${name} sign-in state does not match`))); - } - - const code = queryParam(request, 'code'); - if (!code) { - return $(Err(ApiErr.validationError(`${name} sign-in ended without a code: ${queryParam(request, 'error') ?? 'unknown'}`))); - } - - const server = $(serverAtCallback(payload)); - const signIn = $(await exchangeMcpCode(server, {clientId: payload.clientId, redirectUri: redirectUri(appUrl), code, verifier: payload.verifier})); - const made = $(await flow.connection(signIn, server)); - const orgId = $(await requireUserOrgId(request)); - return $(await saveMcpConnection(orgId, integrationId, {...made, ...(payload.name === undefined ? {} : {name: payload.name})})).id; - }); - } - - /** - * `GET /api/connections//callback` — where the sign-in comes back to; lands on the - * connection, saying it was added, or on the welcome flow's page for it when it started there. - */ - async function handleCallback(request: AuthenticatedRequest, res: Response): Promise { - const appUrl = env.APP_URL; - if (!appUrl) { - res.status(500).json({error: 'app_url_not_configured'}); - return; - } - - // The handshake is single-use: cleared however the callback ends. - res.clearCookie(stateCookieName, stateCookieOptions()); - const payload = readStateCookie(request); - const returnTo = payload.isOk() ? payload.value.returnTo : undefined; - const connectionId = payload.isErr() ? Err(payload.error) : await finishConnect(request, appUrl, payload.value); - if (connectionId.isErr()) { - redirectToConnectError(res, appUrl, returnTo, connectionId.error); - return; - } - if (returnTo === 'welcome') { - res.redirect(`${appUrl}/welcome/connections/${connectionId.value}`); - return; - } - res.redirect(`${appUrl}/connections/${connectionId.value}?added`); - } - - return {beginSignIn, handleCallback, redirectToConnectError}; -} - /** The routes for a server known in advance, as Granola's is: the start, a top-level navigation, and the callback. */ -export function mcpConnectRoutes(args: {integrationId: IntegrationId; server: McpServer; connection: MakeConnection}): {handleStart: Route; handleCallback: Route} { - const flow = mcpSignInRoutes({...args, name: args.server.name}); - - /** `GET /api/connections//start` — registers with the server and sends the browser to its sign-in. */ - async function handleStart(request: AuthenticatedRequest, res: Response): Promise { - const appUrl = env.APP_URL; - if (!appUrl) { - res.status(500).json({error: 'app_url_not_configured'}); - return; - } - - const returnTo = returnToSchema.safeParse(queryParam(request, 'returnTo')).data; - const begun = await flow.beginSignIn(res, {userId: request.user.userId, server: args.server, returnTo}); - if (begun.isErr()) { - flow.redirectToConnectError(res, appUrl, returnTo, begun.error); - return; - } - res.redirect(begun.value); - } - - return {handleStart, handleCallback: flow.handleCallback}; +export function mcpConnectRoutes(args: { + integrationId: IntegrationId; + server: McpServer; + // The connection a finished sign-in makes: two sign-ins to the same account are the same one. + connection: (signIn: McpSignIn, server: McpServer) => Promise, ApiError>>; +}) { + const {server} = args; + return signInRoutes({ + integrationId: args.integrationId, + name: server.name, + keptSchema: z.object({clientId: z.string()}), + start: (request) => startMcpSignIn(server, request, {}), + finish: ({redirectUri, code, verifier, kept}) => + Do(async ($) => { + const signIn = $(await exchangeMcpCode(server, {clientId: kept.clientId, redirectUri, code, verifier})); + return $(await args.connection(signIn, server)); + }), + }); } diff --git a/packages/api/src/connections/mcp/mcp-oauth.ts b/packages/api/src/connections/mcp/mcp-oauth.ts index ea2631d6..5808dc83 100644 --- a/packages/api/src/connections/mcp/mcp-oauth.ts +++ b/packages/api/src/connections/mcp/mcp-oauth.ts @@ -1,5 +1,3 @@ -import {createHash, randomBytes} from 'node:crypto'; - import {Err, type Result} from 'ts-results-es'; import {z} from 'zod'; @@ -113,12 +111,6 @@ export async function registerMcpClient(server: McpServer, redirectUri: string): return registration.map(({client_id}) => client_id).mapErr((error) => ApiErr.providerUnreachable(error)); } -/** A PKCE verifier and the challenge the server is sent in its place. */ -export function makePkce(): {verifier: string; challenge: string} { - const verifier = randomBytes(32).toString('base64url'); - return {verifier, challenge: createHash('sha256').update(verifier).digest('base64url')}; -} - /** The server's sign-in page, coming back to `redirectUri` with `state`. */ export function mcpAuthorizeUrl(server: McpServer, args: {clientId: string; redirectUri: string; state: string; challenge: string}): string { const url = new URL(server.authorizeUrl); From ed64050c351f5c9aa7be71082b05f652f147e9d7 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Tue, 13 Oct 2026 20:26:55 +0000 Subject: [PATCH 2/6] connect google and read drive, docs, sheets and slides as the files collection --- .env.example | 4 +- AGENTS.md | 6 +- packages/api/src/auth/google.ts | 7 +- .../api/src/connections/google/drive-api.ts | 119 +++++ .../connections/google/google-client.test.ts | 131 ++++++ .../src/connections/google/google-client.ts | 120 +++++ .../src/connections/google/google-oauth.ts | 152 +++++++ .../api/src/connections/google/route.test.ts | 198 +++++++++ packages/api/src/connections/google/route.ts | 29 ++ .../api/src/connections/google/sheets-api.ts | 74 ++++ packages/api/src/mcp/route.test.ts | 2 +- packages/api/src/records/connectors.ts | 2 + .../api/src/records/google-connector.test.ts | 394 +++++++++++++++++ packages/api/src/records/google-connector.ts | 28 ++ packages/api/src/records/google-files.ts | 417 ++++++++++++++++++ packages/api/src/server/create-api-router.ts | 3 + packages/api/src/utils/env-schema.ts | 5 +- packages/api/src/utils/google-oauth-client.ts | 15 + packages/db/scripts/seed.ts | 40 ++ packages/integrations/src/catalog.ts | 64 +++ packages/integrations/src/tools.test.ts | 17 + packages/integrations/src/types.ts | 2 +- packages/web/public/integrations/google.svg | 1 + .../src/components/integration-catalog.tsx | 2 +- packages/web/src/lib/integrations.ts | 1 + 25 files changed, 1821 insertions(+), 12 deletions(-) create mode 100644 packages/api/src/connections/google/drive-api.ts create mode 100644 packages/api/src/connections/google/google-client.test.ts create mode 100644 packages/api/src/connections/google/google-client.ts create mode 100644 packages/api/src/connections/google/google-oauth.ts create mode 100644 packages/api/src/connections/google/route.test.ts create mode 100644 packages/api/src/connections/google/route.ts create mode 100644 packages/api/src/connections/google/sheets-api.ts create mode 100644 packages/api/src/records/google-connector.test.ts create mode 100644 packages/api/src/records/google-connector.ts create mode 100644 packages/api/src/records/google-files.ts create mode 100644 packages/api/src/utils/google-oauth-client.ts create mode 100644 packages/web/public/integrations/google.svg diff --git a/.env.example b/.env.example index b3d6e00e..9b9b0440 100644 --- a/.env.example +++ b/.env.example @@ -13,7 +13,9 @@ AUTH_SECRET="dev-only-secret" # Changing it makes everything stored with it unreadable. ENCRYPTION_KEY="ZGV2LW9ubHktZW5jcnlwdGlvbi1rZXktMzItYnl0ZXM=" -# Google sign-in. Register http://localhost:5173/auth/google/callback as a redirect URI. +# Google sign-in, and connecting a Google account. Register http://localhost:5173/auth/google/callback +# and http://localhost:5173/api/connections/google/callback as redirect URIs, and enable the Gmail, +# Drive, Docs, Sheets and Slides APIs. # GOOGLE_CLIENT_ID="" # GOOGLE_CLIENT_SECRET="" diff --git a/AGENTS.md b/AGENTS.md index 6605554a..98e33583 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,17 +25,17 @@ - `packages/utils` - `@proxy/utils`, the Result helpers shared by every package: `Do`, `requirePresent` and `parseSchema` (`src/parse.ts`), and `httpRequest`, the fetch wrapper that returns a `FetchError` union instead of throwing (`src/fetch.ts`). Also `ApiError`/`ApiErr` and `wrapDb` (`src/api-error.ts`), the error every server-side function returns, and the session cookie's name and parsing (`src/session-cookie.ts`) - `packages/integrations` - `@proxy/integrations`, the catalog the api and the web app share: each integration's actions, collections and fields (`src/catalog.ts`), the agents a login can be made for (`src/agent-providers.ts`), the email providers' servers (`src/email-providers.ts`), and access (`src/access.ts`): per action of a connection, an agent's own setting, else the connection's default, an action counting only with the one it `requires`, and tools (`src/tools.ts`): what an agent can do with a connection, generated from the catalog, one per list, open, write and command of each collection (`emails_list`, `emails_archive`), each with the action it needs and a JSON Schema of its params. A custom MCP server (`mcp` in the catalog, `custom`) has no actions of its own: `integrationFor(connection)` (`src/mcp-tools.ts`) builds its integration from the tools the connection keeps (`McpTool`, as the server lists them), one action per tool named by the tool, its risk from the tool's annotations (low when it only reads, medium when what it changes can be undone, high when it can't, and `unknown`, shown without a label, when it says nothing), and `integrationTools` passes them on as `call` tools with the server's own input schema (`ServerTool`, beside the catalog's `CollectionTool`). The web app adds the icons (`src/lib/integrations.ts`, whose `integrationOf(connection)` is what every page uses) - `packages/db` - `@proxy/db`: Prisma 6 on Postgres. `src/index.ts` is the client; each module (`auth.ts`, `user.ts`, `organization.ts`, `connection.ts`, `info.ts`, `agent.ts`, `audit.ts`) exports functions returning `Result` through `wrapDb`. A connection's `tools` column holds a custom MCP server's tools as JSON, and `name` what the person called it; both null for every other integration. `relationMode = "prisma"`: no foreign keys in the database, so Prisma runs the cascades and every relation column has its own index. Locally the database is Prisma's embedded Postgres (PGlite, `scripts/dev-db.ts`), which needs `connection_limit=1&pgbouncer=true` in `DATABASE_URL` -- `packages/api` - `@proxy/api`: Express on Bun. Routes are `handleXRoute` functions mounted in `src/server/create-api-router.ts`; JSON routes return `Result` through `withAuthResult`, which puts the signed-in user on `req.user`. Every environment variable is in `src/utils/env-schema.ts` with its docs, which the docs' Environment variables page is built from; add new ones there. Personal Agent Proxy is one origin everywhere: when `packages/web/dist` exists, as in the Docker image, the api serves the web app's build too (`src/server/serve-web.ts`), and every address the api builds starts from `APP_URL`. Browser-called writes take `requireBrowserOrigin` (CSRF); routes that read a cookie have no CORS. `src/connections/` lists, deletes and sets defaults on an organization's connections, connects a mailbox with an app password after signing in to its IMAP and SMTP servers (`email/`), and connects Granola, Notion and Linear by signing in to their MCP servers (`mcp/`, shared by all three: OAuth with PKCE against the server's authorization server, a client registered for each sign-in so nothing is set up beforehand, the handshake in an encrypted cookie, `mcpConnectRoutes` making both routes; tokens are refreshed as they run out, one refresh at a time per connection on each server, read again first so ones another call or server saved meanwhile are used rather than refreshed twice, and saved only over the ones they replace, whatever else the credential holds kept beside them, by `connections/token-session.ts`, which Google's connection will share; each tool call is one stateless `tools/call`, `mcp-client.ts`, which also lists a server's tools and, for a server that may keep a session (`handshake`), initializes one around each call as the spec asks). Every request to an MCP server, its metadata or its authorization server goes through `utils/checked-request.ts`, which checks the address as a database's host is checked (`utils/host-check.ts`: nothing private unless `ALLOW_PRIVATE_HOSTS`, and `https` unless the address is private) before each request, since a custom server and its endpoints are wherever it said, and never follows a redirect. A custom MCP server is connected by its address (`custom/`): `POST /api/connections/mcp` with `{name, url, token?, returnTo?}` (the name is the person's, kept through the sign-in in its cookie, since a server says nothing about itself that would do) asks the server for its tools, connects a server that answers at once, with no sign-in or the token typed in as a bearer token (as written when it has its scheme in front, `Basic …`), and for one that turns a request without a token down begins its sign-in there, under the browser's origin check, answering `{kind: 'signIn', url}` with the sign-in page the browser goes to: `mcp-discovery.ts` finds the authorization server from the server's protected resource metadata (the `WWW-Authenticate` header's `resource_metadata`, else the well-known paths), its endpoints from that server's metadata (else `/authorize`, `/token` and `/register` under it) and the scopes the server's own metadata names, and `mcpSignInRoutes` (`mcp-connect.ts`, beside `mcpConnectRoutes` for the servers known in advance) keeps the endpoints in the handshake cookie and in the credential beside the tokens. The connection's account is the address without its scheme (`mcp.example.com/mcp`), connecting the same address again replaces its credential and tools, a new connection's defaults are the tools that say they only read, `POST /api/connections/:id/tools` asks the server for its tools again (replacing the tools drops the defaults and agents' settings for tools the server no longer lists, `updateConnectionTools`), and a tool is run as it is (`custom-mcp.ts`): its params go to the server as given, a tool that fails answers the agent with the server's own text as a `validation_error`, and the run is logged under the collection `tools` by the tool's name, with its params as JSON, cut to 200 characters. Granola's notes and transcripts are on by default, Notion's Read pages and Linear's Read issues alone; a Notion or Linear sign-in is for one workspace, and a Notion one lasts at most 180 days, however often it is refreshed. `src/agents/` makes, revokes and deletes agent logins (the password is sent once, only its argon2id hash is kept) and sets an agent's own settings per action of a connection where they differ from the default. `src/records/` reads and writes a collection's records through the integration's `Connector` (`connectors.ts`): Information's live in the database (`info-connector.ts`), a mailbox's are fetched over IMAP on every request (`email-connector.ts`), which merges the inbox and the folders the server marks as Drafts and Sent into one list (ids start with the folder: `inbox-…`, `drafts-…`, `sent-…`) and runs the commands: flags, moves to the folders the server marks as Archive (Gmail's All Mail) or Trash, and sending over SMTP, a draft or a new email, filing a copy in Sent where the server doesn't (Gmail does). Received emails are never edited or deleted for good; only drafts are. Granola's notes are fetched from its MCP server on every request (`granola-connector.ts`; the reading of the XML-like text Granola answers in is in `connections/granola/granola-mcp.ts`): its Notes and Transcripts collections are the same meetings by Granola's meeting id, and since Granola's tools have no pages, a page is 30 days of meetings; one that finds none skips back to the newest meeting before it, so the list ends where the meetings do. Granola's dates and attendees are shown as Granola writes them. Notion's workspace is fetched from its MCP server the same way (`notion-connector.ts`, the reading of its answers in `connections/notion/notion-mcp.ts`), as the person who signed in, as a tree in one nested collection: Notion has no list of every page, so the top is their Recents and a search (Notion's own, AI search where the workspace has it) finds the rest, one page of 50; inside a page are the pages and databases in its content, inside a database its saved views (`view-…` ids), or the rows of its only one, and inside a view its rows, paged with Notion's cursor (view queries have no plan quota; Notion's row and SQL queries do). A page reads as Notion-flavored Markdown, a row's properties beside it as "Name: value" lines (`notion-properties.ts`), and only a page's, database's or view's id is ever sent to Notion's fetch; a new page is a private draft at the top, or goes inside the page, database or view it is made in, a row's title in the database's title property; an edit sends the title, the properties written differently and the content, each only where it changed, and pages are never deleted. Notion's MCP server has no tool that deletes or trashes a page. Linear's issues are fetched from its MCP server the same way (`linear-connector.ts`, the reading of its answers in `connections/linear/linear-mcp.ts`), as the person who signed in, by their identifier (`ENG-123`): a list is the issues updated last first, 50 to a page with Linear's cursor, narrowed by the type of status (`?statusType=Started`) and searched in titles and descriptions, and an issue opens with its comments, oldest first. Teams, statuses, people, projects and labels are written by name, as Linear shows them and finds them, its refusals passed on to the agent; an edit sends only the fields written differently, an empty one clearing it. Linear's MCP server has no tool that archives or deletes an issue. PostgreSQL and MySQL (MariaDB too) are two integrations on one code path: a database is connected with its host, port, database, user, password and encryption (`connections/sql/`: `route.ts` makes both routes per dialect, `sql-client.ts` opens one connection per request with `Bun.sql`, to the address `utils/host-check.ts` resolves the host to, refusing private addresses unless `ALLOW_PRIVATE_HOSTS` is set, and `sql-statement.ts` checks an agent's query is one statement that reads), and `records/sql-connector.ts` reads and writes it through `records/sql-dialect.ts`, where the two differ: the tables are one nested collection, a table holding its rows, a row read and written as "column: value" lines and found again by its primary key (ids `public.users:42`, `pairs:3,4`; a table without a key is never written to), and a query, a second collection that is only ever run, goes through a read-only transaction with a time limit, a cursor on Postgres and `sql_select_limit` on MySQL, 200 rows at most, answered as JSON lines. MySQL's read-only transaction doesn't stop DDL, so the statement check is what keeps a query to reading; the docs tell people to make a read-only database user as well. Lists take `?search=` (only where the collection has a `searchHint`; Granola has none), `?filter=` (the collection's `filterField`, such as `?filter=Draft`), `?parent=` (only in a `nested` collection: the list inside a record, answered with its `trail`, the records above it; a record made there goes in it) and `?page=` (`list-query.ts`) and answer a page, newest first (a mailbox's 50 emails, Granola's 30 days), with the next page's token; a mailbox's token holds where each folder goes on, so paging through the merged list shows every email once, and its search uses Gmail's own syntax on Gmail (`X-GM-EXT-1`) and IMAP text search elsewhere. Agents sign in at `/agent-auth/login` with a cookie of their own (`src/agent-auth/`); `src/agent-side/` serves them: the connections an agent can use, the tools it has with each, and `runAgentTool` (`tools.ts`), the one place a tool runs, which checks it against the agent's access as it is now, validates its params against the tool's schema (an edit changes only the fields given), calls the connector, or a custom server's tool as it is, answering `{text}`, and logs it, denied ones too, to the activity log that `src/activity/` lists. Over HTTP that is `/api/agent/me`, `GET /api/agent/connections/:id/tools` and `POST /api/agent/connections/:id/tools/:tool` with `{params}`; `src/mcp/` serves the same over MCP at `/mcp` (Streamable HTTP without sessions, JSON answers, no stream): four tools whatever the login may do, `list_connections` (the entry point), `list_tools`, `read` (only tools that read, marked read-only) and `run`, and server instructions naming the agent's connections. MCP clients sign in with OAuth 2.1 (`src/oauth/`), Personal Agent Proxy being its own authorization server: metadata under `/.well-known/`, Dynamic Client Registration at `/oauth/register` (open to anyone, each registration first deleting the clients that registered over a day ago and never finished a sign-in), `/oauth/authorize` sealing the request into the web app's `/connect-agent` page, where the person chooses the agent login the client works as or makes one, and `/oauth/token` with PKCE (S256 only) and refresh tokens replaced on every use; codes and tokens are kept as their SHA-256, and revoking an agent ends its clients' tokens. These endpoints take any origin (CORS), since none reads a cookie; credentials and Information records are stored encrypted with `ENCRYPTION_KEY` (`src/utils/secret-crypto.ts`). Has its own `.prettierrc.json` with a 200-column print width +- `packages/api` - `@proxy/api`: Express on Bun. Routes are `handleXRoute` functions mounted in `src/server/create-api-router.ts`; JSON routes return `Result` through `withAuthResult`, which puts the signed-in user on `req.user`. Every environment variable is in `src/utils/env-schema.ts` with its docs, which the docs' Environment variables page is built from; add new ones there. Personal Agent Proxy is one origin everywhere: when `packages/web/dist` exists, as in the Docker image, the api serves the web app's build too (`src/server/serve-web.ts`), and every address the api builds starts from `APP_URL`. Browser-called writes take `requireBrowserOrigin` (CSRF); routes that read a cookie have no CORS. `src/connections/` lists, deletes and sets defaults on an organization's connections, connects a mailbox with an app password after signing in to its IMAP and SMTP servers (`email/`), and connects Granola, Notion and Linear by signing in to their MCP servers (`mcp/`, shared by all three: OAuth with PKCE against the server's authorization server, a client registered for each sign-in so nothing is set up beforehand, the handshake in an encrypted cookie, `mcpConnectRoutes` making both routes on the sign-in flow every OAuth integration shares, `connect-flow.ts`; tokens are refreshed as they run out, one refresh at a time per connection on each server, read again first so ones another call or server saved meanwhile are used rather than refreshed twice, and saved only over the ones they replace, whatever else the credential holds kept beside them, by `connections/token-session.ts`, which Google's connection will share; each tool call is one stateless `tools/call`, `mcp-client.ts`, which also lists a server's tools and, for a server that may keep a session (`handshake`), initializes one around each call as the spec asks). Every request to an MCP server, its metadata or its authorization server goes through `utils/checked-request.ts`, which checks the address as a database's host is checked (`utils/host-check.ts`: nothing private unless `ALLOW_PRIVATE_HOSTS`, and `https` unless the address is private) before each request, since a custom server and its endpoints are wherever it said, and never follows a redirect. A custom MCP server is connected by its address (`custom/`): `POST /api/connections/mcp` with `{name, url, token?, returnTo?}` (the name is the person's, kept through the sign-in in its cookie, since a server says nothing about itself that would do) asks the server for its tools, connects a server that answers at once, with no sign-in or the token typed in as a bearer token (as written when it has its scheme in front, `Basic …`), and for one that turns a request without a token down begins its sign-in there, under the browser's origin check, answering `{kind: 'signIn', url}` with the sign-in page the browser goes to: `mcp-discovery.ts` finds the authorization server from the server's protected resource metadata (the `WWW-Authenticate` header's `resource_metadata`, else the well-known paths), its endpoints from that server's metadata (else `/authorize`, `/token` and `/register` under it) and the scopes the server's own metadata names, and the sign-in, begun with `signInFlow` (`connect-flow.ts`, which `mcpConnectRoutes` builds on for the servers known in advance), keeps the endpoints in the handshake cookie and in the credential beside the tokens. The connection's account is the address without its scheme (`mcp.example.com/mcp`), connecting the same address again replaces its credential and tools, a new connection's defaults are the tools that say they only read, `POST /api/connections/:id/tools` asks the server for its tools again (replacing the tools drops the defaults and agents' settings for tools the server no longer lists, `updateConnectionTools`), and a tool is run as it is (`custom-mcp.ts`): its params go to the server as given, a tool that fails answers the agent with the server's own text as a `validation_error`, and the run is logged under the collection `tools` by the tool's name, with its params as JSON, cut to 200 characters. Granola's notes and transcripts are on by default, Notion's Read pages and Linear's Read issues alone; a Notion or Linear sign-in is for one workspace, and a Notion one lasts at most 180 days, however often it is refreshed. `src/agents/` makes, revokes and deletes agent logins (the password is sent once, only its argon2id hash is kept) and sets an agent's own settings per action of a connection where they differ from the default. `src/records/` reads and writes a collection's records through the integration's `Connector` (`connectors.ts`): Information's live in the database (`info-connector.ts`), a mailbox's are fetched over IMAP on every request (`email-connector.ts`), which merges the inbox and the folders the server marks as Drafts and Sent into one list (ids start with the folder: `inbox-…`, `drafts-…`, `sent-…`) and runs the commands: flags, moves to the folders the server marks as Archive (Gmail's All Mail) or Trash, and sending over SMTP, a draft or a new email, filing a copy in Sent where the server doesn't (Gmail does). Received emails are never edited or deleted for good; only drafts are. Granola's notes are fetched from its MCP server on every request (`granola-connector.ts`; the reading of the XML-like text Granola answers in is in `connections/granola/granola-mcp.ts`): its Notes and Transcripts collections are the same meetings by Granola's meeting id, and since Granola's tools have no pages, a page is 30 days of meetings; one that finds none skips back to the newest meeting before it, so the list ends where the meetings do. Granola's dates and attendees are shown as Granola writes them. Notion's workspace is fetched from its MCP server the same way (`notion-connector.ts`, the reading of its answers in `connections/notion/notion-mcp.ts`), as the person who signed in, as a tree in one nested collection: Notion has no list of every page, so the top is their Recents and a search (Notion's own, AI search where the workspace has it) finds the rest, one page of 50; inside a page are the pages and databases in its content, inside a database its saved views (`view-…` ids), or the rows of its only one, and inside a view its rows, paged with Notion's cursor (view queries have no plan quota; Notion's row and SQL queries do). A page reads as Notion-flavored Markdown, a row's properties beside it as "Name: value" lines (`notion-properties.ts`), and only a page's, database's or view's id is ever sent to Notion's fetch; a new page is a private draft at the top, or goes inside the page, database or view it is made in, a row's title in the database's title property; an edit sends the title, the properties written differently and the content, each only where it changed, and pages are never deleted. Notion's MCP server has no tool that deletes or trashes a page. Linear's issues are fetched from its MCP server the same way (`linear-connector.ts`, the reading of its answers in `connections/linear/linear-mcp.ts`), as the person who signed in, by their identifier (`ENG-123`): a list is the issues updated last first, 50 to a page with Linear's cursor, narrowed by the type of status (`?statusType=Started`) and searched in titles and descriptions, and an issue opens with its comments, oldest first. Teams, statuses, people, projects and labels are written by name, as Linear shows them and finds them, its refusals passed on to the agent; an edit sends only the fields written differently, an empty one clearing it. Linear's MCP server has no tool that archives or deletes an issue. Google is connected by signing in to Google with the app's own Google client, the one Google sign-in uses (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, a second redirect URI `/api/connections/google/callback`), asking for every scope at once, Gmail, Drive, Docs, Sheets and Slides, with offline access (`connections/google/`: `google-oauth.ts` the sign-in and refresh, with the client's secret, keeping the refresh token Google doesn't send again; `google-client.ts` the calls with a bearer token and Google's errors, a disabled API passing Google's own message on, which names the API and where to enable it); the credential holds the tokens, the scopes Google granted, since the consent screen lets a person untick some, and the address, which is the account, so signing in again with it renews the same connection. Its Drive is one nested collection, `files` (`records/google-files.ts`, behind `google-connector.ts`, which dispatches on the collection), fetched live from Drive's and Sheets' APIs (`drive-api.ts`, `sheets-api.ts`): at the top four places, My Drive, Shared with me, Shared drives (`drive-…` ids) and Recent, inside them and inside folders their files, folders first, 50 to a page with Drive's page token, and a search is Drive's full-text one, anywhere or inside the folder opened; the trail is walked up a file's parents to My Drive, its shared drive, or, where the account can't see further up, Shared with me. A document opens as Markdown and a presentation as text, as Drive exports them, up to its 10 MB export cap, beyond which the agent is told the file is too large; a spreadsheet opens into its tabs (`sheet--`) and a tab into its rows (`row---`), 50 to a page, the page token the next row's number, each row's values "Header: value" lines under the tab's first row. Drive isn't written to yet; writes, and Gmail, come later. PostgreSQL and MySQL (MariaDB too) are two integrations on one code path: a database is connected with its host, port, database, user, password and encryption (`connections/sql/`: `route.ts` makes both routes per dialect, `sql-client.ts` opens one connection per request with `Bun.sql`, to the address `utils/host-check.ts` resolves the host to, refusing private addresses unless `ALLOW_PRIVATE_HOSTS` is set, and `sql-statement.ts` checks an agent's query is one statement that reads), and `records/sql-connector.ts` reads and writes it through `records/sql-dialect.ts`, where the two differ: the tables are one nested collection, a table holding its rows, a row read and written as "column: value" lines and found again by its primary key (ids `public.users:42`, `pairs:3,4`; a table without a key is never written to), and a query, a second collection that is only ever run, goes through a read-only transaction with a time limit, a cursor on Postgres and `sql_select_limit` on MySQL, 200 rows at most, answered as JSON lines. MySQL's read-only transaction doesn't stop DDL, so the statement check is what keeps a query to reading; the docs tell people to make a read-only database user as well. Lists take `?search=` (only where the collection has a `searchHint`; Granola has none), `?filter=` (the collection's `filterField`, such as `?filter=Draft`), `?parent=` (only in a `nested` collection: the list inside a record, answered with its `trail`, the records above it; a record made there goes in it) and `?page=` (`list-query.ts`) and answer a page, newest first (a mailbox's 50 emails, Granola's 30 days), with the next page's token; a mailbox's token holds where each folder goes on, so paging through the merged list shows every email once, and its search uses Gmail's own syntax on Gmail (`X-GM-EXT-1`) and IMAP text search elsewhere. Agents sign in at `/agent-auth/login` with a cookie of their own (`src/agent-auth/`); `src/agent-side/` serves them: the connections an agent can use, the tools it has with each, and `runAgentTool` (`tools.ts`), the one place a tool runs, which checks it against the agent's access as it is now, validates its params against the tool's schema (an edit changes only the fields given), calls the connector, or a custom server's tool as it is, answering `{text}`, and logs it, denied ones too, to the activity log that `src/activity/` lists. Over HTTP that is `/api/agent/me`, `GET /api/agent/connections/:id/tools` and `POST /api/agent/connections/:id/tools/:tool` with `{params}`; `src/mcp/` serves the same over MCP at `/mcp` (Streamable HTTP without sessions, JSON answers, no stream): four tools whatever the login may do, `list_connections` (the entry point), `list_tools`, `read` (only tools that read, marked read-only) and `run`, and server instructions naming the agent's connections. MCP clients sign in with OAuth 2.1 (`src/oauth/`), Personal Agent Proxy being its own authorization server: metadata under `/.well-known/`, Dynamic Client Registration at `/oauth/register` (open to anyone, each registration first deleting the clients that registered over a day ago and never finished a sign-in), `/oauth/authorize` sealing the request into the web app's `/connect-agent` page, where the person chooses the agent login the client works as or makes one, and `/oauth/token` with PKCE (S256 only) and refresh tokens replaced on every use; codes and tokens are kept as their SHA-256, and revoking an agent ends its clients' tokens. These endpoints take any origin (CORS), since none reads a cookie; credentials and Information records are stored encrypted with `ENCRYPTION_KEY` (`src/utils/secret-crypto.ts`). Has its own `.prettierrc.json` with a 200-column print width - `skills/` - Agent skills, symlinked into `.claude/skills` and `.codex/skills` by `bun run setup` ## The app Personal Agent Proxy holds the integrations and personal information an AI agent needs, so every agent company does not have to rebuild them and you do not have to trust each one's security. A human connects services and hands out agent logins; an agent signs in with one and works through what it was given. Everything is served by `packages/api`; the web app has no data of its own. -- **Model**: an integration from the catalog (`@proxy/integrations`: Email over IMAP and SMTP, Granola's meeting notes, Notion's pages and Linear's issues over their MCP servers, PostgreSQL and MySQL over their own protocols) has the actions an agent can be allowed with a connection of it (Read, Archive, Write drafts, Send) and collections, each a list of fields with the actions reading and writing it need, and the commands it offers beyond editing (Mark as read, Archive, Send), each needing one action. A command runs on a `record`, where its `where` picks out which (Archive only in the inbox, Send only on a draft), or on values typed in (`new`: sending a new email). A mailbox is one collection, its folder a field the agent side filters by. A `nested` collection is a tree, as Notion's pages are: a record can hold others of the collection (`hasChildren`), listed inside it with the list tool's `parent` and made inside it with the create tool's, and access is the same anywhere in it. A collection without a `read` action is never listed or opened, only run, as a database's queries are, and a `new` command marked `readOnly` counts as a read for MCP clients. Integrations join the catalog once they work end to end; Google Workspace comes later. Any other service with an MCP server is connected as a custom MCP server: the server's tools are the actions, each allowed on its own, and there are no collections; an agent runs a tool with the params the server describes and gets back what it answered. A connection is a connected integration and its account, owned by the organization; an integration can be connected more than once, such as two mailboxes. Information is the built-in `info` connection (addresses, payment cards, notes), made with the organization, so access and the agent side treat it like any other. Nothing is copied from a provider: every request goes to it live. Fields marked `system` are set by the provider, never typed in. +- **Model**: an integration from the catalog (`@proxy/integrations`: Email over IMAP and SMTP, Google's Drive, Docs, Sheets and Slides over Google's APIs, one sign-in for a Google account, Granola's meeting notes, Notion's pages and Linear's issues over their MCP servers, PostgreSQL and MySQL over their own protocols) has the actions an agent can be allowed with a connection of it (Read, Archive, Write drafts, Send) and collections, each a list of fields with the actions reading and writing it need, and the commands it offers beyond editing (Mark as read, Archive, Send), each needing one action. A command runs on a `record`, where its `where` picks out which (Archive only in the inbox, Send only on a draft), or on values typed in (`new`: sending a new email). A mailbox is one collection, its folder a field the agent side filters by. A `nested` collection is a tree, as Notion's pages are: a record can hold others of the collection (`hasChildren`), listed inside it with the list tool's `parent` and made inside it with the create tool's, and access is the same anywhere in it. A collection without a `read` action is never listed or opened, only run, as a database's queries are, and a `new` command marked `readOnly` counts as a read for MCP clients. Integrations join the catalog once they work end to end. Any other service with an MCP server is connected as a custom MCP server: the server's tools are the actions, each allowed on its own, and there are no collections; an agent runs a tool with the params the server describes and gets back what it answered. A connection is a connected integration and its account, owned by the organization; an integration can be connected more than once, such as two mailboxes. Information is the built-in `info` connection (addresses, payment cards, notes), made with the organization, so access and the agent side treat it like any other. Nothing is copied from a provider: every request goes to it live. Fields marked `system` are set by the provider, never typed in. - **Access** is one list of actions per connection, and every action the catalog lists can be set: what Personal Agent Proxy can do with the provider is the limit. Two layers: the connection's default (a `ConnectionDefault` row per action that is on) and an agent's own setting, on or off, where it differs (`AgentGrant`). An agent gets its own setting, else the default as it is now; an action that `requires` another counts only with it (Archive needs Read; Send needs nothing, so an agent can send without reading the mailbox). A collection's `writes` name the action creating, editing and deleting need, and `editable` which records they apply to (drafts). An agent login is for one agent from `AGENT_PROVIDERS`; each organization can have one login per company, named by it, and any number of custom MCP agents (`custom`, for any MCP client not in the list), each named by the person and told apart by their names (`Agent` is unique by organization, provider and name). It has a generated username and a password that is shown once. Every request on the agent side is an audit entry, allowed or denied, with what a list searched for. - **Accounts**: a human signs in with Google or a magic link (`src/auth/` in the api); the sign-in page says that continuing accepts the terms and the privacy policy, on the website, and a sign-up records it (`User.termsAcceptedAt`); accounts from before the notice have none until they are asked. A session is a `Session` row and an httpOnly cookie; the web app's `/api/me` loader on the human side's route redirects to `/login?callbackUrl=…` without one. Every user gets one `Organization` at sign-up, created with the user. `ALLOWED_SIGNUP_EMAILS` limits who can sign up; the login page asks `/auth/methods` whether to offer Google. In development the login page has a Create dev user button; in development, or without `RESEND_KEY`, the api prints magic links instead of emailing them. Cookies are secure when `APP_URL` is HTTPS. A new organization first goes through the welcome flow (`/welcome`, outside the human side's layout; the human side redirects there until `Organization.onboardedAt` is set, which `POST /api/me/onboarded` does when the flow is finished or skipped): step one is `AgentSetup` (`src/components/agent-setup.tsx`, which New agent and setting an agent up again use too): it picks the agent from `AGENT_PROVIDERS`, the choice in the address (`?agent=claude`) so the browser's back button returns to it, and does what its `connects` says, an `mcp` agent (Claude, Claude Code, Claude Desktop, Hermes Agent, ChatGPT, Poke) shown the server address and that agent's steps (Claude Code and Hermes Agent use terminal commands) while the page polls for the login its sign-in makes, and over http, as in development, only a `local` agent (Claude Code, Claude Desktop, Hermes Agent) can be picked, the rest greyed out with a pointer to self-hosting on a server with a domain, a `login` agent given a login at once, as a message to paste to it with the sign-in page, username and password, or, for a `vault` agent (Instinct), which won't take a password in chat, as three steps: a message asking it for a link to its vault, the login to fill in on the page it sends, and a message telling it to sign in; step two (`/welcome/connections`) is the catalog, a plus per integration with how many accounts are connected: a mailbox goes to `/welcome/connections/email`, the form on a page with Continue once signed in, and a sign-in started with `?returnTo=welcome` comes back to `/welcome/connections/:id`, the new connection's permissions (its defaults, saved as they are ticked) with Confirm back to the catalog; a database goes to `/welcome/connections/postgres` or `/welcome/connections/mysql` the same way as a mailbox. Organizations from before the flow were marked done by its migration. -- **Human side** behind `/login`: `/connections` (a right click on a row renames the connection, `PUT /api/connections/:id/name`, an empty name being none of its own, which a custom MCP server can't have, or deletes it, each behind a dialog), `/connections/new` (the catalog, a plus per integration with how many accounts are connected: `/connections/new/email` connects a mailbox with an app password, with a Test connection that signs in and saves nothing, `/connections/new/postgres` and `/connections/new/mysql` a database the same way, `/connections/new/mcp` a custom MCP server under a name of the person's, by its address, with an access token where the server takes one instead of a sign-in (`ConnectMcpForm`, which sends the browser to the server's sign-in when the api says it asks for one), and Granola, Notion and Linear are connected by signing in to them; `ConnectEmailForm`, `ConnectDatabaseForm`, `ConnectMcpForm` and `IntegrationCatalog` serve the welcome flow too, at `/welcome/connections/`), `/connections/:id` (titled by the name the person gave it where they did, as the list and the agent side are, the defaults under Permissions, with how many agents differ per action, a custom MCP server's tools being its actions, with a button that asks the server for them again, and a "Connection added successfully" when `?added` says it was just connected), `/info`, `/agents`, `/agents/new` (the welcome flow's agent step, among the companies without a login yet and Custom MCP agent, on to the agent's page), `/agents/:id/setup` (the same for one agent, to hand it a new password or add the connector again; the only place a password shows, once), `/agents/:id` (what the agent can reach and did, the MCP clients working as it when there are any; access laid out like an editor's settings: one checkbox per action of each connection, the only control for it, each connection folded by default to a row saying Default or Changed; a setting of the agent's own is marked with a bar, a button that makes it the connection's default for every agent, and one that resets it, and a filter and Changed only narrow the list; `ConnectionAccess` serves both pages), `/activity` (filters in the URL, so the agent and connection pages link to their part of the log; a custom server's tool reads "Ran Search docs with {…}"). Each page has a loader in `src/loaders.ts` and revalidates after a change. Layout follows the spanish repo: the shadcn sidebar as a rail, an `h-12` header, one `max-w-3xl` column, one-line `RowList` rows. Type has one size, `text-sm`, in rows, summaries and hints alike, except column headers (`RowHeader`), which are `text-xs`: the thing itself in the normal colour, what is said about it (column headers, times, summaries, descriptions) in `text-muted-foreground`, and `font-medium` only for page and section titles. +- **Human side** behind `/login`: `/connections` (a right click on a row renames the connection, `PUT /api/connections/:id/name`, an empty name being none of its own, which a custom MCP server can't have, or deletes it, each behind a dialog), `/connections/new` (the catalog, a plus per integration with how many accounts are connected: `/connections/new/email` connects a mailbox with an app password, with a Test connection that signs in and saves nothing, `/connections/new/postgres` and `/connections/new/mysql` a database the same way, `/connections/new/mcp` a custom MCP server under a name of the person's, by its address, with an access token where the server takes one instead of a sign-in (`ConnectMcpForm`, which sends the browser to the server's sign-in when the api says it asks for one), and Google, Granola, Notion and Linear are connected by signing in to them; `ConnectEmailForm`, `ConnectDatabaseForm`, `ConnectMcpForm` and `IntegrationCatalog` serve the welcome flow too, at `/welcome/connections/`), `/connections/:id` (titled by the name the person gave it where they did, as the list and the agent side are, the defaults under Permissions, with how many agents differ per action, a custom MCP server's tools being its actions, with a button that asks the server for them again, and a "Connection added successfully" when `?added` says it was just connected), `/info`, `/agents`, `/agents/new` (the welcome flow's agent step, among the companies without a login yet and Custom MCP agent, on to the agent's page), `/agents/:id/setup` (the same for one agent, to hand it a new password or add the connector again; the only place a password shows, once), `/agents/:id` (what the agent can reach and did, the MCP clients working as it when there are any; access laid out like an editor's settings: one checkbox per action of each connection, the only control for it, each connection folded by default to a row saying Default or Changed; a setting of the agent's own is marked with a bar, a button that makes it the connection's default for every agent, and one that resets it, and a filter and Changed only narrow the list; `ConnectionAccess` serves both pages), `/activity` (filters in the URL, so the agent and connection pages link to their part of the log; a custom server's tool reads "Ran Search docs with {…}"). Each page has a loader in `src/loaders.ts` and revalidates after a change. Layout follows the spanish repo: the shadcn sidebar as a rail, an `h-12` header, one `max-w-3xl` column, one-line `RowList` rows. Type has one size, `text-sm`, in rows, summaries and hints alike, except column headers (`RowHeader`), which are `text-xs`: the thing itself in the normal colour, what is said about it (column headers, times, summaries, descriptions) in `text-muted-foreground`, and `font-medium` only for page and section titles. - **MCP**: an agent that speaks MCP (Poke, Claude, Claude Code, Claude Desktop, Hermes Agent, ChatGPT, or any other as a Custom MCP agent) adds `/mcp` as a custom connector and signs in; the person picks which agent login it works as on `/connect-agent`, or makes one there: a client whose name points at no company's agent is offered a new custom agent named after it, which the person can rename. Setting up a custom agent waits for one that wasn't there before to sign in, since there can be several. The agent page shows the server's address and the MCP clients working as the login, each with a Disconnect; the activity log marks requests that came in over MCP (`AuditEntry.via`). - **Agent side** behind `/agent/login`, with its own cookie: `/agent` lists what the login can do with each connection and the lists it can open, `/agent/:connection/tool/:tool` one of a custom MCP server's tools, a form of its params built from the tool's schema and what it answered below (the home page lists the tools the login may run), `/agent/:connection/:collection` a collection's records (newest first, 50 to a page with an Older link, narrowed by its `filterField`, such as the folder, in the address as the list tool takes it, `?folder=Draft`, and searched where the collection has a `searchHint`, which says what the search takes; in a nested collection `?parent=` opens it inside a record, with the records above it as breadcrumbs, a record holding others opening into them, and New making one there), `…/new` and `…/:record` the generic `RecordForm` and `RecordFields`, with a button per command the agent may run on the record, Send beside Save as draft on a new email, and a command that answers something, as running a query does, shown under the form (loaders in `src/agent-loaders.ts`). Every page runs a tool (`toolName` in `@proxy/integrations`) through the api. It is built for a model driving a browser: no sidebar, no icon-only buttons, nothing that only shows on hover. Lists it can neither read nor add to are left out, and one it can only add to (sending without Read) opens on its form; asking for one anyway shows the api's refusal, which it logs as denied. - **Testing email locally**: any IMAP server with TLS on 993 and SMTP on 465 or 587 works through Other. A self-signed one needs `NODE_TLS_REJECT_UNAUTHORIZED=0` on the api, never in production. The server's hostname needs a dot and can't be an IP address: `localhost4.localdomain4` reaches 127.0.0.1 on most Linux machines. diff --git a/packages/api/src/auth/google.ts b/packages/api/src/auth/google.ts index 8e0959fe..0be84d21 100644 --- a/packages/api/src/auth/google.ts +++ b/packages/api/src/auth/google.ts @@ -10,6 +10,7 @@ import {formatFetchError, httpRequest, parseCookieHeader} from '@proxy/utils'; import {log, serializeError} from '../observability/log'; import {env} from '../utils/env'; +import {googleOAuthClient} from '../utils/google-oauth-client'; import {canSignUp, establishSessionAndRedirect, queryParam, redirectToLoginError, sanitizeCallbackUrl, timingSafeEqualString, useSecureCookies} from './session'; /** @@ -45,7 +46,8 @@ type OAuthConfig = { }; function getOAuthConfig(): OAuthConfig | null { - if (!env.GOOGLE_CLIENT_ID || !env.GOOGLE_CLIENT_SECRET) { + const client = googleOAuthClient(); + if (!client) { return null; } @@ -54,8 +56,7 @@ function getOAuthConfig(): OAuthConfig | null { } return { - clientId: env.GOOGLE_CLIENT_ID, - clientSecret: env.GOOGLE_CLIENT_SECRET, + ...client, appUrl: env.APP_URL, redirectUri: `${env.APP_URL}/auth/google/callback`, }; diff --git a/packages/api/src/connections/google/drive-api.ts b/packages/api/src/connections/google/drive-api.ts new file mode 100644 index 00000000..b5e9bea7 --- /dev/null +++ b/packages/api/src/connections/google/drive-api.ts @@ -0,0 +1,119 @@ +import type {Result} from 'ts-results-es'; +import {z} from 'zod'; + +import type {ApiError} from '@proxy/utils'; + +import {googleRequest, googleText} from './google-client'; + +/** + * The Drive v3 calls the `files` collection makes, and the reading of their answers. Every call + * takes shared drives too. + */ + +const DRIVE = 'https://www.googleapis.com/drive/v3'; + +export const PAGE_SIZE = 50; + +/** Drive's ids, as every id an agent sends is checked against before it reaches an address or a query. */ +export const DRIVE_ID = /^[A-Za-z0-9_-]+$/; + +export const MIME_TYPES = { + folder: 'application/vnd.google-apps.folder', + document: 'application/vnd.google-apps.document', + spreadsheet: 'application/vnd.google-apps.spreadsheet', + presentation: 'application/vnd.google-apps.presentation', +}; + +const FILE_FIELDS = 'id,name,mimeType,modifiedTime,owners(displayName,emailAddress),webViewLink,parents,driveId'; + +const driveFileSchema = z.object({ + id: z.string(), + name: z.string(), + mimeType: z.string(), + modifiedTime: z.string().optional(), + // None in a shared drive, which owns its files. + owners: z.array(z.object({displayName: z.string().optional(), emailAddress: z.string().optional()})).optional(), + webViewLink: z.string().optional(), + parents: z.array(z.string()).optional(), + driveId: z.string().optional(), +}); + +export type DriveFile = z.infer; + +const fileListSchema = z.object({files: z.array(driveFileSchema), nextPageToken: z.string().optional()}); + +/** A file's place in the tree: its name, the folder it is in, and the shared drive it is in. */ +const fileParentsSchema = z.object({id: z.string(), name: z.string(), parents: z.array(z.string()).optional(), driveId: z.string().optional()}); + +export type FileParents = z.infer; + +const sharedDriveSchema = z.object({id: z.string(), name: z.string()}); + +export type SharedDrive = z.infer; + +const driveListSchema = z.object({drives: z.array(sharedDriveSchema), nextPageToken: z.string().optional()}); + +export type Page = {items: T[]; nextPage: string | null}; + +function driveUrl(path: string, params: Record): URL { + const url = new URL(`${DRIVE}${path}`); + for (const [name, value] of Object.entries(params)) { + url.searchParams.set(name, value); + } + return url; +} + +/** A term in Drive's query language, quoted: a backslash and a quote escaped with a backslash. */ +export function driveQuoted(term: string): string { + return `'${term.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`; +} + +/** + * One page of files matching `q`, from the page a previous one pointed at. `params` narrows the + * list further, such as to one shared drive (`corpora`, `driveId`) or by an order (`orderBy`). + */ +export async function listFiles(accessToken: string, q: string, params: Record, pageToken: string | null): Promise, ApiError>> { + const url = driveUrl('/files', { + q, + pageSize: String(PAGE_SIZE), + fields: `nextPageToken,files(${FILE_FIELDS})`, + supportsAllDrives: 'true', + includeItemsFromAllDrives: 'true', + ...params, + ...(pageToken ? {pageToken} : {}), + }); + const list = await googleRequest(accessToken, url, fileListSchema); + return list.map(({files, nextPageToken}) => ({items: files, nextPage: nextPageToken ?? null})); +} + +/** A file's metadata. */ +export function getFile(accessToken: string, fileId: string): Promise> { + return googleRequest(accessToken, driveUrl(`/files/${fileId}`, {fields: FILE_FIELDS, supportsAllDrives: 'true'}), driveFileSchema); +} + +/** Where a file is: only what walking up the tree needs. */ +export function getFileParents(accessToken: string, fileId: string): Promise> { + return googleRequest(accessToken, driveUrl(`/files/${fileId}`, {fields: 'id,name,parents,driveId', supportsAllDrives: 'true'}), fileParentsSchema); +} + +/** The id of My Drive itself, which Drive calls `root` in queries but names by its own id in `parents`. */ +export async function getMyDriveId(accessToken: string): Promise> { + const root = await googleRequest(accessToken, driveUrl('/files/root', {fields: 'id'}), z.object({id: z.string()})); + return root.map(({id}) => id); +} + +/** A Google document, spreadsheet or presentation exported as text, such as `text/markdown`. */ +export function exportFile(accessToken: string, fileId: string, mimeType: string): Promise> { + return googleText(accessToken, driveUrl(`/files/${fileId}/export`, {mimeType})); +} + +/** One page of the shared drives the account is a member of, those whose names contain `search` when given. */ +export async function listSharedDrives(accessToken: string, search: string | null, pageToken: string | null): Promise, ApiError>> { + const url = driveUrl('/drives', {pageSize: String(PAGE_SIZE), ...(search ? {q: `name contains ${driveQuoted(search)}`} : {}), ...(pageToken ? {pageToken} : {})}); + const list = await googleRequest(accessToken, url, driveListSchema); + return list.map(({drives, nextPageToken}) => ({items: drives, nextPage: nextPageToken ?? null})); +} + +export function getSharedDrive(accessToken: string, driveId: string): Promise> { + return googleRequest(accessToken, driveUrl(`/drives/${driveId}`, {fields: 'id,name'}), sharedDriveSchema); +} diff --git a/packages/api/src/connections/google/google-client.test.ts b/packages/api/src/connections/google/google-client.test.ts new file mode 100644 index 00000000..e7743702 --- /dev/null +++ b/packages/api/src/connections/google/google-client.test.ts @@ -0,0 +1,131 @@ +import {beforeEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import {Ok, type Result} from 'ts-results-es'; +import {z} from 'zod'; + +import type {ApiError, FetchError} from '@proxy/utils'; + +const updateConnectionCredential = mock(); + +mock.module('@proxy/db/connection', () => ({updateConnectionCredential})); + +mock.module('../../observability/log', () => ({ + log: {debug: mock(), info: mock(), warn: mock(), error: mock()}, + serializeError: mock(() => ({})), +})); + +mock.module('../../utils/env', () => ({env: {NODE_ENV: 'production', APP_URL: 'https://app.example.com', GOOGLE_CLIENT_ID: 'client-1', GOOGLE_CLIENT_SECRET: 'secret-1'}})); + +// Readable stand-ins for encryption, so a test can see what was stored. +mock.module('../../utils/secret-crypto', () => ({ + encryptSecret: (plaintext: string): Result => Ok(`enc:${plaintext}`), + decryptSecret: (stored: string): Result => Ok(stored.replace(/^enc:/, '')), +})); + +// Google's token endpoint and Drive: each request, and the answers given in turn. +const google = {requests: [] as Array<{url: URL; headers: Headers; body: string}>, answers: [] as Response[]}; + +spyOn(globalThis, 'fetch').mockImplementation((async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + google.requests.push({url, headers: new Headers(init?.headers), body: typeof init?.body === 'string' ? init.body : ''}); + return google.answers.shift() ?? Response.json({}); +}) as typeof fetch); + +function httpError(status: number, error: object): FetchError { + return {kind: 'http', url: 'https://www.googleapis.com/drive/v3/files', status, body: {error}}; +} + +const CREDENTIAL = { + accessToken: 'access-old', + refreshToken: 'refresh-1', + expiresAt: new Date(Date.now() - 1000).toISOString(), + scopes: ['openid', 'https://www.googleapis.com/auth/drive'], + email: 'alex@example.com', +}; + +beforeEach(() => { + mock.clearAllMocks(); + google.requests = []; + google.answers = []; + updateConnectionCredential.mockResolvedValue(Ok(true)); +}); + +describe('googleError', () => { + test('a token turned down is to be refreshed, a missing file not found, and Google busy or down out of reach', async () => { + const {googleError} = await import('./google-client'); + + expect(googleError(httpError(401, {code: 401, message: 'Request had invalid authentication credentials.', status: 'UNAUTHENTICATED'})).kind).toBe('credentials_rejected'); + expect(googleError(httpError(404, {code: 404, message: 'File not found: abc.', errors: [{reason: 'notFound'}]})).kind).toBe('not_found'); + expect(googleError(httpError(429, {code: 429, message: 'Quota exceeded'})).kind).toBe('provider_unreachable'); + expect(googleError(httpError(503, {code: 503, message: 'The service is currently unavailable.'})).kind).toBe('provider_unreachable'); + expect(googleError(httpError(403, {code: 403, message: 'User rate limit exceeded.', errors: [{reason: 'userRateLimitExceeded'}]})).kind).toBe('provider_unreachable'); + expect(googleError({kind: 'timeout', url: 'https://www.googleapis.com/drive/v3/files', timeoutMs: 30_000}).kind).toBe('provider_unreachable'); + }); + + test("a disabled API passes Google's own message on, naming the API and where to enable it", async () => { + const {googleError} = await import('./google-client'); + const message = + 'Google Drive API has not been used in project 1234 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/drive.googleapis.com/overview?project=1234 then retry.'; + + expect(googleError(httpError(403, {code: 403, message, status: 'PERMISSION_DENIED', errors: [{reason: 'accessNotConfigured'}]}))).toEqual({kind: 'validation_error', statusCode: 400, message}); + expect(googleError(httpError(403, {code: 403, message, status: 'PERMISSION_DENIED', details: [{'@type': 'type.googleapis.com/google.rpc.ErrorInfo', reason: 'SERVICE_DISABLED'}]}))).toEqual({ + kind: 'validation_error', + statusCode: 400, + message, + }); + }); + + test('a file too large to export says so', async () => { + const {googleError} = await import('./google-client'); + + expect(googleError(httpError(403, {code: 403, message: 'This file is too large to be exported.', errors: [{reason: 'exportSizeLimitExceeded'}]}))).toEqual({ + kind: 'validation_error', + statusCode: 400, + message: 'This file is too large to read; Google exports at most 10 MB', + }); + }); + + test("any other refusal is passed on in Google's words", async () => { + const {googleError} = await import('./google-client'); + + expect(googleError(httpError(400, {code: 400, message: 'Invalid Value', errors: [{reason: 'invalid'}]}))).toMatchObject({kind: 'validation_error', message: 'Google: Invalid Value'}); + }); +}); + +describe('callGoogle', () => { + test('refreshes a running-out token with the client secret, keeping the refresh token Google does not send again', async () => { + const {callGoogle} = await import('./google-client'); + google.answers = [Response.json({access_token: 'access-new', expires_in: 3599, scope: 'openid https://www.googleapis.com/auth/drive', token_type: 'Bearer'})]; + const connection = {id: 'google-1', credential: `enc:${JSON.stringify(CREDENTIAL)}`}; + + const token = (await callGoogle(connection, async (accessToken) => Ok(accessToken))).unwrap(); + + expect(token).toBe('access-new'); + const [refresh] = google.requests; + expect(refresh?.url.href).toBe('https://oauth2.googleapis.com/token'); + expect(Object.fromEntries(new URLSearchParams(refresh?.body))).toEqual({grant_type: 'refresh_token', refresh_token: 'refresh-1', client_id: 'client-1', client_secret: 'secret-1'}); + const saved = updateConnectionCredential.mock.calls[0]?.[0]; + expect(saved).toMatchObject({connectionId: 'google-1', previous: connection.credential}); + expect(JSON.parse(String(saved.credential).replace(/^enc:/, ''))).toMatchObject({accessToken: 'access-new', refreshToken: 'refresh-1', scopes: CREDENTIAL.scopes, email: 'alex@example.com'}); + }); + + test('a refresh Google turns down is a sign-in to make again', async () => { + const {callGoogle} = await import('./google-client'); + google.answers = [Response.json({error: 'invalid_grant', error_description: 'Token has been expired or revoked.'}, {status: 400})]; + const connection = {id: 'google-1', credential: `enc:${JSON.stringify(CREDENTIAL)}`}; + + const result = await callGoogle(connection, async (accessToken) => Ok(accessToken)); + + expect(result.isErr() && result.error.kind).toBe('credentials_rejected'); + expect(updateConnectionCredential).not.toHaveBeenCalled(); + }); + + test('sends the token as a bearer and reads the answer', async () => { + const {googleRequest} = await import('./google-client'); + google.answers = [Response.json({id: 'root-id'})]; + + const answer = await googleRequest('access-1', new URL('https://www.googleapis.com/drive/v3/files/root?fields=id'), z.object({id: z.string()})); + + expect(answer.unwrap()).toEqual({id: 'root-id'}); + expect(google.requests[0]?.headers.get('authorization')).toBe('Bearer access-1'); + }); +}); diff --git a/packages/api/src/connections/google/google-client.ts b/packages/api/src/connections/google/google-client.ts new file mode 100644 index 00000000..0c347e8b --- /dev/null +++ b/packages/api/src/connections/google/google-client.ts @@ -0,0 +1,120 @@ +import {Err, Ok, type Result} from 'ts-results-es'; +import {z} from 'zod'; + +import {ApiErr, type ApiError, Do, type FetchError, httpRequest} from '@proxy/utils'; + +import {googleOAuthClient, type GoogleClient} from '../../utils/google-oauth-client'; +import {callWithToken, type TokenRefresher} from '../token-session'; +import {googleCredentialSchema, isExpiring, refreshGoogleTokens, type GoogleCredential} from './google-oauth'; + +/** + * Google's REST APIs with a connection's access token: JSON in and out, or text where a file is + * exported, and Google's errors turned into what they mean for the agent. A disabled API passes + * Google's own message on, since it names the API and the page enabling it, which is what a + * self-hoster setting up the Cloud project needs. + */ + +const TIMEOUT_MS = 30_000; + +type StoredConnection = {id: string; credential: string | null}; + +const googleErrorSchema = z.object({ + error: z.object({ + message: z.string().optional(), + errors: z.array(z.object({reason: z.string().optional()})).optional(), + // Newer APIs say why in `details`, such as SERVICE_DISABLED. + details: z.array(z.object({reason: z.string().optional()})).optional(), + }), +}); + +// The reasons Google gave for an error, from both places it puts them, and its message. +function readGoogleError(error: Extract): {reasons: string[]; message: string} { + const parsed = googleErrorSchema.safeParse(error.body); + if (!parsed.success) { + return {reasons: [], message: ''}; + } + const {errors = [], details = [], message = ''} = parsed.data.error; + const reasons = [...errors, ...details].flatMap((entry) => (entry.reason ? [entry.reason] : [])); + return {reasons, message}; +} + +const RATE_LIMITED = ['rateLimitExceeded', 'userRateLimitExceeded', 'RATE_LIMIT_EXCEEDED']; + +/** What one of Google's errors means: a token to refresh, a file not there, a refusal to pass on, or Google out of reach. */ +export function googleError(error: FetchError): ApiError { + if (error.kind !== 'http') { + return ApiErr.providerUnreachable(error); + } + if (error.status === 401) { + return ApiErr.credentialsRejected(); + } + if (error.status === 404) { + return ApiErr.notFound('file'); + } + if (error.status === 429 || error.status >= 500) { + return ApiErr.providerUnreachable(error); + } + + const {reasons, message} = readGoogleError(error); + if (reasons.includes('exportSizeLimitExceeded')) { + return ApiErr.validationError('This file is too large to read; Google exports at most 10 MB'); + } + if (reasons.includes('accessNotConfigured') || reasons.includes('SERVICE_DISABLED') || /has not been used|is disabled/.test(message)) { + return ApiErr.validationError(message); + } + // Drive says it is busy with a 403. + if (reasons.some((reason) => RATE_LIMITED.includes(reason))) { + return ApiErr.providerUnreachable(error); + } + // Any other refusal, such as a file the account may not open or a query Google can't read, in + // Google's own words. + if (message !== '') { + return ApiErr.validationError(`Google: ${message}`); + } + return ApiErr.providerUnreachable(error); +} + +function authorized(accessToken: string): RequestInit { + return {headers: {authorization: `Bearer ${accessToken}`, accept: 'application/json'}}; +} + +/** A GET to one of Google's APIs, its answer read with `schema`. */ +export async function googleRequest(accessToken: string, url: URL, schema: S): Promise, ApiError>> { + const answer = await httpRequest(url, authorized(accessToken), {schema, timeoutMs: TIMEOUT_MS}); + return answer.mapErr(googleError); +} + +/** A GET whose answer is text, as a file exported is. */ +export async function googleText(accessToken: string, url: URL): Promise> { + const answer = await httpRequest(url, authorized(accessToken), {parse: 'text', timeoutMs: TIMEOUT_MS}); + return answer.mapErr(googleError); +} + +/** The app's Google client, which signing in and refreshing both need. */ +export function requireGoogleClient(): Result { + const client = googleOAuthClient(); + if (!client) { + return Err(ApiErr.validationError('Google isn’t set up on this server: GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET are missing')); + } + return Ok(client); +} + +/** How a Google connection's tokens are refreshed: with the app's own client. */ +export const googleRefresher: TokenRefresher = { + name: 'Google', + schema: googleCredentialSchema, + isExpiring, + refresh: (credential) => + Do(async ($) => { + return $(await refreshGoogleTokens($(requireGoogleClient()), credential)); + }), +}; + +/** + * Calls Google with a connection's access token: refreshed first when it is running out, and once + * more when Google turns it down anyway. A connection Google won't refresh any more is + * `credentials_rejected`, and has to be signed in again. + */ +export function callGoogle(connection: StoredConnection, call: (accessToken: string) => Promise>): Promise> { + return callWithToken(googleRefresher, connection, (credential) => call(credential.accessToken)); +} diff --git a/packages/api/src/connections/google/google-oauth.ts b/packages/api/src/connections/google/google-oauth.ts new file mode 100644 index 00000000..87b09fcb --- /dev/null +++ b/packages/api/src/connections/google/google-oauth.ts @@ -0,0 +1,152 @@ +import {Err, Result} from 'ts-results-es'; +import {z} from 'zod'; + +import {ApiErr, type ApiError, Do, type FetchError, httpRequest, parseSchema, requirePresent} from '@proxy/utils'; + +import type {GoogleClient} from '../../utils/google-oauth-client'; + +/** + * Signing in to Google for a connection: OAuth 2.0 with PKCE against Google's own endpoints, with + * the client Google sign-in uses (`GOOGLE_CLIENT_ID` and `GOOGLE_CLIENT_SECRET`), which is + * confidential, so its secret goes with every token request. Every scope Personal Agent Proxy uses + * is asked for at once, with offline access for a refresh token; Google's consent screen lets the + * person untick some, so what was granted is kept beside the tokens. + */ + +const AUTHORIZE_URL = 'https://accounts.google.com/o/oauth2/v2/auth'; +const TOKEN_URL = 'https://oauth2.googleapis.com/token'; + +const TIMEOUT_MS = 15_000; + +// Refreshed a minute early, so a token doesn't run out between the check and the call. +const EXPIRY_MARGIN_MS = 60_000; + +/** Every scope a sign-in asks for: the account's address, then Gmail, Drive, Docs, Sheets and Slides. */ +export const GOOGLE_SCOPES = [ + 'openid', + 'https://www.googleapis.com/auth/userinfo.email', + 'https://www.googleapis.com/auth/gmail.modify', + 'https://www.googleapis.com/auth/gmail.send', + 'https://www.googleapis.com/auth/drive', + 'https://www.googleapis.com/auth/documents', + 'https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/presentations', +]; + +/** What a Google connection stores, encrypted: its tokens, the scopes Google granted, and the account. */ +export const googleCredentialSchema = z.object({ + accessToken: z.string(), + refreshToken: z.string().nullable(), + expiresAt: z.string().nullable(), + scopes: z.array(z.string()), + email: z.string(), +}); + +export type GoogleCredential = z.infer; + +const tokensSchema = z.object({ + access_token: z.string().min(1), + refresh_token: z.string().optional(), + expires_in: z.number().optional(), + // Space-separated, as granted. + scope: z.string().optional(), + // Only on the first exchange, since `openid` is asked for. + id_token: z.string().optional(), +}); + +type Tokens = z.infer; + +const idTokenPayloadSchema = z.object({email: z.string().min(1)}); + +/** Google's sign-in page for a connection, coming back to `redirectUri` with `state`. */ +export function googleAuthorizeUrl(client: GoogleClient, args: {redirectUri: string; state: string; challenge: string}): string { + const url = new URL(AUTHORIZE_URL); + url.searchParams.set('response_type', 'code'); + url.searchParams.set('client_id', client.clientId); + url.searchParams.set('redirect_uri', args.redirectUri); + url.searchParams.set('scope', GOOGLE_SCOPES.join(' ')); + url.searchParams.set('state', args.state); + url.searchParams.set('code_challenge', args.challenge); + url.searchParams.set('code_challenge_method', 'S256'); + // A refresh token, and on every sign-in, not only the first: Google only sends one with consent. + url.searchParams.set('access_type', 'offline'); + url.searchParams.set('prompt', 'consent'); + url.searchParams.set('include_granted_scopes', 'true'); + return url.toString(); +} + +// Google turning a token request down is the sign-in revoked or the code spent; anything else is +// Google out of reach. +function tokenError(error: FetchError): ApiError { + if (error.kind === 'http' && error.status >= 400 && error.status < 500) { + return ApiErr.credentialsRejected(); + } + return ApiErr.providerUnreachable(error); +} + +function requestTokens(client: GoogleClient, params: Record): Promise> { + return httpRequest( + TOKEN_URL, + { + method: 'POST', + headers: {'content-type': 'application/x-www-form-urlencoded'}, + body: new URLSearchParams({...params, client_id: client.clientId, client_secret: client.clientSecret}).toString(), + }, + {schema: tokensSchema, timeoutMs: TIMEOUT_MS}, + ).then((tokens) => tokens.mapErr(tokenError)); +} + +function expiresAt(tokens: Tokens): string | null { + return tokens.expires_in === undefined ? null : new Date(Date.now() + tokens.expires_in * 1000).toISOString(); +} + +function grantedScopes(tokens: Tokens): string[] | null { + return tokens.scope === undefined ? null : tokens.scope.split(' ').filter((scope) => scope !== ''); +} + +// The account's address from the id_token's payload. It came straight from Google's token endpoint +// over TLS, so the payload is trusted without checking its signature, as Google sign-in does +// (`auth/google.ts`); that saves asking Google's userinfo endpoint for what the token already says. +function emailOf(idToken: string | undefined): Result { + const unreadable = (cause: unknown): ApiError => ApiErr.providerUnreachable(cause); + return Do(($) => { + const payload = $(requirePresent(idToken?.split('.')[1], unreadable(new Error('Google answered a sign-in without an id_token')))); + const json = $(Result.wrap((): unknown => JSON.parse(Buffer.from(payload, 'base64url').toString('utf8'))).mapErr(unreadable)); + const {email} = $(parseSchema(idTokenPayloadSchema, json).mapErr(unreadable)); + return email.toLowerCase(); + }); +} + +/** Trades the code Google sent back for the connection's first tokens, and the account they are for. */ +export async function exchangeGoogleCode(client: GoogleClient, args: {redirectUri: string; code: string; verifier: string}): Promise> { + const tokens = await requestTokens(client, {grant_type: 'authorization_code', code: args.code, redirect_uri: args.redirectUri, code_verifier: args.verifier}); + return tokens.andThen((value) => + emailOf(value.id_token).map((email) => ({ + accessToken: value.access_token, + refreshToken: value.refresh_token ?? null, + expiresAt: expiresAt(value), + scopes: grantedScopes(value) ?? [], + email, + })), + ); +} + +/** New tokens for a connection whose access token ran out; turned down once the sign-in was revoked. */ +export async function refreshGoogleTokens(client: GoogleClient, credential: GoogleCredential): Promise> { + if (!credential.refreshToken) { + return Err(ApiErr.credentialsRejected()); + } + const tokens = await requestTokens(client, {grant_type: 'refresh_token', refresh_token: credential.refreshToken}); + return tokens.map((value) => ({ + ...credential, + accessToken: value.access_token, + // Google keeps the refresh token as it is, so its answer usually has none. + refreshToken: value.refresh_token ?? credential.refreshToken, + expiresAt: expiresAt(value), + scopes: grantedScopes(value) ?? credential.scopes, + })); +} + +export function isExpiring(credential: GoogleCredential): boolean { + return credential.expiresAt !== null && Date.parse(credential.expiresAt) - EXPIRY_MARGIN_MS < Date.now(); +} diff --git a/packages/api/src/connections/google/route.test.ts b/packages/api/src/connections/google/route.test.ts new file mode 100644 index 00000000..65b69897 --- /dev/null +++ b/packages/api/src/connections/google/route.test.ts @@ -0,0 +1,198 @@ +import {beforeEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import {Ok, type Result} from 'ts-results-es'; + +import type {ApiError} from '@proxy/utils'; + +const listConnections = mock(); +const createConnection = mock(); +const updateConnectionCredential = mock(); + +mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential})); + +const getUserOrgId = mock(); + +mock.module('@proxy/db/organization', () => ({getUserOrgId})); + +mock.module('../../observability/log', () => ({ + log: {debug: mock(), info: mock(), warn: mock(), error: mock()}, + serializeError: mock(() => ({})), +})); + +const env: {NODE_ENV: string; APP_URL: string; GOOGLE_CLIENT_ID?: string; GOOGLE_CLIENT_SECRET?: string} = {NODE_ENV: 'production', APP_URL: 'https://app.example.com'}; + +mock.module('../../utils/env', () => ({env})); + +// Readable stand-ins for encryption, so a test can see what was stored. +mock.module('../../utils/secret-crypto', () => ({ + encryptSecret: (plaintext: string): Result => Ok(`enc:${plaintext}`), + decryptSecret: (stored: string): Result => Ok(stored.replace(/^enc:/, '')), +})); + +const CALLBACK = 'https://app.example.com/api/connections/google/callback'; + +const SCOPES = [ + 'openid', + 'https://www.googleapis.com/auth/userinfo.email', + 'https://www.googleapis.com/auth/gmail.modify', + 'https://www.googleapis.com/auth/gmail.send', + 'https://www.googleapis.com/auth/drive', + 'https://www.googleapis.com/auth/documents', + 'https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/presentations', +]; + +function base64url(value: object): string { + return Buffer.from(JSON.stringify(value)).toString('base64url'); +} + +const ID_TOKEN = `${base64url({alg: 'RS256'})}.${base64url({sub: '1093', email: 'Alex@Example.com', email_verified: true})}.signature`; + +// Google's token endpoint: each request, and its answer. The person unticked Gmail on the consent screen. +const google = {requests: [] as Array<{url: URL; body: URLSearchParams}>}; + +spyOn(globalThis, 'fetch').mockImplementation((async (input: string | URL | Request, init?: RequestInit) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + google.requests.push({url, body: new URLSearchParams(typeof init?.body === 'string' ? init.body : '')}); + return Response.json({ + access_token: 'access-1', + refresh_token: 'refresh-1', + expires_in: 3599, + token_type: 'Bearer', + scope: 'openid https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/documents', + id_token: ID_TOKEN, + }); +}) as typeof fetch); + +function makeRes() { + const cookies: Array<{name: string; value: string; options: Record}> = []; + const res = { + cookies, + redirect: mock(), + cookie: mock((name: string, value: string, options: Record) => { + cookies.push({name, value, options}); + }), + clearCookie: mock(), + status: mock(() => res), + json: mock(() => res), + }; + return res; +} + +const user = {userId: 'user-1', email: 'alex@example.com'}; + +async function start(query: Record = {}) { + const {handleStartGoogleRoute} = await import('./route'); + const res = makeRes(); + await handleStartGoogleRoute({user, query, headers: {}} as never, res as never); + const cookie = res.cookies.find((candidate) => candidate.name === 'proxy.google-state'); + const redirect = String(res.redirect.mock.calls[0]?.[0]); + const authorizeUrl = URL.canParse(redirect) ? new URL(redirect) : null; + return {res, cookie, authorizeUrl, state: authorizeUrl?.searchParams.get('state') ?? ''}; +} + +async function callback(args: {cookie: string; query: Record}) { + const {handleGoogleCallbackRoute} = await import('./route'); + const res = makeRes(); + await handleGoogleCallbackRoute({user, query: args.query, headers: {cookie: `proxy.google-state=${args.cookie}`}} as never, res as never); + return res; +} + +beforeEach(() => { + mock.clearAllMocks(); + google.requests = []; + env.GOOGLE_CLIENT_ID = 'client-1.apps.googleusercontent.com'; + env.GOOGLE_CLIENT_SECRET = 'secret-1'; + getUserOrgId.mockResolvedValue(Ok('org-1')); + listConnections.mockResolvedValue(Ok([])); + createConnection.mockResolvedValue(Ok({id: 'conn-1'})); + updateConnectionCredential.mockResolvedValue(Ok(true)); +}); + +describe('handleStartGoogleRoute', () => { + test('sends the browser to Google asking for every scope at once, offline, with PKCE', async () => { + const {cookie, authorizeUrl} = await start(); + + expect(authorizeUrl?.origin + (authorizeUrl?.pathname ?? '')).toBe('https://accounts.google.com/o/oauth2/v2/auth'); + const params = authorizeUrl?.searchParams; + expect(params?.get('client_id')).toBe('client-1.apps.googleusercontent.com'); + expect(params?.get('redirect_uri')).toBe(CALLBACK); + expect(params?.get('scope')?.split(' ')).toEqual(SCOPES); + expect(params?.get('access_type')).toBe('offline'); + expect(params?.get('prompt')).toBe('consent'); + expect(params?.get('include_granted_scopes')).toBe('true'); + expect(params?.get('code_challenge_method')).toBe('S256'); + expect(params?.get('code_challenge')).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(cookie?.options).toMatchObject({httpOnly: true, sameSite: 'lax', path: '/api/connections/google'}); + expect(google.requests).toEqual([]); + }); + + test('without a Google client set up, goes back to the catalog with the error', async () => { + env.GOOGLE_CLIENT_ID = undefined; + const {res, cookie} = await start(); + + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/new?error=google'); + expect(cookie).toBeUndefined(); + }); +}); + +describe('handleGoogleCallbackRoute', () => { + test('trades the code with the secret and verifier, and makes the connection for the address, agents reading files', async () => { + const {cookie, state} = await start(); + const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + const [exchange] = google.requests; + expect(exchange?.url.href).toBe('https://oauth2.googleapis.com/token'); + expect(Object.fromEntries(exchange?.body ?? [])).toMatchObject({ + grant_type: 'authorization_code', + code: 'code-1', + client_id: 'client-1.apps.googleusercontent.com', + client_secret: 'secret-1', + redirect_uri: CALLBACK, + code_verifier: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/), + }); + const created = createConnection.mock.calls[0]?.[0]; + expect(created).toMatchObject({orgId: 'org-1', integrationId: 'google', account: 'alex@example.com', defaults: ['readFiles']}); + expect(JSON.parse(String(created.credential).replace(/^enc:/, ''))).toEqual({ + accessToken: 'access-1', + refreshToken: 'refresh-1', + expiresAt: expect.any(String), + scopes: ['openid', 'https://www.googleapis.com/auth/userinfo.email', 'https://www.googleapis.com/auth/drive', 'https://www.googleapis.com/auth/documents'], + email: 'alex@example.com', + }); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-1?added'); + }); + + test('signing in again with the same address gives its connection the new tokens', async () => { + listConnections.mockResolvedValue(Ok([{id: 'conn-0', integrationId: 'google', account: 'alex@example.com'}])); + const {cookie, state} = await start(); + const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(createConnection).not.toHaveBeenCalled(); + expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0?added'); + }); + + test('started from the welcome flow, ends on its page for the connection', async () => { + const {cookie, state} = await start({returnTo: 'welcome'}); + const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/welcome/connections/conn-1'); + }); + + test('a state that does not match is turned down without asking Google', async () => { + const {cookie} = await start({returnTo: 'welcome'}); + const res = await callback({cookie: cookie?.value ?? '', query: {state: 'forged', code: 'code-1'}}); + + expect(google.requests).toEqual([]); + expect(createConnection).not.toHaveBeenCalled(); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/welcome/connections?error=google'); + }); + + test('a sign-in turned down at Google goes back to the catalog', async () => { + const {cookie, state} = await start(); + const res = await callback({cookie: cookie?.value ?? '', query: {state, error: 'access_denied'}}); + + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/new?error=google'); + expect(createConnection).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/api/src/connections/google/route.ts b/packages/api/src/connections/google/route.ts new file mode 100644 index 00000000..efefc074 --- /dev/null +++ b/packages/api/src/connections/google/route.ts @@ -0,0 +1,29 @@ +import {z} from 'zod'; + +import {Do} from '@proxy/utils'; + +import {signInRoutes} from '../connect-flow'; +import {requireGoogleClient} from './google-client'; +import {exchangeGoogleCode, googleAuthorizeUrl} from './google-oauth'; + +/** + * Connecting a Google account by signing in to Google with the app's own Google client, the one + * Google sign-in uses, asking for every scope at once. The account is the address signed in with, + * so signing in again with it gives its connection the new tokens. A new Google connection lets + * agents read Drive by default. Without a Google client set up, the start goes back to the catalog + * with the error. + */ +export const {handleStart: handleStartGoogleRoute, handleCallback: handleGoogleCallbackRoute} = signInRoutes({ + integrationId: 'google', + name: 'Google', + keptSchema: z.null(), + start: (request) => + Do(async ($) => { + return {url: googleAuthorizeUrl($(requireGoogleClient()), request), kept: null}; + }), + finish: ({redirectUri, code, verifier}) => + Do(async ($) => { + const credential = $(await exchangeGoogleCode($(requireGoogleClient()), {redirectUri, code, verifier})); + return {account: credential.email, credential, defaults: ['readFiles']}; + }), +}); diff --git a/packages/api/src/connections/google/sheets-api.ts b/packages/api/src/connections/google/sheets-api.ts new file mode 100644 index 00000000..9b64ba9b --- /dev/null +++ b/packages/api/src/connections/google/sheets-api.ts @@ -0,0 +1,74 @@ +import type {Result} from 'ts-results-es'; +import {z} from 'zod'; + +import type {ApiError} from '@proxy/utils'; + +import {googleRequest} from './google-client'; + +/** + * The Sheets v4 calls the `files` collection makes, and the reading of their answers: a + * spreadsheet's tabs, and the values in ranges of one. + */ + +const SHEETS = 'https://sheets.googleapis.com/v4/spreadsheets'; + +/** The last column read: ZZ, the 702nd. */ +const LAST_COLUMN = 'ZZ'; + +const sheetSchema = z.object({ + properties: z.object({ + sheetId: z.number(), + title: z.string(), + // The grid's size, empty cells included. A chart's tab has none. + gridProperties: z.object({rowCount: z.number().optional(), columnCount: z.number().optional()}).optional(), + }), +}); + +const spreadsheetSchema = z.object({sheets: z.array(sheetSchema).optional()}); + +/** One tab of a spreadsheet and the size of its grid. */ +export type Sheet = {sheetId: number; title: string; rowCount: number; columnCount: number}; + +// Formatted values are text; a cell left empty is an empty string, and empty cells at the end of a +// row are left out. +const cellSchema = z.union([z.string(), z.number(), z.boolean()]).transform(String); + +const valueRangeSchema = z.object({values: z.array(z.array(cellSchema)).optional()}); + +const batchSchema = z.object({valueRanges: z.array(valueRangeSchema)}); + +/** A spreadsheet's tabs, in order. */ +export async function getSheets(accessToken: string, spreadsheetId: string): Promise> { + const url = new URL(`${SHEETS}/${spreadsheetId}`); + url.searchParams.set('fields', 'sheets.properties(sheetId,title,gridProperties(rowCount,columnCount))'); + const spreadsheet = await googleRequest(accessToken, url, spreadsheetSchema); + return spreadsheet.map(({sheets = []}) => + sheets.map(({properties}) => ({ + sheetId: properties.sheetId, + title: properties.title, + rowCount: properties.gridProperties?.rowCount ?? 0, + columnCount: properties.gridProperties?.columnCount ?? 0, + })), + ); +} + +/** Rows `from` to `to` of a tab, every column, as A1 notation writes them: `'Q1 Sales'!A2:ZZ51`. */ +export function rowsRange(title: string, from: number, to: number): string { + return `'${title.replace(/'/g, "''")}'!A${from}:${LAST_COLUMN}${to}`; +} + +/** The values in each range, in the order asked, one list of cells a row. */ +export async function getRanges(accessToken: string, spreadsheetId: string, ranges: string[]): Promise> { + const url = new URL(`${SHEETS}/${spreadsheetId}/values:batchGet`); + for (const range of ranges) { + url.searchParams.append('ranges', range); + } + const batch = await googleRequest(accessToken, url, batchSchema); + return batch.map(({valueRanges}) => valueRanges.map((range) => range.values ?? [])); +} + +/** A column's letters, from its position counting from 0: A, Z, AA. */ +export function columnLetters(index: number): string { + const letter = String.fromCharCode(65 + (index % 26)); + return index < 26 ? letter : `${columnLetters(Math.floor(index / 26) - 1)}${letter}`; +} diff --git a/packages/api/src/mcp/route.test.ts b/packages/api/src/mcp/route.test.ts index ebd57bdf..4e3ee188 100644 --- a/packages/api/src/mcp/route.test.ts +++ b/packages/api/src/mcp/route.test.ts @@ -85,7 +85,7 @@ describe('handleMcpRoute', () => { expect(result.instructions).toContain('Start with list_connections'); expect(result.instructions).toContain('- Email (alex@example.com), connection mail-1: Read, Archive'); expect(result.instructions).toContain('check Personal Agent Proxy first'); - expect(result.instructions).toContain('Personal Agent Proxy connects Email, Granola, Notion, Linear, PostgreSQL, MySQL and any custom MCP server'); + expect(result.instructions).toContain('Personal Agent Proxy connects Email, Google, Granola, Notion, Linear, PostgreSQL, MySQL and any custom MCP server'); expect(result.instructions).toContain('https://app.example.com/connections/new'); }); diff --git a/packages/api/src/records/connectors.ts b/packages/api/src/records/connectors.ts index 9d36c71a..9023f903 100644 --- a/packages/api/src/records/connectors.ts +++ b/packages/api/src/records/connectors.ts @@ -2,6 +2,7 @@ import {findIntegration, type IntegrationId} from '@proxy/integrations'; import type {Connector} from './connector'; import {emailConnector} from './email-connector'; +import {googleConnector} from './google-connector'; import {granolaConnector} from './granola-connector'; import {infoConnector} from './info-connector'; import {linearConnector} from './linear-connector'; @@ -17,6 +18,7 @@ const CONNECTORS: Record, Connector> = { linear: linearConnector, postgres: sqlConnector('postgres'), mysql: sqlConnector('mysql'), + google: googleConnector, }; export function connectorFor(integrationId: string): Connector | undefined { diff --git a/packages/api/src/records/google-connector.test.ts b/packages/api/src/records/google-connector.test.ts new file mode 100644 index 00000000..c82a55a2 --- /dev/null +++ b/packages/api/src/records/google-connector.test.ts @@ -0,0 +1,394 @@ +import {beforeEach, describe, expect, mock, spyOn, test} from 'bun:test'; +import {Ok, type Result} from 'ts-results-es'; + +import type {ApiError} from '@proxy/utils'; +import {findCollection, findIntegration, type Collection} from '@proxy/integrations'; + +mock.module('@proxy/db/connection', () => ({updateConnectionCredential: mock()})); + +mock.module('../observability/log', () => ({ + log: {debug: mock(), info: mock(), warn: mock(), error: mock()}, + serializeError: mock(() => ({})), +})); + +mock.module('../utils/env', () => ({env: {NODE_ENV: 'production', APP_URL: 'https://app.example.com', GOOGLE_CLIENT_ID: 'client-1', GOOGLE_CLIENT_SECRET: 'secret-1'}})); + +mock.module('../utils/secret-crypto', () => ({ + encryptSecret: (plaintext: string): Result => Ok(`enc:${plaintext}`), + decryptSecret: (stored: string): Result => Ok(stored.replace(/^enc:/, '')), +})); + +function files(): Collection { + const integration = findIntegration('google'); + const found = integration && findCollection(integration, 'files'); + if (!found) { + throw new Error('No files collection'); + } + return found; +} + +const CREDENTIAL = {accessToken: 'access-1', refreshToken: 'refresh-1', expiresAt: new Date(Date.now() + 3_600_000).toISOString(), scopes: [], email: 'alex@example.com'}; + +const target = { + connection: {id: 'google-1', integrationId: 'google', account: 'alex@example.com', createdAt: new Date(), tools: null, name: null, defaults: [], credential: `enc:${JSON.stringify(CREDENTIAL)}`}, + collection: files(), +}; + +const query = (changes: object = {}) => ({search: null, page: null, filter: null, parent: null, ...changes}); + +// A Drive as Google answers it: My Drive holding Projects, which holds a plan, a budget and a deck. +const MY_DRIVE_ID = '0AFmyDriveRootUk9PVA'; +const OWNER = [{displayName: 'Alex Kim', emailAddress: 'alex@example.com'}]; + +const DRIVE_FILES: Record = { + '1Projects': { + id: '1Projects', + name: 'Projects', + mimeType: 'application/vnd.google-apps.folder', + modifiedTime: '2026-10-01T09:00:00.000Z', + owners: OWNER, + webViewLink: 'https://drive.google.com/drive/folders/1Projects', + parents: [MY_DRIVE_ID], + }, + '1Plan': { + id: '1Plan', + name: 'Launch plan', + mimeType: 'application/vnd.google-apps.document', + modifiedTime: '2026-10-08T08:16:23.137Z', + owners: OWNER, + webViewLink: 'https://docs.google.com/document/d/1Plan/edit?usp=drivesdk', + parents: ['1Projects'], + }, + '1Budget': { + id: '1Budget', + name: 'Budget 2026', + mimeType: 'application/vnd.google-apps.spreadsheet', + modifiedTime: '2026-10-07T12:00:00.000Z', + owners: OWNER, + webViewLink: 'https://docs.google.com/spreadsheets/d/1Budget/edit?usp=drivesdk', + parents: ['1Projects'], + }, + '1Huge': { + id: '1Huge', + name: 'Everything ever written', + mimeType: 'application/vnd.google-apps.document', + modifiedTime: '2026-09-01T12:00:00.000Z', + owners: OWNER, + webViewLink: 'https://docs.google.com/document/d/1Huge/edit?usp=drivesdk', + parents: [MY_DRIVE_ID], + }, +}; + +const SHEETS = { + sheets: [ + {properties: {sheetId: 0, title: 'Q1', gridProperties: {rowCount: 120, columnCount: 8}}}, + {properties: {sheetId: 1834, title: "Alex's notes", gridProperties: {rowCount: 1000, columnCount: 26}}}, + ], +}; + +function rows(from: number, count: number): string[][] { + return Array.from({length: count}, (_, index) => [`Item ${from + index}`, String((from + index) * 10), index === 0 ? 'Paid' : '']); +} + +function googleError(status: number, message: string, reason: string): Response { + return Response.json({error: {code: status, message, errors: [{message, domain: 'global', reason}]}}, {status}); +} + +const drive = {requests: [] as URL[], answer: (_url: URL): Response | null => null}; + +function answer(url: URL): Response { + const custom = drive.answer(url); + if (custom) { + return custom; + } + const path = url.pathname; + if (path === '/drive/v3/files/root') { + return Response.json({id: MY_DRIVE_ID}); + } + if (path === '/drive/v3/files') { + return Response.json({nextPageToken: 'page-2', files: [DRIVE_FILES['1Projects'], DRIVE_FILES['1Plan']]}); + } + if (path === '/drive/v3/files/1Plan/export') { + return new Response('# Launch plan\n\n- Ship it', {headers: {'content-type': 'text/markdown'}}); + } + if (path === '/drive/v3/files/1Huge/export') { + return googleError(403, 'This file is too large to be exported.', 'exportSizeLimitExceeded'); + } + const file = /^\/drive\/v3\/files\/([^/]+)$/.exec(path)?.[1]; + if (file) { + return DRIVE_FILES[file] ? Response.json(DRIVE_FILES[file]) : googleError(404, `File not found: ${file}.`, 'notFound'); + } + if (path === '/v4/spreadsheets/1Budget') { + return Response.json(SHEETS); + } + if (path === '/v4/spreadsheets/1Budget/values:batchGet') { + const [header, range] = url.searchParams.getAll('ranges'); + const [, from = '2', to = '2'] = /!A(\d+):ZZ(\d+)$/.exec(range ?? '') ?? []; + const count = Number(from) === 52 ? 3 : Number(to) - Number(from) + 1; + return Response.json({ + spreadsheetId: '1Budget', + valueRanges: [ + {range: header, majorDimension: 'ROWS', values: [['Item', 'Amount', '']]}, + {range, majorDimension: 'ROWS', values: rows(Number(from), count)}, + ], + }); + } + return googleError(404, 'Not found', 'notFound'); +} + +spyOn(globalThis, 'fetch').mockImplementation((async (input: string | URL | Request) => { + const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + drive.requests.push(url); + return answer(url); +}) as typeof fetch); + +function requested(path: string): URL[] { + return drive.requests.filter((url) => url.pathname === path); +} + +beforeEach(() => { + drive.requests = []; + drive.answer = () => null; +}); + +describe('googleConnector.list', () => { + test('the top is the four places, each holding files', async () => { + const {googleConnector} = await import('./google-connector'); + const page = (await googleConnector.list(target, query())).unwrap(); + + expect(page).toEqual({ + records: [ + {id: 'root', values: {name: 'My Drive', kind: 'Place'}, updatedAt: '', hasChildren: true}, + {id: 'shared-with-me', values: {name: 'Shared with me', kind: 'Place'}, updatedAt: '', hasChildren: true}, + {id: 'shared-drives', values: {name: 'Shared drives', kind: 'Place'}, updatedAt: '', hasChildren: true}, + {id: 'recent', values: {name: 'Recent', kind: 'Place'}, updatedAt: '', hasChildren: true}, + ], + nextPage: null, + }); + expect(drive.requests).toEqual([]); + }); + + test('inside a folder, its files, folders first, from the page token given, with the next one and the trail', async () => { + const {googleConnector} = await import('./google-connector'); + const page = (await googleConnector.list(target, query({parent: '1Projects', page: 'page-1'}))).unwrap(); + + const [list] = requested('/drive/v3/files'); + expect(Object.fromEntries(list?.searchParams ?? [])).toEqual({ + q: "'1Projects' in parents and trashed=false", + orderBy: 'folder,modifiedTime desc', + pageSize: '50', + pageToken: 'page-1', + fields: 'nextPageToken,files(id,name,mimeType,modifiedTime,owners(displayName,emailAddress),webViewLink,parents,driveId)', + supportsAllDrives: 'true', + includeItemsFromAllDrives: 'true', + }); + expect(requested('/drive/v3/files/1Projects')[0]?.searchParams.get('supportsAllDrives')).toBe('true'); + expect(page.nextPage).toBe('page-2'); + expect(page.records).toEqual([ + { + id: '1Projects', + values: {name: 'Projects', kind: 'Folder', modified: '2026-10-01T09:00:00.000Z', owner: 'Alex Kim', link: 'https://drive.google.com/drive/folders/1Projects'}, + updatedAt: '2026-10-01T09:00:00.000Z', + hasChildren: true, + }, + { + id: '1Plan', + values: {name: 'Launch plan', kind: 'Document', modified: '2026-10-08T08:16:23.137Z', owner: 'Alex Kim', link: 'https://docs.google.com/document/d/1Plan/edit?usp=drivesdk'}, + updatedAt: '2026-10-08T08:16:23.137Z', + }, + ]); + expect(page.trail).toEqual([ + {id: 'root', title: 'My Drive'}, + {id: '1Projects', title: 'Projects'}, + ]); + }); + + test('My Drive lists what is at its top, Shared with me and Recent in their own order', async () => { + const {googleConnector} = await import('./google-connector'); + await googleConnector.list(target, query({parent: 'root'})); + await googleConnector.list(target, query({parent: 'shared-with-me'})); + const recent = (await googleConnector.list(target, query({parent: 'recent'}))).unwrap(); + + expect(requested('/drive/v3/files').map((url) => [url.searchParams.get('q'), url.searchParams.get('orderBy')])).toEqual([ + ["'root' in parents and trashed=false", 'folder,modifiedTime desc'], + ['sharedWithMe=true and trashed=false', 'sharedWithMeTime desc'], + ['trashed=false', 'viewedByMeTime desc'], + ]); + expect(recent.trail).toEqual([{id: 'recent', title: 'Recent'}]); + }); + + test('shared drives are listed as folders, and inside one its own files', async () => { + const {googleConnector} = await import('./google-connector'); + drive.answer = (url) => { + if (url.pathname === '/drive/v3/drives') { + return Response.json({drives: [{id: '0ATeamDrive', name: 'Team', kind: 'drive#drive'}]}); + } + if (url.pathname === '/drive/v3/drives/0ATeamDrive') { + return Response.json({id: '0ATeamDrive', name: 'Team'}); + } + return null; + }; + + const drives = (await googleConnector.list(target, query({parent: 'shared-drives'}))).unwrap(); + const inside = (await googleConnector.list(target, query({parent: 'drive-0ATeamDrive'}))).unwrap(); + + expect(drives.records).toEqual([{id: 'drive-0ATeamDrive', values: {name: 'Team', kind: 'Folder'}, updatedAt: '', hasChildren: true}]); + const [list] = requested('/drive/v3/files'); + expect(list?.searchParams.get('q')).toBe("'0ATeamDrive' in parents and trashed=false"); + expect(list?.searchParams.get('corpora')).toBe('drive'); + expect(list?.searchParams.get('driveId')).toBe('0ATeamDrive'); + expect(inside.trail).toEqual([ + {id: 'shared-drives', title: 'Shared drives'}, + {id: 'drive-0ATeamDrive', title: 'Team'}, + ]); + }); + + test('a search goes through all of Drive, its words quoted as Drive needs, in the order Drive finds best', async () => { + const {googleConnector} = await import('./google-connector'); + await googleConnector.list(target, query({search: "Alex's \\ plan"})); + await googleConnector.list(target, query({search: 'plan', parent: '1Projects'})); + + const [everywhere, inFolder] = requested('/drive/v3/files'); + expect(everywhere?.searchParams.get('q')).toBe("fullText contains 'Alex\\'s \\\\ plan' and trashed=false"); + expect(everywhere?.searchParams.get('corpora')).toBe('allDrives'); + expect(everywhere?.searchParams.get('orderBy')).toBeNull(); + expect(inFolder?.searchParams.get('q')).toBe("fullText contains 'plan' and '1Projects' in parents and trashed=false"); + }); + + test("an id that isn't one of Drive's never reaches Google", async () => { + const {googleConnector} = await import('./google-connector'); + const result = await googleConnector.list(target, query({parent: "x' or name contains '"})); + + expect(result.isErr() && result.error.kind).toBe('not_found'); + expect(drive.requests).toEqual([]); + }); + + test("a spreadsheet holds its tabs, and a tab its rows, 50 to a page under the first row's headers", async () => { + const {googleConnector} = await import('./google-connector'); + const tabs = (await googleConnector.list(target, query({parent: '1Budget'}))).unwrap(); + + expect(tabs.records.map((record) => [record.id, record.values.name, record.values.content, record.hasChildren])).toEqual([ + ['sheet-1Budget-0', 'Q1', '120 rows, 8 columns', true], + ['sheet-1Budget-1834', "Alex's notes", '1000 rows, 26 columns', true], + ]); + expect(tabs.trail).toEqual([ + {id: 'root', title: 'My Drive'}, + {id: '1Projects', title: 'Projects'}, + {id: '1Budget', title: 'Budget 2026'}, + ]); + + const first = (await googleConnector.list(target, query({parent: 'sheet-1Budget-1834'}))).unwrap(); + expect(requested('/v4/spreadsheets/1Budget/values:batchGet')[0]?.searchParams.getAll('ranges')).toEqual(["'Alex''s notes'!A1:ZZ1", "'Alex''s notes'!A2:ZZ51"]); + expect(first.records).toHaveLength(50); + expect(first.records[0]).toEqual({ + id: 'row-1Budget-1834-2', + values: { + name: 'Item 2', + kind: 'Row', + modified: '2026-10-07T12:00:00.000Z', + owner: 'Alex Kim', + link: 'https://docs.google.com/spreadsheets/d/1Budget/edit?usp=drivesdk#gid=1834', + content: 'Item: Item 2\nAmount: 20\nColumn C: Paid', + }, + updatedAt: '2026-10-07T12:00:00.000Z', + }); + expect(first.nextPage).toBe('52'); + expect(first.trail?.at(-1)).toEqual({id: 'sheet-1Budget-1834', title: "Alex's notes"}); + + const last = (await googleConnector.list(target, query({parent: 'sheet-1Budget-1834', page: '52'}))).unwrap(); + expect(last.records.map((record) => record.id)).toEqual(['row-1Budget-1834-52', 'row-1Budget-1834-53', 'row-1Budget-1834-54']); + expect(last.nextPage).toBeNull(); + }); + + test('a page of rows that is not a row number is refused', async () => { + const {googleConnector} = await import('./google-connector'); + const result = await googleConnector.list(target, query({parent: 'sheet-1Budget-0', page: 'A1'})); + + expect(result.isErr() && result.error.kind).toBe('validation_error'); + }); + + test('a folder whose folder the account cannot see is under Shared with me', async () => { + const {googleConnector} = await import('./google-connector'); + drive.answer = (url) => { + if (url.pathname === '/drive/v3/files/1Shared') { + return Response.json({id: '1Shared', name: 'Shared plans', mimeType: 'application/vnd.google-apps.folder', parents: ['1Hidden']}); + } + return null; + }; + const page = (await googleConnector.list(target, query({parent: '1Shared'}))).unwrap(); + + expect(page.trail).toEqual([ + {id: 'shared-with-me', title: 'Shared with me'}, + {id: '1Shared', title: 'Shared plans'}, + ]); + }); +}); + +describe('googleConnector.get', () => { + test('a document opens as Markdown', async () => { + const {googleConnector} = await import('./google-connector'); + const record = (await googleConnector.get(target, '1Plan')).unwrap(); + + expect(requested('/drive/v3/files/1Plan/export')[0]?.searchParams.get('mimeType')).toBe('text/markdown'); + expect(record).toEqual({ + id: '1Plan', + values: { + name: 'Launch plan', + kind: 'Document', + modified: '2026-10-08T08:16:23.137Z', + owner: 'Alex Kim', + link: 'https://docs.google.com/document/d/1Plan/edit?usp=drivesdk', + content: '# Launch plan\n\n- Ship it', + }, + updatedAt: '2026-10-08T08:16:23.137Z', + }); + }); + + test('a spreadsheet opens with its tabs one a line', async () => { + const {googleConnector} = await import('./google-connector'); + const record = (await googleConnector.get(target, '1Budget')).unwrap(); + + expect(record.values.content).toBe("Q1: 120 rows, 8 columns\nAlex's notes: 1000 rows, 26 columns"); + expect(record.hasChildren).toBe(true); + }); + + test('a row opens with its values under the headers', async () => { + const {googleConnector} = await import('./google-connector'); + const record = (await googleConnector.get(target, 'row-1Budget-0-7')).unwrap(); + + expect(requested('/v4/spreadsheets/1Budget/values:batchGet')[0]?.searchParams.getAll('ranges')).toEqual(["'Q1'!A1:ZZ1", "'Q1'!A7:ZZ7"]); + expect(record.values).toMatchObject({name: 'Item 7', kind: 'Row', content: 'Item: Item 7\nAmount: 70\nColumn C: Paid'}); + }); + + test('a file too large to export says so', async () => { + const {googleConnector} = await import('./google-connector'); + const result = await googleConnector.get(target, '1Huge'); + + expect(result.isErr() && result.error).toEqual({kind: 'validation_error', statusCode: 400, message: 'This file is too large to read; Google exports at most 10 MB'}); + }); + + test("a disabled API is Google's own message, saying where to enable it", async () => { + const {googleConnector} = await import('./google-connector'); + const message = + 'Google Sheets API has not been used in project 1234 before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/sheets.googleapis.com/overview?project=1234 then retry.'; + drive.answer = (url) => (url.hostname === 'sheets.googleapis.com' ? googleError(403, message, 'accessNotConfigured') : null); + + const result = await googleConnector.get(target, '1Budget'); + + expect(result.isErr() && result.error).toEqual({kind: 'validation_error', statusCode: 400, message}); + }); + + test('a file Drive does not have is not found by its id', async () => { + const {googleConnector} = await import('./google-connector'); + const result = await googleConnector.get(target, '1Gone'); + + expect(result.isErr() && result.error).toMatchObject({kind: 'not_found', resource: 'record', id: '1Gone'}); + }); +}); + +test('files are not written to yet', async () => { + const {googleConnector} = await import('./google-connector'); + const result = await googleConnector.update(target, '1Plan', {name: 'New name'}); + + expect(result.isErr() && result.error.kind).toBe('validation_error'); +}); diff --git a/packages/api/src/records/google-connector.ts b/packages/api/src/records/google-connector.ts new file mode 100644 index 00000000..1e95a78c --- /dev/null +++ b/packages/api/src/records/google-connector.ts @@ -0,0 +1,28 @@ +import {Err, type Result} from 'ts-results-es'; + +import {ApiErr, type ApiError} from '@proxy/utils'; + +import type {Connector, RecordTarget} from './connector'; +import {getFilesRecord, listFilesTree} from './google-files'; + +/** + * A Google account, fetched live from Google's APIs with the connection's tokens, one collection + * per product: `files`, Drive with Docs, Sheets and Slides (`google-files.ts`). Nothing is written + * yet: the collections offer no writes, and the connector turns any down. + */ + +function unknownCollection(target: RecordTarget): Promise> { + return Promise.resolve(Err(ApiErr.notFound('collection', target.collection.id))); +} + +function notWritten(): Promise> { + return Promise.resolve(Err(ApiErr.validationError('Files in Google Drive can’t be changed here yet'))); +} + +export const googleConnector: Connector = { + list: (target, query) => (target.collection.id === 'files' ? listFilesTree(target, query) : unknownCollection(target)), + get: (target, recordId) => (target.collection.id === 'files' ? getFilesRecord(target, recordId) : unknownCollection(target)), + create: notWritten, + update: notWritten, + remove: notWritten, +}; diff --git a/packages/api/src/records/google-files.ts b/packages/api/src/records/google-files.ts new file mode 100644 index 00000000..95396c5e --- /dev/null +++ b/packages/api/src/records/google-files.ts @@ -0,0 +1,417 @@ +import {Err, Ok, type Result} from 'ts-results-es'; + +import {ApiErr, type ApiError, Do} from '@proxy/utils'; + +import { + DRIVE_ID, + driveQuoted, + exportFile, + getFile, + getFileParents, + getMyDriveId, + getSharedDrive, + listFiles, + listSharedDrives, + MIME_TYPES, + PAGE_SIZE, + type DriveFile, + type FileParents, + type SharedDrive, +} from '../connections/google/drive-api'; +import {callGoogle} from '../connections/google/google-client'; +import {columnLetters, getRanges, getSheets, rowsRange, type Sheet} from '../connections/google/sheets-api'; +import type {Crumb, DataRecord, ListQuery, RecordPage, RecordTarget} from './connector'; +import {valueLines} from './value-lines'; + +/** + * A Google account's Drive as the `files` collection: a tree, fetched live. At the top are four + * places, My Drive, Shared with me, Shared drives and Recent; inside them, and inside folders and + * shared drives, are files, folders first, 50 to a page with Drive's own page token. A search is + * Drive's, in names and contents, anywhere or inside the folder or place opened. + * + * A document opens as Markdown and a presentation as text, as Drive exports them, up to Drive's + * 10 MB; a spreadsheet opens into its tabs, and a tab into its rows, 50 to a page, each row's values + * as "Header: value" lines, the headers from the tab's first row. Other files open as what Drive + * knows of them, with a link. Nothing is written yet. + */ + +const PLACES = [ + {id: 'root', name: 'My Drive'}, + {id: 'shared-with-me', name: 'Shared with me'}, + {id: 'shared-drives', name: 'Shared drives'}, + {id: 'recent', name: 'Recent'}, +] as const; + +type Place = (typeof PLACES)[number]; + +const [MY_DRIVE, SHARED_WITH_ME, SHARED_DRIVES] = PLACES; + +// Folders first, then the last changed, as Drive shows a folder. +const FOLDER_ORDER = 'folder,modifiedTime desc'; + +// How far up the folders a trail goes before it stops looking. +const MAX_DEPTH = 10; + +/** + * What a record id names: a place, a shared drive (`drive-`), a Drive file by its own id, a + * spreadsheet's tab (`sheet--`), or a row of one (`row---`, its number + * in the tab, from 2 since row 1 holds the headers). + */ +type FileRef = + | {kind: 'place'; place: Place} + | {kind: 'drive'; driveId: string} + | {kind: 'file'; fileId: string} + | {kind: 'sheet'; fileId: string; sheetId: number} + | {kind: 'row'; fileId: string; sheetId: number; row: number}; + +function parseRef(recordId: string): FileRef | null { + const place = PLACES.find((candidate) => candidate.id === recordId); + if (place) { + return {kind: 'place', place}; + } + const drive = /^drive-([A-Za-z0-9_-]+)$/.exec(recordId); + if (drive?.[1]) { + return {kind: 'drive', driveId: drive[1]}; + } + const sheet = /^sheet-([A-Za-z0-9_-]+)-(\d{1,12})$/.exec(recordId); + if (sheet?.[1] && sheet[2]) { + return {kind: 'sheet', fileId: sheet[1], sheetId: Number(sheet[2])}; + } + const row = /^row-([A-Za-z0-9_-]+)-(\d{1,12})-(\d{1,7})$/.exec(recordId); + if (row?.[1] && row[2] && row[3] && Number(row[3]) >= 2) { + return {kind: 'row', fileId: row[1], sheetId: Number(row[2]), row: Number(row[3])}; + } + if (DRIVE_ID.test(recordId)) { + return {kind: 'file', fileId: recordId}; + } + return null; +} + +function requireRef(recordId: string): Result { + const ref = parseRef(recordId); + if (!ref) { + return Err(ApiErr.notFound('record', recordId)); + } + return Ok(ref); +} + +function notFoundAs(recordId: string) { + return (error: ApiError): ApiError => (error.kind === 'not_found' ? ApiErr.notFound('record', recordId) : error); +} + +/** One request's way into Drive: the access token, and My Drive's own id, asked for once at most. */ +type Drive = {token: string; myDriveId: () => Promise>}; + +function withDrive(target: RecordTarget, run: (drive: Drive) => Promise>): Promise> { + return callGoogle(target.connection, (token) => { + let myDriveId: Promise> | null = null; + return run({token, myDriveId: () => (myDriveId ??= getMyDriveId(token))}); + }); +} + +function kindOf(mimeType: string): string { + if (mimeType === MIME_TYPES.folder) { + return 'Folder'; + } + if (mimeType === MIME_TYPES.document) { + return 'Document'; + } + if (mimeType === MIME_TYPES.spreadsheet) { + return 'Spreadsheet'; + } + if (mimeType === MIME_TYPES.presentation) { + return 'Presentation'; + } + // Shortcuts too: they open as themselves, not as what they point at. + return 'File'; +} + +function fileValues(file: DriveFile): Record { + const owner = file.owners?.[0]; + return {name: file.name, kind: kindOf(file.mimeType), modified: file.modifiedTime ?? '', owner: owner?.displayName ?? owner?.emailAddress ?? '', link: file.webViewLink ?? ''}; +} + +function fileRecord(file: DriveFile): DataRecord { + const kind = kindOf(file.mimeType); + return {id: file.id, values: fileValues(file), updatedAt: file.modifiedTime ?? '', ...(kind === 'Folder' || kind === 'Spreadsheet' ? {hasChildren: true} : {})}; +} + +function placeRecord(place: Place): DataRecord { + return {id: place.id, values: {name: place.name, kind: 'Place'}, updatedAt: '', hasChildren: true}; +} + +function driveRecord(drive: SharedDrive): DataRecord { + return {id: `drive-${drive.id}`, values: {name: drive.name, kind: 'Folder'}, updatedAt: '', hasChildren: true}; +} + +function sheetId(file: DriveFile, sheet: Sheet): string { + return `sheet-${file.id}-${sheet.sheetId}`; +} + +function sheetLink(file: DriveFile, sheet: Sheet): string { + return file.webViewLink ? `${file.webViewLink}#gid=${sheet.sheetId}` : ''; +} + +// "Sheet1: 120 rows, 8 columns", the tab's grid, empty cells included. +function sheetSize(sheet: Sheet): string { + return `${sheet.rowCount} rows, ${sheet.columnCount} columns`; +} + +function sheetRecord(file: DriveFile, sheet: Sheet): DataRecord { + return { + id: sheetId(file, sheet), + values: {...fileValues(file), name: sheet.title, kind: 'Sheet', link: sheetLink(file, sheet), content: sheetSize(sheet)}, + updatedAt: file.modifiedTime ?? '', + hasChildren: true, + }; +} + +// A row's values under the tab's headers; a column without one goes by its letter. +function rowRecord(file: DriveFile, sheet: Sheet, headers: string[], row: number, cells: string[]): DataRecord { + const content = valueLines(cells.map((cell, column): [string, string] => [headers[column] || `Column ${columnLetters(column)}`, cell])); + return { + id: `row-${file.id}-${sheet.sheetId}-${row}`, + // Rows have no time of their own; the spreadsheet's is when anything in it changed. + values: {...fileValues(file), name: cells[0] || `Row ${row}`, kind: 'Row', link: sheetLink(file, sheet), content}, + updatedAt: file.modifiedTime ?? '', + }; +} + +function isBlank(cells: string[]): boolean { + return cells.every((cell) => cell === ''); +} + +/** A spreadsheet and its tab, the tab found by its id among the spreadsheet's. */ +function getSheet(drive: Drive, fileId: string, id: number): Promise> { + return Do(async ($) => { + const [file, sheets] = await Promise.all([getFile(drive.token, fileId), getSheets(drive.token, fileId)]); + const sheet = $(sheets).find((candidate) => candidate.sheetId === id); + if (!sheet) { + return $(Err(ApiErr.notFound('sheet'))); + } + return {file: $(file), sheet}; + }); +} + +/** + * The records above a file, from the top: where it is, then its folders, the nearest last. Drive + * gives each file the folder it is in; walking up stops at My Drive, at the top of a shared drive + * (whose folder has the drive's own id), or where the account can't see further up, as with a + * folder someone shared, which is then under Shared with me. Past 10 folders it stops looking, and + * the trail starts at the 10th. + */ +function crumbsAbove(drive: Drive, file: FileParents): Promise> { + return Do(async ($) => { + const sharedDriveId = file.driveId; + const myDriveId = sharedDriveId ? null : $(await drive.myDriveId()); + const folders: Crumb[] = []; + let parentId = file.parents?.[0]; + for (let depth = 0; depth < MAX_DEPTH; depth += 1) { + if (parentId === undefined) { + return [{id: SHARED_WITH_ME.id, title: SHARED_WITH_ME.name}, ...folders]; + } + if (parentId === myDriveId) { + return [{id: MY_DRIVE.id, title: MY_DRIVE.name}, ...folders]; + } + if (parentId === sharedDriveId) { + const shared = $(await getSharedDrive(drive.token, sharedDriveId)); + return [{id: SHARED_DRIVES.id, title: SHARED_DRIVES.name}, {id: `drive-${shared.id}`, title: shared.name}, ...folders]; + } + const parent = await getFileParents(drive.token, parentId); + if (parent.isErr() && parent.error.kind === 'not_found') { + return [{id: SHARED_WITH_ME.id, title: SHARED_WITH_ME.name}, ...folders]; + } + const {id, name, parents} = $(parent); + folders.unshift({id, title: name}); + parentId = parents?.[0]; + } + return folders; + }); +} + +/** The trail to a file: the records above it, then the file. */ +function trailTo(drive: Drive, file: DriveFile): Promise> { + return Do(async ($) => [...$(await crumbsAbove(drive, file)), {id: file.id, title: file.name}]); +} + +/** + * One page of files matching `terms`, in `order`, or, searched, in the order Drive finds best, + * since Drive sorts no full-text search. Trashed files are never listed. + */ +function filesPage(drive: Drive, query: ListQuery, args: {terms: string[]; order: string | null; params?: Record}): Promise> { + return Do(async ($) => { + const search = query.search === null ? [] : [`fullText contains ${driveQuoted(query.search)}`]; + const q = [...search, ...args.terms, 'trashed=false'].join(' and '); + const order: Record = query.search === null && args.order ? {orderBy: args.order} : {}; + const page = $(await listFiles(drive.token, q, {...order, ...args.params}, query.page)); + return {records: page.items.map(fileRecord), nextPage: page.nextPage}; + }); +} + +function inParents(id: string): string { + return `${driveQuoted(id)} in parents`; +} + +function listPlace(drive: Drive, place: Place, query: ListQuery): Promise> { + return Do(async ($) => { + const trail = [{id: place.id, title: place.name}]; + if (place.id === 'root') { + return {...$(await filesPage(drive, query, {terms: [inParents('root')], order: FOLDER_ORDER})), trail}; + } + if (place.id === 'shared-with-me') { + return {...$(await filesPage(drive, query, {terms: ['sharedWithMe=true'], order: 'sharedWithMeTime desc'})), trail}; + } + if (place.id === 'recent') { + return {...$(await filesPage(drive, query, {terms: [], order: 'viewedByMeTime desc'})), trail}; + } + // Shared drives are searched by their names. + const drives = $(await listSharedDrives(drive.token, query.search, query.page)); + return {records: drives.items.map(driveRecord), nextPage: drives.nextPage, trail}; + }); +} + +function listSharedDrive(drive: Drive, driveId: string, query: ListQuery): Promise> { + return Do(async ($) => { + const [shared, page] = await Promise.all([getSharedDrive(drive.token, driveId), filesPage(drive, query, {terms: [inParents(driveId)], order: FOLDER_ORDER, params: {corpora: 'drive', driveId}})]); + const {id, name} = $(shared); + return { + ...$(page), + trail: [ + {id: SHARED_DRIVES.id, title: SHARED_DRIVES.name}, + {id: `drive-${id}`, title: name}, + ], + }; + }); +} + +/** A tab's rows, 50 to a page, the page token the number of the row it starts at. */ +function listRows(drive: Drive, ref: {fileId: string; sheetId: number}, query: ListQuery): Promise> { + return Do(async ($) => { + if (query.page !== null && !/^\d{1,7}$/.test(query.page)) { + return $(Err(ApiErr.validationError(`Not a page of rows: ${query.page}`))); + } + const from = Math.max(Number(query.page ?? 2), 2); + const to = from + PAGE_SIZE - 1; + const {file, sheet} = $(await getSheet(drive, ref.fileId, ref.sheetId)); + const [ranges, trail] = await Promise.all([getRanges(drive.token, file.id, [rowsRange(sheet.title, 1, 1), rowsRange(sheet.title, from, to)]), trailTo(drive, file)]); + const [[headers = []] = [], rows = []] = $(ranges); + const records = rows.flatMap((cells, index) => (isBlank(cells) ? [] : [rowRecord(file, sheet, headers, from + index, cells)])); + // Sheets leaves out the empty rows at the end, so a page short of 50 is the last. + const more = rows.length === PAGE_SIZE && to < sheet.rowCount; + return {records, nextPage: more ? String(to + 1) : null, trail: [...$(trail), {id: sheetId(file, sheet), title: sheet.title}]}; + }); +} + +const NOT_SEARCHED = 'Only Drive, its folders and shared drives are searched, not a spreadsheet’s tabs or rows'; + +// What a list opened in a file shows: a folder's files, a spreadsheet's tabs; other files hold nothing. +function listFile(drive: Drive, fileId: string, query: ListQuery): Promise> { + return Do(async ($) => { + const file = $(await getFile(drive.token, fileId)); + const kind = kindOf(file.mimeType); + if (kind === 'Spreadsheet' && query.search !== null) { + return $(Err(ApiErr.validationError(NOT_SEARCHED))); + } + const trail = $(await trailTo(drive, file)); + if (kind === 'Folder') { + const params: Record = file.driveId ? {corpora: 'drive', driveId: file.driveId} : {}; + return {...$(await filesPage(drive, query, {terms: [inParents(file.id)], order: FOLDER_ORDER, params})), trail}; + } + if (kind === 'Spreadsheet') { + const sheets = $(await getSheets(drive.token, file.id)); + return {records: sheets.map((sheet) => sheetRecord(file, sheet)), nextPage: null, trail}; + } + return {records: [], nextPage: null, trail}; + }); +} + +function listInside(drive: Drive, ref: FileRef, query: ListQuery): Promise> { + if (ref.kind === 'place') { + return listPlace(drive, ref.place, query); + } + if (ref.kind === 'drive') { + return listSharedDrive(drive, ref.driveId, query); + } + if (ref.kind === 'file') { + return listFile(drive, ref.fileId, query); + } + if (query.search !== null) { + return Promise.resolve(Err(ApiErr.validationError(NOT_SEARCHED))); + } + if (ref.kind === 'sheet') { + return listRows(drive, ref, query); + } + // A row holds nothing. + return Do(async ($) => { + const {file, sheet} = $(await getSheet(drive, ref.fileId, ref.sheetId)); + const trail = [...$(await trailTo(drive, file)), {id: sheetId(file, sheet), title: sheet.title}, {id: `row-${file.id}-${sheet.sheetId}-${ref.row}`, title: `Row ${ref.row}`}]; + return {records: [], nextPage: null, trail}; + }); +} + +export function listFilesTree(target: RecordTarget, query: ListQuery): Promise> { + return withDrive(target, (drive) => + Do(async ($) => { + if (query.parent !== null) { + const ref = $(requireRef(query.parent)); + return $((await listInside(drive, ref, query)).mapErr(notFoundAs(query.parent))); + } + if (query.search !== null) { + return $(await filesPage(drive, query, {terms: [], order: null, params: {corpora: 'allDrives'}})); + } + return {records: PLACES.map(placeRecord), nextPage: null}; + }), + ); +} + +// A document as Markdown, a presentation as text, a spreadsheet's tabs one a line; other files have none. +function contentOf(drive: Drive, file: DriveFile): Promise> { + return Do(async ($) => { + const kind = kindOf(file.mimeType); + if (kind === 'Document') { + return $(await exportFile(drive.token, file.id, 'text/markdown')); + } + if (kind === 'Presentation') { + return $(await exportFile(drive.token, file.id, 'text/plain')); + } + if (kind === 'Spreadsheet') { + const sheets = $(await getSheets(drive.token, file.id)); + return sheets.map((sheet) => `${sheet.title}: ${sheetSize(sheet)}`).join('\n'); + } + return ''; + }); +} + +function getRef(drive: Drive, ref: FileRef): Promise> { + return Do(async ($) => { + if (ref.kind === 'place') { + return placeRecord(ref.place); + } + if (ref.kind === 'drive') { + return driveRecord($(await getSharedDrive(drive.token, ref.driveId))); + } + if (ref.kind === 'file') { + const file = $(await getFile(drive.token, ref.fileId)); + const record = fileRecord(file); + return {...record, values: {...record.values, content: $(await contentOf(drive, file))}}; + } + const {file, sheet} = $(await getSheet(drive, ref.fileId, ref.sheetId)); + if (ref.kind === 'sheet') { + return sheetRecord(file, sheet); + } + const [[headers = []] = [], [cells = []] = []] = $(await getRanges(drive.token, file.id, [rowsRange(sheet.title, 1, 1), rowsRange(sheet.title, ref.row, ref.row)])); + if (isBlank(cells)) { + return $(Err(ApiErr.notFound('row'))); + } + return rowRecord(file, sheet, headers, ref.row, cells); + }); +} + +export function getFilesRecord(target: RecordTarget, recordId: string): Promise> { + return withDrive(target, (drive) => + Do(async ($) => { + const ref = $(requireRef(recordId)); + return $((await getRef(drive, ref)).mapErr(notFoundAs(recordId))); + }), + ); +} diff --git a/packages/api/src/server/create-api-router.ts b/packages/api/src/server/create-api-router.ts index 8b1eaf1a..0c2f9c1d 100644 --- a/packages/api/src/server/create-api-router.ts +++ b/packages/api/src/server/create-api-router.ts @@ -20,6 +20,7 @@ import {handleGoogleCallbackRoute, handleGoogleStartRoute, handleSignInMethodsRo import {handleSignOutRoute} from '../auth/signout'; import {handleConnectCustomMcpRoute, handleCustomMcpCallbackRoute, handleRefreshCustomMcpToolsRoute} from '../connections/custom/route'; import {handleConnectEmailRoute, handleTestEmailRoute} from '../connections/email/route'; +import {handleGoogleCallbackRoute as handleConnectGoogleCallbackRoute, handleStartGoogleRoute} from '../connections/google/route'; import {handleGranolaCallbackRoute, handleStartGranolaRoute} from '../connections/granola/route'; import {handleLinearCallbackRoute, handleStartLinearRoute} from '../connections/linear/route'; import {handleConnectMysqlRoute, handleTestMysqlRoute} from '../connections/mysql/route'; @@ -64,6 +65,8 @@ export function createApiRouter(): express.Router { router.get('/api/connections', withAuthResult('List connections route', handleListConnectionsRoute)); router.post('/api/connections/email', withAuthResult('Connect email route', handleConnectEmailRoute)); router.post('/api/connections/email/test', withAuthResult('Test email route', handleTestEmailRoute)); + router.get('/api/connections/google/start', withAuth('Start Google sign-in route', handleStartGoogleRoute)); + router.get('/api/connections/google/callback', withAuth('Google sign-in callback route', handleConnectGoogleCallbackRoute)); router.get('/api/connections/granola/start', withAuth('Start Granola sign-in route', handleStartGranolaRoute)); router.get('/api/connections/granola/callback', withAuth('Granola sign-in callback route', handleGranolaCallbackRoute)); router.get('/api/connections/notion/start', withAuth('Start Notion sign-in route', handleStartNotionRoute)); diff --git a/packages/api/src/utils/env-schema.ts b/packages/api/src/utils/env-schema.ts index 2c9bff80..1139ee0f 100644 --- a/packages/api/src/utils/env-schema.ts +++ b/packages/api/src/utils/env-schema.ts @@ -58,7 +58,7 @@ export const envSchema = z.object({ group: 'Core', required: true, description: - 'Encrypts the app passwords and Information records Personal Agent Proxy stores: 32 random bytes, base64 (`openssl rand -base64 32`). Back it up: changing or losing it makes everything stored with it unreadable.', + 'Encrypts the app passwords, OAuth tokens and Information records Personal Agent Proxy stores: 32 random bytes, base64 (`openssl rand -base64 32`). Back it up: changing or losing it makes everything stored with it unreadable.', }, ), @@ -69,7 +69,8 @@ export const envSchema = z.object({ }), GOOGLE_CLIENT_ID: documented(z.string().min(1).optional(), { group: 'Sign-in', - description: 'Turns on Google sign-in, with `GOOGLE_CLIENT_SECRET`. Register `/auth/google/callback` as the redirect URI.', + description: + 'Turns on Google sign-in, with `GOOGLE_CLIENT_SECRET`, and connecting a Google account (Gmail, Drive, Docs, Sheets and Slides). Register `/auth/google/callback` and `/api/connections/google/callback` as redirect URIs, enable the Gmail, Drive, Docs, Sheets and Slides APIs in the client’s Cloud project, and set the app’s publishing status to In production: in Testing, Google ends a connection’s sign-in after 7 days.', }), GOOGLE_CLIENT_SECRET: documented(z.string().min(1).optional(), { group: 'Sign-in', diff --git a/packages/api/src/utils/google-oauth-client.ts b/packages/api/src/utils/google-oauth-client.ts new file mode 100644 index 00000000..3d674528 --- /dev/null +++ b/packages/api/src/utils/google-oauth-client.ts @@ -0,0 +1,15 @@ +import {env} from './env'; + +export type GoogleClient = {clientId: string; clientSecret: string}; + +/** + * The app's Google OAuth client, when one is set up: Google sign-in uses it (`auth/google.ts`), and + * so does connecting a Google account (`connections/google/`), each with a redirect URI of its own. + */ +export function googleOAuthClient(): GoogleClient | null { + if (!env.GOOGLE_CLIENT_ID || !env.GOOGLE_CLIENT_SECRET) { + return null; + } + + return {clientId: env.GOOGLE_CLIENT_ID, clientSecret: env.GOOGLE_CLIENT_SECRET}; +} diff --git a/packages/db/scripts/seed.ts b/packages/db/scripts/seed.ts index a33d5429..f6c5bc7e 100644 --- a/packages/db/scripts/seed.ts +++ b/packages/db/scripts/seed.ts @@ -204,6 +204,32 @@ const tracker = await connect({ defaults: ['read', 'create'], }); +// Made-up tokens, like Linear's: opening a file fails. +const drive = await connect({ + integrationId: 'google', + account: 'demo.user@gmail.com', + credential: encrypt( + JSON.stringify({ + accessToken: 'demo-access-token', + refreshToken: null, + expiresAt: null, + scopes: [ + 'openid', + 'https://www.googleapis.com/auth/userinfo.email', + 'https://www.googleapis.com/auth/gmail.modify', + 'https://www.googleapis.com/auth/gmail.send', + 'https://www.googleapis.com/auth/drive', + 'https://www.googleapis.com/auth/documents', + 'https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/presentations', + ], + email: 'demo.user@gmail.com', + }), + ), + daysAgo: 1, + defaults: ['readFiles'], +}); + const connections: [Connection, ...Connection[]] = [ info, personal, @@ -213,6 +239,7 @@ const connections: [Connection, ...Connection[]] = [ meetings, wiki, tracker, + drive, ]; // ---- Information records -------------------------------------------------------------------- @@ -426,6 +453,15 @@ const ISSUE_TITLES: [string, ...string[]] = [ 'Slow search on large workspaces', ]; +// Titles for Google Drive's files. +const FILE_TITLES: [string, ...string[]] = [ + 'Budget 2027', + 'Launch plan', + 'Board deck — October', + 'Contracts', + 'Trip itinerary', +]; + // What agents search for. const SEARCHES: [string, ...string[]] = [ 'invoice', @@ -453,6 +489,10 @@ function recordTitle(connection: Connection, collectionId: string): string { return pick(ISSUE_TITLES); } + if (connection.integrationId === 'google') { + return pick(FILE_TITLES); + } + const titles = (INFO_RECORDS[collectionId] ?? []).map((values) => values.label ?? values.title); return titles[Math.floor(random() * titles.length)] ?? 'Untitled'; } diff --git a/packages/integrations/src/catalog.ts b/packages/integrations/src/catalog.ts index d6bea7f6..0d7541a3 100644 --- a/packages/integrations/src/catalog.ts +++ b/packages/integrations/src/catalog.ts @@ -509,6 +509,69 @@ const postgres = sqlIntegration( const mysql = sqlIntegration('mysql', 'MySQL', 'Tables and queries in a MySQL or MariaDB database'); +// A Google account, signed in to once for everything Personal Agent Proxy reads and writes there: +// Gmail, Drive, Docs, Sheets and Slides. One connection holds every product, so action ids carry +// the product, as Information's `readCards` and `writeCards` do. Drive is one tree: My Drive, +// Shared with me, Shared drives and Recent at the top, folders holding files, a spreadsheet holding +// its tabs and a tab its rows. Documents read as Markdown, presentations as text, rows as "Header: +// value" lines. Nothing is copied: every request goes to Google. +const google: Integration = { + id: 'google', + name: 'Google', + description: 'Gmail, Drive, Docs, Sheets and Slides for one Google account', + actions: [ + { + id: 'readFiles', + label: 'Read files', + description: + 'Browse and search Drive, and open files, documents, spreadsheets and presentations', + risk: 'low', + }, + ], + collections: [ + { + id: 'files', + name: 'Files', + singular: 'file', + titleField: 'name', + summaryField: 'kind', + searchHint: + 'Words in file names and contents, anywhere in Drive, or inside the folder opened.', + nested: true, + read: 'readFiles', + writes: {}, + fields: [ + {key: 'name', label: 'Name', type: 'text'}, + { + key: 'kind', + label: 'Kind', + type: 'select', + // A place is one of the four at the top: My Drive, Shared with me, Shared drives, Recent. + options: [ + 'Place', + 'Folder', + 'Document', + 'Spreadsheet', + 'Presentation', + 'Sheet', + 'Row', + 'File', + ], + system: true, + }, + {key: 'modified', label: 'Modified', type: 'datetime', system: true}, + // Whoever owns the file in Drive; files in a shared drive have none. + {key: 'owner', label: 'Owner', type: 'text', system: true}, + // The file in Drive. + {key: 'link', label: 'Link', type: 'text', system: true}, + // A document as Markdown, a presentation as text, a spreadsheet's tabs one a line with their + // size, a row's values one "Header: value" a line, the headers from the tab's first row. + {key: 'content', label: 'Content', type: 'longtext'}, + ], + }, + ], +}; + // Information's actions for one of its collections, such as `readCards` and `writeCards`: reading // it, and adding, editing and deleting as one. function infoActions(collectionId: string, plural: string): Action[] { @@ -612,6 +675,7 @@ const mcp: Integration = { export const INTEGRATIONS: Integration[] = [ info, email, + google, granola, notion, linear, diff --git a/packages/integrations/src/tools.test.ts b/packages/integrations/src/tools.test.ts index eed57a53..90f4d8ef 100644 --- a/packages/integrations/src/tools.test.ts +++ b/packages/integrations/src/tools.test.ts @@ -195,3 +195,20 @@ describe('postgres and mysql', () => { }); }); }); + +describe('google', () => { + const google = integration('google'); + + test('files are a tree that is listed and opened, never written to yet', () => { + expect(integrationTools(google).map((tool) => tool.name)).toEqual(['files_list', 'files_get']); + }); + + test('a list takes the folder to list inside and a search', () => { + const list = findTool(google, 'files_list'); + + expect(Object.keys(list?.inputSchema.properties ?? {})).toEqual(['parent', 'search', 'page']); + expect(list?.description).toContain('hasChildren'); + expect(list?.action).toBe('readFiles'); + expect(list?.readOnly).toBe(true); + }); +}); diff --git a/packages/integrations/src/types.ts b/packages/integrations/src/types.ts index 384ca94e..0e0dbd64 100644 --- a/packages/integrations/src/types.ts +++ b/packages/integrations/src/types.ts @@ -80,7 +80,7 @@ export type Collection = { }; export type IntegrationId = - 'info' | 'email' | 'granola' | 'notion' | 'linear' | 'postgres' | 'mysql' | 'mcp'; + 'info' | 'email' | 'granola' | 'notion' | 'linear' | 'postgres' | 'mysql' | 'mcp' | 'google'; export type Integration = { id: IntegrationId; diff --git a/packages/web/public/integrations/google.svg b/packages/web/public/integrations/google.svg new file mode 100644 index 00000000..58150186 --- /dev/null +++ b/packages/web/public/integrations/google.svg @@ -0,0 +1 @@ + diff --git a/packages/web/src/components/integration-catalog.tsx b/packages/web/src/components/integration-catalog.tsx index 4b453563..c3df3333 100644 --- a/packages/web/src/components/integration-catalog.tsx +++ b/packages/web/src/components/integration-catalog.tsx @@ -13,7 +13,7 @@ import type {Connection} from '@/lib/types'; // `?returnTo=welcome`, it comes back to the welcome flow's page for the connection, or its catalog. // The rest are connected on a form of their own: a mailbox with an app password, a database with // a user's name and password, a custom MCP server by its address. -const SIGN_IN: IntegrationId[] = ['granola', 'notion', 'linear']; +const SIGN_IN: IntegrationId[] = ['google', 'granola', 'notion', 'linear']; // The whole row is the link or button; its plus is styled as a ghost icon button that lights up // with the row. diff --git a/packages/web/src/lib/integrations.ts b/packages/web/src/lib/integrations.ts index 5634b13b..1c7eb96b 100644 --- a/packages/web/src/lib/integrations.ts +++ b/packages/web/src/lib/integrations.ts @@ -18,6 +18,7 @@ export type Integration = CatalogIntegration & {logo: LucideIcon | IntegrationLo const LOGOS: Record = { info: IdCardIcon, email: MailIcon, + google: {src: '/integrations/google.svg'}, granola: {src: '/integrations/granola.png', fill: true}, notion: {src: '/integrations/notion.svg'}, linear: {src: '/integrations/linear.svg', fill: true}, From 5365f341663e3a7a2315761225bca240df6a3c31 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:14:11 +0000 Subject: [PATCH 3/6] page rows past a page ending in empty ones, take a shared drive the account can't see as shared with me, list only opened files in recent, keep google's reason for a token error, refuse unverified addresses, and don't say a connection signed in to again was added --- packages/api/src/connections/connect-flow.ts | 31 +++++---- packages/api/src/connections/custom/route.ts | 2 +- .../connections/google/google-client.test.ts | 2 +- .../src/connections/google/google-oauth.ts | 24 ++++--- .../api/src/connections/google/route.test.ts | 36 ++++++++-- .../api/src/connections/granola/route.test.ts | 2 +- .../api/src/connections/linear/route.test.ts | 2 +- .../api/src/records/google-connector.test.ts | 69 +++++++++++++++---- packages/api/src/records/google-files.ts | 46 +++++++++++-- 9 files changed, 162 insertions(+), 52 deletions(-) diff --git a/packages/api/src/connections/connect-flow.ts b/packages/api/src/connections/connect-flow.ts index 9df6edd5..c6e4e0e1 100644 --- a/packages/api/src/connections/connect-flow.ts +++ b/packages/api/src/connections/connect-flow.ts @@ -63,27 +63,27 @@ export function makePkce(): {verifier: string; challenge: string} { /** * Saves a connection: a new one, or an existing one of the same account given the new credential, - * and the tools and the name, where it has its own. + * and the tools and the name, where it has its own; `added` unless it was already there. */ -export function saveConnection(orgId: string, integrationId: IntegrationId, connected: Connected): Promise> { +export function saveConnection(orgId: string, integrationId: IntegrationId, connected: Connected): Promise> { return Do(async ($) => { const encrypted = $(encryptSecret(JSON.stringify(connected.credential))); const existing = $(await listConnections(orgId)).find((connection) => connection.integrationId === integrationId && connection.account === connected.account); const name = connected.name === undefined ? {} : {name: connected.name}; if (!existing) { const tools = connected.tools === undefined ? {} : {tools: JSON.stringify(connected.tools)}; - return $(await createConnection({orgId, integrationId, account: connected.account, credential: encrypted, defaults: connected.defaults, ...tools, ...name})); + return {row: $(await createConnection({orgId, integrationId, account: connected.account, credential: encrypted, defaults: connected.defaults, ...tools, ...name})), added: true}; } $(await updateConnectionCredential({connectionId: existing.id, credential: encrypted})); if (connected.name !== undefined) { $(await updateConnectionName({connectionId: existing.id, name: connected.name})); } if (connected.tools === undefined) { - return {...existing, ...name}; + return {row: {...existing, ...name}, added: false}; } const tools = JSON.stringify(connected.tools); $(await updateConnectionTools({connectionId: existing.id, tools, actions: connected.tools.map((tool) => tool.name)})); - return {...existing, tools, ...name}; + return {row: {...existing, tools, ...name}, added: false}; }); } @@ -155,8 +155,8 @@ export function signInFlow(args: SignInArgs) { } // The handshake the cookie holds, checked against the provider's answer, then the connection - // made or given its new tokens. - function finishConnect(request: AuthenticatedRequest, appUrl: string, payload: StatePayload): Promise> { + // made, or given its new tokens when it was already there. + function finishConnect(request: AuthenticatedRequest, appUrl: string, payload: StatePayload): Promise> { return Do(async ($) => { const state = queryParam(request, 'state') ?? ''; if (!timingSafeEqualString(state, payload.state) || payload.userId !== request.user.userId) { @@ -170,13 +170,15 @@ export function signInFlow(args: SignInArgs) { const connected = $(await args.finish({redirectUri: redirectUri(appUrl), code, verifier: payload.verifier, kept: payload.kept})); const orgId = $(await requireUserOrgId(request)); - return $(await saveConnection(orgId, integrationId, connected)).id; + const {row, added} = $(await saveConnection(orgId, integrationId, connected)); + return {id: row.id, added}; }); } /** * `GET /api/connections//callback` — where the sign-in comes back to; lands on the - * connection, saying it was added, or on the welcome flow's page for it when it started there. + * connection, saying it was added unless it was only signed in to again, or on the welcome + * flow's page for it when it started there. */ async function handleCallback(request: AuthenticatedRequest, res: Response): Promise { const appUrl = env.APP_URL; @@ -189,16 +191,17 @@ export function signInFlow(args: SignInArgs) { res.clearCookie(stateCookieName, stateCookieOptions()); const payload = readStateCookie(request); const returnTo = payload.isOk() ? payload.value.returnTo : undefined; - const connectionId = payload.isErr() ? Err(payload.error) : await finishConnect(request, appUrl, payload.value); - if (connectionId.isErr()) { - redirectToConnectError(res, appUrl, returnTo, connectionId.error); + const finished = payload.isErr() ? Err(payload.error) : await finishConnect(request, appUrl, payload.value); + if (finished.isErr()) { + redirectToConnectError(res, appUrl, returnTo, finished.error); return; } + const {id, added} = finished.value; if (returnTo === 'welcome') { - res.redirect(`${appUrl}/welcome/connections/${connectionId.value}`); + res.redirect(`${appUrl}/welcome/connections/${id}`); return; } - res.redirect(`${appUrl}/connections/${connectionId.value}?added`); + res.redirect(`${appUrl}/connections/${id}${added ? '?added' : ''}`); } return {beginSignIn, handleCallback, redirectToConnectError}; diff --git a/packages/api/src/connections/custom/route.ts b/packages/api/src/connections/custom/route.ts index 7f1ebcb4..2393a0fe 100644 --- a/packages/api/src/connections/custom/route.ts +++ b/packages/api/src/connections/custom/route.ts @@ -78,7 +78,7 @@ function connectCustomMcp(request: AuthenticatedRequest, res: Response): Promise } const credential: CustomCredential = token === null ? {auth: 'none', url: url.toString()} : {auth: 'token', url: url.toString(), token}; const orgId = $(await requireUserOrgId(request)); - const row = $(await saveConnection(orgId, 'mcp', {account: mcpAccount(url), credential, defaults: readOnlyToolNames(probe.tools), tools: probe.tools, name: body.name})); + const {row} = $(await saveConnection(orgId, 'mcp', {account: mcpAccount(url), credential, defaults: readOnlyToolNames(probe.tools), tools: probe.tools, name: body.name})); return {kind: 'connected' as const, connection: toConnectionResponse(row)}; }); } diff --git a/packages/api/src/connections/google/google-client.test.ts b/packages/api/src/connections/google/google-client.test.ts index e7743702..4bfb1eca 100644 --- a/packages/api/src/connections/google/google-client.test.ts +++ b/packages/api/src/connections/google/google-client.test.ts @@ -6,7 +6,7 @@ import type {ApiError, FetchError} from '@proxy/utils'; const updateConnectionCredential = mock(); -mock.module('@proxy/db/connection', () => ({updateConnectionCredential})); +mock.module('@proxy/db/connection', () => ({updateConnectionCredential, getConnectionCredential: mock(async () => Ok(null))})); mock.module('../../observability/log', () => ({ log: {debug: mock(), info: mock(), warn: mock(), error: mock()}, diff --git a/packages/api/src/connections/google/google-oauth.ts b/packages/api/src/connections/google/google-oauth.ts index 87b09fcb..6ac1cbb0 100644 --- a/packages/api/src/connections/google/google-oauth.ts +++ b/packages/api/src/connections/google/google-oauth.ts @@ -1,4 +1,4 @@ -import {Err, Result} from 'ts-results-es'; +import {Err, Ok, Result} from 'ts-results-es'; import {z} from 'zod'; import {ApiErr, type ApiError, Do, type FetchError, httpRequest, parseSchema, requirePresent} from '@proxy/utils'; @@ -56,7 +56,7 @@ const tokensSchema = z.object({ type Tokens = z.infer; -const idTokenPayloadSchema = z.object({email: z.string().min(1)}); +const idTokenPayloadSchema = z.object({email: z.string().min(1), email_verified: z.boolean().optional()}); /** Google's sign-in page for a connection, coming back to `redirectUri` with `state`. */ export function googleAuthorizeUrl(client: GoogleClient, args: {redirectUri: string; state: string; challenge: string}): string { @@ -75,10 +75,13 @@ export function googleAuthorizeUrl(client: GoogleClient, args: {redirectUri: str return url.toString(); } -// Google turning a token request down is the sign-in revoked or the code spent; anything else is -// Google out of reach. +const tokenErrorSchema = z.object({error: z.string()}); + +// Google answering a token request with `invalid_grant` is the sign-in revoked or the code spent; +// anything else, such as a wrong client secret (`invalid_client`) or too many requests, keeps +// Google's answer as its cause, for the logs. function tokenError(error: FetchError): ApiError { - if (error.kind === 'http' && error.status >= 400 && error.status < 500) { + if (error.kind === 'http' && tokenErrorSchema.safeParse(error.body).data?.error === 'invalid_grant') { return ApiErr.credentialsRejected(); } return ApiErr.providerUnreachable(error); @@ -109,11 +112,16 @@ function grantedScopes(tokens: Tokens): string[] | null { // (`auth/google.ts`); that saves asking Google's userinfo endpoint for what the token already says. function emailOf(idToken: string | undefined): Result { const unreadable = (cause: unknown): ApiError => ApiErr.providerUnreachable(cause); - return Do(($) => { + const claims = Do, ApiError>(($) => { const payload = $(requirePresent(idToken?.split('.')[1], unreadable(new Error('Google answered a sign-in without an id_token')))); const json = $(Result.wrap((): unknown => JSON.parse(Buffer.from(payload, 'base64url').toString('utf8'))).mapErr(unreadable)); - const {email} = $(parseSchema(idTokenPayloadSchema, json).mapErr(unreadable)); - return email.toLowerCase(); + return $(parseSchema(idTokenPayloadSchema, json).mapErr(unreadable)); + }); + return claims.andThen(({email, email_verified: verified}) => { + if (verified === false) { + return Err(ApiErr.validationError(`Google hasn't verified ${email}`)); + } + return Ok(email.toLowerCase()); }); } diff --git a/packages/api/src/connections/google/route.test.ts b/packages/api/src/connections/google/route.test.ts index 65b69897..c97c6678 100644 --- a/packages/api/src/connections/google/route.test.ts +++ b/packages/api/src/connections/google/route.test.ts @@ -7,7 +7,7 @@ const listConnections = mock(); const createConnection = mock(); const updateConnectionCredential = mock(); -mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential})); +mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential, getConnectionCredential: mock(async () => Ok(null)), updateConnectionTools: mock(), updateConnectionName: mock()})); const getUserOrgId = mock(); @@ -45,21 +45,28 @@ function base64url(value: object): string { return Buffer.from(JSON.stringify(value)).toString('base64url'); } -const ID_TOKEN = `${base64url({alg: 'RS256'})}.${base64url({sub: '1093', email: 'Alex@Example.com', email_verified: true})}.signature`; +function idToken(payload: object): string { + return `${base64url({alg: 'RS256'})}.${base64url(payload)}.signature`; +} + +const ID_TOKEN = idToken({sub: '1093', email: 'Alex@Example.com', email_verified: true}); // Google's token endpoint: each request, and its answer. The person unticked Gmail on the consent screen. -const google = {requests: [] as Array<{url: URL; body: URLSearchParams}>}; +const google = {requests: [] as Array<{url: URL; body: URLSearchParams}>, idToken: ID_TOKEN, refusal: null as object | null}; spyOn(globalThis, 'fetch').mockImplementation((async (input: string | URL | Request, init?: RequestInit) => { const url = new URL(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); google.requests.push({url, body: new URLSearchParams(typeof init?.body === 'string' ? init.body : '')}); + if (google.refusal) { + return Response.json(google.refusal, {status: 401}); + } return Response.json({ access_token: 'access-1', refresh_token: 'refresh-1', expires_in: 3599, token_type: 'Bearer', scope: 'openid https://www.googleapis.com/auth/userinfo.email https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/documents', - id_token: ID_TOKEN, + id_token: google.idToken, }); }) as typeof fetch); @@ -100,6 +107,8 @@ async function callback(args: {cookie: string; query: Record}) { beforeEach(() => { mock.clearAllMocks(); google.requests = []; + google.idToken = ID_TOKEN; + google.refusal = null; env.GOOGLE_CLIENT_ID = 'client-1.apps.googleusercontent.com'; env.GOOGLE_CLIENT_SECRET = 'secret-1'; getUserOrgId.mockResolvedValue(Ok('org-1')); @@ -169,7 +178,24 @@ describe('handleGoogleCallbackRoute', () => { expect(createConnection).not.toHaveBeenCalled(); expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); - expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0?added'); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0'); + }); + + test('an address Google has not verified is turned down', async () => { + google.idToken = idToken({sub: '1093', email: 'alex@example.com', email_verified: false}); + const {cookie, state} = await start(); + const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(createConnection).not.toHaveBeenCalled(); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/new?error=google'); + }); + + test("a client Google doesn't know is logged with Google's answer, not taken for a revoked sign-in", async () => { + google.refusal = {error: 'invalid_client', error_description: 'The OAuth client was not found.'}; + const {exchangeGoogleCode} = await import('./google-oauth'); + const result = await exchangeGoogleCode({clientId: 'client-1', clientSecret: 'wrong'}, {redirectUri: CALLBACK, code: 'code-1', verifier: 'verifier'}); + + expect(result.isErr() && result.error).toMatchObject({kind: 'provider_unreachable', cause: {kind: 'http', status: 401, body: {error: 'invalid_client'}}}); }); test('started from the welcome flow, ends on its page for the connection', async () => { diff --git a/packages/api/src/connections/granola/route.test.ts b/packages/api/src/connections/granola/route.test.ts index 70242262..abdb1719 100644 --- a/packages/api/src/connections/granola/route.test.ts +++ b/packages/api/src/connections/granola/route.test.ts @@ -133,7 +133,7 @@ describe('handleGranolaCallbackRoute', () => { expect(createConnection).not.toHaveBeenCalled(); expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-old'}); - expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-old?added'); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-old'); }); test('a state that does not match, or another user, connects nothing', async () => { diff --git a/packages/api/src/connections/linear/route.test.ts b/packages/api/src/connections/linear/route.test.ts index e3ccf522..c29e78f3 100644 --- a/packages/api/src/connections/linear/route.test.ts +++ b/packages/api/src/connections/linear/route.test.ts @@ -137,7 +137,7 @@ describe('handleLinearCallbackRoute', () => { expect(createConnection).not.toHaveBeenCalled(); expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); - expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0?added'); + expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0'); }); test('a sign-in turned down at Linear goes back to the catalog', async () => { diff --git a/packages/api/src/records/google-connector.test.ts b/packages/api/src/records/google-connector.test.ts index c82a55a2..c0cd8327 100644 --- a/packages/api/src/records/google-connector.test.ts +++ b/packages/api/src/records/google-connector.test.ts @@ -4,7 +4,7 @@ import {Ok, type Result} from 'ts-results-es'; import type {ApiError} from '@proxy/utils'; import {findCollection, findIntegration, type Collection} from '@proxy/integrations'; -mock.module('@proxy/db/connection', () => ({updateConnectionCredential: mock()})); +mock.module('@proxy/db/connection', () => ({updateConnectionCredential: mock(), getConnectionCredential: mock(async () => Ok(null))})); mock.module('../observability/log', () => ({ log: {debug: mock(), info: mock(), warn: mock(), error: mock()}, @@ -86,15 +86,26 @@ const SHEETS = { ], }; -function rows(from: number, count: number): string[][] { - return Array.from({length: count}, (_, index) => [`Item ${from + index}`, String((from + index) * 10), index === 0 ? 'Paid' : '']); +// The rows of a tab as Google keeps them, by number; a row without any is empty. +function storedRow(row: number): string[] | null { + if (row < 2 || row > 54) { + return null; + } + return [`Item ${row}`, String(row * 10), row % 5 === 2 ? 'Paid' : '']; +} + +// A range's rows as Sheets answers them: the empty ones at its end left out. +function rowsBetween(from: number, to: number): string[][] { + const all = Array.from({length: to - from + 1}, (_, index) => drive.row(from + index)); + const last = all.findLastIndex((row) => row !== null); + return all.slice(0, last + 1).map((row) => row ?? []); } function googleError(status: number, message: string, reason: string): Response { return Response.json({error: {code: status, message, errors: [{message, domain: 'global', reason}]}}, {status}); } -const drive = {requests: [] as URL[], answer: (_url: URL): Response | null => null}; +const drive = {requests: [] as URL[], answer: (_url: URL): Response | null => null, row: storedRow}; function answer(url: URL): Response { const custom = drive.answer(url); @@ -122,16 +133,12 @@ function answer(url: URL): Response { return Response.json(SHEETS); } if (path === '/v4/spreadsheets/1Budget/values:batchGet') { - const [header, range] = url.searchParams.getAll('ranges'); - const [, from = '2', to = '2'] = /!A(\d+):ZZ(\d+)$/.exec(range ?? '') ?? []; - const count = Number(from) === 52 ? 3 : Number(to) - Number(from) + 1; - return Response.json({ - spreadsheetId: '1Budget', - valueRanges: [ - {range: header, majorDimension: 'ROWS', values: [['Item', 'Amount', '']]}, - {range, majorDimension: 'ROWS', values: rows(Number(from), count)}, - ], + const valueRanges = url.searchParams.getAll('ranges').map((range) => { + const [, from = '1', to = '1'] = /!A(\d+):ZZ(\d+)$/.exec(range) ?? []; + const values = from === '1' ? [['Item', 'Amount', '']] : rowsBetween(Number(from), Number(to)); + return {range, majorDimension: 'ROWS', ...(values.length > 0 ? {values} : {})}; }); + return Response.json({spreadsheetId: '1Budget', valueRanges}); } return googleError(404, 'Not found', 'notFound'); } @@ -149,6 +156,7 @@ function requested(path: string): URL[] { beforeEach(() => { drive.requests = []; drive.answer = () => null; + drive.row = storedRow; }); describe('googleConnector.list', () => { @@ -212,8 +220,9 @@ describe('googleConnector.list', () => { expect(requested('/drive/v3/files').map((url) => [url.searchParams.get('q'), url.searchParams.get('orderBy')])).toEqual([ ["'root' in parents and trashed=false", 'folder,modifiedTime desc'], ['sharedWithMe=true and trashed=false', 'sharedWithMeTime desc'], - ['trashed=false', 'viewedByMeTime desc'], + ["viewedByMeTime > '1970-01-01T00:00:00' and mimeType != 'application/vnd.google-apps.folder' and trashed=false", 'viewedByMeTime desc'], ]); + expect(requested('/drive/v3/files')[2]?.searchParams.get('corpora')).toBe('allDrives'); expect(recent.trail).toEqual([{id: 'recent', title: 'Recent'}]); }); @@ -298,6 +307,19 @@ describe('googleConnector.list', () => { const last = (await googleConnector.list(target, query({parent: 'sheet-1Budget-1834', page: '52'}))).unwrap(); expect(last.records.map((record) => record.id)).toEqual(['row-1Budget-1834-52', 'row-1Budget-1834-53', 'row-1Budget-1834-54']); expect(last.nextPage).toBeNull(); + expect(requested('/v4/spreadsheets/1Budget/values:batchGet').at(-1)?.searchParams.getAll('ranges')).toEqual(["'Alex''s notes'!A102:ZZ1000"]); + }); + + test('a page of rows ending in empty ones goes on at the next row that has any', async () => { + drive.row = (row) => (row <= 40 || (row >= 75 && row <= 80) ? storedRow(Math.min(row, 54)) : null); + const {googleConnector} = await import('./google-connector'); + const first = (await googleConnector.list(target, query({parent: 'sheet-1Budget-0'}))).unwrap(); + + expect(first.records).toHaveLength(39); + expect(first.nextPage).toBe('75'); + const second = (await googleConnector.list(target, query({parent: 'sheet-1Budget-0', page: '75'}))).unwrap(); + expect(second.records).toHaveLength(6); + expect(second.nextPage).toBeNull(); }); test('a page of rows that is not a row number is refused', async () => { @@ -324,6 +346,25 @@ describe('googleConnector.list', () => { }); }); +test('a file at the top of a shared drive the account is not a member of is under Shared with me', async () => { + const {googleConnector} = await import('./google-connector'); + drive.answer = (url) => { + if (url.pathname === '/drive/v3/files/1Theirs') { + return Response.json({id: '1Theirs', name: 'Their plans', mimeType: 'application/vnd.google-apps.folder', parents: ['0ATheirDrive'], driveId: '0ATheirDrive'}); + } + if (url.pathname === '/drive/v3/drives/0ATheirDrive') { + return googleError(404, 'Shared drive not found: 0ATheirDrive', 'notFound'); + } + return null; + }; + const page = (await googleConnector.list(target, query({parent: '1Theirs'}))).unwrap(); + + expect(page.trail).toEqual([ + {id: 'shared-with-me', title: 'Shared with me'}, + {id: '1Theirs', title: 'Their plans'}, + ]); +}); + describe('googleConnector.get', () => { test('a document opens as Markdown', async () => { const {googleConnector} = await import('./google-connector'); diff --git a/packages/api/src/records/google-files.ts b/packages/api/src/records/google-files.ts index 95396c5e..6e78489f 100644 --- a/packages/api/src/records/google-files.ts +++ b/packages/api/src/records/google-files.ts @@ -52,6 +52,9 @@ const FOLDER_ORDER = 'folder,modifiedTime desc'; // How far up the folders a trail goes before it stops looking. const MAX_DEPTH = 10; +// Rows looked through past a page of rows that came back short, for the next one holding anything. +const LOOKAHEAD_ROWS = 1000; + /** * What a record id names: a place, a shared drive (`drive-`), a Drive file by its own id, a * spreadsheet's tab (`sheet--`), or a row of one (`row---`, its number @@ -197,7 +200,7 @@ function getSheet(drive: Drive, fileId: string, id: number): Promise> { @@ -214,8 +217,13 @@ function crumbsAbove(drive: Drive, file: FileParents): Promise '1970-01-01T00:00:00'", `mimeType != '${MIME_TYPES.folder}'`]; + return {...$(await filesPage(drive, query, {terms, order: 'viewedByMeTime desc', params: {corpora: 'allDrives'}})), trail}; } // Shared drives are searched by their names. const drives = $(await listSharedDrives(drive.token, query.search, query.page)); @@ -296,9 +306,31 @@ function listRows(drive: Drive, ref: {fileId: string; sheetId: number}, query: L const [ranges, trail] = await Promise.all([getRanges(drive.token, file.id, [rowsRange(sheet.title, 1, 1), rowsRange(sheet.title, from, to)]), trailTo(drive, file)]); const [[headers = []] = [], rows = []] = $(ranges); const records = rows.flatMap((cells, index) => (isBlank(cells) ? [] : [rowRecord(file, sheet, headers, from + index, cells)])); - // Sheets leaves out the empty rows at the end, so a page short of 50 is the last. - const more = rows.length === PAGE_SIZE && to < sheet.rowCount; - return {records, nextPage: more ? String(to + 1) : null, trail: [...$(trail), {id: sheetId(file, sheet), title: sheet.title}]}; + const nextPage = $(await nextRowsPage(drive, file.id, sheet, to, rows.length)); + return {records, nextPage, trail: [...$(trail), {id: sheetId(file, sheet), title: sheet.title}]}; + }); +} + +/** + * Where the page after rows up to `to` starts. Sheets leaves out the empty rows at the end of each + * range asked for, so a page short of 50 may only have ended in empty rows: the rows after it are + * looked through for the next that isn't, 1000 at a time. + */ +function nextRowsPage(drive: Drive, fileId: string, sheet: Sheet, to: number, read: number): Promise> { + return Do(async ($) => { + if (to >= sheet.rowCount) { + return null; + } + if (read === PAGE_SIZE) { + return String(to + 1); + } + const end = Math.min(sheet.rowCount, to + LOOKAHEAD_ROWS); + const [after = []] = $(await getRanges(drive.token, fileId, [rowsRange(sheet.title, to + 1, end)])); + const next = after.findIndex((cells) => !isBlank(cells)); + if (next !== -1) { + return String(to + 1 + next); + } + return end < sheet.rowCount ? String(end + 1) : null; }); } From cee64e81f0c22bd75e3cb4286b7c9e5099246f34 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:36:52 +0000 Subject: [PATCH 4/6] find a google connection signed in to again by google's id for the account, so it is renewed, and takes the new address, after the address changed --- AGENTS.md | 2 +- packages/api/src/connections/connect-flow.ts | 35 ++++---- .../api/src/connections/custom/route.test.ts | 83 +++++++++++++++---- .../src/connections/google/google-oauth.ts | 36 +++++--- .../api/src/connections/google/route.test.ts | 40 ++++++--- packages/api/src/connections/google/route.ts | 9 +- .../api/src/connections/granola/route.test.ts | 25 ++++-- .../api/src/connections/linear/route.test.ts | 24 ++++-- .../api/src/connections/notion/route.test.ts | 20 +++-- .../migration.sql | 7 ++ packages/db/prisma/schema.prisma | 4 + packages/db/src/connection.ts | 48 +++++++++++ 12 files changed, 254 insertions(+), 79 deletions(-) create mode 100644 packages/db/prisma/migrations/v015_add_connection_provider_account_id/migration.sql diff --git a/AGENTS.md b/AGENTS.md index 98e33583..9f3ed40b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ - `packages/utils` - `@proxy/utils`, the Result helpers shared by every package: `Do`, `requirePresent` and `parseSchema` (`src/parse.ts`), and `httpRequest`, the fetch wrapper that returns a `FetchError` union instead of throwing (`src/fetch.ts`). Also `ApiError`/`ApiErr` and `wrapDb` (`src/api-error.ts`), the error every server-side function returns, and the session cookie's name and parsing (`src/session-cookie.ts`) - `packages/integrations` - `@proxy/integrations`, the catalog the api and the web app share: each integration's actions, collections and fields (`src/catalog.ts`), the agents a login can be made for (`src/agent-providers.ts`), the email providers' servers (`src/email-providers.ts`), and access (`src/access.ts`): per action of a connection, an agent's own setting, else the connection's default, an action counting only with the one it `requires`, and tools (`src/tools.ts`): what an agent can do with a connection, generated from the catalog, one per list, open, write and command of each collection (`emails_list`, `emails_archive`), each with the action it needs and a JSON Schema of its params. A custom MCP server (`mcp` in the catalog, `custom`) has no actions of its own: `integrationFor(connection)` (`src/mcp-tools.ts`) builds its integration from the tools the connection keeps (`McpTool`, as the server lists them), one action per tool named by the tool, its risk from the tool's annotations (low when it only reads, medium when what it changes can be undone, high when it can't, and `unknown`, shown without a label, when it says nothing), and `integrationTools` passes them on as `call` tools with the server's own input schema (`ServerTool`, beside the catalog's `CollectionTool`). The web app adds the icons (`src/lib/integrations.ts`, whose `integrationOf(connection)` is what every page uses) - `packages/db` - `@proxy/db`: Prisma 6 on Postgres. `src/index.ts` is the client; each module (`auth.ts`, `user.ts`, `organization.ts`, `connection.ts`, `info.ts`, `agent.ts`, `audit.ts`) exports functions returning `Result` through `wrapDb`. A connection's `tools` column holds a custom MCP server's tools as JSON, and `name` what the person called it; both null for every other integration. `relationMode = "prisma"`: no foreign keys in the database, so Prisma runs the cascades and every relation column has its own index. Locally the database is Prisma's embedded Postgres (PGlite, `scripts/dev-db.ts`), which needs `connection_limit=1&pgbouncer=true` in `DATABASE_URL` -- `packages/api` - `@proxy/api`: Express on Bun. Routes are `handleXRoute` functions mounted in `src/server/create-api-router.ts`; JSON routes return `Result` through `withAuthResult`, which puts the signed-in user on `req.user`. Every environment variable is in `src/utils/env-schema.ts` with its docs, which the docs' Environment variables page is built from; add new ones there. Personal Agent Proxy is one origin everywhere: when `packages/web/dist` exists, as in the Docker image, the api serves the web app's build too (`src/server/serve-web.ts`), and every address the api builds starts from `APP_URL`. Browser-called writes take `requireBrowserOrigin` (CSRF); routes that read a cookie have no CORS. `src/connections/` lists, deletes and sets defaults on an organization's connections, connects a mailbox with an app password after signing in to its IMAP and SMTP servers (`email/`), and connects Granola, Notion and Linear by signing in to their MCP servers (`mcp/`, shared by all three: OAuth with PKCE against the server's authorization server, a client registered for each sign-in so nothing is set up beforehand, the handshake in an encrypted cookie, `mcpConnectRoutes` making both routes on the sign-in flow every OAuth integration shares, `connect-flow.ts`; tokens are refreshed as they run out, one refresh at a time per connection on each server, read again first so ones another call or server saved meanwhile are used rather than refreshed twice, and saved only over the ones they replace, whatever else the credential holds kept beside them, by `connections/token-session.ts`, which Google's connection will share; each tool call is one stateless `tools/call`, `mcp-client.ts`, which also lists a server's tools and, for a server that may keep a session (`handshake`), initializes one around each call as the spec asks). Every request to an MCP server, its metadata or its authorization server goes through `utils/checked-request.ts`, which checks the address as a database's host is checked (`utils/host-check.ts`: nothing private unless `ALLOW_PRIVATE_HOSTS`, and `https` unless the address is private) before each request, since a custom server and its endpoints are wherever it said, and never follows a redirect. A custom MCP server is connected by its address (`custom/`): `POST /api/connections/mcp` with `{name, url, token?, returnTo?}` (the name is the person's, kept through the sign-in in its cookie, since a server says nothing about itself that would do) asks the server for its tools, connects a server that answers at once, with no sign-in or the token typed in as a bearer token (as written when it has its scheme in front, `Basic …`), and for one that turns a request without a token down begins its sign-in there, under the browser's origin check, answering `{kind: 'signIn', url}` with the sign-in page the browser goes to: `mcp-discovery.ts` finds the authorization server from the server's protected resource metadata (the `WWW-Authenticate` header's `resource_metadata`, else the well-known paths), its endpoints from that server's metadata (else `/authorize`, `/token` and `/register` under it) and the scopes the server's own metadata names, and the sign-in, begun with `signInFlow` (`connect-flow.ts`, which `mcpConnectRoutes` builds on for the servers known in advance), keeps the endpoints in the handshake cookie and in the credential beside the tokens. The connection's account is the address without its scheme (`mcp.example.com/mcp`), connecting the same address again replaces its credential and tools, a new connection's defaults are the tools that say they only read, `POST /api/connections/:id/tools` asks the server for its tools again (replacing the tools drops the defaults and agents' settings for tools the server no longer lists, `updateConnectionTools`), and a tool is run as it is (`custom-mcp.ts`): its params go to the server as given, a tool that fails answers the agent with the server's own text as a `validation_error`, and the run is logged under the collection `tools` by the tool's name, with its params as JSON, cut to 200 characters. Granola's notes and transcripts are on by default, Notion's Read pages and Linear's Read issues alone; a Notion or Linear sign-in is for one workspace, and a Notion one lasts at most 180 days, however often it is refreshed. `src/agents/` makes, revokes and deletes agent logins (the password is sent once, only its argon2id hash is kept) and sets an agent's own settings per action of a connection where they differ from the default. `src/records/` reads and writes a collection's records through the integration's `Connector` (`connectors.ts`): Information's live in the database (`info-connector.ts`), a mailbox's are fetched over IMAP on every request (`email-connector.ts`), which merges the inbox and the folders the server marks as Drafts and Sent into one list (ids start with the folder: `inbox-…`, `drafts-…`, `sent-…`) and runs the commands: flags, moves to the folders the server marks as Archive (Gmail's All Mail) or Trash, and sending over SMTP, a draft or a new email, filing a copy in Sent where the server doesn't (Gmail does). Received emails are never edited or deleted for good; only drafts are. Granola's notes are fetched from its MCP server on every request (`granola-connector.ts`; the reading of the XML-like text Granola answers in is in `connections/granola/granola-mcp.ts`): its Notes and Transcripts collections are the same meetings by Granola's meeting id, and since Granola's tools have no pages, a page is 30 days of meetings; one that finds none skips back to the newest meeting before it, so the list ends where the meetings do. Granola's dates and attendees are shown as Granola writes them. Notion's workspace is fetched from its MCP server the same way (`notion-connector.ts`, the reading of its answers in `connections/notion/notion-mcp.ts`), as the person who signed in, as a tree in one nested collection: Notion has no list of every page, so the top is their Recents and a search (Notion's own, AI search where the workspace has it) finds the rest, one page of 50; inside a page are the pages and databases in its content, inside a database its saved views (`view-…` ids), or the rows of its only one, and inside a view its rows, paged with Notion's cursor (view queries have no plan quota; Notion's row and SQL queries do). A page reads as Notion-flavored Markdown, a row's properties beside it as "Name: value" lines (`notion-properties.ts`), and only a page's, database's or view's id is ever sent to Notion's fetch; a new page is a private draft at the top, or goes inside the page, database or view it is made in, a row's title in the database's title property; an edit sends the title, the properties written differently and the content, each only where it changed, and pages are never deleted. Notion's MCP server has no tool that deletes or trashes a page. Linear's issues are fetched from its MCP server the same way (`linear-connector.ts`, the reading of its answers in `connections/linear/linear-mcp.ts`), as the person who signed in, by their identifier (`ENG-123`): a list is the issues updated last first, 50 to a page with Linear's cursor, narrowed by the type of status (`?statusType=Started`) and searched in titles and descriptions, and an issue opens with its comments, oldest first. Teams, statuses, people, projects and labels are written by name, as Linear shows them and finds them, its refusals passed on to the agent; an edit sends only the fields written differently, an empty one clearing it. Linear's MCP server has no tool that archives or deletes an issue. Google is connected by signing in to Google with the app's own Google client, the one Google sign-in uses (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, a second redirect URI `/api/connections/google/callback`), asking for every scope at once, Gmail, Drive, Docs, Sheets and Slides, with offline access (`connections/google/`: `google-oauth.ts` the sign-in and refresh, with the client's secret, keeping the refresh token Google doesn't send again; `google-client.ts` the calls with a bearer token and Google's errors, a disabled API passing Google's own message on, which names the API and where to enable it); the credential holds the tokens, the scopes Google granted, since the consent screen lets a person untick some, and the address, which is the account, so signing in again with it renews the same connection. Its Drive is one nested collection, `files` (`records/google-files.ts`, behind `google-connector.ts`, which dispatches on the collection), fetched live from Drive's and Sheets' APIs (`drive-api.ts`, `sheets-api.ts`): at the top four places, My Drive, Shared with me, Shared drives (`drive-…` ids) and Recent, inside them and inside folders their files, folders first, 50 to a page with Drive's page token, and a search is Drive's full-text one, anywhere or inside the folder opened; the trail is walked up a file's parents to My Drive, its shared drive, or, where the account can't see further up, Shared with me. A document opens as Markdown and a presentation as text, as Drive exports them, up to its 10 MB export cap, beyond which the agent is told the file is too large; a spreadsheet opens into its tabs (`sheet--`) and a tab into its rows (`row---`), 50 to a page, the page token the next row's number, each row's values "Header: value" lines under the tab's first row. Drive isn't written to yet; writes, and Gmail, come later. PostgreSQL and MySQL (MariaDB too) are two integrations on one code path: a database is connected with its host, port, database, user, password and encryption (`connections/sql/`: `route.ts` makes both routes per dialect, `sql-client.ts` opens one connection per request with `Bun.sql`, to the address `utils/host-check.ts` resolves the host to, refusing private addresses unless `ALLOW_PRIVATE_HOSTS` is set, and `sql-statement.ts` checks an agent's query is one statement that reads), and `records/sql-connector.ts` reads and writes it through `records/sql-dialect.ts`, where the two differ: the tables are one nested collection, a table holding its rows, a row read and written as "column: value" lines and found again by its primary key (ids `public.users:42`, `pairs:3,4`; a table without a key is never written to), and a query, a second collection that is only ever run, goes through a read-only transaction with a time limit, a cursor on Postgres and `sql_select_limit` on MySQL, 200 rows at most, answered as JSON lines. MySQL's read-only transaction doesn't stop DDL, so the statement check is what keeps a query to reading; the docs tell people to make a read-only database user as well. Lists take `?search=` (only where the collection has a `searchHint`; Granola has none), `?filter=` (the collection's `filterField`, such as `?filter=Draft`), `?parent=` (only in a `nested` collection: the list inside a record, answered with its `trail`, the records above it; a record made there goes in it) and `?page=` (`list-query.ts`) and answer a page, newest first (a mailbox's 50 emails, Granola's 30 days), with the next page's token; a mailbox's token holds where each folder goes on, so paging through the merged list shows every email once, and its search uses Gmail's own syntax on Gmail (`X-GM-EXT-1`) and IMAP text search elsewhere. Agents sign in at `/agent-auth/login` with a cookie of their own (`src/agent-auth/`); `src/agent-side/` serves them: the connections an agent can use, the tools it has with each, and `runAgentTool` (`tools.ts`), the one place a tool runs, which checks it against the agent's access as it is now, validates its params against the tool's schema (an edit changes only the fields given), calls the connector, or a custom server's tool as it is, answering `{text}`, and logs it, denied ones too, to the activity log that `src/activity/` lists. Over HTTP that is `/api/agent/me`, `GET /api/agent/connections/:id/tools` and `POST /api/agent/connections/:id/tools/:tool` with `{params}`; `src/mcp/` serves the same over MCP at `/mcp` (Streamable HTTP without sessions, JSON answers, no stream): four tools whatever the login may do, `list_connections` (the entry point), `list_tools`, `read` (only tools that read, marked read-only) and `run`, and server instructions naming the agent's connections. MCP clients sign in with OAuth 2.1 (`src/oauth/`), Personal Agent Proxy being its own authorization server: metadata under `/.well-known/`, Dynamic Client Registration at `/oauth/register` (open to anyone, each registration first deleting the clients that registered over a day ago and never finished a sign-in), `/oauth/authorize` sealing the request into the web app's `/connect-agent` page, where the person chooses the agent login the client works as or makes one, and `/oauth/token` with PKCE (S256 only) and refresh tokens replaced on every use; codes and tokens are kept as their SHA-256, and revoking an agent ends its clients' tokens. These endpoints take any origin (CORS), since none reads a cookie; credentials and Information records are stored encrypted with `ENCRYPTION_KEY` (`src/utils/secret-crypto.ts`). Has its own `.prettierrc.json` with a 200-column print width +- `packages/api` - `@proxy/api`: Express on Bun. Routes are `handleXRoute` functions mounted in `src/server/create-api-router.ts`; JSON routes return `Result` through `withAuthResult`, which puts the signed-in user on `req.user`. Every environment variable is in `src/utils/env-schema.ts` with its docs, which the docs' Environment variables page is built from; add new ones there. Personal Agent Proxy is one origin everywhere: when `packages/web/dist` exists, as in the Docker image, the api serves the web app's build too (`src/server/serve-web.ts`), and every address the api builds starts from `APP_URL`. Browser-called writes take `requireBrowserOrigin` (CSRF); routes that read a cookie have no CORS. `src/connections/` lists, deletes and sets defaults on an organization's connections, connects a mailbox with an app password after signing in to its IMAP and SMTP servers (`email/`), and connects Granola, Notion and Linear by signing in to their MCP servers (`mcp/`, shared by all three: OAuth with PKCE against the server's authorization server, a client registered for each sign-in so nothing is set up beforehand, the handshake in an encrypted cookie, `mcpConnectRoutes` making both routes on the sign-in flow every OAuth integration shares, `connect-flow.ts`; tokens are refreshed as they run out, one refresh at a time per connection on each server, read again first so ones another call or server saved meanwhile are used rather than refreshed twice, and saved only over the ones they replace, whatever else the credential holds kept beside them, by `connections/token-session.ts`, which Google's connection will share; each tool call is one stateless `tools/call`, `mcp-client.ts`, which also lists a server's tools and, for a server that may keep a session (`handshake`), initializes one around each call as the spec asks). Every request to an MCP server, its metadata or its authorization server goes through `utils/checked-request.ts`, which checks the address as a database's host is checked (`utils/host-check.ts`: nothing private unless `ALLOW_PRIVATE_HOSTS`, and `https` unless the address is private) before each request, since a custom server and its endpoints are wherever it said, and never follows a redirect. A custom MCP server is connected by its address (`custom/`): `POST /api/connections/mcp` with `{name, url, token?, returnTo?}` (the name is the person's, kept through the sign-in in its cookie, since a server says nothing about itself that would do) asks the server for its tools, connects a server that answers at once, with no sign-in or the token typed in as a bearer token (as written when it has its scheme in front, `Basic …`), and for one that turns a request without a token down begins its sign-in there, under the browser's origin check, answering `{kind: 'signIn', url}` with the sign-in page the browser goes to: `mcp-discovery.ts` finds the authorization server from the server's protected resource metadata (the `WWW-Authenticate` header's `resource_metadata`, else the well-known paths), its endpoints from that server's metadata (else `/authorize`, `/token` and `/register` under it) and the scopes the server's own metadata names, and the sign-in, begun with `signInFlow` (`connect-flow.ts`, which `mcpConnectRoutes` builds on for the servers known in advance), keeps the endpoints in the handshake cookie and in the credential beside the tokens. The connection's account is the address without its scheme (`mcp.example.com/mcp`), connecting the same address again replaces its credential and tools, a new connection's defaults are the tools that say they only read, `POST /api/connections/:id/tools` asks the server for its tools again (replacing the tools drops the defaults and agents' settings for tools the server no longer lists, `updateConnectionTools`), and a tool is run as it is (`custom-mcp.ts`): its params go to the server as given, a tool that fails answers the agent with the server's own text as a `validation_error`, and the run is logged under the collection `tools` by the tool's name, with its params as JSON, cut to 200 characters. Granola's notes and transcripts are on by default, Notion's Read pages and Linear's Read issues alone; a Notion or Linear sign-in is for one workspace, and a Notion one lasts at most 180 days, however often it is refreshed. `src/agents/` makes, revokes and deletes agent logins (the password is sent once, only its argon2id hash is kept) and sets an agent's own settings per action of a connection where they differ from the default. `src/records/` reads and writes a collection's records through the integration's `Connector` (`connectors.ts`): Information's live in the database (`info-connector.ts`), a mailbox's are fetched over IMAP on every request (`email-connector.ts`), which merges the inbox and the folders the server marks as Drafts and Sent into one list (ids start with the folder: `inbox-…`, `drafts-…`, `sent-…`) and runs the commands: flags, moves to the folders the server marks as Archive (Gmail's All Mail) or Trash, and sending over SMTP, a draft or a new email, filing a copy in Sent where the server doesn't (Gmail does). Received emails are never edited or deleted for good; only drafts are. Granola's notes are fetched from its MCP server on every request (`granola-connector.ts`; the reading of the XML-like text Granola answers in is in `connections/granola/granola-mcp.ts`): its Notes and Transcripts collections are the same meetings by Granola's meeting id, and since Granola's tools have no pages, a page is 30 days of meetings; one that finds none skips back to the newest meeting before it, so the list ends where the meetings do. Granola's dates and attendees are shown as Granola writes them. Notion's workspace is fetched from its MCP server the same way (`notion-connector.ts`, the reading of its answers in `connections/notion/notion-mcp.ts`), as the person who signed in, as a tree in one nested collection: Notion has no list of every page, so the top is their Recents and a search (Notion's own, AI search where the workspace has it) finds the rest, one page of 50; inside a page are the pages and databases in its content, inside a database its saved views (`view-…` ids), or the rows of its only one, and inside a view its rows, paged with Notion's cursor (view queries have no plan quota; Notion's row and SQL queries do). A page reads as Notion-flavored Markdown, a row's properties beside it as "Name: value" lines (`notion-properties.ts`), and only a page's, database's or view's id is ever sent to Notion's fetch; a new page is a private draft at the top, or goes inside the page, database or view it is made in, a row's title in the database's title property; an edit sends the title, the properties written differently and the content, each only where it changed, and pages are never deleted. Notion's MCP server has no tool that deletes or trashes a page. Linear's issues are fetched from its MCP server the same way (`linear-connector.ts`, the reading of its answers in `connections/linear/linear-mcp.ts`), as the person who signed in, by their identifier (`ENG-123`): a list is the issues updated last first, 50 to a page with Linear's cursor, narrowed by the type of status (`?statusType=Started`) and searched in titles and descriptions, and an issue opens with its comments, oldest first. Teams, statuses, people, projects and labels are written by name, as Linear shows them and finds them, its refusals passed on to the agent; an edit sends only the fields written differently, an empty one clearing it. Linear's MCP server has no tool that archives or deletes an issue. Google is connected by signing in to Google with the app's own Google client, the one Google sign-in uses (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, a second redirect URI `/api/connections/google/callback`), asking for every scope at once, Gmail, Drive, Docs, Sheets and Slides, with offline access (`connections/google/`: `google-oauth.ts` the sign-in and refresh, with the client's secret, keeping the refresh token Google doesn't send again; `google-client.ts` the calls with a bearer token and Google's errors, a disabled API passing Google's own message on, which names the API and where to enable it); the credential holds the tokens, the scopes Google granted, since the consent screen lets a person untick some, and the address; the connection keeps Google's id for the account (`Connection.providerAccountId`, its `sub`), so signing in again renews the same connection, and its address, even after the address changed (`findAccountConnection`, by address for one without an id and for the MCP integrations). Its Drive is one nested collection, `files` (`records/google-files.ts`, behind `google-connector.ts`, which dispatches on the collection), fetched live from Drive's and Sheets' APIs (`drive-api.ts`, `sheets-api.ts`): at the top four places, My Drive, Shared with me, Shared drives (`drive-…` ids, of the kind Shared drive, which is never edited or trashed) and Recent, inside them and inside folders their files, folders first, 50 to a page with Drive's page token, and a search is Drive's full-text one, anywhere or inside the folder opened; the trail is walked up a file's parents to My Drive, its shared drive, or, where the account can't see further up, Shared with me. A document opens as Markdown and a presentation as text, as Drive exports them, up to its 10 MB export cap, beyond which its content says the file is too large, so it can still be renamed or trashed; a spreadsheet opens into its tabs (`sheet--`) and a tab into its rows (`row---`), 50 to a page, the page token the next row's number, each row's values "Header: value" lines under the tab's first row, a header repeated or a column without one going by its letter (`Status (E)`). Drive isn't written to yet; writes, and Gmail, come later. PostgreSQL and MySQL (MariaDB too) are two integrations on one code path: a database is connected with its host, port, database, user, password and encryption (`connections/sql/`: `route.ts` makes both routes per dialect, `sql-client.ts` opens one connection per request with `Bun.sql`, to the address `utils/host-check.ts` resolves the host to, refusing private addresses unless `ALLOW_PRIVATE_HOSTS` is set, and `sql-statement.ts` checks an agent's query is one statement that reads), and `records/sql-connector.ts` reads and writes it through `records/sql-dialect.ts`, where the two differ: the tables are one nested collection, a table holding its rows, a row read and written as "column: value" lines and found again by its primary key (ids `public.users:42`, `pairs:3,4`; a table without a key is never written to), and a query, a second collection that is only ever run, goes through a read-only transaction with a time limit, a cursor on Postgres and `sql_select_limit` on MySQL, 200 rows at most, answered as JSON lines. MySQL's read-only transaction doesn't stop DDL, so the statement check is what keeps a query to reading; the docs tell people to make a read-only database user as well. Lists take `?search=` (only where the collection has a `searchHint`; Granola has none), `?filter=` (the collection's `filterField`, such as `?filter=Draft`), `?parent=` (only in a `nested` collection: the list inside a record, answered with its `trail`, the records above it; a record made there goes in it) and `?page=` (`list-query.ts`) and answer a page, newest first (a mailbox's 50 emails, Granola's 30 days), with the next page's token; a mailbox's token holds where each folder goes on, so paging through the merged list shows every email once, and its search uses Gmail's own syntax on Gmail (`X-GM-EXT-1`) and IMAP text search elsewhere. Agents sign in at `/agent-auth/login` with a cookie of their own (`src/agent-auth/`); `src/agent-side/` serves them: the connections an agent can use, the tools it has with each, and `runAgentTool` (`tools.ts`), the one place a tool runs, which checks it against the agent's access as it is now, validates its params against the tool's schema (an edit changes only the fields given), calls the connector, or a custom server's tool as it is, answering `{text}`, and logs it, denied ones too, to the activity log that `src/activity/` lists. Over HTTP that is `/api/agent/me`, `GET /api/agent/connections/:id/tools` and `POST /api/agent/connections/:id/tools/:tool` with `{params}`; `src/mcp/` serves the same over MCP at `/mcp` (Streamable HTTP without sessions, JSON answers, no stream): four tools whatever the login may do, `list_connections` (the entry point), `list_tools`, `read` (only tools that read, marked read-only) and `run`, and server instructions naming the agent's connections. MCP clients sign in with OAuth 2.1 (`src/oauth/`), Personal Agent Proxy being its own authorization server: metadata under `/.well-known/`, Dynamic Client Registration at `/oauth/register` (open to anyone, each registration first deleting the clients that registered over a day ago and never finished a sign-in), `/oauth/authorize` sealing the request into the web app's `/connect-agent` page, where the person chooses the agent login the client works as or makes one, and `/oauth/token` with PKCE (S256 only) and refresh tokens replaced on every use; codes and tokens are kept as their SHA-256, and revoking an agent ends its clients' tokens. These endpoints take any origin (CORS), since none reads a cookie; credentials and Information records are stored encrypted with `ENCRYPTION_KEY` (`src/utils/secret-crypto.ts`). Has its own `.prettierrc.json` with a 200-column print width - `skills/` - Agent skills, symlinked into `.claude/skills` and `.codex/skills` by `bun run setup` ## The app diff --git a/packages/api/src/connections/connect-flow.ts b/packages/api/src/connections/connect-flow.ts index c6e4e0e1..02aeffa2 100644 --- a/packages/api/src/connections/connect-flow.ts +++ b/packages/api/src/connections/connect-flow.ts @@ -5,7 +5,7 @@ import {Err, type Result} from 'ts-results-es'; import {z} from 'zod'; import {ApiErr, type ApiError, Do, parseCookieHeader, requirePresent} from '@proxy/utils'; -import {createConnection, listConnections, updateConnectionCredential, updateConnectionName, updateConnectionTools, type ConnectionRow} from '@proxy/db/connection'; +import {createConnection, findAccountConnection, renewConnection, updateConnectionName, updateConnectionTools, type ConnectionRow} from '@proxy/db/connection'; import type {IntegrationId, McpTool} from '@proxy/integrations'; import {queryParam, timingSafeEqualString, useSecureCookies} from '../auth/session'; @@ -50,10 +50,12 @@ type SignInStart = (request: SignInRequest) => Promise = {redirectUri: string; code: string; verifier: string; kept: E}; /** - * A connection to save: the account it shows, its credential, stored encrypted, the actions agents - * get with a new one, and a custom MCP server's tools and the name the person gave it. + * A connection to save: the account it shows, the provider's own id for it where it gives one, + * which stays as the account's address changes (Google's `sub`), its credential, stored encrypted, + * the actions agents get with a new one, and a custom MCP server's tools and the name the person + * gave it. */ -export type Connected = {account: string; credential: C; defaults: string[]; tools?: McpTool[]; name?: string}; +export type Connected = {account: string; accountId?: string; credential: C; defaults: string[]; tools?: McpTool[]; name?: string}; /** A PKCE verifier and the challenge the provider is sent in its place. */ export function makePkce(): {verifier: string; challenge: string} { @@ -62,28 +64,31 @@ export function makePkce(): {verifier: string; challenge: string} { } /** - * Saves a connection: a new one, or an existing one of the same account given the new credential, - * and the tools and the name, where it has its own; `added` unless it was already there. + * Saves a connection: a new one, or an existing one of the same account given the new credential + * and the account as the provider names it now, such as an address changed since, and the tools + * and the name, where it has its own; `added` unless it was already there. */ export function saveConnection(orgId: string, integrationId: IntegrationId, connected: Connected): Promise> { return Do(async ($) => { + const {account, tools} = connected; const encrypted = $(encryptSecret(JSON.stringify(connected.credential))); - const existing = $(await listConnections(orgId)).find((connection) => connection.integrationId === integrationId && connection.account === connected.account); + const providerAccountId = connected.accountId ?? null; const name = connected.name === undefined ? {} : {name: connected.name}; + const existing = $(await findAccountConnection({orgId, integrationId, account, providerAccountId})); if (!existing) { - const tools = connected.tools === undefined ? {} : {tools: JSON.stringify(connected.tools)}; - return {row: $(await createConnection({orgId, integrationId, account: connected.account, credential: encrypted, defaults: connected.defaults, ...tools, ...name})), added: true}; + const listed = tools === undefined ? {} : {tools: JSON.stringify(tools)}; + return {row: $(await createConnection({orgId, integrationId, account, providerAccountId, credential: encrypted, defaults: connected.defaults, ...listed, ...name})), added: true}; } - $(await updateConnectionCredential({connectionId: existing.id, credential: encrypted})); + $(await renewConnection({connectionId: existing.id, account, providerAccountId, credential: encrypted})); if (connected.name !== undefined) { $(await updateConnectionName({connectionId: existing.id, name: connected.name})); } - if (connected.tools === undefined) { - return {row: {...existing, ...name}, added: false}; + if (tools === undefined) { + return {row: {...existing, account, ...name}, added: false}; } - const tools = JSON.stringify(connected.tools); - $(await updateConnectionTools({connectionId: existing.id, tools, actions: connected.tools.map((tool) => tool.name)})); - return {row: {...existing, tools, ...name}, added: false}; + const stored = JSON.stringify(tools); + $(await updateConnectionTools({connectionId: existing.id, tools: stored, actions: tools.map((tool) => tool.name)})); + return {row: {...existing, account, tools: stored, ...name}, added: false}; }); } diff --git a/packages/api/src/connections/custom/route.test.ts b/packages/api/src/connections/custom/route.test.ts index edb11be4..ac0bb11e 100644 --- a/packages/api/src/connections/custom/route.test.ts +++ b/packages/api/src/connections/custom/route.test.ts @@ -3,9 +3,9 @@ import {Err, Ok, type Result} from 'ts-results-es'; import {ApiErr, type ApiError} from '@proxy/utils'; -const listConnections = mock(); +const findAccountConnection = mock(); const createConnection = mock(); -const updateConnectionCredential = mock(); +const renewConnection = mock(); const getConnectionWithCredential = mock(); const updateConnectionTools = mock(); const updateConnectionName = mock(); @@ -23,9 +23,10 @@ afterAll(() => { }); mock.module('@proxy/db/connection', () => ({ - listConnections, + findAccountConnection, createConnection, - updateConnectionCredential, + renewConnection, + updateConnectionCredential: mock(), getConnectionWithCredential, updateConnectionTools, updateConnectionName, @@ -63,6 +64,16 @@ const listCustomTools = mock(); const customMcp = await import('./custom-mcp'); mock.module('./custom-mcp', () => ({...customMcp, listCustomTools})); +const exchangeMcpCode = mock(); + +const mcpOauth = await import('../mcp/mcp-oauth'); +mock.module('../mcp/mcp-oauth', () => ({...mcpOauth, exchangeMcpCode})); + +const listMcpTools = mock(); + +const mcpClient = await import('../mcp/mcp-client'); +mock.module('../mcp/mcp-client', () => ({...mcpClient, listMcpTools})); + const search = {name: 'search_docs', title: 'Search docs', inputSchema: {type: 'object', properties: {query: {type: 'string'}}}, annotations: {readOnlyHint: true}}; const publish = {name: 'publish_doc', inputSchema: {type: 'object'}}; @@ -76,18 +87,23 @@ const row = { defaults: [{actionId: 'search_docs'}], }; -function makeRequest(body: unknown, params: Record = {}) { - return {user: {userId: 'user-1', email: 'alex@example.com'}, params, body, query: {}, headers: {}} as never; +function makeRequest(body: unknown, params: Record = {}, query: Record = {}, headers: Record = {}) { + return {user: {userId: 'user-1', email: 'alex@example.com'}, params, body, query, headers} as never; } -// What the connect route answered, and the cookies it set. +// What the connect route answered, the cookies it set, and where the callback sent the browser. function makeRes() { const res = { - cookies: [] as Array<{name: string; options: Record}>, + cookies: [] as Array<{name: string; value: string; options: Record}>, sent: undefined as unknown, + redirectedTo: undefined as string | undefined, statusCode: 200, - cookie: mock((name: string, _value: string, options: Record) => { - res.cookies.push({name, options}); + cookie: mock((name: string, value: string, options: Record) => { + res.cookies.push({name, value, options}); + }), + clearCookie: mock(), + redirect: mock((url: string) => { + res.redirectedTo = url; }), status: mock((code: number) => { res.statusCode = code; @@ -111,11 +127,11 @@ async function connect(body: unknown) { beforeEach(() => { mock.clearAllMocks(); getUserOrgId.mockResolvedValue(Ok('org-1')); - listConnections.mockResolvedValue(Ok([])); + findAccountConnection.mockResolvedValue(Ok(null)); createConnection.mockImplementation(async (data: {account: string; tools: string; defaults: string[]; name?: string}) => Ok({...row, account: data.account, tools: data.tools, name: data.name ?? null, defaults: data.defaults.map((actionId) => ({actionId}))}), ); - updateConnectionCredential.mockResolvedValue(Ok(true)); + renewConnection.mockResolvedValue(Ok(undefined)); updateConnectionTools.mockResolvedValue(Ok(undefined)); updateConnectionName.mockResolvedValue(Ok(undefined)); probeMcpServer.mockResolvedValue(Ok({kind: 'tools', tools: [search, publish]})); @@ -141,6 +157,7 @@ describe('handleConnectCustomMcpRoute', () => { orgId: 'org-1', integrationId: 'mcp', account: 'mcp.example.com/mcp', + providerAccountId: null, credential: `enc:${JSON.stringify({auth: 'none', url: 'https://mcp.example.com/mcp'})}`, defaults: ['search_docs'], tools: JSON.stringify([search, publish]), @@ -169,12 +186,18 @@ describe('handleConnectCustomMcpRoute', () => { }); test('connecting the same address again replaces its credential and tools, the settings for tools no longer listed dropped', async () => { - listConnections.mockResolvedValue(Ok([row])); + findAccountConnection.mockResolvedValue(Ok(row)); const res = await connect({name: 'Docs again', url: 'https://mcp.example.com/mcp/'}); expect(createConnection).not.toHaveBeenCalled(); expect(updateConnectionName).toHaveBeenCalledWith({connectionId: 'mcp-1', name: 'Docs again'}); - expect(updateConnectionCredential).toHaveBeenCalledWith({connectionId: 'mcp-1', credential: `enc:${JSON.stringify({auth: 'none', url: 'https://mcp.example.com/mcp/'})}`}); + expect(findAccountConnection).toHaveBeenCalledWith({orgId: 'org-1', integrationId: 'mcp', account: 'mcp.example.com/mcp', providerAccountId: null}); + expect(renewConnection).toHaveBeenCalledWith({ + connectionId: 'mcp-1', + account: 'mcp.example.com/mcp', + providerAccountId: null, + credential: `enc:${JSON.stringify({auth: 'none', url: 'https://mcp.example.com/mcp/'})}`, + }); expect(updateConnectionTools).toHaveBeenCalledWith({connectionId: 'mcp-1', tools: JSON.stringify([search, publish]), actions: ['search_docs', 'publish_doc']}); expect(res.sent).toMatchObject({kind: 'connected', connection: {id: 'mcp-1', tools: [search, publish], name: 'Docs again'}}); }); @@ -207,6 +230,38 @@ describe('handleConnectCustomMcpRoute', () => { }); }); +describe('handleCustomMcpCallbackRoute', () => { + test('a sign-in begun on the connect route comes back to the server it found, connected under the name given, its endpoints kept beside the tokens', async () => { + probeMcpServer.mockResolvedValue(Ok({kind: 'signIn'})); + const begun = await connect({name: 'Docs', url: 'https://mcp.example.com/mcp'}); + const state = new URL((begun.sent as {url: string}).url).searchParams.get('state') ?? ''; + const cookie = `proxy.mcp-state=${encodeURIComponent(begun.cookies[0]?.value ?? '')}`; + + const tokens = {clientId: 'client-1', accessToken: 'access-1', refreshToken: 'refresh-1', expiresAt: null}; + exchangeMcpCode.mockResolvedValue(Ok({credential: tokens, workspaceName: null})); + listMcpTools.mockResolvedValue(Ok([search, publish])); + const {handleCustomMcpCallbackRoute} = await import('./route'); + const res = makeRes(); + await handleCustomMcpCallbackRoute(makeRequest(undefined, {}, {state, code: 'code-1'}, {cookie}), res as never); + + const endpoints = { + name: 'mcp.example.com', + url: 'https://mcp.example.com/mcp', + registerUrl: 'https://mcp.example.com/register', + authorizeUrl: 'https://mcp.example.com/authorize', + tokenUrl: 'https://mcp.example.com/token', + scope: '', + handshake: true, + }; + expect(exchangeMcpCode).toHaveBeenCalledWith(endpoints, {clientId: 'client-1', redirectUri: 'https://app.example.com/api/connections/mcp/callback', code: 'code-1', verifier: expect.any(String)}); + expect(listMcpTools).toHaveBeenCalledWith(endpoints, 'access-1'); + const created = createConnection.mock.calls[0]?.[0] as {credential: string}; + expect(created).toMatchObject({account: 'mcp.example.com/mcp', providerAccountId: null, defaults: ['search_docs'], tools: JSON.stringify([search, publish]), name: 'Docs'}); + expect(JSON.parse(created.credential.replace(/^enc:/, ''))).toEqual({auth: 'oauth', url: 'https://mcp.example.com/mcp', server: endpoints, ...tokens}); + expect(res.redirectedTo).toBe('https://app.example.com/connections/mcp-1?added'); + }); +}); + describe('handleRefreshCustomMcpToolsRoute', () => { test('asks the server for its tools again and keeps the new list', async () => { getConnectionWithCredential.mockResolvedValue(Ok({...row, credential: 'enc:{}'})); diff --git a/packages/api/src/connections/google/google-oauth.ts b/packages/api/src/connections/google/google-oauth.ts index 6ac1cbb0..62a80046 100644 --- a/packages/api/src/connections/google/google-oauth.ts +++ b/packages/api/src/connections/google/google-oauth.ts @@ -56,7 +56,11 @@ const tokensSchema = z.object({ type Tokens = z.infer; -const idTokenPayloadSchema = z.object({email: z.string().min(1), email_verified: z.boolean().optional()}); +// `sub` is Google's id for the account, which stays as its address changes. +const idTokenPayloadSchema = z.object({sub: z.string().min(1), email: z.string().min(1), email_verified: z.boolean().optional()}); + +/** Who a sign-in was for: Google's id for the account, and its address as it is now. */ +export type GoogleAccount = {id: string; email: string}; /** Google's sign-in page for a connection, coming back to `redirectUri` with `state`. */ export function googleAuthorizeUrl(client: GoogleClient, args: {redirectUri: string; state: string; challenge: string}): string { @@ -107,34 +111,40 @@ function grantedScopes(tokens: Tokens): string[] | null { return tokens.scope === undefined ? null : tokens.scope.split(' ').filter((scope) => scope !== ''); } -// The account's address from the id_token's payload. It came straight from Google's token endpoint -// over TLS, so the payload is trusted without checking its signature, as Google sign-in does +// The account from the id_token's payload. It came straight from Google's token endpoint over TLS, +// so the payload is trusted without checking its signature, as Google sign-in does // (`auth/google.ts`); that saves asking Google's userinfo endpoint for what the token already says. -function emailOf(idToken: string | undefined): Result { +function accountOf(idToken: string | undefined): Result { const unreadable = (cause: unknown): ApiError => ApiErr.providerUnreachable(cause); const claims = Do, ApiError>(($) => { const payload = $(requirePresent(idToken?.split('.')[1], unreadable(new Error('Google answered a sign-in without an id_token')))); const json = $(Result.wrap((): unknown => JSON.parse(Buffer.from(payload, 'base64url').toString('utf8'))).mapErr(unreadable)); return $(parseSchema(idTokenPayloadSchema, json).mapErr(unreadable)); }); - return claims.andThen(({email, email_verified: verified}) => { + return claims.andThen(({sub, email, email_verified: verified}) => { if (verified === false) { return Err(ApiErr.validationError(`Google hasn't verified ${email}`)); } - return Ok(email.toLowerCase()); + return Ok({id: sub, email: email.toLowerCase()}); }); } /** Trades the code Google sent back for the connection's first tokens, and the account they are for. */ -export async function exchangeGoogleCode(client: GoogleClient, args: {redirectUri: string; code: string; verifier: string}): Promise> { +export async function exchangeGoogleCode( + client: GoogleClient, + args: {redirectUri: string; code: string; verifier: string}, +): Promise> { const tokens = await requestTokens(client, {grant_type: 'authorization_code', code: args.code, redirect_uri: args.redirectUri, code_verifier: args.verifier}); return tokens.andThen((value) => - emailOf(value.id_token).map((email) => ({ - accessToken: value.access_token, - refreshToken: value.refresh_token ?? null, - expiresAt: expiresAt(value), - scopes: grantedScopes(value) ?? [], - email, + accountOf(value.id_token).map((account) => ({ + account, + credential: { + accessToken: value.access_token, + refreshToken: value.refresh_token ?? null, + expiresAt: expiresAt(value), + scopes: grantedScopes(value) ?? [], + email: account.email, + }, })), ); } diff --git a/packages/api/src/connections/google/route.test.ts b/packages/api/src/connections/google/route.test.ts index c97c6678..1c2762e4 100644 --- a/packages/api/src/connections/google/route.test.ts +++ b/packages/api/src/connections/google/route.test.ts @@ -3,11 +3,19 @@ import {Ok, type Result} from 'ts-results-es'; import type {ApiError} from '@proxy/utils'; -const listConnections = mock(); +const findAccountConnection = mock(); const createConnection = mock(); -const updateConnectionCredential = mock(); - -mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential, getConnectionCredential: mock(async () => Ok(null)), updateConnectionTools: mock(), updateConnectionName: mock()})); +const renewConnection = mock(); + +mock.module('@proxy/db/connection', () => ({ + findAccountConnection, + createConnection, + renewConnection, + updateConnectionCredential: mock(), + getConnectionCredential: mock(async () => Ok(null)), + updateConnectionTools: mock(), + updateConnectionName: mock(), +})); const getUserOrgId = mock(); @@ -112,9 +120,9 @@ beforeEach(() => { env.GOOGLE_CLIENT_ID = 'client-1.apps.googleusercontent.com'; env.GOOGLE_CLIENT_SECRET = 'secret-1'; getUserOrgId.mockResolvedValue(Ok('org-1')); - listConnections.mockResolvedValue(Ok([])); + findAccountConnection.mockResolvedValue(Ok(null)); createConnection.mockResolvedValue(Ok({id: 'conn-1'})); - updateConnectionCredential.mockResolvedValue(Ok(true)); + renewConnection.mockResolvedValue(Ok(undefined)); }); describe('handleStartGoogleRoute', () => { @@ -160,7 +168,8 @@ describe('handleGoogleCallbackRoute', () => { code_verifier: expect.stringMatching(/^[A-Za-z0-9_-]{43}$/), }); const created = createConnection.mock.calls[0]?.[0]; - expect(created).toMatchObject({orgId: 'org-1', integrationId: 'google', account: 'alex@example.com', defaults: ['readFiles']}); + expect(findAccountConnection).toHaveBeenCalledWith({orgId: 'org-1', integrationId: 'google', account: 'alex@example.com', providerAccountId: '1093'}); + expect(created).toMatchObject({orgId: 'org-1', integrationId: 'google', account: 'alex@example.com', providerAccountId: '1093', defaults: ['readFiles']}); expect(JSON.parse(String(created.credential).replace(/^enc:/, ''))).toEqual({ accessToken: 'access-1', refreshToken: 'refresh-1', @@ -171,16 +180,27 @@ describe('handleGoogleCallbackRoute', () => { expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-1?added'); }); - test('signing in again with the same address gives its connection the new tokens', async () => { - listConnections.mockResolvedValue(Ok([{id: 'conn-0', integrationId: 'google', account: 'alex@example.com'}])); + test('signing in again with the same account gives its connection the new tokens', async () => { + findAccountConnection.mockResolvedValue(Ok({id: 'conn-0'})); const {cookie, state} = await start(); const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); expect(createConnection).not.toHaveBeenCalled(); - expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); + expect(renewConnection.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0'); }); + test("after the account's address changed, signing in again finds its connection by Google's id and gives it the new address", async () => { + google.idToken = idToken({sub: '1093', email: 'alex.kim@example.com', email_verified: true}); + findAccountConnection.mockResolvedValue(Ok({id: 'conn-0'})); + const {cookie, state} = await start(); + await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); + + expect(findAccountConnection).toHaveBeenCalledWith({orgId: 'org-1', integrationId: 'google', account: 'alex.kim@example.com', providerAccountId: '1093'}); + expect(renewConnection.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0', account: 'alex.kim@example.com', providerAccountId: '1093'}); + expect(JSON.parse(String(renewConnection.mock.calls[0]?.[0].credential).replace(/^enc:/, ''))).toMatchObject({email: 'alex.kim@example.com'}); + }); + test('an address Google has not verified is turned down', async () => { google.idToken = idToken({sub: '1093', email: 'alex@example.com', email_verified: false}); const {cookie, state} = await start(); diff --git a/packages/api/src/connections/google/route.ts b/packages/api/src/connections/google/route.ts index efefc074..d2427db3 100644 --- a/packages/api/src/connections/google/route.ts +++ b/packages/api/src/connections/google/route.ts @@ -8,8 +8,9 @@ import {exchangeGoogleCode, googleAuthorizeUrl} from './google-oauth'; /** * Connecting a Google account by signing in to Google with the app's own Google client, the one - * Google sign-in uses, asking for every scope at once. The account is the address signed in with, - * so signing in again with it gives its connection the new tokens. A new Google connection lets + * Google sign-in uses, asking for every scope at once. The connection shows the address signed in + * with and is found by Google's id for the account, so signing in again gives it the new tokens + * and the address as it is now, even after the address changed. A new Google connection lets * agents read Drive by default. Without a Google client set up, the start goes back to the catalog * with the error. */ @@ -23,7 +24,7 @@ export const {handleStart: handleStartGoogleRoute, handleCallback: handleGoogleC }), finish: ({redirectUri, code, verifier}) => Do(async ($) => { - const credential = $(await exchangeGoogleCode($(requireGoogleClient()), {redirectUri, code, verifier})); - return {account: credential.email, credential, defaults: ['readFiles']}; + const {account, credential} = $(await exchangeGoogleCode($(requireGoogleClient()), {redirectUri, code, verifier})); + return {account: account.email, accountId: account.id, credential, defaults: ['readFiles']}; }), }); diff --git a/packages/api/src/connections/granola/route.test.ts b/packages/api/src/connections/granola/route.test.ts index abdb1719..d1f909c3 100644 --- a/packages/api/src/connections/granola/route.test.ts +++ b/packages/api/src/connections/granola/route.test.ts @@ -5,11 +5,19 @@ import {Ok, type Result} from 'ts-results-es'; import type {ApiError} from '@proxy/utils'; -const listConnections = mock(); +const findAccountConnection = mock(); const createConnection = mock(); -const updateConnectionCredential = mock(); - -mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential, updateConnectionTools: mock(), updateConnectionName: mock(), getConnectionCredential: mock(async () => Ok(null))})); +const renewConnection = mock(); + +mock.module('@proxy/db/connection', () => ({ + findAccountConnection, + createConnection, + renewConnection, + updateConnectionCredential: mock(), + getConnectionCredential: mock(async () => Ok(null)), + updateConnectionTools: mock(), + updateConnectionName: mock(), +})); const getUserOrgId = mock(); @@ -86,9 +94,9 @@ beforeEach(() => { mock.clearAllMocks(); granola.requests = []; getUserOrgId.mockResolvedValue(Ok('org-1')); - listConnections.mockResolvedValue(Ok([])); + findAccountConnection.mockResolvedValue(Ok(null)); createConnection.mockResolvedValue(Ok({id: 'conn-1'})); - updateConnectionCredential.mockResolvedValue(Ok(true)); + renewConnection.mockResolvedValue(Ok(undefined)); }); describe('handleStartGranolaRoute', () => { @@ -127,12 +135,13 @@ describe('handleGranolaCallbackRoute', () => { }); test('signing in to an account already connected gives that connection the new tokens', async () => { - listConnections.mockResolvedValue(Ok([{id: 'conn-old', integrationId: 'granola', account: 'alex@example.com'}])); + findAccountConnection.mockResolvedValue(Ok({id: 'conn-old'})); const {cookie, state} = await start(); const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); expect(createConnection).not.toHaveBeenCalled(); - expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-old'}); + expect(findAccountConnection).toHaveBeenCalledWith({orgId: 'org-1', integrationId: 'granola', account: 'alex@example.com', providerAccountId: null}); + expect(renewConnection.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-old'}); expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-old'); }); diff --git a/packages/api/src/connections/linear/route.test.ts b/packages/api/src/connections/linear/route.test.ts index c29e78f3..82b669ab 100644 --- a/packages/api/src/connections/linear/route.test.ts +++ b/packages/api/src/connections/linear/route.test.ts @@ -3,11 +3,19 @@ import {Ok, type Result} from 'ts-results-es'; import type {ApiError} from '@proxy/utils'; -const listConnections = mock(); +const findAccountConnection = mock(); const createConnection = mock(); -const updateConnectionCredential = mock(); - -mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential, updateConnectionTools: mock(), updateConnectionName: mock(), getConnectionCredential: mock(async () => Ok(null))})); +const renewConnection = mock(); + +mock.module('@proxy/db/connection', () => ({ + findAccountConnection, + createConnection, + renewConnection, + updateConnectionCredential: mock(), + getConnectionCredential: mock(async () => Ok(null)), + updateConnectionTools: mock(), + updateConnectionName: mock(), +})); const getUserOrgId = mock(); @@ -93,9 +101,9 @@ beforeEach(() => { mock.clearAllMocks(); linear.requests = []; getUserOrgId.mockResolvedValue(Ok('org-1')); - listConnections.mockResolvedValue(Ok([])); + findAccountConnection.mockResolvedValue(Ok(null)); createConnection.mockResolvedValue(Ok({id: 'conn-1'})); - updateConnectionCredential.mockResolvedValue(Ok(true)); + renewConnection.mockResolvedValue(Ok(undefined)); }); describe('handleStartLinearRoute', () => { @@ -131,12 +139,12 @@ describe('handleLinearCallbackRoute', () => { }); test('signing in to the same workspace again gives its connection the new tokens', async () => { - listConnections.mockResolvedValue(Ok([{id: 'conn-0', integrationId: 'linear', account: 'alex@example.com · Acme'}])); + findAccountConnection.mockResolvedValue(Ok({id: 'conn-0'})); const {cookie, state} = await start(); const res = await callback({cookie: cookie?.value ?? '', query: {state, code: 'code-1'}}); expect(createConnection).not.toHaveBeenCalled(); - expect(updateConnectionCredential.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); + expect(renewConnection.mock.calls[0]?.[0]).toMatchObject({connectionId: 'conn-0'}); expect(res.redirect).toHaveBeenCalledWith('https://app.example.com/connections/conn-0'); }); diff --git a/packages/api/src/connections/notion/route.test.ts b/packages/api/src/connections/notion/route.test.ts index 4d32ef75..856c09ee 100644 --- a/packages/api/src/connections/notion/route.test.ts +++ b/packages/api/src/connections/notion/route.test.ts @@ -3,11 +3,19 @@ import {Ok, type Result} from 'ts-results-es'; import type {ApiError} from '@proxy/utils'; -const listConnections = mock(); +const findAccountConnection = mock(); const createConnection = mock(); -const updateConnectionCredential = mock(); - -mock.module('@proxy/db/connection', () => ({listConnections, createConnection, updateConnectionCredential, updateConnectionTools: mock(), updateConnectionName: mock(), getConnectionCredential: mock(async () => Ok(null))})); +const renewConnection = mock(); + +mock.module('@proxy/db/connection', () => ({ + findAccountConnection, + createConnection, + renewConnection, + updateConnectionCredential: mock(), + getConnectionCredential: mock(async () => Ok(null)), + updateConnectionTools: mock(), + updateConnectionName: mock(), +})); const getUserOrgId = mock(); @@ -84,9 +92,9 @@ beforeEach(() => { mock.clearAllMocks(); notion.requests = []; getUserOrgId.mockResolvedValue(Ok('org-1')); - listConnections.mockResolvedValue(Ok([])); + findAccountConnection.mockResolvedValue(Ok(null)); createConnection.mockResolvedValue(Ok({id: 'conn-1'})); - updateConnectionCredential.mockResolvedValue(Ok(true)); + renewConnection.mockResolvedValue(Ok(undefined)); }); describe('handleStartNotionRoute', () => { diff --git a/packages/db/prisma/migrations/v015_add_connection_provider_account_id/migration.sql b/packages/db/prisma/migrations/v015_add_connection_provider_account_id/migration.sql new file mode 100644 index 00000000..706a46cb --- /dev/null +++ b/packages/db/prisma/migrations/v015_add_connection_provider_account_id/migration.sql @@ -0,0 +1,7 @@ +-- The provider's own id for a connection's account, which stays as its address changes: Google's +-- `sub`. Signing in again finds the connection by it, and a Google grant is kept while another +-- connection is for the same account. Connections from before it, and the integrations whose +-- providers give none, have none and go by their address. +ALTER TABLE "Connection" ADD COLUMN "providerAccountId" TEXT; + +CREATE INDEX "Connection_integrationId_providerAccountId_idx" ON "Connection"("integrationId", "providerAccountId"); diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma index d874cb1d..7fd11a28 100644 --- a/packages/db/prisma/schema.prisma +++ b/packages/db/prisma/schema.prisma @@ -133,6 +133,9 @@ model Connection { // The account as the provider names it, such as an email address; the organization's name for // Information. account String + // The provider's own id for the account, which stays as its address changes: Google's `sub`. + // None where the provider gives none, or for a connection from before it was kept. + providerAccountId String? // Encrypted with ENCRYPTION_KEY: an app password with its server settings. None for Information. credential String? // A custom MCP server's tools as the server last listed them (`@proxy/integrations`' McpTool), @@ -151,6 +154,7 @@ model Connection { agentGrants AgentGrant[] @@index([orgId]) + @@index([integrationId, providerAccountId]) } // An action of the connection that agents get by default, such as `read` or `archive` on a diff --git a/packages/db/src/connection.ts b/packages/db/src/connection.ts index 16e1a2fe..3b954a36 100644 --- a/packages/db/src/connection.ts +++ b/packages/db/src/connection.ts @@ -80,6 +80,7 @@ export async function createConnection(data: { orgId: string; integrationId: string; account: string; + providerAccountId?: string | null; credential: string; defaults?: string[]; tools?: string; @@ -94,6 +95,53 @@ export async function createConnection(data: { ); } +/** + * The organization's connection of an integration for an account, signed in to again: by the + * provider's own id for the account where it gives one, else by its address, which is also how a + * connection made before ids were kept is found. + */ +export async function findAccountConnection(data: { + orgId: string; + integrationId: string; + account: string; + providerAccountId: string | null; +}): Promise> { + const {orgId, integrationId, account, providerAccountId} = data; + return wrapDb(async () => { + const byId = + providerAccountId === null + ? null + : await db.connection.findFirst({ + where: {orgId, integrationId, providerAccountId}, + select: connectionSelect, + }); + return ( + byId ?? + db.connection.findFirst({ + where: {orgId, integrationId, account, providerAccountId: null}, + select: connectionSelect, + }) + ); + }); +} + +/** + * Gives a connection signed in to again its new credential, and the account as the provider names + * it now, such as an address changed since. + */ +export async function renewConnection(data: { + connectionId: string; + account: string; + providerAccountId: string | null; + credential: string; +}): Promise> { + const {connectionId, ...renewed} = data; + const updated = await wrapDb(() => + db.connection.update({where: {id: connectionId}, data: renewed, select: {id: true}}), + ); + return updated.map(() => undefined); +} + /** * Replaces a connection's encrypted credential, as when Granola hands out new tokens. With * `previous`, only while the credential is still that one, so two servers refreshing at once From ff50b948c6966ee443a0fd27769ddabe0401935e Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Wed, 14 Oct 2026 18:07:09 +0000 Subject: [PATCH 5/6] answer a page of rows past a tab's last row empty and a row past it not found without asking sheets, which refuses a range outside the grid, keep the pkce helper and the sign-in answer to connect-flow, and say what drive does on this branch --- AGENTS.md | 2 +- packages/api/src/connections/connect-flow.ts | 4 ++-- .../api/src/records/google-connector.test.ts | 18 ++++++++++++++++++ packages/api/src/records/google-files.ts | 11 ++++++++++- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9f3ed40b..434db9e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ - `packages/utils` - `@proxy/utils`, the Result helpers shared by every package: `Do`, `requirePresent` and `parseSchema` (`src/parse.ts`), and `httpRequest`, the fetch wrapper that returns a `FetchError` union instead of throwing (`src/fetch.ts`). Also `ApiError`/`ApiErr` and `wrapDb` (`src/api-error.ts`), the error every server-side function returns, and the session cookie's name and parsing (`src/session-cookie.ts`) - `packages/integrations` - `@proxy/integrations`, the catalog the api and the web app share: each integration's actions, collections and fields (`src/catalog.ts`), the agents a login can be made for (`src/agent-providers.ts`), the email providers' servers (`src/email-providers.ts`), and access (`src/access.ts`): per action of a connection, an agent's own setting, else the connection's default, an action counting only with the one it `requires`, and tools (`src/tools.ts`): what an agent can do with a connection, generated from the catalog, one per list, open, write and command of each collection (`emails_list`, `emails_archive`), each with the action it needs and a JSON Schema of its params. A custom MCP server (`mcp` in the catalog, `custom`) has no actions of its own: `integrationFor(connection)` (`src/mcp-tools.ts`) builds its integration from the tools the connection keeps (`McpTool`, as the server lists them), one action per tool named by the tool, its risk from the tool's annotations (low when it only reads, medium when what it changes can be undone, high when it can't, and `unknown`, shown without a label, when it says nothing), and `integrationTools` passes them on as `call` tools with the server's own input schema (`ServerTool`, beside the catalog's `CollectionTool`). The web app adds the icons (`src/lib/integrations.ts`, whose `integrationOf(connection)` is what every page uses) - `packages/db` - `@proxy/db`: Prisma 6 on Postgres. `src/index.ts` is the client; each module (`auth.ts`, `user.ts`, `organization.ts`, `connection.ts`, `info.ts`, `agent.ts`, `audit.ts`) exports functions returning `Result` through `wrapDb`. A connection's `tools` column holds a custom MCP server's tools as JSON, and `name` what the person called it; both null for every other integration. `relationMode = "prisma"`: no foreign keys in the database, so Prisma runs the cascades and every relation column has its own index. Locally the database is Prisma's embedded Postgres (PGlite, `scripts/dev-db.ts`), which needs `connection_limit=1&pgbouncer=true` in `DATABASE_URL` -- `packages/api` - `@proxy/api`: Express on Bun. Routes are `handleXRoute` functions mounted in `src/server/create-api-router.ts`; JSON routes return `Result` through `withAuthResult`, which puts the signed-in user on `req.user`. Every environment variable is in `src/utils/env-schema.ts` with its docs, which the docs' Environment variables page is built from; add new ones there. Personal Agent Proxy is one origin everywhere: when `packages/web/dist` exists, as in the Docker image, the api serves the web app's build too (`src/server/serve-web.ts`), and every address the api builds starts from `APP_URL`. Browser-called writes take `requireBrowserOrigin` (CSRF); routes that read a cookie have no CORS. `src/connections/` lists, deletes and sets defaults on an organization's connections, connects a mailbox with an app password after signing in to its IMAP and SMTP servers (`email/`), and connects Granola, Notion and Linear by signing in to their MCP servers (`mcp/`, shared by all three: OAuth with PKCE against the server's authorization server, a client registered for each sign-in so nothing is set up beforehand, the handshake in an encrypted cookie, `mcpConnectRoutes` making both routes on the sign-in flow every OAuth integration shares, `connect-flow.ts`; tokens are refreshed as they run out, one refresh at a time per connection on each server, read again first so ones another call or server saved meanwhile are used rather than refreshed twice, and saved only over the ones they replace, whatever else the credential holds kept beside them, by `connections/token-session.ts`, which Google's connection will share; each tool call is one stateless `tools/call`, `mcp-client.ts`, which also lists a server's tools and, for a server that may keep a session (`handshake`), initializes one around each call as the spec asks). Every request to an MCP server, its metadata or its authorization server goes through `utils/checked-request.ts`, which checks the address as a database's host is checked (`utils/host-check.ts`: nothing private unless `ALLOW_PRIVATE_HOSTS`, and `https` unless the address is private) before each request, since a custom server and its endpoints are wherever it said, and never follows a redirect. A custom MCP server is connected by its address (`custom/`): `POST /api/connections/mcp` with `{name, url, token?, returnTo?}` (the name is the person's, kept through the sign-in in its cookie, since a server says nothing about itself that would do) asks the server for its tools, connects a server that answers at once, with no sign-in or the token typed in as a bearer token (as written when it has its scheme in front, `Basic …`), and for one that turns a request without a token down begins its sign-in there, under the browser's origin check, answering `{kind: 'signIn', url}` with the sign-in page the browser goes to: `mcp-discovery.ts` finds the authorization server from the server's protected resource metadata (the `WWW-Authenticate` header's `resource_metadata`, else the well-known paths), its endpoints from that server's metadata (else `/authorize`, `/token` and `/register` under it) and the scopes the server's own metadata names, and the sign-in, begun with `signInFlow` (`connect-flow.ts`, which `mcpConnectRoutes` builds on for the servers known in advance), keeps the endpoints in the handshake cookie and in the credential beside the tokens. The connection's account is the address without its scheme (`mcp.example.com/mcp`), connecting the same address again replaces its credential and tools, a new connection's defaults are the tools that say they only read, `POST /api/connections/:id/tools` asks the server for its tools again (replacing the tools drops the defaults and agents' settings for tools the server no longer lists, `updateConnectionTools`), and a tool is run as it is (`custom-mcp.ts`): its params go to the server as given, a tool that fails answers the agent with the server's own text as a `validation_error`, and the run is logged under the collection `tools` by the tool's name, with its params as JSON, cut to 200 characters. Granola's notes and transcripts are on by default, Notion's Read pages and Linear's Read issues alone; a Notion or Linear sign-in is for one workspace, and a Notion one lasts at most 180 days, however often it is refreshed. `src/agents/` makes, revokes and deletes agent logins (the password is sent once, only its argon2id hash is kept) and sets an agent's own settings per action of a connection where they differ from the default. `src/records/` reads and writes a collection's records through the integration's `Connector` (`connectors.ts`): Information's live in the database (`info-connector.ts`), a mailbox's are fetched over IMAP on every request (`email-connector.ts`), which merges the inbox and the folders the server marks as Drafts and Sent into one list (ids start with the folder: `inbox-…`, `drafts-…`, `sent-…`) and runs the commands: flags, moves to the folders the server marks as Archive (Gmail's All Mail) or Trash, and sending over SMTP, a draft or a new email, filing a copy in Sent where the server doesn't (Gmail does). Received emails are never edited or deleted for good; only drafts are. Granola's notes are fetched from its MCP server on every request (`granola-connector.ts`; the reading of the XML-like text Granola answers in is in `connections/granola/granola-mcp.ts`): its Notes and Transcripts collections are the same meetings by Granola's meeting id, and since Granola's tools have no pages, a page is 30 days of meetings; one that finds none skips back to the newest meeting before it, so the list ends where the meetings do. Granola's dates and attendees are shown as Granola writes them. Notion's workspace is fetched from its MCP server the same way (`notion-connector.ts`, the reading of its answers in `connections/notion/notion-mcp.ts`), as the person who signed in, as a tree in one nested collection: Notion has no list of every page, so the top is their Recents and a search (Notion's own, AI search where the workspace has it) finds the rest, one page of 50; inside a page are the pages and databases in its content, inside a database its saved views (`view-…` ids), or the rows of its only one, and inside a view its rows, paged with Notion's cursor (view queries have no plan quota; Notion's row and SQL queries do). A page reads as Notion-flavored Markdown, a row's properties beside it as "Name: value" lines (`notion-properties.ts`), and only a page's, database's or view's id is ever sent to Notion's fetch; a new page is a private draft at the top, or goes inside the page, database or view it is made in, a row's title in the database's title property; an edit sends the title, the properties written differently and the content, each only where it changed, and pages are never deleted. Notion's MCP server has no tool that deletes or trashes a page. Linear's issues are fetched from its MCP server the same way (`linear-connector.ts`, the reading of its answers in `connections/linear/linear-mcp.ts`), as the person who signed in, by their identifier (`ENG-123`): a list is the issues updated last first, 50 to a page with Linear's cursor, narrowed by the type of status (`?statusType=Started`) and searched in titles and descriptions, and an issue opens with its comments, oldest first. Teams, statuses, people, projects and labels are written by name, as Linear shows them and finds them, its refusals passed on to the agent; an edit sends only the fields written differently, an empty one clearing it. Linear's MCP server has no tool that archives or deletes an issue. Google is connected by signing in to Google with the app's own Google client, the one Google sign-in uses (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, a second redirect URI `/api/connections/google/callback`), asking for every scope at once, Gmail, Drive, Docs, Sheets and Slides, with offline access (`connections/google/`: `google-oauth.ts` the sign-in and refresh, with the client's secret, keeping the refresh token Google doesn't send again; `google-client.ts` the calls with a bearer token and Google's errors, a disabled API passing Google's own message on, which names the API and where to enable it); the credential holds the tokens, the scopes Google granted, since the consent screen lets a person untick some, and the address; the connection keeps Google's id for the account (`Connection.providerAccountId`, its `sub`), so signing in again renews the same connection, and its address, even after the address changed (`findAccountConnection`, by address for one without an id and for the MCP integrations). Its Drive is one nested collection, `files` (`records/google-files.ts`, behind `google-connector.ts`, which dispatches on the collection), fetched live from Drive's and Sheets' APIs (`drive-api.ts`, `sheets-api.ts`): at the top four places, My Drive, Shared with me, Shared drives (`drive-…` ids, of the kind Shared drive, which is never edited or trashed) and Recent, inside them and inside folders their files, folders first, 50 to a page with Drive's page token, and a search is Drive's full-text one, anywhere or inside the folder opened; the trail is walked up a file's parents to My Drive, its shared drive, or, where the account can't see further up, Shared with me. A document opens as Markdown and a presentation as text, as Drive exports them, up to its 10 MB export cap, beyond which its content says the file is too large, so it can still be renamed or trashed; a spreadsheet opens into its tabs (`sheet--`) and a tab into its rows (`row---`), 50 to a page, the page token the next row's number, each row's values "Header: value" lines under the tab's first row, a header repeated or a column without one going by its letter (`Status (E)`). Drive isn't written to yet; writes, and Gmail, come later. PostgreSQL and MySQL (MariaDB too) are two integrations on one code path: a database is connected with its host, port, database, user, password and encryption (`connections/sql/`: `route.ts` makes both routes per dialect, `sql-client.ts` opens one connection per request with `Bun.sql`, to the address `utils/host-check.ts` resolves the host to, refusing private addresses unless `ALLOW_PRIVATE_HOSTS` is set, and `sql-statement.ts` checks an agent's query is one statement that reads), and `records/sql-connector.ts` reads and writes it through `records/sql-dialect.ts`, where the two differ: the tables are one nested collection, a table holding its rows, a row read and written as "column: value" lines and found again by its primary key (ids `public.users:42`, `pairs:3,4`; a table without a key is never written to), and a query, a second collection that is only ever run, goes through a read-only transaction with a time limit, a cursor on Postgres and `sql_select_limit` on MySQL, 200 rows at most, answered as JSON lines. MySQL's read-only transaction doesn't stop DDL, so the statement check is what keeps a query to reading; the docs tell people to make a read-only database user as well. Lists take `?search=` (only where the collection has a `searchHint`; Granola has none), `?filter=` (the collection's `filterField`, such as `?filter=Draft`), `?parent=` (only in a `nested` collection: the list inside a record, answered with its `trail`, the records above it; a record made there goes in it) and `?page=` (`list-query.ts`) and answer a page, newest first (a mailbox's 50 emails, Granola's 30 days), with the next page's token; a mailbox's token holds where each folder goes on, so paging through the merged list shows every email once, and its search uses Gmail's own syntax on Gmail (`X-GM-EXT-1`) and IMAP text search elsewhere. Agents sign in at `/agent-auth/login` with a cookie of their own (`src/agent-auth/`); `src/agent-side/` serves them: the connections an agent can use, the tools it has with each, and `runAgentTool` (`tools.ts`), the one place a tool runs, which checks it against the agent's access as it is now, validates its params against the tool's schema (an edit changes only the fields given), calls the connector, or a custom server's tool as it is, answering `{text}`, and logs it, denied ones too, to the activity log that `src/activity/` lists. Over HTTP that is `/api/agent/me`, `GET /api/agent/connections/:id/tools` and `POST /api/agent/connections/:id/tools/:tool` with `{params}`; `src/mcp/` serves the same over MCP at `/mcp` (Streamable HTTP without sessions, JSON answers, no stream): four tools whatever the login may do, `list_connections` (the entry point), `list_tools`, `read` (only tools that read, marked read-only) and `run`, and server instructions naming the agent's connections. MCP clients sign in with OAuth 2.1 (`src/oauth/`), Personal Agent Proxy being its own authorization server: metadata under `/.well-known/`, Dynamic Client Registration at `/oauth/register` (open to anyone, each registration first deleting the clients that registered over a day ago and never finished a sign-in), `/oauth/authorize` sealing the request into the web app's `/connect-agent` page, where the person chooses the agent login the client works as or makes one, and `/oauth/token` with PKCE (S256 only) and refresh tokens replaced on every use; codes and tokens are kept as their SHA-256, and revoking an agent ends its clients' tokens. These endpoints take any origin (CORS), since none reads a cookie; credentials and Information records are stored encrypted with `ENCRYPTION_KEY` (`src/utils/secret-crypto.ts`). Has its own `.prettierrc.json` with a 200-column print width +- `packages/api` - `@proxy/api`: Express on Bun. Routes are `handleXRoute` functions mounted in `src/server/create-api-router.ts`; JSON routes return `Result` through `withAuthResult`, which puts the signed-in user on `req.user`. Every environment variable is in `src/utils/env-schema.ts` with its docs, which the docs' Environment variables page is built from; add new ones there. Personal Agent Proxy is one origin everywhere: when `packages/web/dist` exists, as in the Docker image, the api serves the web app's build too (`src/server/serve-web.ts`), and every address the api builds starts from `APP_URL`. Browser-called writes take `requireBrowserOrigin` (CSRF); routes that read a cookie have no CORS. `src/connections/` lists, deletes and sets defaults on an organization's connections, connects a mailbox with an app password after signing in to its IMAP and SMTP servers (`email/`), and connects Granola, Notion and Linear by signing in to their MCP servers (`mcp/`, shared by all three: OAuth with PKCE against the server's authorization server, a client registered for each sign-in so nothing is set up beforehand, the handshake in an encrypted cookie, `mcpConnectRoutes` making both routes on the sign-in flow every OAuth integration shares, `connect-flow.ts`; tokens are refreshed as they run out, one refresh at a time per connection on each server, read again first so ones another call or server saved meanwhile are used rather than refreshed twice, and saved only over the ones they replace, whatever else the credential holds kept beside them, by `connections/token-session.ts`, which Google's connection will share; each tool call is one stateless `tools/call`, `mcp-client.ts`, which also lists a server's tools and, for a server that may keep a session (`handshake`), initializes one around each call as the spec asks). Every request to an MCP server, its metadata or its authorization server goes through `utils/checked-request.ts`, which checks the address as a database's host is checked (`utils/host-check.ts`: nothing private unless `ALLOW_PRIVATE_HOSTS`, and `https` unless the address is private) before each request, since a custom server and its endpoints are wherever it said, and never follows a redirect. A custom MCP server is connected by its address (`custom/`): `POST /api/connections/mcp` with `{name, url, token?, returnTo?}` (the name is the person's, kept through the sign-in in its cookie, since a server says nothing about itself that would do) asks the server for its tools, connects a server that answers at once, with no sign-in or the token typed in as a bearer token (as written when it has its scheme in front, `Basic …`), and for one that turns a request without a token down begins its sign-in there, under the browser's origin check, answering `{kind: 'signIn', url}` with the sign-in page the browser goes to: `mcp-discovery.ts` finds the authorization server from the server's protected resource metadata (the `WWW-Authenticate` header's `resource_metadata`, else the well-known paths), its endpoints from that server's metadata (else `/authorize`, `/token` and `/register` under it) and the scopes the server's own metadata names, and the sign-in, begun with `signInFlow` (`connect-flow.ts`, which `mcpConnectRoutes` builds on for the servers known in advance), keeps the endpoints in the handshake cookie and in the credential beside the tokens. The connection's account is the address without its scheme (`mcp.example.com/mcp`), connecting the same address again replaces its credential and tools, a new connection's defaults are the tools that say they only read, `POST /api/connections/:id/tools` asks the server for its tools again (replacing the tools drops the defaults and agents' settings for tools the server no longer lists, `updateConnectionTools`), and a tool is run as it is (`custom-mcp.ts`): its params go to the server as given, a tool that fails answers the agent with the server's own text as a `validation_error`, and the run is logged under the collection `tools` by the tool's name, with its params as JSON, cut to 200 characters. Granola's notes and transcripts are on by default, Notion's Read pages and Linear's Read issues alone; a Notion or Linear sign-in is for one workspace, and a Notion one lasts at most 180 days, however often it is refreshed. `src/agents/` makes, revokes and deletes agent logins (the password is sent once, only its argon2id hash is kept) and sets an agent's own settings per action of a connection where they differ from the default. `src/records/` reads and writes a collection's records through the integration's `Connector` (`connectors.ts`): Information's live in the database (`info-connector.ts`), a mailbox's are fetched over IMAP on every request (`email-connector.ts`), which merges the inbox and the folders the server marks as Drafts and Sent into one list (ids start with the folder: `inbox-…`, `drafts-…`, `sent-…`) and runs the commands: flags, moves to the folders the server marks as Archive (Gmail's All Mail) or Trash, and sending over SMTP, a draft or a new email, filing a copy in Sent where the server doesn't (Gmail does). Received emails are never edited or deleted for good; only drafts are. Granola's notes are fetched from its MCP server on every request (`granola-connector.ts`; the reading of the XML-like text Granola answers in is in `connections/granola/granola-mcp.ts`): its Notes and Transcripts collections are the same meetings by Granola's meeting id, and since Granola's tools have no pages, a page is 30 days of meetings; one that finds none skips back to the newest meeting before it, so the list ends where the meetings do. Granola's dates and attendees are shown as Granola writes them. Notion's workspace is fetched from its MCP server the same way (`notion-connector.ts`, the reading of its answers in `connections/notion/notion-mcp.ts`), as the person who signed in, as a tree in one nested collection: Notion has no list of every page, so the top is their Recents and a search (Notion's own, AI search where the workspace has it) finds the rest, one page of 50; inside a page are the pages and databases in its content, inside a database its saved views (`view-…` ids), or the rows of its only one, and inside a view its rows, paged with Notion's cursor (view queries have no plan quota; Notion's row and SQL queries do). A page reads as Notion-flavored Markdown, a row's properties beside it as "Name: value" lines (`notion-properties.ts`), and only a page's, database's or view's id is ever sent to Notion's fetch; a new page is a private draft at the top, or goes inside the page, database or view it is made in, a row's title in the database's title property; an edit sends the title, the properties written differently and the content, each only where it changed, and pages are never deleted. Notion's MCP server has no tool that deletes or trashes a page. Linear's issues are fetched from its MCP server the same way (`linear-connector.ts`, the reading of its answers in `connections/linear/linear-mcp.ts`), as the person who signed in, by their identifier (`ENG-123`): a list is the issues updated last first, 50 to a page with Linear's cursor, narrowed by the type of status (`?statusType=Started`) and searched in titles and descriptions, and an issue opens with its comments, oldest first. Teams, statuses, people, projects and labels are written by name, as Linear shows them and finds them, its refusals passed on to the agent; an edit sends only the fields written differently, an empty one clearing it. Linear's MCP server has no tool that archives or deletes an issue. Google is connected by signing in to Google with the app's own Google client, the one Google sign-in uses (`GOOGLE_CLIENT_ID`, `GOOGLE_CLIENT_SECRET`, a second redirect URI `/api/connections/google/callback`), asking for every scope at once, Gmail, Drive, Docs, Sheets and Slides, with offline access (`connections/google/`: `google-oauth.ts` the sign-in and refresh, with the client's secret, keeping the refresh token Google doesn't send again; `google-client.ts` the calls with a bearer token and Google's errors, a disabled API passing Google's own message on, which names the API and where to enable it); the credential holds the tokens, the scopes Google granted, since the consent screen lets a person untick some, and the address; the connection keeps Google's id for the account (`Connection.providerAccountId`, its `sub`), so signing in again renews the same connection, and its address, even after the address changed (`findAccountConnection`, by address for one without an id and for the MCP integrations). Its Drive is one nested collection, `files` (`records/google-files.ts`, behind `google-connector.ts`, which dispatches on the collection), fetched live from Drive's and Sheets' APIs (`drive-api.ts`, `sheets-api.ts`): at the top four places, My Drive, Shared with me, Shared drives (`drive-…` ids) and Recent, inside them and inside folders their files, folders first, 50 to a page with Drive's page token, and a search is Drive's full-text one, anywhere or inside the folder opened; the trail is walked up a file's parents to My Drive, its shared drive, or, where the account can't see further up, Shared with me. A document opens as Markdown and a presentation as text, as Drive exports them, up to its 10 MB export cap, beyond which opening it says so; a spreadsheet opens into its tabs (`sheet--`) and a tab into its rows (`row---`), 50 to a page, the page token the next row's number, each row's values "Header: value" lines under the tab's first row, a column without a header going by its letter (`Column C`); a page past the tab's last row is empty and a row past it not found without asking Sheets, which refuses a range outside the grid. Drive isn't written to yet; writes, and Gmail, come later. PostgreSQL and MySQL (MariaDB too) are two integrations on one code path: a database is connected with its host, port, database, user, password and encryption (`connections/sql/`: `route.ts` makes both routes per dialect, `sql-client.ts` opens one connection per request with `Bun.sql`, to the address `utils/host-check.ts` resolves the host to, refusing private addresses unless `ALLOW_PRIVATE_HOSTS` is set, and `sql-statement.ts` checks an agent's query is one statement that reads), and `records/sql-connector.ts` reads and writes it through `records/sql-dialect.ts`, where the two differ: the tables are one nested collection, a table holding its rows, a row read and written as "column: value" lines and found again by its primary key (ids `public.users:42`, `pairs:3,4`; a table without a key is never written to), and a query, a second collection that is only ever run, goes through a read-only transaction with a time limit, a cursor on Postgres and `sql_select_limit` on MySQL, 200 rows at most, answered as JSON lines. MySQL's read-only transaction doesn't stop DDL, so the statement check is what keeps a query to reading; the docs tell people to make a read-only database user as well. Lists take `?search=` (only where the collection has a `searchHint`; Granola has none), `?filter=` (the collection's `filterField`, such as `?filter=Draft`), `?parent=` (only in a `nested` collection: the list inside a record, answered with its `trail`, the records above it; a record made there goes in it) and `?page=` (`list-query.ts`) and answer a page, newest first (a mailbox's 50 emails, Granola's 30 days), with the next page's token; a mailbox's token holds where each folder goes on, so paging through the merged list shows every email once, and its search uses Gmail's own syntax on Gmail (`X-GM-EXT-1`) and IMAP text search elsewhere. Agents sign in at `/agent-auth/login` with a cookie of their own (`src/agent-auth/`); `src/agent-side/` serves them: the connections an agent can use, the tools it has with each, and `runAgentTool` (`tools.ts`), the one place a tool runs, which checks it against the agent's access as it is now, validates its params against the tool's schema (an edit changes only the fields given), calls the connector, or a custom server's tool as it is, answering `{text}`, and logs it, denied ones too, to the activity log that `src/activity/` lists. Over HTTP that is `/api/agent/me`, `GET /api/agent/connections/:id/tools` and `POST /api/agent/connections/:id/tools/:tool` with `{params}`; `src/mcp/` serves the same over MCP at `/mcp` (Streamable HTTP without sessions, JSON answers, no stream): four tools whatever the login may do, `list_connections` (the entry point), `list_tools`, `read` (only tools that read, marked read-only) and `run`, and server instructions naming the agent's connections. MCP clients sign in with OAuth 2.1 (`src/oauth/`), Personal Agent Proxy being its own authorization server: metadata under `/.well-known/`, Dynamic Client Registration at `/oauth/register` (open to anyone, each registration first deleting the clients that registered over a day ago and never finished a sign-in), `/oauth/authorize` sealing the request into the web app's `/connect-agent` page, where the person chooses the agent login the client works as or makes one, and `/oauth/token` with PKCE (S256 only) and refresh tokens replaced on every use; codes and tokens are kept as their SHA-256, and revoking an agent ends its clients' tokens. These endpoints take any origin (CORS), since none reads a cookie; credentials and Information records are stored encrypted with `ENCRYPTION_KEY` (`src/utils/secret-crypto.ts`). Has its own `.prettierrc.json` with a 200-column print width - `skills/` - Agent skills, symlinked into `.claude/skills` and `.codex/skills` by `bun run setup` ## The app diff --git a/packages/api/src/connections/connect-flow.ts b/packages/api/src/connections/connect-flow.ts index 02aeffa2..5357d5d7 100644 --- a/packages/api/src/connections/connect-flow.ts +++ b/packages/api/src/connections/connect-flow.ts @@ -47,7 +47,7 @@ export type SignInRequest = {redirectUri: string; state: string; challenge: stri type SignInStart = (request: SignInRequest) => Promise>; /** The provider's answer coming back, with the verifier and what the start kept for it. */ -export type SignInAnswer = {redirectUri: string; code: string; verifier: string; kept: E}; +type SignInAnswer = {redirectUri: string; code: string; verifier: string; kept: E}; /** * A connection to save: the account it shows, the provider's own id for it where it gives one, @@ -58,7 +58,7 @@ export type SignInAnswer = {redirectUri: string; code: string; verifier: stri export type Connected = {account: string; accountId?: string; credential: C; defaults: string[]; tools?: McpTool[]; name?: string}; /** A PKCE verifier and the challenge the provider is sent in its place. */ -export function makePkce(): {verifier: string; challenge: string} { +function makePkce(): {verifier: string; challenge: string} { const verifier = randomBytes(32).toString('base64url'); return {verifier, challenge: createHash('sha256').update(verifier).digest('base64url')}; } diff --git a/packages/api/src/records/google-connector.test.ts b/packages/api/src/records/google-connector.test.ts index c0cd8327..57380cc1 100644 --- a/packages/api/src/records/google-connector.test.ts +++ b/packages/api/src/records/google-connector.test.ts @@ -322,6 +322,16 @@ describe('googleConnector.list', () => { expect(second.nextPage).toBeNull(); }); + test("a page of rows past the tab's last row is empty, without asking Sheets for a range outside its grid", async () => { + const {googleConnector} = await import('./google-connector'); + const page = (await googleConnector.list(target, query({parent: 'sheet-1Budget-0', page: '121'}))).unwrap(); + + expect(page.records).toEqual([]); + expect(page.nextPage).toBeNull(); + expect(page.trail?.at(-1)).toEqual({id: 'sheet-1Budget-0', title: 'Q1'}); + expect(requested('/v4/spreadsheets/1Budget/values:batchGet')).toHaveLength(0); + }); + test('a page of rows that is not a row number is refused', async () => { const {googleConnector} = await import('./google-connector'); const result = await googleConnector.list(target, query({parent: 'sheet-1Budget-0', page: 'A1'})); @@ -401,6 +411,14 @@ describe('googleConnector.get', () => { expect(record.values).toMatchObject({name: 'Item 7', kind: 'Row', content: 'Item: Item 7\nAmount: 70\nColumn C: Paid'}); }); + test("a row past the tab's last row is not found, without asking Sheets for it", async () => { + const {googleConnector} = await import('./google-connector'); + const result = await googleConnector.get(target, 'row-1Budget-0-121'); + + expect(result.isErr() && result.error).toMatchObject({kind: 'not_found', resource: 'record', id: 'row-1Budget-0-121'}); + expect(requested('/v4/spreadsheets/1Budget/values:batchGet')).toHaveLength(0); + }); + test('a file too large to export says so', async () => { const {googleConnector} = await import('./google-connector'); const result = await googleConnector.get(target, '1Huge'); diff --git a/packages/api/src/records/google-files.ts b/packages/api/src/records/google-files.ts index 6e78489f..753020b9 100644 --- a/packages/api/src/records/google-files.ts +++ b/packages/api/src/records/google-files.ts @@ -303,11 +303,16 @@ function listRows(drive: Drive, ref: {fileId: string; sheetId: number}, query: L const from = Math.max(Number(query.page ?? 2), 2); const to = from + PAGE_SIZE - 1; const {file, sheet} = $(await getSheet(drive, ref.fileId, ref.sheetId)); + const tab = {id: sheetId(file, sheet), title: sheet.title}; + // Sheets refuses a range wholly outside the grid, so a page past the tab's last row is empty without asking. + if (from > sheet.rowCount) { + return {records: [], nextPage: null, trail: [...$(await trailTo(drive, file)), tab]}; + } const [ranges, trail] = await Promise.all([getRanges(drive.token, file.id, [rowsRange(sheet.title, 1, 1), rowsRange(sheet.title, from, to)]), trailTo(drive, file)]); const [[headers = []] = [], rows = []] = $(ranges); const records = rows.flatMap((cells, index) => (isBlank(cells) ? [] : [rowRecord(file, sheet, headers, from + index, cells)])); const nextPage = $(await nextRowsPage(drive, file.id, sheet, to, rows.length)); - return {records, nextPage, trail: [...$(trail), {id: sheetId(file, sheet), title: sheet.title}]}; + return {records, nextPage, trail: [...$(trail), tab]}; }); } @@ -431,6 +436,10 @@ function getRef(drive: Drive, ref: FileRef): Promise sheet.rowCount) { + return $(Err(ApiErr.notFound('row'))); + } const [[headers = []] = [], [cells = []] = []] = $(await getRanges(drive.token, file.id, [rowsRange(sheet.title, 1, 1), rowsRange(sheet.title, ref.row, ref.row)])); if (isBlank(cells)) { return $(Err(ApiErr.notFound('row'))); From 0c0e35383dbc7b425921fe3aea1b04086953d225 Mon Sep 17 00:00:00 2001 From: hkonsti <45428767+hkonsti@users.noreply.github.com> Date: Thu, 15 Oct 2026 00:19:10 +0000 Subject: [PATCH 6/6] number the provider account id migration after main's v015 and v016, so it runs after them --- .../migration.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/db/prisma/migrations/{v015_add_connection_provider_account_id => v017_add_connection_provider_account_id}/migration.sql (100%) diff --git a/packages/db/prisma/migrations/v015_add_connection_provider_account_id/migration.sql b/packages/db/prisma/migrations/v017_add_connection_provider_account_id/migration.sql similarity index 100% rename from packages/db/prisma/migrations/v015_add_connection_provider_account_id/migration.sql rename to packages/db/prisma/migrations/v017_add_connection_provider_account_id/migration.sql