show balances and spending from the home page, behind the plaid experiment

packages/api/src/intent/classify.test.ts

@@ -5,6 +5,10 @@ mock.module('../utils/env', () => ({env: mockEnv}));
 
 const text = 'Create a calendar invite with Jon Burke and Jacky Doe at One Boulevard Plaza at 7pm tomorrow';
 
+// An organization with no experiment on, and one with banks.
+const NOBODY = {experiments: []};
+const WITH_BANKS = {experiments: ['plaid' as const]};
+
 // What OpenRouter answers: the model's JSON as the message's text.
 let content = '';
 let status = 200;
@@ -19,7 +23,7 @@ function answer(values: Record<string, string | string[] | string[][] | null>) {
 function sent(): {
 	model: string;
 	messages: Array<{role: string; content: string}>;
-	provider: {order: string[]};
+	provider: {order: string[]; data_collection: string};
 	response_format: {json_schema: {schema: {required: string[]; properties: {widget: {enum: string[]}}}}};
 } {
 	return JSON.parse(String(fetchSpy.mock.calls.at(-1)?.[1]?.body));
@@ -47,11 +51,12 @@ beforeEach(() => {
 describe('classifyIntent', () => {
 	test('asks gpt-oss-120b, on Cerebras first, for the widget and every field in one answer', async () => {
 		const {classifyIntent} = await import('./classify');
-		await classifyIntent(text);
+		await classifyIntent(text, NOBODY);
 
 		const body = sent();
 		expect(body.model).toBe('openai/gpt-oss-120b');
 		expect(body.provider.order).toEqual(['Cerebras', 'Groq']);
+		expect(body.provider.data_collection).toBe('deny');
 		expect(body.messages[1]).toEqual({role: 'user', content: text});
 		expect(body.messages[0]?.content).toContain('- find_file: Find or open a file');
 		const schema = body.response_format.json_schema.schema;
@@ -100,7 +105,7 @@ describe('classifyIntent', () => {
 
 	test("answers the picked widget's fields alone, each value trimmed and once", async () => {
 		const {classifyIntent} = await import('./classify');
-		const intent = (await classifyIntent(text)).unwrap();
+		const intent = (await classifyIntent(text, NOBODY)).unwrap();
 
 		expect(intent.widget).toBe('new_event');
 		expect(intent.fields).toEqual([
@@ -113,7 +118,7 @@ describe('classifyIntent', () => {
 	test('answers search terms as each one\'s words joined by " | "', async () => {
 		answer({widget: 'find_email', from: null, about: [['flight', ' itinerary ', 'e|ticket'], ['SFO'], []]});
 		const {classifyIntent} = await import('./classify');
-		const intent = (await classifyIntent('find the email about my flight to sfo')).unwrap();
+		const intent = (await classifyIntent('find the email about my flight to sfo', NOBODY)).unwrap();
 
 		expect(intent.fields).toEqual([
 			{id: 'from', name: 'From', values: []},
@@ -123,6 +128,39 @@ describe('classifyIntent', () => {
 		expect(schema.properties.about).toEqual({type: 'array', items: {type: 'array', items: {type: 'string'}}});
 	});
 
+	test('offers the bank widgets, and their fields, only to an organization with the plaid experiment', async () => {
+		const {classifyIntent} = await import('./classify');
+		await classifyIntent('how much did I spend on restaurants last month', WITH_BANKS);
+
+		const withBanks = sent();
+		expect(withBanks.messages[0]?.content).toContain('- find_transactions: Find bank or card transactions');
+		expect(withBanks.messages[0]?.content).toContain('FOOD_AND_DRINK (restaurants, groceries');
+		expect(withBanks.response_format.json_schema.schema.properties.widget.enum).toContain('show_balances');
+		expect(withBanks.response_format.json_schema.schema.required).toEqual(expect.arrayContaining(['balanceAccount', 'merchant', 'category', 'flow', 'transactionAccount']));
+
+		await classifyIntent('how much did I spend on restaurants last month', NOBODY);
+		const without = sent();
+		expect(without.messages[0]?.content).not.toContain('find_transactions');
+		expect(without.response_format.json_schema.schema.properties.widget.enum).not.toContain('show_balances');
+		expect(without.response_format.json_schema.schema.required).not.toContain('merchant');
+	});
+
+	test('answers a bank widget to an organization with the experiment, and none to one without', async () => {
+		const {classifyIntent} = await import('./classify');
+		answer({widget: 'find_transactions', merchant: 'uber', category: 'TRANSPORTATION', flow: 'out', transactionAccount: null});
+
+		expect((await classifyIntent('uber charges this month', WITH_BANKS)).unwrap()).toMatchObject({
+			widget: 'find_transactions',
+			fields: [
+				{id: 'merchant', values: ['uber']},
+				{id: 'category', values: ['TRANSPORTATION']},
+				{id: 'flow', values: ['out']},
+				{id: 'transactionAccount', values: []},
+			],
+		});
+		expect((await classifyIntent('uber charges this month', NOBODY)).unwrap()).toMatchObject({widget: 'none', fields: []});
+	});
+
 	test("every field id is one widget's alone, as the model answers them all in one object", async () => {
 		const {WIDGETS} = await import('./widgets');
 		const ids = WIDGETS.flatMap((widget) => widget.fields.map((field) => field.id));
@@ -131,7 +169,7 @@ describe('classifyIntent', () => {
 
 	test('holds a unit to the units there are', async () => {
 		const {classifyIntent} = await import('./classify');
-		await classifyIntent('5 miles in km');
+		await classifyIntent('5 miles in km', NOBODY);
 
 		const schema = sent().response_format.json_schema.schema as unknown as {properties: Record<string, {enum?: Array<string | null>}>};
 		expect(schema.properties.fromUnit?.enum).toContain('mi');
@@ -142,35 +180,35 @@ describe('classifyIntent', () => {
 		const {classifyIntent} = await import('./classify');
 		for (const widget of ['none', 'make_coffee']) {
 			answer({widget});
-			expect((await classifyIntent('hi')).unwrap()).toMatchObject({widget: 'none', fields: []});
+			expect((await classifyIntent('hi', NOBODY)).unwrap()).toMatchObject({widget: 'none', fields: []});
 		}
 	});
 
 	test('tells text still being typed from a whole request no widget does', async () => {
 		const {classifyIntent} = await import('./classify');
 		answer({widget: 'incomplete'});
 
-		expect((await classifyIntent('find the')).unwrap()).toMatchObject({widget: 'incomplete', fields: []});
+		expect((await classifyIntent('find the', NOBODY)).unwrap()).toMatchObject({widget: 'incomplete', fields: []});
 		expect(sent().messages[0]?.content).toContain('- incomplete: the text is cut off');
 	});
 
 	test('is unavailable without a key, and asks nothing', async () => {
 		delete mockEnv.OPENROUTER_KEY;
 		const {classifyIntent} = await import('./classify');
 
-		expect((await classifyIntent(text)).unwrapErr()).toMatchObject({kind: 'unavailable', message: 'The home page is not set up: set `OPENROUTER_KEY`.'});
+		expect((await classifyIntent(text, NOBODY)).unwrapErr()).toMatchObject({kind: 'unavailable', message: 'The home page is not set up: set `OPENROUTER_KEY`.'});
 		expect(fetchSpy).not.toHaveBeenCalled();
 	});
 
 	test('OpenRouter failing, or the model answering outside the form, is a provider out of reach', async () => {
 		const {classifyIntent} = await import('./classify');
 
 		status = 503;
-		expect((await classifyIntent(text)).unwrapErr().kind).toBe('provider_unreachable');
+		expect((await classifyIntent(text, NOBODY)).unwrapErr().kind).toBe('provider_unreachable');
 		status = 200;
 		content = 'Sure! Here is the form:';
-		expect((await classifyIntent(text)).unwrapErr().kind).toBe('provider_unreachable');
+		expect((await classifyIntent(text, NOBODY)).unwrapErr().kind).toBe('provider_unreachable');
 		answer({attendees: ['Jon']});
-		expect((await classifyIntent(text)).unwrapErr().kind).toBe('provider_unreachable');
+		expect((await classifyIntent(text, NOBODY)).unwrapErr().kind).toBe('provider_unreachable');
 	});
 });

packages/api/src/intent/classify.ts

@@ -1,18 +1,20 @@
 import {Err, Ok, Result} from 'ts-results-es';
 import {z} from 'zod';
 
-import {ApiErr, type ApiError, httpRequest, parseSchema} from '@proxy/utils';
+import {ApiErr, type ApiError, type Experiment, httpRequest, parseSchema} from '@proxy/utils';
 
 import {env} from '../utils/env';
-import {INCOMPLETE, NO_WIDGET, WIDGETS, type WidgetField} from './widgets';
+import {INCOMPLETE, NO_WIDGET, offeredWidgets, type Widget, type WidgetField} from './widgets';
 
 /**
  * What a person typed into the home page's input, in one request to gpt-oss-120b over OpenRouter,
  * on Cerebras, the fastest host, else Groq: the widget that fits, and its fields filled in from the
  * text, in about 150 ms. Picked over Jev and GLiNER, one after the other, by trying them on typed
  * requests: as fast, and every field right on nine in ten where GLiNER managed half, since it
  * found no title it wasn't told of, no subject and no file's name. The answer is held to a JSON
- * schema: the widget's id, and every widget's fields, the others left empty.
+ * schema: the widget's id, and every widget's fields, the others left empty. The widgets offered
+ * are the organization's: a widget behind an experiment it doesn't have is in neither the
+ * instructions nor the schema.
  */
 
 const OPENROUTER_URL = 'https://openrouter.ai/api/v1/chat/completions';
@@ -36,15 +38,15 @@ export type Intent = {
 	ms: number;
 };
 
-const FIELDS = WIDGETS.flatMap((widget) => widget.fields);
-
-const INSTRUCTIONS = [
-	'Pick the widget for what the person typed into a command box, and fill in its fields from the text. Copy values from the text, but write the body of an email; never invent a person, place or address. Leave the fields of other widgets empty. Answer with JSON only.',
-	'Widgets:',
-	...WIDGETS.map((widget) => `- ${widget.id}: ${widget.description}. Fields: ${widget.fields.map((field) => `${field.id}${field.many ? ' (list)' : ''}: ${field.hint}`).join('; ')}`),
-	`- ${INCOMPLETE}: the text is cut off or says too little to tell what is wanted yet, as it is while still being typed ("find the", "email", "what's")`,
-	`- ${NO_WIDGET}: a whole request that nothing above does ("summarize my week", "what's the weather tomorrow")`,
-].join('\n');
+function instructions(widgets: Widget[]): string {
+	return [
+		'Pick the widget for what the person typed into a command box, and fill in its fields from the text. Copy values from the text, but write the body of an email; never invent a person, place or address. Leave the fields of other widgets empty. Answer with JSON only.',
+		'Widgets:',
+		...widgets.map((widget) => `- ${widget.id}: ${widget.description}. Fields: ${widget.fields.map((field) => `${field.id}${field.many ? ' (list)' : ''}: ${field.hint}`).join('; ')}`),
+		`- ${INCOMPLETE}: the text is cut off or says too little to tell what is wanted yet, as it is while still being typed ("find the", "email", "what's"); never a whole sentence or question, which ends in a question mark or a full stop`,
+		`- ${NO_WIDGET}: a whole request that nothing above does ("summarize my week", "what's the weather tomorrow")`,
+	].join('\n');
+}
 
 function fieldSchema(field: WidgetField) {
 	if (field.options) {
@@ -59,15 +61,35 @@ function fieldSchema(field: WidgetField) {
 	return {type: ['string', 'null']};
 }
 
-const JSON_SCHEMA = {
-	type: 'object',
-	additionalProperties: false,
-	required: ['widget', ...FIELDS.map((field) => field.id)],
-	properties: {
-		widget: {type: 'string', enum: [...WIDGETS.map((widget) => widget.id), INCOMPLETE, NO_WIDGET]},
-		...Object.fromEntries(FIELDS.map((field) => [field.id, fieldSchema(field)])),
-	},
-};
+function jsonSchema(widgets: Widget[]) {
+	const fields = widgets.flatMap((widget) => widget.fields);
+	return {
+		type: 'object',
+		additionalProperties: false,
+		required: ['widget', ...fields.map((field) => field.id)],
+		properties: {
+			widget: {type: 'string', enum: [...widgets.map((widget) => widget.id), INCOMPLETE, NO_WIDGET]},
+			...Object.fromEntries(fields.map((field) => [field.id, fieldSchema(field)])),
+		},
+	};
+}
+
+type Form = {widgets: Widget[]; instructions: string; schema: ReturnType<typeof jsonSchema>};
+
+// The instructions and schema for each set of widgets offered, built once: there are few sets.
+const forms = new Map<string, Form>();
+
+function formFor(holder: {experiments: Experiment[]}): Form {
+	const widgets = offeredWidgets(holder);
+	const key = widgets.map((widget) => widget.id).join(' ');
+	const kept = forms.get(key);
+	if (kept) {
+		return kept;
+	}
+	const form = {widgets, instructions: instructions(widgets), schema: jsonSchema(widgets)};
+	forms.set(key, form);
+	return form;
+}
 
 const completionSchema = z.object({choices: z.array(z.object({message: z.object({content: z.string()})})).min(1)});
 
@@ -92,12 +114,13 @@ function cleaned(value: AnswerValue | undefined): string[] {
 	return unique(value.map((one) => (Array.isArray(one) ? unique(one.map((word) => word.replace(/\|/g, ' '))).join(' | ') : one)));
 }
 
-/** Which widget fits `text`, and its fields as the model fills them in. */
-export async function classifyIntent(text: string): Promise<Result<Intent, ApiError>> {
+/** Which of the widgets offered to `holder` fits `text`, and its fields as the model fills them in. */
+export async function classifyIntent(text: string, holder: {experiments: Experiment[]}): Promise<Result<Intent, ApiError>> {
 	if (!env.OPENROUTER_KEY) {
 		return Err(ApiErr.unavailable('The home page is not set up: set `OPENROUTER_KEY`.'));
 	}
 
+	const form = formFor(holder);
 	const start = performance.now();
 	const answered = await httpRequest(
 		OPENROUTER_URL,
@@ -107,7 +130,7 @@ export async function classifyIntent(text: string): Promise<Result<Intent, ApiEr
 			body: JSON.stringify({
 				model: MODEL,
 				messages: [
-					{role: 'system', content: INSTRUCTIONS},
+					{role: 'system', content: form.instructions},
 					{role: 'user', content: text},
 				],
 				temperature: 0,
@@ -116,8 +139,11 @@ export async function classifyIntent(text: string): Promise<Result<Intent, ApiEr
 				max_tokens: MAX_TOKENS,
 				// It thinks before answering; a little is enough here, and keeps it fast.
 				reasoning: {effort: 'low'},
-				response_format: {type: 'json_schema', json_schema: {name: 'intent', strict: true, schema: JSON_SCHEMA}},
-				provider: {order: ['Cerebras', 'Groq'], allow_fallbacks: true},
+				response_format: {type: 'json_schema', json_schema: {name: 'intent', strict: true, schema: form.schema}},
+				// Cerebras first, Groq next, and any other host of the model after them, but only a
+				// host OpenRouter records as neither keeping prompts nor training on them: the
+				// text is what the person typed, their bank's name or a merchant's among it.
+				provider: {order: ['Cerebras', 'Groq'], allow_fallbacks: true, data_collection: 'deny'},
 			}),
 		},
 		{schema: completionSchema, timeoutMs: TIMEOUT_MS},
@@ -136,7 +162,7 @@ export async function classifyIntent(text: string): Promise<Result<Intent, ApiEr
 	if (answer.value.widget === INCOMPLETE) {
 		return Ok({widget: INCOMPLETE, fields: [], ms});
 	}
-	const widget = WIDGETS.find((candidate) => candidate.id === answer.value.widget);
+	const widget = form.widgets.find((candidate) => candidate.id === answer.value.widget);
 	if (!widget) {
 		return Ok({widget: NO_WIDGET, fields: [], ms});
 	}

packages/api/src/intent/route.test.ts

@@ -6,7 +6,7 @@ const classifyIntent = mock();
 mock.module('./classify', () => ({classifyIntent}));
 
 function makeRequest(body: unknown) {
-	return {user: {userId: 'user-1'}, body} as never;
+	return {user: {userId: 'user-1', experiments: ['plaid']}, body} as never;
 }
 
 beforeEach(() => {
@@ -15,12 +15,12 @@ beforeEach(() => {
 });
 
 describe('handleClassifyIntentRoute', () => {
-	test('classifies the text, trimmed', async () => {
+	test("classifies the text, trimmed, among the widgets the person's organization is offered", async () => {
 		const {handleClassifyIntentRoute} = await import('./route');
 		const result = await handleClassifyIntentRoute(makeRequest({text: '  who is hugo  '}));
 
 		expect(result.isOk()).toBe(true);
-		expect(classifyIntent).toHaveBeenCalledWith('who is hugo');
+		expect(classifyIntent).toHaveBeenCalledWith('who is hugo', {userId: 'user-1', experiments: ['plaid']});
 	});
 
 	test('refuses empty text, and text past 500 characters, without asking the models', async () => {

packages/api/src/intent/route.ts

@@ -12,6 +12,6 @@ const intentBodySchema = z.object({text: z.string().trim().min(1).max(500)});
 export function handleClassifyIntentRoute(request: AuthenticatedRequest): Promise<Result<Intent, ApiError>> {
 	return Do(async ($) => {
 		const {text} = $(parseSchema(intentBodySchema, request.body));
-		return $(await classifyIntent(text));
+		return $(await classifyIntent(text, request.user));
 	});
 }