From 61a3f9b9564456f0d88d482f4ac795f12c727298 Mon Sep 17 00:00:00 2001 From: Henrik Schumacher Date: Thu, 6 Aug 2026 18:50:35 +0200 Subject: [PATCH 1/2] test(e2e): cover Graph preview persistence --- e2e/scenarios/microsoft-graph-full.test.ts | 42 +++++++++++++++++----- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/e2e/scenarios/microsoft-graph-full.test.ts b/e2e/scenarios/microsoft-graph-full.test.ts index bba32a2c2a..d2b86113cc 100644 --- a/e2e/scenarios/microsoft-graph-full.test.ts +++ b/e2e/scenarios/microsoft-graph-full.test.ts @@ -25,18 +25,25 @@ type ToolView = { const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; const MICROSOFT_FILES_PRESET_ID = "files"; +const MICROSOFT_FILES_SPEC_URL = `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`; const MICROSOFT_FILES_DELEGATED_SCOPES = [ "offline_access", "User.Read", "Files.ReadWrite.All", "Sites.ReadWrite.All", ] as const; +const MICROSOFT_FILES_AUTH_TEMPLATE = microsoftCatalog + .filter((preset) => preset.id === `microsoft-${MICROSOFT_FILES_PRESET_ID}`) + .flatMap((preset) => preset.authTemplate ?? []) + .flatMap((template) => + template.kind === "oauth2" ? [{ ...template, scopes: [...template.scopes] }] : [], + ); -// Adding a catalog service extracts only that service's Microsoft Graph subtree -// and persists a binding per operation. This is the regression guard for both -// former worker pressure sites: the add streams compile and persist, and -// tools/list serves from persisted bindings plus the content-addressed defs blob -// without re-parsing the Graph spec. +// The real add-integration flow previews the selected catalog service before it +// submits the add request. The preview parses the extracted Microsoft Graph +// spec, then the add must still stream-compile and persist one binding per +// operation. This guards that sequence as well as tools/list serving from the +// persisted bindings and content-addressed defs blob without re-parsing Graph. scenario( "Microsoft Graph: the files catalog service adds and serves without re-parsing the spec", { timeout: 300_000 }, @@ -51,18 +58,33 @@ scenario( yield* Effect.ensuring( Effect.gen(function* () { - // Add path, first former OOM site: the Graph spec is fetched and - // stream-compiled into one persisted binding per selected operation. + // Match AddOpenApiIntegration: analyze the URL first, then submit the + // preset's explicit auth template and empty base-URL override. Supplying + // both keeps addSpec on the streaming persistence path instead of having + // it derive defaults by previewing the spec again inside the add call. + const preview = yield* client.openapi.previewSpec({ + payload: { + spec: MICROSOFT_FILES_SPEC_URL, + specFormat: "microsoft-graph", + }, + }); + expect( + preview.operationCount, + "previewing the Microsoft files service parses its focused Graph subtree", + ).toBeGreaterThan(10); + const added = yield* client.openapi.addSpec({ payload: { spec: { kind: "url", - url: `${MICROSOFT_GRAPH_OPENAPI_URL}#preset=${MICROSOFT_FILES_PRESET_ID}`, + url: MICROSOFT_FILES_SPEC_URL, }, slug: integration, name: "Microsoft Graph Files", + baseUrl: "", family: "microsoft", specFormat: "microsoft-graph", + authenticationTemplate: MICROSOFT_FILES_AUTH_TEMPLATE, }, }); expect(added.slug, "the Microsoft files integration keeps the requested slug").toBe( @@ -72,6 +94,10 @@ scenario( added.toolCount, "adding the files catalog service extracts a focused Graph operation subtree", ).toBeGreaterThan(10); + expect( + preview.operationCount, + "preview and streaming persistence apply the same Microsoft workload filter", + ).toBe(added.toolCount); const config = yield* client.openapi.getConfig({ params: { slug: integration } }); const delegatedScopes = config?.authenticationTemplate?.flatMap((template) => From 707fd98f40842bda2f9892ca57062ee4a364c884 Mon Sep 17 00:00:00 2001 From: Henrik Schumacher Date: Fri, 7 Aug 2026 08:03:52 +0200 Subject: [PATCH 2/2] fix(openapi): preserve preview path filtering --- .../plugins/openapi/src/sdk/plugin.test.ts | 79 +++++++++++++++++++ packages/plugins/openapi/src/sdk/plugin.ts | 10 ++- packages/plugins/openapi/src/sdk/preview.ts | 71 ++++++++++++++++- 3 files changed, 154 insertions(+), 6 deletions(-) diff --git a/packages/plugins/openapi/src/sdk/plugin.test.ts b/packages/plugins/openapi/src/sdk/plugin.test.ts index 1617e8b555..9dd0057090 100644 --- a/packages/plugins/openapi/src/sdk/plugin.test.ts +++ b/packages/plugins/openapi/src/sdk/plugin.test.ts @@ -38,6 +38,7 @@ import { } from "@executor-js/sdk/testing"; import { openApiPlugin } from "./plugin"; +import type { SpecFormatAdapter } from "./spec-format"; import { type AuthenticationInput } from "./types"; import { addOpenApiTestConnection, @@ -113,6 +114,56 @@ const testApiSpecText = () => { const MICROSOFT_GRAPH_V1_OPERATION_COUNT = 16_548; +const FILTERED_PREVIEW_SPEC_TEXT = `openapi: 3.0.0 +info: + title: Filtered preview + version: 1.0.0 +servers: + - url: https://api.example.test +paths: + /kept: + get: + operationId: kept.get + tags: + - selected + responses: + "200": + description: OK + content: + application/json: + schema: + $ref: "#/components/schemas/KeptResponse" + /discarded: + get: + operationId: discarded.get + tags: + - unselected + responses: + "200": + description: OK +components: + schemas: + KeptResponse: + type: object + properties: + id: + type: string + UnusedResponse: + type: object + properties: + ignored: + type: string +`; + +const filteredPreviewAdapter: SpecFormatAdapter = { + id: "filtered-preview", + fetch: () => + Effect.succeed({ + specText: FILTERED_PREVIEW_SPEC_TEXT, + keepPathItem: (path, pathItem) => (path === "/kept" ? pathItem : null), + }), +}; + const microsoftGraphScaleSpecText = () => { const paths: Record = {}; for (let index = 0; index < MICROSOFT_GRAPH_V1_OPERATION_COUNT; index += 1) { @@ -387,6 +438,34 @@ describe("OpenAPI Plugin", () => { ), ); + it.effect("previewSpec preserves a format adapter's streaming path filter", () => + Effect.gen(function* () { + const executor = yield* createExecutor( + makeTestConfig({ + plugins: [ + openApiPlugin({ specFormats: [filteredPreviewAdapter] }), + memoryCredentialsPlugin(), + ] as const, + }), + ); + + const preview = yield* executor.openapi.previewSpec({ + spec: "https://spec.example.test/openapi.yaml", + specFormat: filteredPreviewAdapter.id, + }); + + expect(preview.operationCount).toBe(1); + expect(preview.operations.map((operation) => operation.path)).toEqual(["/kept"]); + expect(preview.tags).toEqual(["selected"]); + expect(preview.healthCheckCandidates).toEqual([ + expect.objectContaining({ + operation: "selected.keptGet", + responseFields: [{ path: "id", type: "string" }], + }), + ]); + }), + ); + it.effect("previewSpec discovers OAuth metadata from a URL-hosted bearer spec", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/plugins/openapi/src/sdk/plugin.ts b/packages/plugins/openapi/src/sdk/plugin.ts index 234b163f3c..74f4503a1b 100644 --- a/packages/plugins/openapi/src/sdk/plugin.ts +++ b/packages/plugins/openapi/src/sdk/plugin.ts @@ -36,6 +36,7 @@ import { OAuth2Flows, OAuth2Preset, SecurityScheme, + parsePreviewSpecText, previewSpecText, type SpecPreview, } from "./preview"; @@ -718,6 +719,7 @@ export const openApiPlugin = definePlugin< const enrichPreviewWithDiscoveredOAuth = (input: { readonly specText: string; readonly preview: SpecPreview; + readonly keepPathItem?: ConvertedSpec["keepPathItem"]; readonly specUrl?: string; readonly baseUrl?: string; }): Effect.Effect => @@ -734,7 +736,7 @@ export const openApiPlugin = definePlugin< ); if (!oauth.ok) continue; - const doc = yield* parse(input.specText); + const doc = yield* parsePreviewSpecText(input.specText, input.keepPathItem); const declaredScopes = collectDeclaredSecurityScopes( doc, nonOAuthSecuritySchemeNames(input.preview), @@ -800,11 +802,12 @@ export const openApiPlugin = definePlugin< const needsDerivedAuth = config.authenticationTemplate == null; const preview = needsDerivedBaseUrl || needsDerivedAuth - ? yield* previewSpecText(resolved.specText).pipe( + ? yield* previewSpecText(resolved.specText, resolved.keepPathItem).pipe( Effect.flatMap((rawPreview) => enrichPreviewWithDiscoveredOAuth({ specText: resolved.specText, preview: rawPreview, + keepPathItem: resolved.keepPathItem, specUrl: resolved.specUrl ?? specInputToSpecUrl(config.spec), baseUrl: explicitBaseUrl, }), @@ -1091,10 +1094,11 @@ export const openApiPlugin = definePlugin< }, httpClientLayer, ); - const preview = yield* previewSpecText(resolved.specText); + const preview = yield* previewSpecText(resolved.specText, resolved.keepPathItem); return yield* enrichPreviewWithDiscoveredOAuth({ specText: resolved.specText, preview, + keepPathItem: resolved.keepPathItem, specUrl: resolved.specUrl ?? (spec.kind === "url" ? spec.url : undefined), }); }), diff --git a/packages/plugins/openapi/src/sdk/preview.ts b/packages/plugins/openapi/src/sdk/preview.ts index 56fb3e7038..12af12aa04 100644 --- a/packages/plugins/openapi/src/sdk/preview.ts +++ b/packages/plugins/openapi/src/sdk/preview.ts @@ -9,6 +9,16 @@ import { import { parse, resolveSpecText, type ParsedDocument } from "./parse"; import { extract } from "./extract"; +import { OpenApiExtractionError } from "./errors"; +import { + collectReferencedSchemas, + indexSchemas, + parseEntry, + parseHead, + parseSmallComponents, + structuralSplit, + type KeepPathItem, +} from "./split"; import { compileToolDefinitions } from "./definitions"; import { normalizeOpenApiRefs } from "./backing"; import { DocResolver } from "./openapi-utils"; @@ -520,10 +530,65 @@ const buildPreviewHealthCheckCandidates = ( // Public API // --------------------------------------------------------------------------- +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === "object" && !Array.isArray(value); + +/** + * Parse the document shape needed by preview. Format adapters that provide a + * path filter also opt into the structural path: each path-item is parsed in + * isolation, filtered immediately, and only schemas reachable from the kept + * workload are materialized. This keeps preview on the same bounded-memory + * path as streaming persistence instead of parsing the adapter's full source. + */ +export const parsePreviewSpecText = Effect.fn("OpenApi.parsePreviewSpecText")(function* ( + specText: string, + keepPathItem?: KeepPathItem, +) { + if (!keepPathItem) return yield* parse(specText); + + const structure = structuralSplit(specText); + if (!structure) { + return yield* new OpenApiExtractionError({ + message: + "OpenAPI spec is not in the streamable block-YAML profile (no top-level `paths:` block); cannot stream-preview this adapted spec.", + }); + } + + const paths: Record> = {}; + for (const range of structure.pathItems) { + const entry = parseEntry(structure.text, range, 2); + if (!entry) continue; + const [path, rawPathItem] = entry; + if (!isRecord(rawPathItem)) continue; + const kept = keepPathItem(path, rawPathItem); + if (kept) paths[path] = kept; + } + + const smallComponents = parseSmallComponents(structure); + const schemas = collectReferencedSchemas(structure, indexSchemas(structure), [ + ...Object.values(paths), + smallComponents, + ]); + + // oxlint-disable-next-line executor/no-double-cast -- boundary: the structural parser builds the OpenAPI document subset preview consumes; parseHead/parseSmallComponents deliberately return generic records. + return { + ...parseHead(structure), + paths, + components: { + ...smallComponents, + schemas, + }, + } as unknown as ParsedDocument; +}); + /** Preview already-resolved spec text — extract metadata without registering - * anything and without any HTTP dependency. */ -export const previewSpecText = Effect.fn("OpenApi.previewSpecText")(function* (specText: string) { - const doc: ParsedDocument = yield* parse(specText); + * anything and without any HTTP dependency. When a format adapter supplied a + * path filter, preview structurally reduces the source before extraction. */ +export const previewSpecText = Effect.fn("OpenApi.previewSpecText")(function* ( + specText: string, + keepPathItem?: KeepPathItem, +) { + const doc = yield* parsePreviewSpecText(specText, keepPathItem); const result = yield* extract(doc); const resolver = new DocResolver(doc);