Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
176 changes: 176 additions & 0 deletions docs/plans/execute-result-recovery.md

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions packages/codemode/src/codemode.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ export type ExecutionLimits = {
readonly timeoutMs?: number
/** Maximum number of tool calls admitted by the runtime. No default: absent means unlimited. */
readonly maxToolCalls?: number
/** Total explicit loop-body entries across this execution. Absent means unlimited. */
readonly maxLoopIterations?: number
/** Maximum UTF-8 bytes of model-facing output. No default: absent means no truncation. */
readonly maxOutputBytes?: number
/** 日志按 UTF-8 字节限制留存;业务返回值仍受独立的 maxOutputBytes 控制。 */
Expand All @@ -39,6 +41,7 @@ type ToolTree<R = never> = {
export type ResolvedExecutionLimits = {
readonly timeoutMs: number | undefined
readonly maxToolCalls: number | undefined
readonly maxLoopIterations: number | undefined
readonly maxOutputBytes: number | undefined
readonly maxLogBytes: number | undefined
}
Expand Down Expand Up @@ -89,6 +92,7 @@ export const DiagnosticKind = Schema.Literals([
"InvalidToolOutput",
"InvalidDataValue",
"ToolCallLimitExceeded",
"LoopIterationLimitExceeded",
"TimeoutExceeded",
"ToolFailure",
"ExecutionFailure",
Expand Down Expand Up @@ -156,6 +160,7 @@ const resolveExecutionLimits = (limits?: ExecutionLimits): ResolvedExecutionLimi
// Effect 的计时器只支持有符号32位毫秒;超界不能悄悄变成无期限执行。
timeoutMs: validateLimit("timeoutMs", limits?.timeoutMs, 1, 2_147_483_647),
maxToolCalls: validateLimit("maxToolCalls", limits?.maxToolCalls, 0),
maxLoopIterations: validateLimit("maxLoopIterations", limits?.maxLoopIterations, 0),
maxOutputBytes: validateLimit("maxOutputBytes", limits?.maxOutputBytes, 0),
// 零预算表示只记录省略数;不能按真值把它当成无限额。
maxLogBytes: validateLimit("maxLogBytes", limits?.maxLogBytes, 0),
Expand Down
3 changes: 2 additions & 1 deletion packages/codemode/src/interpreter/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ export type DiagnosticKind =
| "InvalidToolOutput"
| "InvalidDataValue"
| "ToolCallLimitExceeded"
| "LoopIterationLimitExceeded"
| "TimeoutExceeded"
| "ToolFailure"
| "ExecutionFailure"
Expand All @@ -139,7 +140,7 @@ export const OptionalShortCircuit: unique symbol = Symbol("codemode.optional-sho

// 诊断与实际调用面一致,不能再把已支持的链式方法说成禁用。
export const supportedSyntaxMessage =
"Supported orchestration syntax: tools.* calls (they return promises - resolve them with await), data literals, destructuring, optional chaining, template literals, conditionals, switch, loops (incl. for...of and for...in over object/array/tools keys), arrow functions, spread, try/catch, array methods (map/filter/find/findIndex/some/every/reduce/flatMap/forEach/sort/slice/concat/indexOf/lastIndexOf/at/flat/reverse/includes/join), string methods (incl. match/matchAll/replace/split with regular expressions), Date/RegExp/Map/Set/URL/URLSearchParams, URI encoding helpers, Object/Math/JSON helpers, captured console.log/warn/error/dir/table, and Promise.all/allSettled/race/resolve/reject over arrays mixing promises and plain values for parallel tool calls (Promise .then/.catch/.finally accept collection callbacks and adopt returned promises)."
"Supported orchestration syntax: tools.* calls (they return promises - resolve them with await), data literals, destructuring, optional chaining, template literals, conditionals, switch, loops (incl. for...of and for...in over object/array/tools keys), arrow functions, supported builtins as callbacks, spread, try/catch, Array construction and Array.from mapping, array methods (map/filter/find/findIndex/some/every/reduce/flatMap/forEach/sort/slice/concat/indexOf/lastIndexOf/at/flat/reverse/includes/join), string methods (incl. match/matchAll/replace/split with regular expressions), Date/RegExp/Map/Set/URL/URLSearchParams, URI encoding helpers, Object/Math/JSON helpers, captured console.log/warn/error/dir/table, and Promise.all/allSettled/race/resolve/reject over arrays mixing promises and plain values for parallel tool calls (Promise .then/.catch/.finally accept collection callbacks and adopt returned promises)."

export class InterpreterRuntimeError extends Error {
readonly node?: AstNode
Expand Down
181 changes: 118 additions & 63 deletions packages/codemode/src/interpreter/runtime.ts

Large diffs are not rendered by default.

9 changes: 5 additions & 4 deletions packages/codemode/src/stdlib/math.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,9 @@ export const mathMethods = new Set([

export const invokeMathMethod = (name: string, args: Array<unknown>, node: AstNode): number => {
if (!mathMethods.has(name)) throw new InterpreterRuntimeError(`Math.${name} is not available in CodeMode.`, node)
const nums = args.map((arg) => {
if (typeof arg !== "number") throw new InterpreterRuntimeError(`Math.${name} expects number arguments.`, node)
return arg
})
// Fixed-arity builtins ignore callback index/array arguments, as native JS does.
const input = name === "max" || name === "min" || name === "hypot" ? args : args.slice(0, name === "pow" ? 2 : 1)
const nums = input.map(coerceToNumber)
const [a = Number.NaN, b = Number.NaN] = nums
switch (name) {
case "max":
Expand Down Expand Up @@ -63,3 +62,5 @@ export const invokeMathMethod = (name: string, args: Array<unknown>, node: AstNo
throw new InterpreterRuntimeError(`Math.${name} is not available in CodeMode.`, node)
}
import { type AstNode, InterpreterRuntimeError } from "../interpreter/model.js"

import { coerceToNumber } from "./value.js"
6 changes: 3 additions & 3 deletions packages/codemode/src/tool.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ export type SchemaType = Schema.Decoder<unknown> | JsonSchema
export type Definition<R = never> = {
readonly _tag: "CodeModeTool"
readonly description: string
// 一个实际合同同时服务展示和形式推导;input 仍独立决定此层是否解码。
// 模型可见的输入合同;宿主的运行时合同仅用于推导快捷形式。
readonly inputContract: SchemaType
readonly inputForms?: InputForms
readonly input: SchemaType
Expand All @@ -54,7 +54,7 @@ type ResultType<S> = S extends Schema.Decoder<unknown> ? S["Encoded"] : unknown
/** Options for defining one CodeMode tool. */
export type Options<I extends SchemaType, O extends SchemaType | undefined, R = never> = {
readonly description: string
// 适配器传完整原始合同而非派生形状;其实际解码仍由原宿主执行。
// 适配器提供原宿主合同以推导快捷形式,不覆盖 input 的可见字段;解码仍由宿主执行。
readonly inputContract?: Schema.Decoder<unknown>
readonly input: I
readonly output?: O
Expand Down Expand Up @@ -100,7 +100,7 @@ export const make = <I extends SchemaType, const O extends SchemaType | undefine
_tag: "CodeModeTool",
description: options.description,
// 形式推导只读取元数据;执行一次 decoder 来猜输入会重复 default/transform 的效果。
inputContract: contract,
inputContract: options.input,
inputForms: Schema.isSchema(contract) ? inputForms(contract) : undefined,
input: options.input,
output: options.output,
Expand Down
85 changes: 85 additions & 0 deletions packages/codemode/test/codemode.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1654,6 +1654,7 @@ describe("CodeMode public contract", () => {
RangeError,
)
expect(() => CodeMode.execute({ code: "return 1", limits: { maxToolCalls: -1 } })).toThrow(RangeError)
expect(() => CodeMode.execute({ code: "return 1", limits: { maxLoopIterations: -1 } })).toThrow(RangeError)
expect(() => CodeMode.execute({ code: "return 1", limits: { maxOutputBytes: -1 } })).toThrow(RangeError)
// 日志留存预算独立于结果预算;无日志脚本也应在构造执行时拒绝无效配置。
expect(() => CodeMode.execute({ code: "return 1", limits: { maxLogBytes: -1 } })).toThrow(RangeError)
Expand Down Expand Up @@ -1687,6 +1688,41 @@ describe("CodeMode public contract", () => {
if (!result.ok) expect(result.error.kind).toBe("ToolCallLimitExceeded")
})

test("loop budgets count body entries across loop forms and reset per execution", async () => {
for (const code of [
"for (let i = 0; i < 3; i++) console.log(i)",
"let i = 0; while (i < 3) console.log(i++)",
"let i = 0; do { console.log(i++) } while (i < 3)",
"for (const i of [0, 1, 2]) console.log(i)",
"for await (const i of [0, 1, 2]) console.log(i)",
"for (const i in {a: 0, b: 1, c: 2}) console.log(i)",
]) {
const runtime = CodeMode.make({ limits: { maxLoopIterations: 3 } })
for (let run = 0; run < 2; run++) expect(await Effect.runPromise(runtime.execute(code))).toMatchObject({ ok: true })
const result = await Effect.runPromise(CodeMode.execute({ code, limits: { maxLoopIterations: 2 } }))
expect(result).toMatchObject({ ok: false, error: { kind: "LoopIterationLimitExceeded" } })
expect(result.logs).toHaveLength(2)
}
for (const code of [
"for (let i=0; i<2; i++) { for (let j=0; j<2; j++) continue }",
"async function f() { for (let i=0; i<2; i++) { await sleep(0); } } await Promise.all([f(), f()])",
]) expect(await Effect.runPromise(CodeMode.execute({ code, limits: { maxLoopIterations: 3 } })))
.toMatchObject({ ok: false, error: { kind: "LoopIterationLimitExceeded" } })
expect(await Effect.runPromise(CodeMode.execute({ code: "while(false) {}", limits: { maxLoopIterations: 0 } })))
.toMatchObject({ ok: true })
})

test("parallel tool admission cannot exceed the shared tool-call budget", async () => {
let called = 0
const count = Tool.make({ description: "count", input: Schema.Struct({}), output: Schema.Number,
run: () => Effect.sync(() => ++called) })
const result = await Effect.runPromise(CodeMode.execute({ tools: { count }, limits: { maxToolCalls: 2 },
code: "await Promise.all([tools.count({}), tools.count({}), tools.count({})])" }))
expect(result).toMatchObject({ ok: false, error: { kind: "ToolCallLimitExceeded" } })
expect(result.toolCalls).toHaveLength(2)
expect(called).toBe(2)
})

test("timeoutMs and maxToolCalls have no defaults: absent means unlimited", async () => {
// 150 tool calls would have exceeded the old default cap of 100; with no limits
// provided, there is no cap and no timeout - budgets are host policy.
Expand Down Expand Up @@ -2910,3 +2946,52 @@ describe("CodeMode retained environments", () => {
)
})
})

describe("builtin callbacks and array construction", () => {
const execute = (code: string) => Effect.runPromise(CodeMode.make({ tools: {} }).execute(code))

test("uses sandbox builtins as collection, sort, promise and replacement callbacks", async () => {
expect(await execute(`return {
floor: [1.8, 2.2].map(Math.floor), json: ['{"n":1}'].map(JSON.parse),
keys: [{a: 1}, {b: 2}].map(Object.keys), finite: [1, Infinity].filter(Number.isFinite),
sort: [3, -1, 2].sort(Array.of), replace: "a1b2".replace(/\\d/g, Math.floor),
promised: await Promise.resolve(2.8).then(Math.floor)
}`)).toMatchObject({ ok: true, value: {
floor: [1, 2], json: [{ n: 1 }], keys: [["a"], ["b"]], finite: [1],
sort: [3, -1, 2], replace: "a1b2", promised: 2,
} })
})

test("constructs numeric lengths and element lists, preserving sparse callback behavior", async () => {
expect(await execute(`const sparse = Array(3); sparse[2] = 4;
return { empty: Array(), filled: new Array(3).fill(2), one: Array("3"), many: Array(1, 2),
holes: Object.keys(Array(3).map(() => 1)), sortedHoles: Object.keys(Array(3).sort(Array.of)), denseSorted: Object.keys(Array(3).toSorted()), filtered: sparse.filter(Boolean),
reduced: sparse.reduce((a, b) => a + b), right: sparse.reduceRight((a, b) => a + b),
every: Array(3).every(() => false), some: Array(3).some(() => true) }`)).toMatchObject({
ok: true, value: { empty: [], filled: [2, 2, 2], one: ["3"], many: [1, 2],
holes: [], sortedHoles: [], denseSorted: ["0", "1", "2"], filtered: [4], reduced: 4, right: 4, every: true, some: false },
})
expect(await execute('try { Array(-1) } catch (e) { return e.name }')).toMatchObject({ ok: true, value: "RangeError" })
})

test("sparse callback scans honor the existing execution timeout", async () => {
for (const method of ["map", "flatMap", "filter", "some", "every", "forEach", "reduce", "reduceRight"]) {
for (const initial of method.startsWith("reduce") ? ["", ", 0"] : [""]) {
expect(await Effect.runPromise(CodeMode.execute({
code: `return Array(1_000_000).${method}(() => 0${initial})`, limits: { timeoutMs: 10 },
}))).toMatchObject({ ok: false, error: { kind: "TimeoutExceeded" } })
}
}
})

test("Array.from maps values and indices including array-like holes and undefined mapper", async () => {
expect(await execute(`return {
indices: Array.from({length: 3}, (_, i) => i), floor: Array.from([1.8, 2.2], Math.floor),
text: Array.from("ab", (v, i) => v + i), set: Array.from(new Set([2, 3]), v => v * 2),
plain: Array.from([1, 2], undefined), args: Array.from([5], (...args) => args.length)
}`)).toMatchObject({ ok: true, value: {
indices: [0, 1, 2], floor: [1, 2], text: ["a0", "b1"], set: [4, 6], plain: [1, 2], args: [2],
} })
expect(await execute('return Array.from([], 1)')).toMatchObject({ ok: false })
})
})
5 changes: 2 additions & 3 deletions packages/codemode/test/promise.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1208,19 +1208,18 @@ describe("async standard-library callbacks and iteration", () => {
})

// 内建令牌沿用函数的不透明边界,别名和集合回调只能返回受监管的 Promise。
test("sleep aliases and callbacks retain callable identity without exposing global methods", async () => {
test("sleep aliases and callbacks retain callable identity without exposing host properties", async () => {
expect(
await value(`const pause = sleep; const box = { pause };
await box.pause(0.25); await Promise.all([0, 0].map(pause));
new Set([0]).forEach(pause); new Map([[0, 0]]).forEach(pause);
async function nested() { await pause(0); return 3; }
return [typeof pause, await nested(), await pause(0)];`),
).toEqual(["function", 3, null])
// sleep可当回调传递但不暴露宿主函数属性;Math.abs的反例保留原有全局方法不可提取的限制。
// 内建可当回调传递,但不能借此读取宿主函数属性。
for (const code of [
"return sleep.name",
"return sleep.constructor",
"return [1].map(Math.abs)",
"return sleep.call(null, 0)",
]) {
expect((await run(code)).ok).toBe(false)
Expand Down
33 changes: 17 additions & 16 deletions packages/codemode/test/signature.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,9 +320,7 @@ test("search emits the actual encoded input, decoded output and shared definitio
tools: {
call: Tool.make({
description: "Actual contract",
input: { type: "string" },
// 展示输入故意写成string而真实合同是对象;详情必须由inputContract发射,不能沿展示schema取巧。
inputContract: input,
input,
output: Schema.Struct({ amount: Schema.NumberFromString, person }),
run: () => Effect.succeed({ amount: "1", person: { name: "Ada" } }),
}),
Expand Down Expand Up @@ -455,22 +453,19 @@ describe("non-identifier tool paths", () => {
})
})

// 适配器展示 schema 不得覆盖实际标量合同;签名和执行都保留原始类型。
// 标量适配器展示编码类型,执行仍由宿主验证。
test("scalar adapter signature uses its authoritative encoded scalar", async () => {
const runtime = CodeMode.make({
tools: {
call: Tool.make({
description: "Scalar adapter",
input: {
type: "object",
properties: { misleading: { type: "boolean" } },
},
input: { type: "string" },
inputContract: Schema.String,
run: (value) => Effect.succeed(Schema.decodeUnknownSync(Schema.String)(value)),
}),
},
})
// 显示对象与实际字符串合同相冲突,签名使用string且调用拒绝包装对象,两个出口共同约束合同来源。
// 签名使用 string,宿主仍拒绝包装对象。
expect(runtime.catalog()[0].signature).toBe("tools.call(input: string): Promise<unknown>")
expect(await Effect.runPromise(runtime.execute('return await tools.call("yes")'))).toMatchObject({
ok: true,
Expand All @@ -481,7 +476,7 @@ test("scalar adapter signature uses its authoritative encoded scalar", async ()
})
})

// 签名、发现和实际校验必须读取同一对象合同,不能被适配器展示字段替代。
// 宿主合同保留快捷形式和校验,可见 schema 独立控制签名、搜索和错误建议。
for (const [actual, expected, input, value] of [
[
Schema.Struct({
Expand All @@ -499,14 +494,14 @@ for (const [actual, expected, input, value] of [
{ realKey: "yes", count: 2 },
],
] as const) {
test(`adapter contract remains authoritative for ${expected}`, async () => {
test(`adapter presentation preserves runtime decoding for ${expected}`, async () => {
let decodes = 0
const runtime = CodeMode.make({
tools: {
adapter: {
call: Tool.make({
description: "Adapter",
input: { type: "object", properties: { misleading: { type: "boolean" } } },
input: { type: "object", properties: { visible: { type: "boolean", description: "visibleneedle" } } },
inputContract: actual,
run: (input) =>
Effect.sync(() => {
Expand All @@ -517,18 +512,24 @@ for (const [actual, expected, input, value] of [
},
},
})
const signature = `tools.adapter.call(input: ${expected}): Promise<unknown>`
const signature = "tools.adapter.call(input: { visible?: boolean }): Promise<unknown>"
expect(runtime.catalog()[0].signature).toBe(signature)
expect(runtime.instructions()).toContain(signature)
// 字段索引同样取真实来源,不能签名已修而 search 仍只索引虚假展示字段。
// 所有发现出口遵守公开 schema,宿主内部字段不能重新进入模型上下文。
expect(
await Effect.runPromise(runtime.execute('return search({query:"contractneedle"}).items[0].signature')),
await Effect.runPromise(runtime.execute('return search({query:"visibleneedle"}).items[0].signature')),
).toMatchObject({ ok: true, value: signature })
expect(await Effect.runPromise(runtime.execute("return tools.adapter()"))).toMatchObject({
ok: false,
error: { suggestions: [signature] },
})
// 目录、搜索和错误建议都已生成,计数仍为0才能说明这些发现出口没有执行宿主decoder。
expect((await search(runtime)).items[0].inputSchema).toEqual({
type: "object", properties: { visible: { type: "boolean", description: "visibleneedle" } },
})
expect(await Effect.runPromise(runtime.execute('return search({query:"contractneedle"}).items'))).toMatchObject({
ok: true, value: [],
})
expect(decodes).toBe(0)
// 多必填字段没有快捷形式也必须保留完整合同,真实解码只发生在调用本身。
expect(await Effect.runPromise(runtime.execute(`return await tools.adapter.call(${input})`))).toMatchObject({
Expand All @@ -539,7 +540,7 @@ for (const [actual, expected, input, value] of [
expect(
await Effect.runPromise(runtime.execute("return await tools.adapter.call({misleading:true})")),
).toMatchObject({ ok: false })
// 第二次传的是展示schema里的虚假字段;宿主decoder仍需被调用并拒绝,不能因目录展示成功而放行。
// 公开 schema 不替代宿主校验,无效输入仍由原 decoder 拒绝。
expect(decodes).toBe(2)
})
}
Expand Down
Loading
Loading