Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/coding-agent/src/core/evolution-runtime.ts
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,7 @@ class CodingAgentSnapshotSource implements EvolutionSnapshotSource {
},
pipelineVersion: VERSION,
promptVersions: {
reflector: "reflector-v1",
reflector: "reflector-v2",
curator: "curator-v1",
validator: "validator-v1",
writer: "writer-v1",
Expand Down
1 change: 1 addition & 0 deletions packages/evolution/src/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,7 @@ export interface ReflectionNode {
export type LessonKind =
| "counterexample"
| "direct-instruction"
| "failed-approach"
| "failure-workaround"
| "harmful-memory"
| "incorrect-source-of-truth"
Expand Down
14 changes: 14 additions & 0 deletions packages/evolution/src/reflector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ const DEFAULT_STAGE_DEADLINE_MS = 1_800_000;
const LESSON_KINDS = new Set<LessonKind>([
"counterexample",
"direct-instruction",
"failed-approach",
"failure-workaround",
"harmful-memory",
"incorrect-source-of-truth",
Expand Down Expand Up @@ -1329,6 +1330,19 @@ export class RecursiveReflector implements Reflector {
: this.#maxLessonsPerSynthesisNode,
lessonSelection:
"Return only the highest-impact, non-overlapping lessons that would change future behavior.",
lessonPolicy: {
targetNeutral: true,
distinctFindingTypes: [
{ findingType: "successful-strategy", lessonKinds: ["successful-strategy"] },
{ findingType: "failed-approach", lessonKinds: ["failed-approach", "failure-workaround"] },
{ findingType: "verification-gap", lessonKinds: ["missing-verification"] },
{ findingType: "tool-behavior", lessonKinds: ["tool-behavior"] },
{ findingType: "counterexample", lessonKinds: ["counterexample"] },
],
separationRule:
"Keep supported finding types as separate evidence-grounded lessons; do not merge a success, failure, verification gap, tool behavior, or counterexample merely because they occurred in the same task.",
excludedDecisions: ["downstream-target", "filesystem-destination", "final-skill-frontmatter"],
},
behaviorChangingLessonsOnly: true,
requireUserTaskOrMeaningfulFailure: true,
requireCausalAgentBehavior: true,
Expand Down
62 changes: 62 additions & 0 deletions packages/evolution/test/reflector.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,68 @@ describe("recursive reflector", () => {
for (const schema of schemas) expect(findUnsupportedCodexSchemaKeywords(schema)).toEqual([]);
});

it("preserves failed approaches as evidence-grounded lessons", async () => {
const { reflector } = await reflectorFixture();
const input = reflectInput({
async execute(request) {
if (request.stage !== "reflector") throw new Error("Unexpected Curator request");
const lesson = { ...lessonForRequest(request.input), kind: "failed-approach" };
const lessonSchema = jsonObject(
jsonObject(jsonObject(request.outputSchema.properties, "schema properties").lessons, "lessons").items,
"lesson schema",
);
expect(jsonObject(lessonSchema.properties, "lesson properties").kind).toMatchObject({
enum: expect.arrayContaining(["failed-approach", "missing-verification", "successful-strategy"]),
});
return {
output: { complete: true, lessons: [lesson] },
model: { provider: "faux", model: "fixture" },
};
},
});

const result = await reflector.reflect(input);

expect(result.lessons).not.toHaveLength(0);
expect(result.lessons.every((lesson) => lesson.kind === "failed-approach")).toBe(true);
expect(result.lessons[0]).toMatchObject({
preconditions: ["Evidence is materialized"],
observedOutcome: "The result remains traceable",
evidenceIds: expect.any(Array),
counterevidenceIds: [],
applicabilityBoundaries: ["Repository-scoped tasks"],
uncertainty: null,
actionableTakeaway: "Read only through the bound evidence registry",
});
});

it("keeps reflection target-neutral and separates distinct findings", async () => {
const { reflector } = await reflectorFixture();
const input = reflectInput({
async execute(request) {
if (request.stage !== "reflector") throw new Error("Unexpected Curator request");
expect(jsonObject(request.input.control, "control").lessonPolicy).toEqual({
targetNeutral: true,
distinctFindingTypes: [
{ findingType: "successful-strategy", lessonKinds: ["successful-strategy"] },
{ findingType: "failed-approach", lessonKinds: ["failed-approach", "failure-workaround"] },
{ findingType: "verification-gap", lessonKinds: ["missing-verification"] },
{ findingType: "tool-behavior", lessonKinds: ["tool-behavior"] },
{ findingType: "counterexample", lessonKinds: ["counterexample"] },
],
separationRule: expect.stringContaining("separate evidence-grounded lessons"),
excludedDecisions: ["downstream-target", "filesystem-destination", "final-skill-frontmatter"],
});
return {
output: { complete: true, lessons: [lessonForRequest(request.input)] },
model: { provider: "faux", model: "fixture" },
};
},
});

await expect(reflector.reflect(input)).resolves.toMatchObject({ finalLessonIds: [expect.any(String)] });
});

it("limits each leaf to three high-impact lessons", async () => {
const { persistence } = await reflectorFixture();
const reflector = new RecursiveReflector(
Expand Down
Loading