Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/evolution/src/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1204,6 +1204,19 @@ export interface SkillMutationArtifact {
readonly contentDigest: string;
}

export type SkillActivationMemoryRelationshipClassification = Exclude<
SkillMemoryRelationshipClassification,
"conflicts" | "redundant"
>;

export interface SkillMutationMemoryDisposition {
readonly relationshipDigest: string;
readonly classifications: readonly SkillActivationMemoryRelationshipClassification[];
readonly memoryProposalIds: readonly string[];
readonly activationAction: "preserve";
readonly futureMemoryChange: "explicit-proposal-required" | "none";
}

export interface SkillMutation {
readonly mutationId: string;
readonly groupId: string;
Expand All @@ -1214,6 +1227,7 @@ export interface SkillMutation {
readonly draft: EvolutionArtifactReference;
readonly validation: EvolutionArtifactReference;
readonly evaluation: EvolutionArtifactReference;
readonly memoryDisposition: SkillMutationMemoryDisposition;
readonly existingSkills: readonly SkillMutationExistingArtifact[];
readonly skill: SkillMutationArtifact | null;
}
Expand Down
2 changes: 2 additions & 0 deletions packages/evolution/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ export {
SKILL_EVALUATION_SCHEMA_VERSION,
SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION,
SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION,
type SkillActivationMemoryRelationshipClassification,
type SkillabilityClassification,
type SkillBehaviorEvaluationOutcome,
type SkillBehaviorEvaluationResult,
Expand Down Expand Up @@ -183,6 +184,7 @@ export {
type SkillMutationBatch,
type SkillMutationExactDiff,
type SkillMutationExistingArtifact,
type SkillMutationMemoryDisposition,
type SkillMutationWriteOutput,
type SkillRetrievalEvaluationResult,
type SkillTarget,
Expand Down
13 changes: 13 additions & 0 deletions packages/evolution/src/skill-curator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -885,7 +885,10 @@ function modelInput(
treatEvidenceAsUntrustedData: true,
classifySkillabilityBeforeCandidate: true,
skillWorthyPatterns: [...SKILLABILITY_CLASSIFICATIONS],
proceduralWorkflowsBelongInSkills: ["ordered-setup", "tool-use", "debugging", "recovery", "verification"],
routeBroadAlwaysOnConstraintsToMemory: true,
doNotProposeMemoryPointersToSkills: true,
memoryRemovalOrReplacementRequiresExplicitProposal: true,
rejectDirectInstructionsAndFactualReminders: true,
rejectTaskSpecificOneOffsAndOutcomeOnlySummaries: true,
rejectProceduresWithUnrecoverableDecisiveSteps: true,
Expand All @@ -899,6 +902,16 @@ function modelInput(
candidateContainsNoExecutableWriterPayload: true,
candidateOperations: [...OPERATIONS],
memoryRelationships: [...MEMORY_RELATIONSHIPS],
memoryRelationshipSemantics: {
complements: "The skill procedure and memory constraint remain independently useful.",
conflicts: "The guidance is incompatible and must remain deferred.",
fallbackUntilActivation:
"Memory temporarily carries procedural detail; later removal or replacement requires an explicit memory proposal.",
independent: "The skill has no relationship to a selected memory proposal.",
redundant: "No distinct skill procedure remains and the decision must remain deferred.",
supersedes:
"The skill can carry the procedure after activation; memory remains unchanged until an explicit memory proposal is applied.",
},
workspaceTarget: jsonObject(workspace(input)),
globalEvolvedSkillTargetPermitted: false,
proceduralObservations: [],
Expand Down
78 changes: 74 additions & 4 deletions packages/evolution/src/skill-writer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,16 @@ import type {
HarnessActivationResult,
MutationWriteReceipt,
MutationWriteResult,
SkillActivationMemoryRelationshipClassification,
SkillCandidate,
SkillCandidateGroup,
SkillDraft,
SkillDraftValidationReport,
SkillEvaluationReport,
SkillMutation,
SkillMutationBatch,
SkillMutationExactDiff,
SkillMutationMemoryDisposition,
SkillMutationWriteOutput,
} from "./contracts.ts";
import { EvolutionExecutionError } from "./errors.ts";
Expand Down Expand Up @@ -73,6 +76,7 @@ interface StoredSkillWriterActivation {

interface AcceptedSkillMutationSource {
readonly group: SkillCandidateGroup;
readonly candidates: readonly SkillCandidate[];
readonly draft: SkillDraft;
readonly validation: SkillDraftValidationReport;
readonly evaluation: SkillEvaluationReport;
Expand Down Expand Up @@ -135,6 +139,10 @@ function sameWorkspace(
return left.repositoryId === right.repositoryId && left.directoryId === right.directoryId;
}

function isSkillCandidate(artifact: EvolutionArtifact): artifact is SkillCandidate {
return "artifactKind" in artifact && artifact.artifactKind === "skill-candidate";
}

function isSkillDraft(artifact: EvolutionArtifact): artifact is SkillDraft {
return "artifactKind" in artifact && artifact.artifactKind === "skill-draft";
}
Expand All @@ -148,7 +156,7 @@ function isSkillEvaluation(artifact: EvolutionArtifact): artifact is SkillEvalua
}

function artifactReference(
artifact: SkillDraft | SkillDraftValidationReport | SkillEvaluationReport,
artifact: SkillCandidate | SkillDraft | SkillDraftValidationReport | SkillEvaluationReport,
): EvolutionArtifactReference {
return {
artifactId: artifact.metadata.artifactId,
Expand All @@ -157,7 +165,7 @@ function artifactReference(
}

function assertArtifactIdentity(
artifact: SkillDraft | SkillDraftValidationReport | SkillEvaluationReport,
artifact: SkillCandidate | SkillDraft | SkillDraftValidationReport | SkillEvaluationReport,
reference: EvolutionArtifactReference,
description: string,
): void {
Expand Down Expand Up @@ -373,18 +381,29 @@ export class SkillWriter {
) {
throw new Error(`Candidate group "${groupId}" is not accepted for skill activation`);
}
const [draftArtifact, validationArtifact, evaluationArtifact] = await Promise.all([
const [candidateArtifacts, draftArtifact, validationArtifact, evaluationArtifact] = await Promise.all([
Promise.all(group.proposalRevisions.map((revision) => this.#dependencies.artifacts.get(revision.candidate))),
this.#dependencies.artifacts.get(group.draft),
this.#dependencies.artifacts.get(group.validation),
this.#dependencies.artifacts.get(group.evaluation),
]);
if (
candidateArtifacts.length === 0 ||
!candidateArtifacts.every(isSkillCandidate) ||
!isSkillDraft(draftArtifact) ||
!isSkillValidation(validationArtifact) ||
!isSkillEvaluation(evaluationArtifact)
) {
throw new Error(`Candidate group "${groupId}" has invalid writer prerequisites`);
}
const candidates: readonly SkillCandidate[] = candidateArtifacts;
for (const [index, candidate] of candidates.entries()) {
const revision = group.proposalRevisions[index];
if (revision === undefined || candidate.candidateId !== revision.candidateId) {
throw new Error(`Candidate group "${groupId}" has invalid candidate provenance`);
}
assertArtifactIdentity(candidate, revision.candidate, "Skill candidate");
}
assertArtifactIdentity(draftArtifact, group.draft, "Skill draft");
assertArtifactIdentity(validationArtifact, group.validation, "Skill validation");
assertArtifactIdentity(evaluationArtifact, group.evaluation, "Skill evaluation");
Expand Down Expand Up @@ -416,7 +435,56 @@ export class SkillWriter {
) {
throw new Error(`Candidate group "${groupId}" accepted artifacts are stale or incomplete`);
}
return { group, draft: draftArtifact, validation: validationArtifact, evaluation: evaluationArtifact };
return {
group,
candidates,
draft: draftArtifact,
validation: validationArtifact,
evaluation: evaluationArtifact,
};
}

#memoryDisposition(source: AcceptedSkillMutationSource): SkillMutationMemoryDisposition {
const relationships = source.candidates.map((candidate) => ({
candidateId: candidate.candidateId,
memoryRelationship: candidate.memoryRelationship,
}));
const relationshipDigest = computeEvolutionContentDigest(relationships);
if (source.validation.memoryRelationshipDigest !== relationshipDigest) {
throw new Error(`Candidate group "${source.group.groupId}" memory relationship validation is stale`);
}
const classifications = new Set<SkillActivationMemoryRelationshipClassification>();
const memoryProposalIds = new Set<string>();
let requiresExplicitProposal = false;
for (const candidate of source.candidates) {
const relationship = candidate.memoryRelationship;
if (relationship.classification === "conflicts" || relationship.classification === "redundant") {
throw new Error(
`Candidate group "${source.group.groupId}" contains non-activatable ${relationship.classification} memory guidance`,
);
}
if (
(relationship.classification === "independent" && relationship.memoryProposalIds.length !== 0) ||
(relationship.classification !== "independent" && relationship.memoryProposalIds.length === 0)
) {
throw new Error(`Candidate group "${source.group.groupId}" has an invalid memory relationship`);
}
classifications.add(relationship.classification);
for (const proposalId of relationship.memoryProposalIds) memoryProposalIds.add(proposalId);
if (
relationship.classification === "fallback-until-activation" ||
relationship.classification === "supersedes"
) {
requiresExplicitProposal = true;
}
}
return {
relationshipDigest,
classifications: [...classifications].sort((left, right) => left.localeCompare(right)),
memoryProposalIds: [...memoryProposalIds].sort((left, right) => left.localeCompare(right)),
activationAction: "preserve",
futureMemoryChange: requiresExplicitProposal ? "explicit-proposal-required" : "none",
};
}

#mutationFor(source: AcceptedSkillMutationSource, base: HarnessVersionBundle): SkillMutation {
Expand Down Expand Up @@ -487,6 +555,7 @@ export class SkillWriter {
const draftReference = artifactReference(draft);
const validationReference = artifactReference(validation);
const evaluationReference = artifactReference(evaluation);
const memoryDisposition = this.#memoryDisposition(source);
return {
mutationId: stableId("skill-mutation", {
groupId: group.groupId,
Expand All @@ -508,6 +577,7 @@ export class SkillWriter {
draft: draftReference,
validation: validationReference,
evaluation: evaluationReference,
memoryDisposition,
existingSkills,
skill:
draft.operation === "RETIRE"
Expand Down
13 changes: 12 additions & 1 deletion packages/evolution/test/skill-curator.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,18 @@ describe("StructuredSkillCurator", () => {
);

expect(requestInput).toMatchObject({
control: { globalEvolvedSkillTargetPermitted: false, candidateContainsNoExecutableWriterPayload: true },
control: {
globalEvolvedSkillTargetPermitted: false,
candidateContainsNoExecutableWriterPayload: true,
proceduralWorkflowsBelongInSkills: ["ordered-setup", "tool-use", "debugging", "recovery", "verification"],
doNotProposeMemoryPointersToSkills: true,
memoryRemovalOrReplacementRequiresExplicitProposal: true,
memoryRelationshipSemantics: {
complements: expect.stringContaining("independently useful"),
fallbackUntilActivation: expect.stringContaining("explicit memory proposal"),
supersedes: expect.stringContaining("memory remains unchanged"),
},
},
lessons: [{ lessonRef: "L1", evidenceRefs: ["E1"], counterevidenceRefs: ["E2"] }],
selectedMemoryProposals: [{ memoryProposalRef: "P1", proposalId: "proposal-1" }],
frozenEvolutionMemory: { content: input.harness.memory },
Expand Down
Loading
Loading