Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions packages/evolution/COUNTER.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,13 @@ The observation and report contracts carry the applicable policy version without
This reservation does not add observation persistence, aggregation, attribution, scoring, pruning, trace selection metadata, or memory mutation.
A future implementation may provide the existing `PreReflectionStage` port, persist raw observations in a dedicated append-only store, and emit a versioned `CounterReport`.
That implementation must preserve the frozen job input, checkpoint artifact format, and explicit `counterReport` input already consumed by Reflector.

## Skill lifecycle observations

Skill lifecycle outcomes use a separate append-only store under evolution state. They are never written into `SKILL.md`, `EVOLVE.md`, or a harness revision, so recording a later outcome cannot rewrite the active or historical harness.

Each observation binds one exact skill ID, version, content digest, harness revision, workspace identity, and top-level task lineage. Observable invocation events remain distinct (`available`, `listed`, `selected`, `explicitly-referenced`, `read`, and `force-loaded`); none is represented as inferred skill use. Co-occurrence can record only an `unknown` outcome and cannot carry a lifecycle issue classification.

Final observations are unique per workspace, exact skill version, and top-level task lineage. Direct user attribution and deterministic test, tool, replay, or environment outcomes receive more report weight than model self-assessment or inference. Candidate observations remain auditable but do not contribute to final outcome weights or recommendations.

Lifecycle reports consider recommendations only for the exact evolved skill versions active in the supplied workspace snapshot. Outcomes for the same immutable skill version remain applicable when that version reappears in a later harness revision, while the observation retains its original harness attribution. Different skill versions and installed skills remain available for audit but cannot target the evolved-skill writer. Reports can recommend narrowing triggers, revising a procedure, merging active evolved duplicates, deactivating a harmful skill, or retiring obsolete guidance. A recommendation is not a mutation: it requires a new Skill Curator candidate and the existing deterministic validation, retrieval evaluation, and behavioral evaluation gates before the skill writer can act.
102 changes: 102 additions & 0 deletions packages/evolution/src/contracts.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ export const COUNTER_OBSERVATION_SCHEMA_VERSION = 1 as const;
export const COUNTER_REPORT_SCHEMA_VERSION = 1 as const;
export const SKILL_CANDIDATE_REGISTRY_SCHEMA_VERSION = 1 as const;
export const SKILL_EVALUATION_SCHEMA_VERSION = 1 as const;
export const SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION = 1 as const;
export const SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION = 1 as const;

export type JsonPrimitive = boolean | number | string | null;
export type JsonValue = JsonPrimitive | JsonObject | readonly JsonValue[];
Expand Down Expand Up @@ -109,6 +111,106 @@ export interface CounterReport {
readonly trends: readonly CounterReportTrend[];
}

export interface SkillLifecycleWorkspaceIdentity {
readonly repositoryId: string;
readonly directoryId?: string;
}

export interface SkillLifecycleSkillVersion {
readonly skillId: string;
readonly version: string;
readonly contentDigest: string;
readonly sourceKind: EvolutionSkillSourceKind;
readonly harnessRevision: HarnessVersion;
readonly workspace: SkillLifecycleWorkspaceIdentity;
}

export type SkillLifecycleInvocationEvent =
| "available"
| "explicitly-referenced"
| "force-loaded"
| "listed"
| "read"
| "selected";

export type SkillLifecycleIssueKind =
| "duplicate-guidance"
| "obsolete-guidance"
| "procedure-defect"
| "trigger-mismatch";

export interface SkillLifecycleObservation {
readonly schemaVersion: typeof SKILL_LIFECYCLE_OBSERVATION_SCHEMA_VERSION;
readonly observationId: string;
readonly scope: EvolutionScope;
readonly skill: SkillLifecycleSkillVersion;
readonly relatedSkills: readonly SkillLifecycleSkillVersion[];
readonly invocationEvents: readonly SkillLifecycleInvocationEvent[];
readonly topLevelTaskLineageId: string;
readonly traceIds: readonly string[];
readonly evidenceIds: readonly string[];
readonly signal: CounterSignal;
readonly signalSource: CounterSignalSource;
readonly attributionMethod: CounterAttributionMethod;
readonly issueKind: SkillLifecycleIssueKind | null;
readonly status: CounterObservationStatus;
readonly policyVersion: string;
readonly timestamp: string;
readonly idempotencyKey: string;
}

export interface SkillLifecycleSignalWeights {
readonly helpful: number;
readonly harmful: number;
readonly neutral: number;
readonly unknown: number;
}

export interface SkillLifecycleOutcomeSummary {
readonly skill: SkillLifecycleSkillVersion;
readonly finalObservationIds: readonly string[];
readonly candidateObservationIds: readonly string[];
readonly topLevelTaskLineageIds: readonly string[];
readonly traceIds: readonly string[];
readonly evidenceIds: readonly string[];
readonly weightedSignals: SkillLifecycleSignalWeights;
}

export type SkillLifecycleRecommendationAction =
| "deactivate-harmful"
| "merge-duplicates"
| "narrow-triggers"
| "retire-obsolete"
| "revise-procedure";

export interface SkillLifecycleRecommendation {
readonly recommendationId: string;
readonly action: SkillLifecycleRecommendationAction;
readonly candidateOperation: Extract<SkillCandidateOperation, "MERGE" | "RETIRE" | "UPDATE">;
readonly skillVersions: readonly SkillLifecycleSkillVersion[];
readonly observationIds: readonly string[];
readonly topLevelTaskLineageIds: readonly string[];
readonly traceIds: readonly string[];
readonly evidenceIds: readonly string[];
readonly evidenceWeight: number;
readonly reasonCodes: readonly string[];
readonly handoff: {
readonly kind: "skill-curator-candidate";
readonly requiredGates: readonly ["deterministic-validation", "retrieval-evaluation", "behavioral-evaluation"];
};
}

export interface SkillLifecycleReport {
readonly artifactKind: "skill-lifecycle-report";
readonly schemaVersion: typeof SKILL_LIFECYCLE_REPORT_SCHEMA_VERSION;
readonly metadata: EvolutionArtifactMetadata;
readonly policyVersion: string;
readonly activeSkillVersions: readonly SkillLifecycleSkillVersion[];
readonly summaries: readonly SkillLifecycleOutcomeSummary[];
readonly recommendations: readonly SkillLifecycleRecommendation[];
readonly staleObservationIds: readonly string[];
}

export interface ImmutableHarnessSnapshot {
readonly harnessVersion: HarnessVersion;
readonly baseVersion: HarnessVersion | null;
Expand Down
Loading
Loading