From dde597979a941d6fb5549e9228a10b61e8f5860d Mon Sep 17 00:00:00 2001 From: Fengzdadi <453788063@qq.com> Date: Fri, 13 Nov 2026 03:06:47 -0500 Subject: [PATCH] fix(atomic-memory): let long candidates find short related memories Lexical related recall admitted a stored memory only when the shared terms met the candidate's own coverage threshold. The requirement grew with the candidate, so a long candidate could need more shared terms than a short memory of the same fact contains, and the two were never compared. Related enumeration now also admits a memory whose shared terms number at least the minimum match count and cover the coverage fraction of the memory's own indexed words. The per-row requirement is computed from the existing searchable text on SQLite, seekDB and OceanBase; the native OceanBase path adds one MATCH block limited to memories short enough to qualify below the candidate threshold. Earlier window candidates use the same rule. Search, prepare-time admission and stored projections are unchanged. The workflow docs also drop an incorrect claim that related enumeration is prefiltered by Owner columns. The OceanBase long-query and native FTS CI steps get the case ceilings for the three new OceanBase cases. --- .github/workflows/e2e-harness.yml | 13 +- docs/en/docs/operate/configuration.md | 10 +- docs/en/docs/workflows/atomic-memory.md | 7 +- docs/zh/docs/operate/configuration.md | 10 +- docs/zh/docs/workflows/atomic-memory.md | 7 +- release/upgrade-support.json | 4 +- src/powercontext/builtin/artifacts/search.py | 23 ++-- .../persistence/atomic_memory_index.py | 53 ++++++++ .../oceanbase/atomic_memory_index.py | 79 ++++++++---- .../persistence/sqlite/atomic_memory_index.py | 70 +++++------ .../runtime/atomic_memory_processing.py | 12 +- tests/e2e/test_atomic_memory_fts_coverage.py | 118 ++++++++++++++++++ .../e2e/test_atomic_memory_long_fts_query.py | 4 +- ..._atomic_memory_unrelated_reconciliation.py | 77 ++++++++++++ 14 files changed, 396 insertions(+), 91 deletions(-) diff --git a/.github/workflows/e2e-harness.yml b/.github/workflows/e2e-harness.yml index db46b7c81b..11818610f4 100644 --- a/.github/workflows/e2e-harness.yml +++ b/.github/workflows/e2e-harness.yml @@ -427,9 +427,9 @@ jobs: - name: Test long-query retrieval and admission without skips env: POWERCONTEXT_TEST_ATOMIC_MEMORY_FTS_PATH: legacy - # The count cap keeps 13 cases within 7 per shard, independent of duration estimates. - # 7 * (300s + 10s grace) + 190s collection < 40m. - timeout-minutes: 40 + # The count cap keeps 16 cases within 8 per shard, independent of duration estimates. + # 8 * (300s + 10s grace) + 190s collection < 45m. + timeout-minutes: 45 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ oceanbase --test-timeout 300 --shard "${{ matrix.shard }}/2" --durations .github/oceanbase-migration-durations.json --output "$MIGRATION_REPORTS/long-query" --tests \ @@ -486,8 +486,8 @@ jobs: ${{ !cancelled() && contains(fromJSON('["schedule","workflow_dispatch"]'), github.event_name) && (github.event_name != 'workflow_dispatch' || inputs.suite != 'recall-effort') && ((inputs.suites == '' && needs.changes.result != 'success') || fromJSON(inputs.suites || needs.changes.outputs.suites || '{}').database) }} runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'powercontext-ci-standard' || 'ubuntu-24.04' }} - # Five cases (including three parser fallbacks), bounded setup, and diagnostics. - timeout-minutes: 65 + # Eight cases (including three parser fallbacks), bounded setup, and diagnostics. + timeout-minutes: 80 env: OCEANBASE_CI_CONTAINER: atomic-memory-native-fts-oceanbase OCEANBASE_IMAGE: ghcr.io/oceanbase/oceanbase-ce:4.4.2.1-101000022026050611 @@ -528,7 +528,8 @@ jobs: --output "$MIGRATION_REPORTS/startup" - name: Test native FTS retrieval and admission without skips - timeout-minutes: 30 + # 8 * (300s + 10s grace) + 190s collection < 45m. + timeout-minutes: 45 run: | uv run --locked python scripts/run_atomic_memory_clean_migration.py \ oceanbase --test-timeout 300 --output "$MIGRATION_REPORTS/native-fts" --tests \ diff --git a/docs/en/docs/operate/configuration.md b/docs/en/docs/operate/configuration.md index 26defed8e8..1c2a6e1d51 100644 --- a/docs/en/docs/operate/configuration.md +++ b/docs/en/docs/operate/configuration.md @@ -47,7 +47,11 @@ budget does not replay a write whose commit outcome is unknown after a connectio Source processing generates candidates, then enumerates related active memories satisfying permission, tag and retrieval eligibility. Fts enumerates all qualifying matches; vector enumerates all results within its threshold; -hybrid combines both. Neither comparison batch size nor token budget truncates the total candidates. Oversized +hybrid combines both. Lexical relatedness counts both sides: a memory qualifies when the distinct candidate terms it +contains meet the lexical admission threshold for the candidate's distinct terms, or number at least the minimum match +count and cover the coverage fraction of the memory's own indexed words, including its kind. A long candidate therefore +still finds a short memory of the same fact. Ordinary search and `prepare_context` keep the query-only threshold. +Neither comparison batch size nor token budget truncates the total candidates. Oversized comparison input reduces the batch until every candidate is processed. A single object or complete extraction input that exceeds the budget fails the window without partial publication or cursor advancement. Fallback changes recall evidence and requires an explicit choice; by default errors retain the window for diagnosis and retry. @@ -644,6 +648,10 @@ Atomic Memory uses native `minimum_should_match` filtering when the connected Oc Older engines retain the whole-word `LOCATE` coverage check. Capability detection does not treat query timeouts, permission failures, or connection errors as an unsupported feature. Both paths keep the same analyzed, deduplicated query terms and admission threshold, including recovery probes and complete related-memory enumeration. +For related-memory enumeration, the native path also matches memories that share at least the minimum match count of +terms and are short enough to qualify below the candidate's threshold, then applies whole-word coverage to them. For a +long candidate almost every memory is that short, so the whole-word check runs on nearly every memory sharing that +many terms. Queries with more than 256 distinct terms use native filtering only on identified OceanBase 4.6 or newer engines. Earlier OceanBase versions, seekDB, and unrecognized version strings retain whole-word coverage for these queries: OceanBase CE 4.4.2.1 accepts the native syntax but can ignore the threshold above that term count. diff --git a/docs/en/docs/workflows/atomic-memory.md b/docs/en/docs/workflows/atomic-memory.md index fee09467b3..267c6965d0 100644 --- a/docs/en/docs/workflows/atomic-memory.md +++ b/docs/en/docs/workflows/atomic-memory.md @@ -267,9 +267,10 @@ Client methods for creation, replacement and exact historical reads. Search and list require `scope.read`, following the Memory and Topic Memory Scope boundary. An individual Artifact share permits its exact get and history reads; it does not grant a Scope inventory or search. If Scope read is revoked before reranking or context assembly, the request returns `403` even when an individual share remains. -Source extraction enumerates the complete same-Scope threshold set owned by its principal, then checks the -configured authorizer for Artifact read and write before comparing a memory. Denied candidates are skipped; -authorization failures stop processing. The immutable Owner columns are only an extraction prefilter. +Source extraction enumerates every active same-Scope memory that meets the related threshold, without an owner +prefilter, then checks the configured authorizer for Artifact read and write before comparing a memory. Denied +candidates are skipped; authorization failures stop processing. In fts and hybrid related modes the lexical threshold +also measures the memory's own indexed words, so a long candidate still reaches a short memory of the same fact. Scope checks using the same database with supported Builtin or Casbin providers share the business snapshot. Independent decision stores and custom providers retain their configured read consistency contracts. diff --git a/docs/zh/docs/operate/configuration.md b/docs/zh/docs/operate/configuration.md index c12ff5d8d5..e82b65c970 100644 --- a/docs/zh/docs/operate/configuration.md +++ b/docs/zh/docs/operate/configuration.md @@ -40,7 +40,10 @@ PowerContext 进程启动时从环境变量读取配置。当前工作目录存 带 token 的恢复遇到变化会返回 `preview_stale`,不会自动重新解释意图;重试预算也不处理提交结果未知的连接失败。 Source 处理先生成候选,再枚举满足权限、标签及检索资格的相关 active 记忆。fts 枚举全部合格匹配,vector 枚举 -阈值内全部结果,hybrid 合并两者。总候选不会按 comparison batch size 或 token 限额截断。 +阈值内全部结果,hybrid 合并两者。词法相关性同时衡量两侧:记忆包含的候选不同词数达到候选不同词数的词法 +准入阈值,或者不少于最少匹配词数且覆盖该记忆自身索引词(含 kind)的覆盖比例,即算相关,因此较长的候选 +仍能找到同一事实的较短记忆。普通搜索和 `prepare_context` 仍只按查询词计算阈值。总候选不会按 comparison +batch size 或 token 限额截断。 比较输入过大时缩小单批,直到所有候选都处理完;连单个对象或完整抽取输入都超预算时窗口失败,不部分提交或推进 cursor。 开启 fallback 会改变相关召回依据,应显式选择;默认错误会保留当前窗口供排查和重试。 @@ -581,7 +584,10 @@ URL 必须使用 `mysql+aoceanbase` 驱动、明确的端口,并设置 `charse Atomic Memory 会在连接的 OceanBase 或 seekDB 引擎支持时,使用原生 `minimum_should_match` 过滤;旧版引擎继续使用 完整词 `LOCATE` 覆盖检查。能力检测不会把查询超时、权限错误或连接错误当作功能不支持。两条路径使用相同的分析、去重后的 -查询词和准入阈值,也适用于恢复探测与完整关联记忆枚举。超过 256 个不同词的查询,仅在确认版本为 OceanBase 4.6 或更新版本时使用 +查询词和准入阈值,也适用于恢复探测与完整关联记忆枚举。关联记忆枚举的原生路径还会匹配与候选共享至少 +最少匹配词数、且短到可低于候选阈值入选的记忆,再对这些记忆做完整词覆盖检查。候选较长时,几乎所有记忆 +都满足这个长度条件,因此几乎每条共享这么多词的记忆都要做完整词覆盖检查。 +超过 256 个不同词的查询,仅在确认版本为 OceanBase 4.6 或更新版本时使用 原生过滤;更早的 OceanBase、seekDB 和无法识别版本的引擎继续使用完整词覆盖检查,因为 OceanBase CE 4.4.2.1 虽然接受原生语法, 但超过该词数时可能忽略匹配阈值。查询包含 `com` 等被 SPACE 分词器省略的 Analyzer 词时,检索、完整枚举和恢复探测 也会保留完整词覆盖检查,阈值仍按所有不同的 Analyzer 词计算。该选择无需新增配置、表或索引迁移,沿用现有 `SPACE` 全文索引。 diff --git a/docs/zh/docs/workflows/atomic-memory.md b/docs/zh/docs/workflows/atomic-memory.md index 2efa2a3d6f..740fbefbb2 100644 --- a/docs/zh/docs/workflows/atomic-memory.md +++ b/docs/zh/docs/workflows/atomic-memory.md @@ -247,9 +247,10 @@ unsupported 通过 `ServerResponseError.status_code`、`.code`、`.details` 检 搜索和列表沿用 Memory、Topic Memory 的 Scope 边界,要求 `scope.read`。单条 Artifact 分享允许 精确读取和历史读取,不授予整个 Scope 的列表或搜索权限。重排或上下文组装前若 Scope 读取权限已被 -撤销,请求返回 `403`,即使单条分享仍有效。Source 提取会完整枚举同 Scope 中满足阈值、 -属于当前主体的记忆,再由实际配置的授权服务检查 Artifact 读取和写入权限后交给模型比较。明确拒绝的 -候选会被跳过;授权服务故障会终止处理。不可变的 Owner 列只用于提取时的所有者预筛选。 +撤销,请求返回 `403`,即使单条分享仍有效。Source 提取会完整枚举同 Scope 中满足相关阈值的 +active 记忆,不按所有者预筛选,再由实际配置的授权服务检查 Artifact 读取和写入权限后交给模型比较。明确拒绝的 +候选会被跳过;授权服务故障会终止处理。在 fts 和 hybrid 关联模式下,词法阈值也会衡量记忆自身的索引词, +较长的候选仍能找到同一事实的较短记忆。 使用同一数据库的受支持 Builtin 或 Casbin 服务进行 Scope 检查时,授权读取与业务数据共享快照。 独立的决策存储和自定义服务保留各自配置的读取一致性契约。 diff --git a/release/upgrade-support.json b/release/upgrade-support.json index 7887c05ed8..0ccd83dfae 100644 --- a/release/upgrade-support.json +++ b/release/upgrade-support.json @@ -3,8 +3,8 @@ "target_version": "1.5.1", "mode": "upgrade-required", "schema_review": { - "persistence_sha256": "b12151736cda3bf917c7d6e858c88d54562b420c64f39b7e7b3c642b34e3df02", - "rationale": "Topic Memory processing failures attach bounded, content-free stage, reason and budget diagnostics to in-flight exceptions. Rejected reservations retain their existing durable charges; readback omits token-specific fields unless committed evidence establishes the limiting dimension, and repeated rejected calls cannot reinterpret prior evidence. Source structure and history-count guards do not imply context overflow. Missing FTS capability is configuration, and incomplete vector projections are business validation. No database schema, migration resource, registered table layout, frozen revision checksum, row encoding, persisted journal format or worker retry behavior is changed by these diagnostic annotations. The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Revision v1_5_0_002 follows v1_5_0_001 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_003 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker.. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged. Ownership recovery preflight derives candidate-family filters from the exact migration identities, including Prompt Candidates that do not have standalone Artifact owners. Static Scope permission previews now use the same complete internal receipt keys as bootstrap for long Scope IDs. These authorization-only changes add no stored fields, row encoding, schema, or migration registration. Static Scope preset keys retain existing short identities and use SHA-256 for oversized relationship keys. Bootstrap and read-only preview recognize both bounded keys and historical receipts, including revoked bindings. No schema or migration registration changes. Managed observability reads existing Context Bootstrap receipt metadata in authorized Scope batches and aggregates it by integration and receipt state. This adds no schema revision, stored row changes, migration resources, or data encoding changes; receipt identities, historical evidence and the declared upgrade acceptance matrix remain unchanged. SQLite connection failures drain the aiosqlite 0.22.1 worker before propagating the error or cancellation, including failures before pool admission. This connection-lifecycle fix changes no registered schema, historical revision checksum, stored value, or upgrade source policy. Prepared-context reuse proofs revalidate cited code files by reading their current worktree digest through the existing no-follow capture reader and exclusion rules. This adds a read-only lookup; it changes no code index cache layout, manifest, graph rows, publication journal, SQL layout, migration resource or checksum. Revision v1_5_0_004 adds the optional pc_artifact_source_locators relation keyed by the existing Artifact Revision and Source-edge ordinal for both registered source releases and vector/Code profiles. Explicit maintenance creates the empty relation without rewriting historical lineage, payloads, Source identities, earlier revision resources or checksums. Missing locators remain omitted in legacy encodings; located citations survive persistence, HTTP/MCP and portable export/restore, and Source sequence mismatches are rejected without redirecting identity. Atomic Memory content and vector bindings are unchanged, so locator metadata does not require re-embedding. Local SQLite migration, restart, citation and portable round-trip tests pass; MySQL layout checks do not constitute native seekDB/OceanBase or installed-release acceptance. The upgrade-required policy, required automatic matrix and publication restrictions remain unchanged. Projection verification and repair read only the exact authoritative head content and state, avoiding target-only Source locator lineage tables at earlier registered migration checkpoints. Frozen resources and revision checksums remain unchanged. Source locator line-count and heading-path limits are centralized without changing their values, accepted payloads or stored encodings. OceanBase Atomic Memory lexical retrieval uses native MATCH with minimum_should_match and SCORE when a controlled capability probe confirms support, and retains the existing MATCH AGAINST plus complete-word coverage path for unsupported backends. Capability decisions are held only in memory per connection pool. Search, complete related enumeration and recovery preserve Analyzer term deduplication, coverage thresholds, scope and content filters, vector completeness checks, same-statement authority heads and the existing oceanbase_match raw-score contract. This query-only change adds no tables, public interfaces, schema revisions, stored fields or row encodings; existing SPACE index definitions, projection formats, historical migration resources and revision checksums remain unchanged. No schema migration or projection rebuild is required for this query implementation. The upgrade-required policy, registered sources and complete installed-release acceptance matrix remain unchanged; this review does not certify installed-release upgrades. Queries above 256 distinct terms retain whole-word coverage unless the engine is identified as OceanBase 4.6 or newer, because earlier native engines can accept MATCH while ignoring that threshold. This compatibility guard changes query selection only. Native capability caching obtains the pool through run_sync, including pinned historical migration connections, without acquiring another connection or releasing maintenance locks. Queries containing SPACE stopwords retain Analyzer-based whole-word admission in all lexical operations. These corrections change neither stored projections nor registered schema resources. SQLite Topic Memory full-text helper version 2 assigns each Scope's helper rows to a hashed 2^32-rowid range above every rowid that version 1 assigned, so Topic replacement and search bound their FTS5 cursors to one Scope's range. Virtual table definitions, registered schema fingerprints, migration resources and authoritative active Topic rows are unchanged. Unmanaged initialization and the existing s1400 domain step rebuild version-1 helper rows from the active tables with exact identities and revisions; a managed database that still holds version-1 helper rows keeps the previous identity filters until the helper is rebuilt. bm25 statistics remain table-wide, so search hits, ordering and scores are unchanged. The standalone seekdb maintenance identity is derived from @@server_uuid and DATABASE() when ob_compatibility_mode is missing, NULL, or empty and VERSION() contains -OceanBase seekdb-. The db-migrate identity probe and the server-run tenant gate share one compatibility classifier; ORACLE and non-seekdb servers remain rejected. This is an admission-only change and does not alter registered table layouts, migration resources, persisted row encodings, or public interfaces. Source writes and text retrieval reuse parameterized SQL structures with fresh Scope, identity, revision, binding and increment values on every execution. Only the fixed built-in Content manifest construction and exact validated payload bytes reuse cached metadata; database registration and indexed-identity checks remain current, and callers receive deep copies. MySQL/OceanBase journal allocation retains SELECT FOR UPDATE; SQLite retains its write reservation. Concurrent Content captures coalesce already queued requests into bounded transactions, isolate individual failures after batch rollback, replay stable identities on cancellation and drain admitted callers during shutdown. No schema, migration, stored encoding, Source identity, journal ordering, processing counter semantics or upgrade support claims change." + "persistence_sha256": "61a85a7e06fc42aace60031505280f5f8ac044e88544e07ff9b740d39a43c6c0", + "rationale": "Topic Memory processing failures attach bounded, content-free stage, reason and budget diagnostics to in-flight exceptions. Rejected reservations retain their existing durable charges; readback omits token-specific fields unless committed evidence establishes the limiting dimension, and repeated rejected calls cannot reinterpret prior evidence. Source structure and history-count guards do not imply context overflow. Missing FTS capability is configuration, and incomplete vector projections are business validation. No database schema, migration resource, registered table layout, frozen revision checksum, row encoding, persisted journal format or worker retry behavior is changed by these diagnostic annotations. The reusable Server migration framework registers actual PowerContext 1.3.0 through s1300 -> s1400 -> s1401 -> s1402 and actual 1.4.0 through the separate immutable r1400 -> s1400 -> s1401 -> s1402 source bridge. Original 1.3 resources and checksums remain byte-identical. Real 1.4.0 SQLite, embedded seekDB and OceanBase structures were captured from the official wheel in all twelve vector/Code configurations; provenance and exact source layouts are packaged with the bridge. The 1.4 bridge preserves current Candidate, Artifact and task data, adds the missing Topic budget/block structure, revokes old lease fences and joins the existing archive/portable-storage path. Empty retained tables align the final schema without inventing 1.3 Candidate history. Source histories retain their identity through subsequent upgrades and recovery; shared head names never authorize adopting unknown layouts. Vector profiles remain bound to historical model, profile ID, dimension, unit normalization and L2 distance, with no provider calls during maintenance. Complete schema, domain, projection, profile and retained-history checks precede startup and partial maintenance. Local SQLite and embedded seekDB startup reject an observable active maintenance lock through a read-only probe. SQLite runtime admission validates the default adjacent journal, rejecting unfinished full or partial maintenance, changed recorded revision checksums and missing required retained tables. Absent journals do not invent retention obligations; custom evidence directories require explicit operator verification and remote coordinator locks remain subject to deployment orchestration. Native read-only readiness uses a consistent snapshot and does not acquire Atomic Memory head locks; normal business transaction isolation is unchanged. Offline projection repairs retain the original locked state comparison and write-side CAS protection. Target locks, backup choices, original recovery evidence and explicit remote acknowledgments remain enforced. The installed-artifact catalog retains twelve 1.3.0 source/backend/vector/Code combinations. Embedded seekDB uses pylibseekdb 1.4.0.post2 or newer with the recovered-transaction lock fix. Main and publication require all twelve configurations, including embedded seekDB full-server-code and full-server-vector-code. No configuration is waived; incomplete, failed or skipped evidence rejects acceptance. The seekdb-code diagnostic selection cannot authorize publication. The registered 1.4.0 bridge remains implemented but is outside the CI upgrade acceptance matrix; passing this gate does not certify that path. The gate preserves source-specific Processing counters and isolates OceanBase source databases. Any failed, skipped or missing automatic case rejects publication. The policy remains upgrade-required until the exact target artifact passes the required automatic matrix and the reviewed support declaration is updated. Captured source structure, local tests and checkpoint replay are distinct from installed-release acceptance, native interruption/observer recovery and backup restoration. The Candidate generation prototype observes actual SQLite/seekDB/OceanBase migration checkpoints outside the production bundle; it does not replace frozen registrations or weaken recovery checks. Existing target schema and data formats, including batched persistence interfaces, remain governed by the shared registered path. The synchronized master adds blank-text validation at Source creation and managed worker logging. It changes neither registered table layout nor migration data encodings; existing stored Source values are not rewritten. The source bridges and frozen history remain unchanged. Compatibility commands identify Server-owned storage before opening a writable legacy profile and delegate source admission to the shared framework, including actual 1.4.0 and rejection of unknown layouts. Partial Processing and Atomic Memory projection maintenance verify the original recorded revision checksums; native admission also validates typed journal identity and its completed position against the database revision. Persisted vector profiles reject unsupported partial maintenance before plans, backups or operation evidence, including when the supplied configuration omits embeddings. These are admission-only changes; frozen migration resources, registered layouts, historical checksums and data transformations remain unchanged. PowerContext 1.4.1 assigns release-based presentation names v1_4_1_001 through v1_4_1_003 to the existing shared upgrade steps; the 1.3.0 and 1.4.0 source baselines remain independently registered. Existing revision IDs, bundle and revision checksums, accepted plan identities and journal schemas are unchanged. New Server revisions use software-version-plus-sequence names directly; software releases without storage changes may reuse the same database revision. Naming does not grant acceptance or replace source registration. Full Server status, plan, apply and verify reject unfinished Processing configuration and Atomic Memory projection journals through the same typed admission check used by partial maintenance and SQLite startup. The owning domain runner can still resume its own journal. This only changes admission; frozen revisions, journal formats, database layouts and data transformations remain unchanged. Revision v1_4_1_004 registers the pc_access_owner_recovery outbox on the shared Server chain for both installed-release sources and all registered vector/Code profiles. It creates an empty table through explicit maintenance for existing managed databases and through the registered bootstrap for fresh databases. Earlier revision resources, fingerprints, checksums and existing domain row encodings remain unchanged. External relationship writers use version-1 Artifact and Candidate intents committed with domain data; replay retains the original deployment, principal, resource or target, and idempotency identity. Stable operation and audit event identities prevent lost acknowledgements from duplicating ownership or audit. Conflicting intents are rejected and unknown historical ownership is not inferred. Unregistered outbox layouts are not adopted merely from a table name. This registration does not establish installed-release or external-Provider upgrade acceptance. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Database selection now defaults to powercontext for seekDB and for OceanBase URLs without a database, while explicit database names remain authoritative. An implicit seekDB selection is rejected before initialization when the legacy test database contains pc_ tables; operators can explicitly select database=test to retain those records, and no database is moved or deleted automatically. Bootstrap reuses an accessible selected database without CREATE privilege and creates only a confirmed missing database, with idempotent creation and explicit permission errors. Read-only maintenance on an initialized MySQL-compatible engine requires the selected database and never provisions it; an absent or empty seekDB instance remains a no-engine initial plan. MySQL initialization creates only missing requested tables and indexes, and sorts only those requested tables; existing registered tables are preserved. Code graph connections use the selected database, retain historical graph identities for test, and distinguish other database identities. Server identity initialization avoids redundant MySQL DDL on an existing table. Frozen migration resources, revision identifiers and checksums, registered table definitions, journal formats, and existing row encodings are unchanged. These runtime selection and admission changes do not establish an accepted upgrade path; the policy remains upgrade-required and the installed-release acceptance gate still applies. MySQL recovery rejects different registered table sets before normalization. Backend-default normalization copies only the column and index maps it changes and compares each table without cloning whole inventories. Admitted layouts, registered transitions, recovery checks, schema equality and caller-owned inventories remain preserved. SeekDB status and verify distinguish a missing configured database from an uninitialized engine and denied access. Read-only plan can represent a missing selected database as an uninitialized target without creating it; apply provisions that selected database only after the operator accepts its bound plan, then rechecks the target under the maintenance lock. Installed-release seekDB acceptance explicitly retains the historical test database and separately verifies that omitted selection is refused. Registered schema resources, revision checksums, row encodings and write-side provisioning remain unchanged. Server startup can apply an uninitialized plan to provision a missing selected seekDB database; plan, status and verify still never provision it. OceanBase 4.6 SHOW CREATE TABLE can reflect FTS_INDEX_TYPE=MATCH or PHRASE_MATCH on FULLTEXT indexes. Reflection accepts these known types without changing the logical inventory; unknown values remain rejected and indexed columns, parser identity and token-size properties remain significant. PHRASE_MATCH additionally supports phrase matching; the registered layouts require term matching and do not declare a phrase-only contract. Frozen migration resources, checksums, recovery evidence and persisted layouts remain unchanged. Native 4.6.3.1 DDL was captured from a real installed 1.3.0 database; replay of this fixture does not establish complete native upgrade acceptance. The CI database image and master full automatic upgrade matrix remain unchanged. Revision v1_4_1_005 adds nullable Atomic Memory description projection metadata after the registered owner-recovery revision, through explicit SQLite and MySQL schema transitions for both source chains and all vector/Code profiles. Earlier revision resources, fingerprints and checksums remain unchanged. The additive change neither rewrites authoritative Artifact history nor generates summaries or scans stored bodies. Existing managed databases require explicit migration; existing descriptions remain null until a new write or explicit projection repair copies authoritative metadata. Intermediate projection validation uses only columns available at that revision and still rejects inconsistent existing fields. Unmanaged SDK initialization retains its nullable-column compatibility path. The installed-release acceptance matrix and publication restrictions remain required; local regression checks do not establish native upgrade acceptance. Artifact plugin storage extends the shared Server chain from v1_4_1_006 to v1_5_0_001 with registration columns, family declarations, search state and projections, index coverage and changes, and vector-space/index registrations. SQLite also registers portable vectors and FTS auxiliary structures. Native registration columns are added in one ALTER with a matching recovery checkpoint; final column definitions and constraints are unchanged. Native vector tables are created online from registered spaces and are admitted to later maintenance inventories only after exact layout validation. Historical migration resources, retained objects, and existing data are preserved. This review does not expand the supported upgrade matrix or certify installed-release acceptance. Source write plugin registrations use the additive v1_4_1_006 revision after v1_4_1_005 on both source chains. It creates only pc_source_write_plugins; all earlier master revision scripts, schema resources and checksums remain unchanged. Revision v1_5_0_002 follows v1_5_0_001 and adds retained Source evidence bindings, exact Atomic Memory evidence snapshots and their schema marker on all registered backends. A new versioned domain step verifies retained evidence for lexical and vector profiles; earlier released domain resources and recorded revision checksums remain byte-identical. Existing Sources and Memory revisions without accepted declarations stay neutral, with no trust backfill. Current projection maintenance and rebuild explicitly require evidence validation. This additive registration does not establish installed-release acceptance; the upgrade-required gate and publication restrictions are unchanged. Revision v1_5_0_003 adds the disposable exact-revision pc_atomic_memory_evidence projection through the existing index adapters. Transactional body writes publish it from accepted evidence; scoped portable restore and maintenance rebuild derive it from retained history without current-registry trust backfill or model calls. Domain conformance compares historical and derived values, including neutral legacy declarations, on lexical and vector layouts. Earlier revision resources and fingerprints remain unchanged. Local SQLite recovery and real OceanBase 4.4.2.1 evidence conformance were exercised; these do not certify installed-release upgrades or change publication restrictions. Source evidence maintenance now accepts the deployment's own Source definition registry, reports one bounded neutralization reason through operational logs, batches exact-revision reads and caches retained-schema validation per manifest fingerprint. Indexed repair accepts that registry explicitly; the migration bundle still admits evidence against the builtin registry and says so. These changes alter neither registered table layouts nor retained encodings, and the runtime and Server evidence DDL remain byte-identical. The policy remains upgrade-required with the full automatic matrix and publication restrictions described above. Source definition manifests derive every field whose default is null from the validation schema, so an already accepted Definition keeps its fingerprint and re-registers after upgrade; populating such a field requires the nested model to serialize by alias. Registered layouts and retained encodings are unchanged. The runtime MySQL-family table fast path now runs the registered additive evidence migration before creating other tables, so an unmanaged OceanBase or seekDB database cannot retain evidence tables without their p0005 marker.. Code publication recovery adds an optional per-binding publication.json journal that records a generation identifier before its database commit and final directory rename. Caches created before this journal existed remain readable without synthesizing historical publication records; identifiers for already orphaned rows are not inferred. New seekDB generations retain the existing identifier, graph descriptor, manifest and row encodings, while reclamation validates the recorded identifier and scopes deletion to its binding. Live or reader-retained generation directories and the in-flight publication are protected from journal-window reclamation. SQLite graph rows remain owned by their generation directory. This additive local cache journal changes recovery behavior but does not change registered SQL table layouts, migration resources, revision identifiers, historical checksums or the Server migration journal. The policy remains upgrade-required, and the complete installed-release acceptance matrix remains mandatory; this compatibility review is not evidence of completed native upgrade acceptance. Managed Ops admission validates a private main-file and WAL snapshot through the exact target Server bundle, retaining the original database identity and maintenance evidence. Snapshot admission is read-only and cannot initialize, migrate or recover storage. It adds no schema revision, changes no persisted row or journal format, and does not expand supported upgrade declarations or the installed-release acceptance matrix. PreparedContext Receipt records the actual FTS mode on Experience search outcomes. This adds only an in-memory observation field; it does not change stored Experience rows, SQL selection, schema registrations, migration resources, or data encodings. Unmanaged historical Server databases are admitted by required structure and domain state, then normalized before adopting an existing checkpoint. Historical Memory conversion, Scope preservation, SQLite rollback and native DDL recovery use the shared maintenance flow. Independent extension objects and harmless DDL differences do not require clean-install fingerprints. Existing immutable revision resources and the installed-release acceptance matrix remain unchanged. Local historical tests do not certify arbitrary tenant defaults, every vector/Code configuration or future releases. SQLite compatibility validation includes effective primary-key collations and rejects conflicting identity semantics. Historical normalization and the frozen s1400 rebuild preserve generated extension definitions while copying only writable columns. Both virtual and stored generated values remain computed by SQLite; immutable revision resources and the native backend paths are unchanged. SQLite table rebuilds preserve declared collations on ordinary and generated extension columns, including their inherited index comparison semantics. Expression-local COLLATE clauses remain part of their expressions and do not change the column collation. This changes only SQLite compatibility normalization; frozen revisions and native backend execution remain unchanged. Historical Memory admission recognizes lineage ownership columns and default non-merge references while rejecting unmapped payloads. Native compatibility journals record temporary Processing counter ownership before DDL and clear it after cleanup, including completed-phase recovery. SQLite extension preservation also covers the registered r1400 bridge. Frozen migration resources and the installed-release acceptance matrix remain unchanged. Native historical Atomic conversion creates nullable description metadata with the registered UTF-8 binary collation, allowing the later frozen ADD transition to recognize its completed postcondition without duplicate DDL. SQLite historical normalization and registered source bridges preserve dependent views and their triggers during same-name table replacement. Legacy rename behavior is limited to the exact staging-to-final rename statements and the original connection setting is restored; genuine table renames still update dependent references. Frozen migration resources and native backend behavior remain unchanged. SQLite admission validates required extension defaults using SQLite deterministic-expression restrictions and a non-NULL evaluation on an isolated connection. Connection-state, random and clock-dependent expressions are rejected; direct CURRENT_DATE, CURRENT_TIME and CURRENT_TIMESTAMP defaults remain supported alongside usable constant expressions. Regression coverage verifies consecutive Source writes in distinct scopes within the same transaction. Read-only verification and startup admission share this rule; supported defaults remain usable through Source write and readback. Independent extension tables remain preserved through managed upgrades and initialization recovery; missing recovery evidence and missing vector configuration still block maintenance. Registered native column additions and modifications materialize the charset and collation declared by their immutable postcondition when the SQL would otherwise inherit a historical table default. This preserves GBK-origin upgrade recovery without changing frozen resources, existing table defaults, independent extension encodings or readiness requirements. SQLite Candidate extension columns and their values survive both stages of the registered Dream rebuild, including generated columns and dependent views. Frozen source introspection projects the already-validated managed columns without altering stored extension data. Independent indexes use SQLite-rewritten definitions after genuine table renames. Native CHECK removal resolves the actual constraint name from the registered expression and enforcement semantics, including recovery after committed DDL. Frozen resources, registered revision checksums and the installed-release acceptance matrix remain unchanged. SQLite admission rejects additional NOT NULL generated columns because computed values can reject valid Source writes. Nullable virtual and stored extensions remain preserved across historical upgrades and support Source write/readback. Frozen migrations and native backend behavior are unchanged. Ownership recovery preflight derives candidate-family filters from the exact migration identities, including Prompt Candidates that do not have standalone Artifact owners. Static Scope permission previews now use the same complete internal receipt keys as bootstrap for long Scope IDs. These authorization-only changes add no stored fields, row encoding, schema, or migration registration. Static Scope preset keys retain existing short identities and use SHA-256 for oversized relationship keys. Bootstrap and read-only preview recognize both bounded keys and historical receipts, including revoked bindings. No schema or migration registration changes. Managed observability reads existing Context Bootstrap receipt metadata in authorized Scope batches and aggregates it by integration and receipt state. This adds no schema revision, stored row changes, migration resources, or data encoding changes; receipt identities, historical evidence and the declared upgrade acceptance matrix remain unchanged. SQLite connection failures drain the aiosqlite 0.22.1 worker before propagating the error or cancellation, including failures before pool admission. This connection-lifecycle fix changes no registered schema, historical revision checksum, stored value, or upgrade source policy. Prepared-context reuse proofs revalidate cited code files by reading their current worktree digest through the existing no-follow capture reader and exclusion rules. This adds a read-only lookup; it changes no code index cache layout, manifest, graph rows, publication journal, SQL layout, migration resource or checksum. Revision v1_5_0_004 adds the optional pc_artifact_source_locators relation keyed by the existing Artifact Revision and Source-edge ordinal for both registered source releases and vector/Code profiles. Explicit maintenance creates the empty relation without rewriting historical lineage, payloads, Source identities, earlier revision resources or checksums. Missing locators remain omitted in legacy encodings; located citations survive persistence, HTTP/MCP and portable export/restore, and Source sequence mismatches are rejected without redirecting identity. Atomic Memory content and vector bindings are unchanged, so locator metadata does not require re-embedding. Local SQLite migration, restart, citation and portable round-trip tests pass; MySQL layout checks do not constitute native seekDB/OceanBase or installed-release acceptance. The upgrade-required policy, required automatic matrix and publication restrictions remain unchanged. Projection verification and repair read only the exact authoritative head content and state, avoiding target-only Source locator lineage tables at earlier registered migration checkpoints. Frozen resources and revision checksums remain unchanged. Source locator line-count and heading-path limits are centralized without changing their values, accepted payloads or stored encodings. OceanBase Atomic Memory lexical retrieval uses native MATCH with minimum_should_match and SCORE when a controlled capability probe confirms support, and retains the existing MATCH AGAINST plus complete-word coverage path for unsupported backends. Capability decisions are held only in memory per connection pool. Search, complete related enumeration and recovery preserve Analyzer term deduplication, coverage thresholds, scope and content filters, vector completeness checks, same-statement authority heads and the existing oceanbase_match raw-score contract. This query-only change adds no tables, public interfaces, schema revisions, stored fields or row encodings; existing SPACE index definitions, projection formats, historical migration resources and revision checksums remain unchanged. No schema migration or projection rebuild is required for this query implementation. The upgrade-required policy, registered sources and complete installed-release acceptance matrix remain unchanged; this review does not certify installed-release upgrades. Queries above 256 distinct terms retain whole-word coverage unless the engine is identified as OceanBase 4.6 or newer, because earlier native engines can accept MATCH while ignoring that threshold. This compatibility guard changes query selection only. Native capability caching obtains the pool through run_sync, including pinned historical migration connections, without acquiring another connection or releasing maintenance locks. Queries containing SPACE stopwords retain Analyzer-based whole-word admission in all lexical operations. These corrections change neither stored projections nor registered schema resources. SQLite Topic Memory full-text helper version 2 assigns each Scope's helper rows to a hashed 2^32-rowid range above every rowid that version 1 assigned, so Topic replacement and search bound their FTS5 cursors to one Scope's range. Virtual table definitions, registered schema fingerprints, migration resources and authoritative active Topic rows are unchanged. Unmanaged initialization and the existing s1400 domain step rebuild version-1 helper rows from the active tables with exact identities and revisions; a managed database that still holds version-1 helper rows keeps the previous identity filters until the helper is rebuilt. bm25 statistics remain table-wide, so search hits, ordering and scores are unchanged. The standalone seekdb maintenance identity is derived from @@server_uuid and DATABASE() when ob_compatibility_mode is missing, NULL, or empty and VERSION() contains -OceanBase seekdb-. The db-migrate identity probe and the server-run tenant gate share one compatibility classifier; ORACLE and non-seekdb servers remain rejected. This is an admission-only change and does not alter registered table layouts, migration resources, persisted row encodings, or public interfaces. Source writes and text retrieval reuse parameterized SQL structures with fresh Scope, identity, revision, binding and increment values on every execution. Only the fixed built-in Content manifest construction and exact validated payload bytes reuse cached metadata; database registration and indexed-identity checks remain current, and callers receive deep copies. MySQL/OceanBase journal allocation retains SELECT FOR UPDATE; SQLite retains its write reservation. Concurrent Content captures coalesce already queued requests into bounded transactions, isolate individual failures after batch rollback, replay stable identities on cancellation and drain admitted callers during shutdown. No schema, migration, stored encoding, Source identity, journal ordering, processing counter semantics or upgrade support claims change. Complete Atomic Memory related enumeration admits a memory when the distinct candidate terms it contains meet the lexical threshold of the candidate's distinct terms, or number at least the minimum match count and cover the coverage fraction of the memory's own indexed words, so a long candidate reaches a short memory of the same fact; window siblings use the same rule. The per-row requirement is computed from the existing searchable text in SQL on SQLite, seekDB and OceanBase, and the native OceanBase path adds one MATCH block that applies whole-word coverage only to memories short enough to qualify below the candidate threshold. Search, recovery probes, stored projections, registered schema resources and migrations are unchanged." }, "cases": [], "automatic_sources": { diff --git a/src/powercontext/builtin/artifacts/search.py b/src/powercontext/builtin/artifacts/search.py index cd4b6bddbc..39f7991454 100644 --- a/src/powercontext/builtin/artifacts/search.py +++ b/src/powercontext/builtin/artifacts/search.py @@ -381,17 +381,21 @@ def fts_query_requirements(value: str, /, *, floor: AdmissionFloor | None = None query_terms = tuple(sorted(set(analyze_fts_query(value).split()))) if not query_terms: return (), 0 + return query_terms, fts_required_matches(len(query_terms), floor=floor) + + +def fts_required_matches(term_count: int, /, *, floor: AdmissionFloor | None = None) -> int: + """Return how many shared terms the admission threshold requires of a set of ``term_count`` terms. + + A set of at most two terms requires one match; a larger set requires + ``max(lexical_min_matched_terms, ceil(lexical_coverage * term_count))``. + """ + + if term_count <= _FTS_SHORT_QUERY_MAX_TERMS: + return 1 coverage = _FTS_MIN_QUERY_COVERAGE if floor is None else floor.lexical_coverage min_matched = _FTS_MIN_MATCHED_TERMS if floor is None else floor.lexical_min_matched_terms - required_matches = ( - 1 - if len(query_terms) <= _FTS_SHORT_QUERY_MAX_TERMS - else max( - min_matched, - math.ceil(len(query_terms) * coverage), - ) - ) - return query_terms, required_matches + return max(min_matched, math.ceil(term_count * coverage)) def fts_match_query(value: str) -> str | None: @@ -434,6 +438,7 @@ def admits_fts_projection(query: str, projection: str, /, *, floor: AdmissionFlo "analyze_text_with_spans", "fts_match_query", "fts_query_requirements", + "fts_required_matches", "lexical_search_score", "unit_l2_cosine_similarity", ] diff --git a/src/powercontext/builtin/persistence/atomic_memory_index.py b/src/powercontext/builtin/persistence/atomic_memory_index.py index e9d0e97b36..1081087fb4 100644 --- a/src/powercontext/builtin/persistence/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/atomic_memory_index.py @@ -40,6 +40,7 @@ RecallChannelWeights, analyze_text, fts_query_requirements, + fts_required_matches, lexical_search_score, ) from powercontext.builtin.inference import EmbeddingModel @@ -424,6 +425,56 @@ def oceanbase_fts_coverage_sql(terms: tuple[str, ...], /) -> tuple[str, dict[str return coverage[0] if coverage else "0", parameters +def _memory_words_requirement(word_count: int, floor: AdmissionFloor | None) -> int: + floor = DEFAULT_ADMISSION_FLOOR if floor is None else floor + return max(floor.lexical_min_matched_terms, math.ceil(word_count * floor.lexical_coverage)) + + +def atomic_memory_related_requirement( + term_count: int, word_count: int, /, *, floor: AdmissionFloor | None = None +) -> int: + """Return how many distinct candidate terms a stored memory must contain to be related. + + The shared terms must meet the lexical admission threshold for the candidate's ``term_count`` distinct terms, or + number at least the minimum match count and cover the coverage fraction of the memory's own ``word_count`` indexed + words. The requirement never grows beyond what a short memory can supply, so a long candidate still finds a short + memory of the same fact. Prepare-time search keeps the query-only threshold. + """ + + return min(fts_required_matches(term_count, floor=floor), _memory_words_requirement(word_count, floor)) + + +def atomic_memory_related_requirement_sql( + term_count: int, dialect: Literal["sqlite", "mysql"], /, *, floor: AdmissionFloor | None = None +) -> tuple[str, dict[str, object], int]: + """Render ``atomic_memory_related_requirement`` over each row's indexed words. + + Returns the SQL expression, its parameters, and the lowest requirement any row can have. A shared-term count is an + integer, so comparing it with ``words * coverage`` equals comparing it with that product's ceiling. Both backends + multiply in double precision, as Python does, so SQL and Python admit the same rows. + """ + + floor = DEFAULT_ADMISSION_FLOOR if floor is None else floor + required = fts_required_matches(term_count, floor=floor) + parameters: dict[str, object] = {"fts_related_required": required} + lowest = min(required, floor.lexical_min_matched_terms) + if lowest == required: + return ":fts_related_required", parameters, lowest + length, least, greatest = ("length", "MIN", "MAX") if dialect == "sqlite" else ("CHAR_LENGTH", "LEAST", "GREATEST") + # Analyzed text joins encoded words with single spaces; encoded words never contain one. + words = ( + f"(CASE WHEN searchable_text = '' THEN 0 ELSE {length}(searchable_text) - " + f"{length}(REPLACE(searchable_text, ' ', '')) + 1 END)" + ) + parameters.update( + fts_related_min_matched=floor.lexical_min_matched_terms, fts_related_coverage=float(floor.lexical_coverage) + ) + expression = ( + f"{least}(:fts_related_required, {greatest}(:fts_related_min_matched, {words} * :fts_related_coverage))" + ) + return expression, parameters, lowest + + def atomic_memory_filter_sql( filters: AtomicMemoryIndexFilter, dialect: Literal["sqlite", "mysql"], / ) -> tuple[str, dict[str, object]]: @@ -901,6 +952,8 @@ def validate_request(request: AtomicMemorySearchRequest | AtomicMemoryRelatedReq "atomic_memory_embedding_input_hash", "atomic_memory_filter_sql", "atomic_memory_profile_fingerprint", + "atomic_memory_related_requirement", + "atomic_memory_related_requirement_sql", "combine_atomic_memory_channels", "load_atomic_memory_tags", "oceanbase_fts_coverage_sql", diff --git a/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py b/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py index 8c7b3ac38f..84ea490e07 100644 --- a/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/oceanbase/atomic_memory_index.py @@ -39,6 +39,7 @@ atomic_memory_channel_hits, atomic_memory_filter_sql, atomic_memory_profile_fingerprint, + atomic_memory_related_requirement_sql, atomic_memory_vector_sql, combine_atomic_memory_channels, oceanbase_fts_coverage_sql, @@ -367,24 +368,25 @@ async def enumerate_related_with_heads( # noqa: C901 -- Keep admission, vector vector = request.mode in {"vector", "hybrid"} max_distance = request.max_distance terms, required = fts_query_requirements(request.query, floor=request.admission) if lexical else ((), 0) - if required > len(terms): - terms = () + requirement, requirement_parameters, lowest = ( + atomic_memory_related_requirement_sql(len(terms), "mysql", floor=request.admission) + if terms + else ("1", {}, 0) + ) + if lowest > len(terms): + # No memory can contain enough distinct query terms to be lexically related. + terms, requirement = (), "1" if not terms and not vector: return (), {} native = bool(terms) and await self._supports_native_fts(connection, terms=terms) + coverage = "0" + score = "0.0" if terms: - parameters["fts_query"] = " ".join(terms) - parameters["fts_required"] = required - if native: - parameters["fts_options"] = f"operator=or;minimum_should_match={required}" - coverage, score = ":fts_required", "SCORE()" - else: + parameters.update(requirement_parameters, fts_query=" ".join(terms), fts_required=required) + if not native or lowest < required: coverage, coverage_parameters = oceanbase_fts_coverage_sql(terms) parameters.update(coverage_parameters) - score = "MATCH(searchable_text) AGAINST (:fts_query)" - else: - coverage = "0" - score = "0.0" + score = "SCORE()" if native else "MATCH(searchable_text) AGAINST (:fts_query)" profile = self.profile if vector: query_vector = self._require_vectors(request) @@ -411,34 +413,59 @@ async def enumerate_related_with_heads( # noqa: C901 -- Keep admission, vector # Native MATCH qualifies its own query block. Keep vector readiness # over every eligible row, including rows outside lexical admission. # Both channels and their authority heads remain one SQL snapshot. - query = ( - f"SELECT {_HIT_COLUMNS}, {score} AS score, {score} AS raw_score, " # noqa: S608 - "'oceanbase_match' AS score_metric, :fts_required AS coverage, " - "NULL AS distance, 0 AS invalid_vectors FROM pc_atomic_memory_current " - f"WHERE scope_id = :scope_id AND ({eligibility}) AND {_NATIVE_FTS_MATCH}" - ) + blocks = [] + if required <= len(terms): + parameters["fts_options"] = f"operator=or;minimum_should_match={required}" + blocks.append( + f"SELECT {_HIT_COLUMNS}, {score} AS score, {score} AS raw_score, " # noqa: S608 + "'oceanbase_match' AS score_metric, :fts_required AS coverage, " + ":fts_required AS fts_requirement, NULL AS distance, 0 AS invalid_vectors " + f"FROM pc_atomic_memory_current WHERE scope_id = :scope_id AND ({eligibility}) " + f"AND {_NATIVE_FTS_MATCH}" + ) + if lowest < required: + # A memory shorter than the candidate can be related below the candidate's own + # threshold. Only such rows pay for the whole-word count, after sharing the + # fewest terms any memory needs. The unbounded LIMIT keeps OceanBase from + # merging this native MATCH block. + parameters["fts_reverse_options"] = f"operator=or;minimum_should_match={lowest}" + reverse = ( + f"SELECT {_HIT_COLUMNS}, {score} AS score, {score} AS raw_score, " # noqa: S608 + f"'oceanbase_match' AS score_metric, {coverage} AS coverage, " + f"{requirement} AS fts_requirement, NULL AS distance, 0 AS invalid_vectors " + f"FROM pc_atomic_memory_current WHERE scope_id = :scope_id AND ({eligibility}) " + "AND MATCH(searchable_text^1, :fts_query, :fts_reverse_options) " + f"AND {requirement} < :fts_required LIMIT 18446744073709551615" + ) + blocks.append( + f"SELECT * FROM ({reverse}) AS reverse_hits " # noqa: S608 + "WHERE reverse_hits.coverage >= reverse_hits.fts_requirement " + "AND reverse_hits.coverage < :fts_required" + ) if vector: vectors = ( f"SELECT {_HIT_COLUMNS}, 0.0 AS score, 0.0 AS raw_score, " # noqa: S608 - "'oceanbase_match' AS score_metric, 0 AS coverage, " + "'oceanbase_match' AS score_metric, 0 AS coverage, 1 AS fts_requirement, " f"{distance} AS distance, {readiness} AS invalid_vectors " f"FROM pc_atomic_memory_current WHERE scope_id = :scope_id AND ({eligibility})" ) - query += ( - f" UNION ALL SELECT * FROM ({vectors}) AS vectors " # noqa: S608 + blocks.append( + f"SELECT * FROM ({vectors}) AS vectors " # noqa: S608 "WHERE vectors.distance <= :max_distance OR vectors.invalid_vectors > 0" ) + query = " UNION ALL ".join(f"({block})" if len(blocks) > 1 else block for block in blocks) else: query = ( f"SELECT {_HIT_COLUMNS}, {score} AS score, {score} AS raw_score, " # noqa: S608 "'oceanbase_match' AS score_metric, " - f"{coverage} AS coverage, {distance} AS distance, {readiness} AS invalid_vectors " + f"{coverage} AS coverage, {requirement} AS fts_requirement, " + f"{distance} AS distance, {readiness} AS invalid_vectors " "FROM pc_atomic_memory_current WHERE scope_id = :scope_id " f"AND ({eligibility})" ) admission = [] if terms: - admission.append("(scored.score > 0 AND scored.coverage >= :fts_required)") + admission.append("(scored.score > 0 AND scored.coverage >= scored.fts_requirement)") if vector: admission.append("scored.distance <= :max_distance") admission.append("scored.invalid_vectors > 0") @@ -472,7 +499,11 @@ async def enumerate_related_with_heads( # noqa: C901 -- Keep admission, vector "incomplete-vector", "Eligible Atomic Memory vectors are incomplete or use another profile" ) fts_rows = sorted( - (row for row in rows if terms and float(row["score"] or 0) > 0 and int(row["coverage"]) >= required), + ( + row + for row in rows + if terms and float(row["score"] or 0) > 0 and int(row["coverage"]) >= float(row["fts_requirement"]) + ), key=lambda row: (-float(row["score"]), str(row["artifact_id"])), ) vector_rows = sorted( diff --git a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py index c3d39537fb..22f54525f7 100644 --- a/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py +++ b/src/powercontext/builtin/persistence/sqlite/atomic_memory_index.py @@ -40,6 +40,7 @@ atomic_memory_channel_hits, atomic_memory_filter_sql, atomic_memory_profile_fingerprint, + atomic_memory_related_requirement_sql, atomic_memory_vector_sql, combine_atomic_memory_channels, sqlite_fts_coverage_sql, @@ -329,24 +330,7 @@ async def _related_single( match_query = fts_match_query(request.query) if match_query is None: return (), {} - terms, required = fts_query_requirements(request.query, floor=request.admission) - coverage, coverage_parameters = sqlite_fts_coverage_sql(terms) - parameters.update(coverage_parameters) - grouped = f"searchable_text:({match_query})" - parameters.update( - fts_query=grouped, - fts_scope_query=f'({grouped}) AND identity_token:"i{scope_id.encode("utf-8").hex()}x"*', - fts_required=required, - ) - rank = f"bm25({_FTS_TABLE}, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0)" - query_sql = ( - f"SELECT {_HIT_COLUMNS}, -{rank} AS score, {rank} AS raw_score, " # noqa: S608 - f"'sqlite_bm25' AS score_metric FROM {_FTS_TABLE} " - f"WHERE {_FTS_TABLE} MATCH (CASE WHEN EXISTS (SELECT 1 " - "FROM pc_atomic_memory_current WHERE scope_id = :scope_id LIMIT 1 OFFSET 64) " - "THEN :fts_query ELSE :fts_scope_query END) AND scope_id = :scope_id " - f"AND ({eligibility}) AND {coverage} >= :fts_required" - ) + query_sql = self._related_fts_sql(scope_id, request, match_query, eligibility, parameters) rows, heads = await self._related_rows(connection, query_sql, parameters) sorted_rows = sorted(rows, key=lambda row: (-row["score"], row["artifact_id"])) return atomic_memory_channel_hits(sorted_rows), heads @@ -371,6 +355,37 @@ async def _related_single( ) return atomic_memory_channel_hits(sorted_rows, vector=True), heads + @staticmethod + def _related_fts_sql( + scope_id: str, + request: AtomicMemoryRelatedRequest, + match_query: str, + eligibility: str, + parameters: dict[str, object], + ) -> str: + """Enumerate every memory sharing a term, then keep those meeting the related threshold.""" + terms, required = fts_query_requirements(request.query, floor=request.admission) + coverage, coverage_parameters = sqlite_fts_coverage_sql(terms) + requirement, requirement_parameters, _ = atomic_memory_related_requirement_sql( + len(terms), "sqlite", floor=request.admission + ) + parameters.update(coverage_parameters, **requirement_parameters) + grouped = f"searchable_text:({match_query})" + parameters.update( + fts_query=grouped, + fts_scope_query=f'({grouped}) AND identity_token:"i{scope_id.encode("utf-8").hex()}x"*', + fts_required=required, + ) + rank = f"bm25({_FTS_TABLE}, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0)" + return ( + f"SELECT {_HIT_COLUMNS}, -{rank} AS score, {rank} AS raw_score, " # noqa: S608 + f"'sqlite_bm25' AS score_metric FROM {_FTS_TABLE} " + f"WHERE {_FTS_TABLE} MATCH (CASE WHEN EXISTS (SELECT 1 " + "FROM pc_atomic_memory_current WHERE scope_id = :scope_id LIMIT 1 OFFSET 64) " + "THEN :fts_query ELSE :fts_scope_query END) AND scope_id = :scope_id " + f"AND ({eligibility}) AND {coverage} >= {requirement}" + ) + async def _related_rows( self, connection: AsyncConnection, query_sql: str, parameters: dict[str, object] ) -> tuple[tuple[RowMapping, ...], dict[str, dict[str, object]]]: @@ -478,15 +493,7 @@ async def _related_hybrid_channels( match_query = fts_match_query(request.query) if match_query is None: return None - terms, required = fts_query_requirements(request.query, floor=request.admission) - coverage, coverage_parameters = sqlite_fts_coverage_sql(terms) - parameters.update(coverage_parameters) - grouped = f"searchable_text:({match_query})" - parameters.update( - fts_query=grouped, - fts_scope_query=f'({grouped}) AND identity_token:"i{scope_id.encode("utf-8").hex()}x"*', - fts_required=required, - ) + fts_sql = self._related_fts_sql(scope_id, request, match_query, eligibility, parameters) query_vector = self._require_vectors(request) profile = self.profile if profile is None: @@ -501,15 +508,6 @@ async def _related_hybrid_channels( DEFAULT_ADMISSION_FLOOR if request.admission is None else request.admission ).min_semantic_similarity, ) - rank = f"bm25({_FTS_TABLE}, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0)" - fts_sql = ( - f"SELECT {_HIT_COLUMNS}, -{rank} AS score, {rank} AS raw_score, " # noqa: S608 - f"'sqlite_bm25' AS score_metric FROM {_FTS_TABLE} " - f"WHERE {_FTS_TABLE} MATCH (CASE WHEN EXISTS (SELECT 1 " - "FROM pc_atomic_memory_current WHERE scope_id = :scope_id LIMIT 1 OFFSET 64) " - "THEN :fts_query ELSE :fts_scope_query END) AND scope_id = :scope_id " - f"AND ({eligibility}) AND {coverage} >= :fts_required" - ) vector_sql = atomic_memory_vector_sql(eligibility, "sqlite", bounded=False, threshold=True) channel_sql = ( "SELECT 'fts' AS channel, f.*, NULL AS distance, 0 AS invalid_vectors " # noqa: S608 diff --git a/src/powercontext/builtin/runtime/atomic_memory_processing.py b/src/powercontext/builtin/runtime/atomic_memory_processing.py index 61b2355af6..c1758fb2a0 100644 --- a/src/powercontext/builtin/runtime/atomic_memory_processing.py +++ b/src/powercontext/builtin/runtime/atomic_memory_processing.py @@ -68,6 +68,7 @@ AtomicMemoryIndexFilter, AtomicMemoryRelatedRequest, atomic_memory_embedding_input, + atomic_memory_related_requirement, ) from powercontext.builtin.persistence.cursors import SourceCursorRepository from powercontext.builtin.persistence.decision_observations import DecisionObservationRepository @@ -1025,13 +1026,16 @@ def related_window_match( ) -> bool: """Apply the related-enumeration threshold to one unindexed item, as the backends do for indexed rows. - Lexical coverage counts whole analyzed terms of the projection's searchable text; the vector channel compares the - item's document embedding by exact L2 distance. Hybrid admits an item that meets either channel. + Lexical coverage counts whole analyzed terms of the projection's searchable text against the related threshold, + which also measures the item's own indexed words; the vector channel compares the item's document embedding by + exact L2 distance. Hybrid admits an item that meets either channel. """ if request.mode in {"fts", "hybrid"}: - terms, required = fts_query_requirements(request.query, floor=request.admission) - searchable = f" {analyze_text(atomic_memory_embedding_input(kind, body))} " + terms, _ = fts_query_requirements(request.query, floor=request.admission) + words = analyze_text(atomic_memory_embedding_input(kind, body)) + searchable = f" {words} " + required = atomic_memory_related_requirement(len(terms), len(words.split()), floor=request.admission) if terms and sum(f" {term} " in searchable for term in terms) >= required: return True if request.mode in {"vector", "hybrid"}: diff --git a/tests/e2e/test_atomic_memory_fts_coverage.py b/tests/e2e/test_atomic_memory_fts_coverage.py index de8b713248..f3f7a9d886 100644 --- a/tests/e2e/test_atomic_memory_fts_coverage.py +++ b/tests/e2e/test_atomic_memory_fts_coverage.py @@ -34,6 +34,7 @@ AtomicMemorySearchRequest, ) from powercontext.builtin.persistence.migrations.pinned import finish, pinned +from powercontext.builtin.runtime.atomic_memory_processing import related_window_match from powercontext.builtin.tags import TagFilter from tests.e2e.atomic_memory_migration_backend import MigrationBackend, migration_index, migration_profile from tests.e2e.test_atomic_memory_vector_index import Embedding, _initialize_related_authority @@ -277,3 +278,120 @@ async def scenario(): assert not await index.probe_recoverable(connection, "scope", strict, AdmissionFloor()) asyncio.run(scenario()) + + +@pytest.mark.parametrize("mode", ["fts", "hybrid"]) +def test_related_recall_finds_a_shorter_memory_of_the_same_fact(migration_backend: MigrationBackend, mode: str): + # Related recall also measures the stored memory's own words, so a long candidate reaches a short memory of the same + # fact while a memory sharing one incidental term stays out. + short = "The billing database is PostgreSQL 15." + long = ( + "After last week's maintenance window, the billing team upgraded the billing database cluster " + "from PostgreSQL 15 to PostgreSQL 16 to get logical replication improvements, faster vacuum, " + "and better monitoring of slow checkout queries during the holiday traffic peak." + ) + rows = ( + ("same-fact", short), + ("one-shared-term", "The holiday party moved to Friday."), + ("unrelated", "Deploys to production never run on Friday afternoons."), + ) + + async def scenario(): + # Every stored vector is far from the query vector, so hybrid relies on its lexical channel alone. + model = Embedding() if mode == "hybrid" else None + index = migration_index(migration_backend.config, None if model is None else model.profile) + preparer = AtomicMemoryProjectionPublisher(index, embedding_model=model) + async with migration_profile(migration_backend.config, load_vector_extension=model is not None) as profile: + async with profile.database.transaction() as connection: + await index.initialize(connection) + await _initialize_related_authority(connection) + for identity, body in rows: + prepared = await preparer.prepare(AtomicMemoryContent(kind="fact", text=body)) + await index.replace_many( + connection, + "scope", + ( + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id=identity, revision=1), + state_version=0, + prepared=prepared, + tag_keys=(), + ), + ), + ) + filters = AtomicMemoryIndexFilter() + + def related_request(query: str) -> AtomicMemoryRelatedRequest: + if model is None: + return AtomicMemoryRelatedRequest(query, filters) + return AtomicMemoryRelatedRequest( + query, + filters, + mode="hybrid", + query_vector=(0.0, 1.0, 0.0), + embedding_profile=model.profile, + max_distance=0.1, + ) + + async with profile.database.transaction() as connection: + related, _ = await index.enumerate_related_with_heads(connection, "scope", related_request(long)) + assert [hit.artifact_ref.artifact_id for hit in related] == ["same-fact"] + reverse, _ = await index.enumerate_related_with_heads(connection, "scope", related_request(short)) + assert [hit.artifact_ref.artifact_id for hit in reverse] == ["same-fact"] + # Search answers a query, so its admission still measures coverage of the query alone. + searched = await index.search(connection, "scope", AtomicMemorySearchRequest(long, filters)) + assert not searched.fts + + asyncio.run(scenario()) + + +def test_related_recall_admits_what_the_related_threshold_admits(migration_backend: MigrationBackend): + # Earlier window candidates are matched in Python; indexed memories must be admitted by the same threshold. + terms = tuple(f"t{number:02}" for number in range(12)) + query = " ".join(terms) + bodies = { + f"shared-{shared}-filler-{filler}": " ".join((*terms[:shared], *(f"f{number:02}" for number in range(filler)))) + for shared in range(1, 7) + for filler in (0, 1, 3, 7, 15, 30) + } + floors = ( + None, + AdmissionFloor(lexical_coverage=0.3, lexical_min_matched_terms=1), + AdmissionFloor(lexical_coverage=0.1, lexical_min_matched_terms=3), + AdmissionFloor(lexical_coverage=0.7), + AdmissionFloor(lexical_coverage=0.0), + ) + + async def scenario(): + index = migration_index(migration_backend.config) + preparer = AtomicMemoryProjectionPublisher(index) + async with migration_profile(migration_backend.config) as profile: + async with profile.database.transaction() as connection: + await index.initialize(connection) + await _initialize_related_authority(connection) + for identity, body in bodies.items(): + prepared = await preparer.prepare(AtomicMemoryContent(kind="fact", text=body)) + await index.replace_many( + connection, + "scope", + ( + AtomicMemoryProjection( + artifact_ref=ArtifactRef(family="atomic-memory", artifact_id=identity, revision=1), + state_version=0, + prepared=prepared, + tag_keys=(), + ), + ), + ) + async with profile.database.transaction() as connection: + for floor in floors: + request = AtomicMemoryRelatedRequest(query, AtomicMemoryIndexFilter(), admission=floor) + hits, _ = await index.enumerate_related_with_heads(connection, "scope", request) + expected = { + identity + for identity, body in bodies.items() + if related_window_match(request, "fact", body, None) + } + assert {hit.artifact_ref.artifact_id for hit in hits} == expected, floor + + asyncio.run(scenario()) diff --git a/tests/e2e/test_atomic_memory_long_fts_query.py b/tests/e2e/test_atomic_memory_long_fts_query.py index 43490657b4..525ccd3f25 100644 --- a/tests/e2e/test_atomic_memory_long_fts_query.py +++ b/tests/e2e/test_atomic_memory_long_fts_query.py @@ -34,6 +34,8 @@ ADMITTED = "Alpha " + " ".join(f"q{index:04}" for index in range(750, 1000)) # Repetition raises BM25 without increasing the distinct matched-term count. BELOW_THRESHOLD = "Alpha " + " ".join(f"q{index:04}" for index in range(249) for _ in range(3)) +# Related recall also measures the memory's own words, so unmatched words keep its own share below the threshold. +BELOW_RELATED_THRESHOLD = BELOW_THRESHOLD + " " + " ".join(f"other{index:04}" for index in range(300)) @pytest.fixture(autouse=True) @@ -126,7 +128,7 @@ async def scenario(): async with open_builtin_contexts(BuiltinConfig(database=migration_backend.config)) as contexts: await contexts.get("project") await contexts.records.create_atomic_memories( - "project", tuple({"kind": "fact", "text": body} for body in (ADMITTED, BELOW_THRESHOLD)) + "project", tuple({"kind": "fact", "text": body} for body in (ADMITTED, BELOW_RELATED_THRESHOLD)) ) async with contexts.database.transaction() as connection: hits, _ = await contexts.atomic_memory.index.enumerate_related_with_heads( diff --git a/tests/e2e/test_atomic_memory_unrelated_reconciliation.py b/tests/e2e/test_atomic_memory_unrelated_reconciliation.py index bff939db3e..0e9b5b890f 100644 --- a/tests/e2e/test_atomic_memory_unrelated_reconciliation.py +++ b/tests/e2e/test_atomic_memory_unrelated_reconciliation.py @@ -243,3 +243,80 @@ async def scenario(): assert [ (request["proposal"]["text"], [item["text"] for item in request["related"]]) for request in reconciliations ] == [(texts[2], [texts[0]])] + + +def test_builtin_reconciliation_compares_a_long_candidate_with_a_short_memory_of_the_same_fact(tmp_path, monkeypatch): + # Lexical related recall also measures the stored side, so a long candidate reaches a short published memory or an + # earlier sibling of the same fact. + short = "The billing database is PostgreSQL 15." + unrelated = "Deploys to production never run on Friday afternoons." + long = ( + "After last week's maintenance window, the billing team upgraded the billing database cluster " + "from PostgreSQL 15 to PostgreSQL 16 to get logical replication improvements, faster vacuum, " + "and better monitoring of slow checkout queries during the holiday traffic peak." + ) + windows = [(short, unrelated), (long,), ("Release notes are written in English.", short, long)] + reconciliations = [] + + def respond(messages, _info) -> ModelResponse: + prompt = next( + part.content + for message in reversed(messages) + for part in message.parts + if isinstance(part, UserPromptPart) and isinstance(part.content, str) + ) + if prompt == "Reply with one token.": + return ModelResponse(parts=[TextPart("ok")]) + request = json.loads(prompt) + if "proposal" in request: + reconciliations.append(request) + proposal = request["proposal"] + value = { + "action": "create", + "compared_ids": [item["item_id"] for item in request["related"]], + "content": {key: proposal[key] for key in ("kind", "text", "description")}, + "evidence_ids": proposal["evidence_ids"], + "reason": "Recorded for the test.", + } + else: + evidence_id = request["evidence"][0]["evidence_id"] + value = { + "candidates": [ + {"kind": "fact", "text": text, "description": text[:60], "evidence_ids": [evidence_id]} + for text in windows.pop(0) + ] + } + return ModelResponse(parts=[TextPart(json.dumps(value))]) + + monkeypatch.setattr("pydantic_ai.models.infer_model", lambda _name, **_kwargs: FunctionModel(respond)) + + async def flush(scope_id: str, runtime, number: int) -> None: + await runtime.sources.for_scope(scope_id).capture( + CaptureSource(source_id=f"note-{number}", content=f"Billing note {number}.", metadata={}) + ) + await runtime.atomic_memory.for_scope(scope_id).flush() + + async def scenario(): + async with open_builtin_runtime( + BuiltinConfig( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'builtin.db'}"), + inference=InferenceConfig(generation_model="test"), + ), + scheduler_path=tmp_path / "scheduler.db", + ) as runtime: + assert runtime.scopes is not None + published = ( + await runtime.scopes.create(ScopeDraft(title="Billing", summary="Billing", idempotency_key="published")) + ).scope_id + await flush(published, runtime, 1) + await flush(published, runtime, 2) + window = ( + await runtime.scopes.create(ScopeDraft(title="Window", summary="Window", idempotency_key="window")) + ).scope_id + await flush(window, runtime, 3) + + asyncio.run(scenario()) + + assert [ + (request["proposal"]["text"], [item["text"] for item in request["related"]]) for request in reconciliations + ] == [(long, [short]), (long, [short])]